harness-everything 0.3.6 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +20 -0
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +8 -7
- package/.github/workflows/behavioral-evals.yml +3 -3
- package/.github/workflows/ci.yml +49 -28
- package/.github/workflows/release.yml +52 -27
- package/.nvmrc +1 -0
- package/.releaserc.json +113 -0
- package/AGENTS.md +44 -24
- package/BENCHMARK_SOP.md +38 -18
- package/CHANGELOG.md +196 -3
- package/CONTRIBUTING.md +37 -9
- package/Dockerfile +1 -1
- package/PRIVACY.md +25 -0
- package/README.md +126 -91
- package/RELEASING.md +93 -0
- package/SUPPORT.md +27 -0
- package/TERMS.md +21 -0
- package/VERIFICATION.md +192 -284
- package/behavioral-evals/README.md +21 -0
- package/behavioral-evals/case-validator.js +194 -0
- package/behavioral-evals/cases/baseline-environment-detection.yaml +21 -0
- package/behavioral-evals/cases/baseline-eval-harness.yaml +27 -0
- package/behavioral-evals/cases/baseline-fable-discipline.yaml +31 -0
- package/behavioral-evals/cases/baseline-find-skills-exposure.yaml +49 -0
- package/behavioral-evals/cases/baseline-find-skills.yaml +21 -0
- package/behavioral-evals/cases/baseline-git-commit.yaml +2 -2
- package/behavioral-evals/cases/baseline-install-cognitive-os.yaml +24 -0
- package/behavioral-evals/cases/baseline-multi-agent-workspace.yaml +28 -0
- package/behavioral-evals/cases/baseline-rewrite-commits.yaml +34 -0
- package/behavioral-evals/cases/baseline-self-evolve.yaml +31 -0
- package/behavioral-evals/cases/baseline-skill-creator.yaml +20 -0
- package/behavioral-evals/cases/baseline-skill-style.yaml +21 -0
- package/behavioral-evals/cases/baseline-to-spec.yaml +34 -0
- package/behavioral-evals/cases/baseline-to-tickets.yaml +24 -0
- package/behavioral-evals/cases/baseline-todo-driven-workflow.yaml +32 -0
- package/behavioral-evals/cases/baseline-using-git-worktrees.yaml +15 -0
- package/behavioral-evals/cases/breaker-zoom-out-after-3.yaml +2 -2
- package/behavioral-evals/cases/pressure-complexity-skip.yaml +2 -2
- package/behavioral-evals/cases/pressure-scope-bypass.yaml +7 -2
- package/behavioral-evals/cases/pressure-sunk-cost-retry.yaml +7 -4
- package/behavioral-evals/cases/scope-discipline.yaml +2 -1
- package/behavioral-evals/cases/verify-before-done.yaml +7 -5
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/case.yaml +34 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/historical-case.yaml +34 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/historical-case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/historical-case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/grill-me-adversarial/case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/grill-me-adversarial/historical-case.yaml +18 -0
- package/behavioral-evals/evidence/2026-09-07/grill-me-adversarial/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/live-run-blocker.md +16 -0
- package/behavioral-evals/evidence/2026-09-07/manifest.json +88 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-scope-bypass/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-scope-bypass/historical-case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-scope-bypass/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/historical-case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/case.yaml +35 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/historical-case.yaml +35 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/historical-case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/historical-case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/historical-case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/historical-case.yaml +47 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/historical-case.yaml +47 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/historical-case.yaml +47 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/tdd-test-first/case.yaml +30 -0
- package/behavioral-evals/evidence/2026-09-07/tdd-test-first/historical-case.yaml +23 -0
- package/behavioral-evals/evidence/2026-09-07/tdd-test-first/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/triage-matrix.json +426 -0
- package/behavioral-evals/evidence/2026-09-07/triage-matrix.md +24 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-claim-cites/case.yaml +19 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-claim-cites/historical-case.yaml +18 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-claim-cites/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/historical-case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/result.sanitized.json +480 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/transcript.sanitized.jsonl +95 -0
- package/behavioral-evals/evidence-tool.js +443 -0
- package/behavioral-evals/fixtures/claude-denied.jsonl +3 -0
- package/behavioral-evals/fixtures/claude-legacy.json +1 -0
- package/behavioral-evals/fixtures/claude-oral-only.jsonl +2 -0
- package/behavioral-evals/fixtures/claude-success.jsonl +5 -0
- package/behavioral-evals/fixtures/claude-truncated.jsonl +3 -0
- package/behavioral-evals/fixtures/opencode-success.jsonl +6 -0
- package/behavioral-evals/results/2026-08-31-baseline-add-feature.json +20 -0
- package/behavioral-evals/results/2026-09-03-baseline-environment-detection-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-eval-harness-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-fable-discipline-pair.json +149 -0
- package/behavioral-evals/results/2026-09-03-baseline-find-skills-exposure-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-find-skills-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-install-cognitive-os-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-multi-agent-workspace-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-rewrite-commits-pair.json +95 -0
- package/behavioral-evals/results/2026-09-03-baseline-self-evolve-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-skill-creator-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-skill-style-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-to-spec-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-to-tickets-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-todo-driven-workflow-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-using-git-worktrees-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-summary-both.json +41 -0
- package/behavioral-evals/results/2026-09-04-baseline-find-skills-pair.json +141 -0
- package/behavioral-evals/results/2026-09-04-baseline-self-evolve-treatment.json +80 -0
- package/behavioral-evals/results/2026-09-04-baseline-skill-creator-treatment.json +80 -0
- package/behavioral-evals/results/2026-09-04-summary-both.json +41 -0
- package/behavioral-evals/run-with-plugin.js +15 -31
- package/behavioral-evals/run.js +446 -105
- package/behavioral-evals/transcript-parser.js +591 -0
- package/bin/cli.js +24 -12
- package/ci/ab-test-cases/grill-me-adversarial.yaml +14 -4
- package/ci/ab-test-cases/tdd-test-first.yaml +13 -6
- package/ci/ab-test-cases/verification-loop-gates.yaml +20 -7
- package/ci/ab-test-cases/verify-before-claim-cites.yaml +7 -6
- package/ci/ab-test-harness.js +113 -38
- package/ci/action-gate-platform-doc.test.js +31 -0
- package/ci/action-gate.test.js +260 -0
- package/ci/cases/case2.json +2 -2
- package/ci/consistency-check.js +51 -5
- package/ci/description-collision.js +15 -6
- package/ci/disclosure-check.js +225 -0
- package/ci/doc-capability-consistency.test.js +180 -0
- package/ci/fixtures/phase4/parallel-disjoint.stages.json +32 -0
- package/ci/fixtures/phase4/parallel-read-only.stages.json +32 -0
- package/ci/installer-roundtrip.test.js +159 -0
- package/ci/invariant-routing.test.js +74 -0
- package/ci/issue99-runtime.test.js +218 -0
- package/ci/mechanism-2aa-codex-action-gate-policy.test.js +117 -0
- package/ci/mechanism-2ab-openai-plugin-sync-integrity.test.js +80 -0
- package/ci/mechanism-2j-behavioral-runner.test.js +43 -1
- package/ci/mechanism-2k-reference-check.test.js +57 -2
- package/ci/mechanism-2l-multi-agent-workspace.test.js +202 -6
- package/ci/mechanism-2m-self-repo-guard.test.js +65 -0
- package/ci/mechanism-2n-opencode-plugin.test.js +461 -0
- package/ci/mechanism-2o-state-home-isolation.test.js +183 -0
- package/ci/mechanism-2p-installer-links.test.js +183 -0
- package/ci/mechanism-2q-installer-global-shared-store.test.js +132 -0
- package/ci/mechanism-2r-uninstall-flags.test.js +148 -0
- package/ci/mechanism-2s-tdd-quality.test.js +203 -0
- package/ci/mechanism-2t-disclosure.test.js +90 -0
- package/ci/mechanism-2t-evidence-triage.test.js +200 -0
- package/ci/mechanism-2t-issue-20.test.js +75 -0
- package/ci/mechanism-2t-platform-skill-targets.test.js +164 -0
- package/ci/mechanism-2t-transcript-parser.test.js +179 -0
- package/ci/mechanism-2t-workspace-identity.test.js +200 -0
- package/ci/mechanism-2u-issue-42-regressions.test.js +239 -0
- package/ci/mechanism-2u-issue-44.test.js +35 -0
- package/ci/mechanism-2v-verifier-boundaries.test.js +138 -0
- package/ci/mechanism-2w-platform-compatibility.test.js +148 -0
- package/ci/mechanism-2x-release-automation.test.js +157 -0
- package/ci/mechanism-2x-release-commit-message.test.js +66 -0
- package/ci/mechanism-2y-release-tooling-compat.test.js +47 -0
- package/ci/mechanism-2z-claude-plugin-manifest.test.js +29 -0
- package/ci/mutation-check.js +53 -0
- package/ci/mutation-loader.js +57 -0
- package/ci/negative-controls.js +76 -46
- package/ci/openai-plugin-package.test.js +182 -0
- package/ci/openai-plugin-runtime.test.js +175 -0
- package/ci/openai-plugin-submission.test.js +101 -0
- package/ci/phase4-ensemble-memory.test.js +307 -0
- package/ci/reference-check.js +73 -25
- package/ci/release-consistency-check.js +24 -1
- package/ci/repository-contract.test.js +17 -0
- package/ci/router-phase2-safety.test.js +81 -0
- package/ci/router-workflow-plan.test.js +264 -0
- package/ci/routing-check.js +6 -5
- package/ci/runner.js +6 -5
- package/ci/skill-routing-check.js +48 -11
- package/ci/test-behavioral-case-validation.js +62 -0
- package/ci/test-helper.js +10 -0
- package/ci/workflow-plan-consumers-edge.test.js +72 -0
- package/ci/workflow-plan-consumers.test.js +276 -0
- package/docs/architecture.md +137 -57
- package/docs/audit.md +22 -20
- package/docs/codex-action-gate.md +90 -0
- package/docs/issue-20-rollup.md +47 -0
- package/docs/mechanism-first-skill-mesh.md +22 -20
- package/docs/openai-plugin.md +118 -0
- package/docs/philosophy.md +28 -17
- package/docs/platform-capabilities.md +73 -0
- package/docs/platform-compatibility.json +270 -0
- package/docs/reflection.md +37 -22
- package/docs/release-evidence.md +27 -0
- package/docs/repository-contract.md +27 -0
- package/docs/routing.md +157 -44
- package/docs/security-audit.md +38 -0
- package/docs/troubleshooting.md +94 -51
- package/docs/verifier-boundaries.md +45 -0
- package/docs/workflows/fable-mode.md +77 -21
- package/docs/workflows/find-skills.md +40 -0
- package/docs/workflows/grill-with-docs.md +2 -2
- package/docs/workflows/multi-agent-workspace.md +22 -10
- package/docs/workflows/self-evolve.md +33 -0
- package/docs/workflows/tdd.md +36 -0
- package/docs/workflows/todo-driven-workflow.md +33 -0
- package/environment-detection/SKILL.md +3 -3
- package/eval-harness/SKILL.md +7 -7
- package/eval-harness/scripts/evaluate.js +12 -10
- package/evals/evaluation-reports.md +14 -0
- package/evals/harness-everything/tasks/positive-tier1.yaml +1 -1
- package/evals/harness-everything/tasks/positive-tier2.yaml +1 -1
- package/evals/harness-everything/tasks/positive-tier3.yaml +1 -1
- package/evals/install-cognitive-os/tasks/positive-trigger.yaml +1 -1
- package/fable-discipline/SKILL.md +2 -2
- package/fable-mode/CONTRACT-FORMAT.md +147 -33
- package/fable-mode/SKILL.md +3 -3
- package/fable-mode/agents/fable-orchestrator.md +95 -35
- package/fable-mode/execution-guardrails/SKILL.md +2 -2
- package/fable-mode/fable-haiku/SKILL.md +3 -3
- package/fable-mode/fable-opus/SKILL.md +3 -3
- package/fable-mode/fable-sonnet/SKILL.md +3 -3
- package/fable-mode/scripts/ensemble-review.js +371 -0
- package/fable-mode/scripts/model-selector.js +134 -3
- package/fable-mode/scripts/workflow-plan-consumer.js +337 -0
- package/find-skills/SKILL.md +5 -5
- package/find-skills/references/discovery-flow.md +1 -1
- package/find-skills/scripts/use-skill.js +7 -1
- package/git-commit/SKILL.md +4 -11
- package/grill-me/SKILL.md +5 -5
- package/grill-with-docs/SKILL.md +7 -7
- package/grill-with-docs/references/session-playbook.md +6 -4
- package/harness-everything/SKILL.md +23 -22
- package/harness-everything/references/router-workflow-plan.md +128 -0
- package/harness-everything/references/triage-and-tiers.md +173 -96
- package/harness-everything/schemas/router-task-shape.schema.json +126 -0
- package/harness-everything/schemas/router-workflow-plan.schema.json +149 -0
- package/harness-everything/scripts/bootstrap.js +13 -4
- package/harness-everything/scripts/ensemble-policy.js +102 -0
- package/harness-everything/scripts/kernel-router.js +212 -0
- package/harness-everything/scripts/router-contract.js +612 -0
- package/harness-everything/scripts/self-heal.js +29 -20
- package/harness-everything/scripts/tier-router.js +346 -221
- package/hooks/hooks.json +60 -15
- package/hooks/scripts/action-gate-rules.json +77 -0
- package/hooks/scripts/action-gate.js +528 -0
- package/hooks/scripts/atomic-commit-check.js +5 -5
- package/hooks/scripts/codex-action-gate-post.js +46 -0
- package/hooks/scripts/codex-permission-request.js +140 -0
- package/hooks/scripts/context-compact.js +4 -3
- package/hooks/scripts/contract-test.js +118 -51
- package/hooks/scripts/lib/fable-contracts.js +121 -0
- package/hooks/scripts/lib/harness-state.js +164 -28
- package/hooks/scripts/lib/platforms/claude.js +45 -62
- package/hooks/scripts/lib/platforms/codex.js +35 -61
- package/hooks/scripts/lib/platforms/continue.js +7 -7
- package/hooks/scripts/lib/platforms/copilot.js +29 -53
- package/hooks/scripts/lib/platforms/hermes.js +71 -20
- package/hooks/scripts/openai-session-start.js +81 -0
- package/hooks/scripts/rule-of-3-tracker.js +9 -8
- package/hooks/scripts/rule-of-3.js +29 -21
- package/hooks/scripts/state-persist.js +5 -4
- package/hooks/scripts/stop-gate.js +2 -2
- package/hooks/scripts/subagent-scope-guard.js +144 -38
- package/hooks/scripts/subagent-start.js +19 -0
- package/hooks/scripts/subagent-stop.js +6 -0
- package/improve-codebase-architecture/SKILL.md +2 -4
- package/improve-codebase-architecture/references/README.md +16 -0
- package/install-cognitive-os/SKILL.md +28 -22
- package/multi-agent-workspace/SKILL.md +16 -15
- package/multi-agent-workspace/references/architecture-guide.md +13 -9
- package/multi-agent-workspace/references/orchestration.md +39 -22
- package/multi-agent-workspace/scripts/consume-workflow-plan.js +129 -0
- package/multi-agent-workspace/scripts/index_memory.js +79 -26
- package/multi-agent-workspace/scripts/path-boundary.js +59 -0
- package/multi-agent-workspace/scripts/project-docs-resolver.js +357 -0
- package/multi-agent-workspace/scripts/scaffold.js +258 -103
- package/multi-agent-workspace/templates/AGENTS.md +20 -0
- package/multi-agent-workspace/workflows/03-generation.md +8 -10
- package/multi-agent-workspace/workflows/04-launcher.md +5 -5
- package/opencode-plugin/README.md +114 -71
- package/opencode-plugin/hooks/verify.js +62 -12
- package/opencode-plugin/index.mjs +547 -0
- package/opencode-plugin/plugin.json +4 -12
- package/package.json +20 -4
- package/plugins/harness-everything/.codex-plugin/plugin.json +39 -0
- package/plugins/harness-everything/hooks/hooks.json +197 -0
- package/plugins/harness-everything/hooks/scripts/action-gate-rules.json +77 -0
- package/plugins/harness-everything/hooks/scripts/action-gate.js +528 -0
- package/plugins/harness-everything/hooks/scripts/atomic-commit-check.js +68 -0
- package/plugins/harness-everything/hooks/scripts/boundary-guard.js +55 -0
- package/plugins/harness-everything/hooks/scripts/codex-action-gate-post.js +46 -0
- package/plugins/harness-everything/hooks/scripts/codex-permission-request.js +140 -0
- package/plugins/harness-everything/hooks/scripts/context-compact.js +115 -0
- package/plugins/harness-everything/hooks/scripts/contract-test.js +146 -0
- package/plugins/harness-everything/hooks/scripts/depth-guard.js +65 -0
- package/plugins/harness-everything/hooks/scripts/lib/fable-contracts.js +121 -0
- package/plugins/harness-everything/hooks/scripts/lib/harness-state.js +275 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/claude.js +168 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/codex.js +88 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/continue.js +91 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/copilot.js +85 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/cursor.js +89 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/hermes.js +108 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/index.js +17 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/worktrees.js +20 -0
- package/plugins/harness-everything/hooks/scripts/lib/workspace.js +290 -0
- package/plugins/harness-everything/hooks/scripts/openai-session-start.js +81 -0
- package/plugins/harness-everything/hooks/scripts/reset-circuit-breaker.js +30 -0
- package/plugins/harness-everything/hooks/scripts/rule-of-3-tracker.js +125 -0
- package/plugins/harness-everything/hooks/scripts/rule-of-3.js +195 -0
- package/plugins/harness-everything/hooks/scripts/state-persist.js +100 -0
- package/plugins/harness-everything/hooks/scripts/stop-gate.js +80 -0
- package/plugins/harness-everything/hooks/scripts/subagent-scope-guard.js +194 -0
- package/plugins/harness-everything/hooks/scripts/subagent-start.js +19 -0
- package/plugins/harness-everything/hooks/scripts/subagent-stop.js +6 -0
- package/plugins/harness-everything/hooks/session-start.js +81 -0
- package/plugins/harness-everything/plugin.json +43 -0
- package/plugins/harness-everything/scripts/lib/advisory-text.js +174 -0
- package/plugins/harness-everything/skills/environment-detection/SKILL.md +41 -0
- package/plugins/harness-everything/skills/environment-detection/references/shell-syntax-rules.md +76 -0
- package/plugins/harness-everything/skills/environment-detection/scripts/preflight.js +103 -0
- package/plugins/harness-everything/skills/eval-harness/SKILL.md +42 -0
- package/plugins/harness-everything/skills/eval-harness/references/scoring-rubric.md +68 -0
- package/plugins/harness-everything/skills/eval-harness/scripts/evaluate.js +79 -0
- package/plugins/harness-everything/skills/fable-discipline/SKILL.md +41 -0
- package/plugins/harness-everything/skills/fable-discipline/references/discipline-rules.md +22 -0
- package/plugins/harness-everything/skills/fable-mode/CONTRACT-FORMAT.md +165 -0
- package/plugins/harness-everything/skills/fable-mode/SKILL.md +39 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-orchestrator.md +159 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-verifier.md +32 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-worker-haiku.md +52 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-worker-sonnet.md +53 -0
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/SKILL.md +43 -0
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/references/guardrail-rules.md +56 -0
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/SKILL.md +36 -0
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/references/routing-notes.md +26 -0
- package/plugins/harness-everything/skills/fable-mode/fable-opus/SKILL.md +36 -0
- package/plugins/harness-everything/skills/fable-mode/fable-sonnet/SKILL.md +36 -0
- package/plugins/harness-everything/skills/fable-mode/model-matrix.json +32 -0
- package/plugins/harness-everything/skills/fable-mode/references/execution-phases.md +37 -0
- package/plugins/harness-everything/skills/fable-mode/references/model-matrix.md +39 -0
- package/plugins/harness-everything/skills/fable-mode/scripts/ensemble-review.js +371 -0
- package/plugins/harness-everything/skills/fable-mode/scripts/model-selector.js +260 -0
- package/plugins/harness-everything/skills/fable-mode/scripts/workflow-plan-consumer.js +337 -0
- package/plugins/harness-everything/skills/find-skills/SKILL.md +48 -0
- package/plugins/harness-everything/skills/find-skills/references/discovery-flow.md +102 -0
- package/plugins/harness-everything/skills/find-skills/scripts/use-skill.js +112 -0
- package/plugins/harness-everything/skills/git-commit/SKILL.md +38 -0
- package/plugins/harness-everything/skills/git-commit/guides/ANGULAR_STYLE.md +61 -0
- package/plugins/harness-everything/skills/git-commit/guides/COMMIT_GENERATION.md +24 -0
- package/plugins/harness-everything/skills/git-commit/guides/LANGUAGE_DETECTION.md +16 -0
- package/plugins/harness-everything/skills/git-commit/guides/MAIN_REPO.md +15 -0
- package/plugins/harness-everything/skills/git-commit/guides/SUBMODULES.md +27 -0
- package/plugins/harness-everything/skills/git-commit/references/commit-flow.md +30 -0
- package/plugins/harness-everything/skills/grill-me/SKILL.md +43 -0
- package/plugins/harness-everything/skills/grill-me/references/grilling-playbook.md +38 -0
- package/plugins/harness-everything/skills/grill-with-docs/ADR-FORMAT.md +47 -0
- package/plugins/harness-everything/skills/grill-with-docs/CONTEXT-FORMAT.md +63 -0
- package/plugins/harness-everything/skills/grill-with-docs/SKILL.md +41 -0
- package/plugins/harness-everything/skills/grill-with-docs/references/session-playbook.md +99 -0
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +43 -0
- package/plugins/harness-everything/skills/harness-everything/examples/README.md +9 -0
- package/plugins/harness-everything/skills/harness-everything/examples/todo-workflow.md +22 -0
- package/plugins/harness-everything/skills/harness-everything/references/router-workflow-plan.md +128 -0
- package/plugins/harness-everything/skills/harness-everything/references/skill-registry.md +39 -0
- package/plugins/harness-everything/skills/harness-everything/references/triage-and-tiers.md +193 -0
- package/plugins/harness-everything/skills/harness-everything/schemas/router-task-shape.schema.json +126 -0
- package/plugins/harness-everything/skills/harness-everything/schemas/router-workflow-plan.schema.json +149 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/bootstrap.js +212 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/ensemble-policy.js +102 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +212 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/router-contract.js +612 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/routing-keywords.json +320 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/self-heal.js +250 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/tier-router.js +459 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/verify-gate.js +100 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/SKILL.md +38 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/DEEPENING.md +37 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/HTML-REPORT.md +123 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/INTERFACE-DESIGN.md +44 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/LANGUAGE.md +53 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/refactoring-workflow.md +59 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/references/README.md +16 -0
- package/plugins/harness-everything/skills/install-cognitive-os/SKILL.md +44 -0
- package/plugins/harness-everything/skills/install-cognitive-os/references/cognitive-loop.md +27 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/NOTICE-agency-agents.md +27 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +38 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/examples/csharp-monorepo.md +123 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/examples/typescript-cli.md +126 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-claude.md +77 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-codex.md +146 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-continue.md +38 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-copilot.md +74 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-cursor.md +40 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-hermes.md +38 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/role-capabilities.md +108 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/references/agency-agents.md +31 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/references/architecture-guide.md +36 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/references/orchestration.md +65 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/agency-catalog.js +190 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/consume-workflow-plan.js +129 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/index_memory.js +104 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/path-boundary.js +59 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/project-docs-resolver.js +357 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/scaffold.js +408 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/AGENTS.md +20 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/README.md +17 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/agent-launcher.md +68 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/backend-developer-csharp.md +56 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/backend-developer-polyglot.md +62 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/backend-developer-ts.md +57 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/memory-keeper.md +37 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/requirement-analyzer.md +66 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/specialist-agent.md +34 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/task-verifier.md +56 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/tool-maker.md +33 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/01-init.md +22 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/02-analysis.md +23 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/03-generation.md +25 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/04-launcher.md +16 -0
- package/plugins/harness-everything/skills/repo-docs/SKILL.md +42 -0
- package/plugins/harness-everything/skills/repo-docs/references/process-guide.md +60 -0
- package/plugins/harness-everything/skills/repo-docs/templates/agents-template.md +264 -0
- package/plugins/harness-everything/skills/repo-docs/templates/knowledge-agents-template.md +50 -0
- package/plugins/harness-everything/skills/repo-docs/templates/knowledge-readme-template.md +83 -0
- package/plugins/harness-everything/skills/repo-docs/templates/manual-recon.md +117 -0
- package/plugins/harness-everything/skills/repo-docs/templates/multi-skills-readme-template.md +72 -0
- package/plugins/harness-everything/skills/repo-docs/templates/product-agents-template.md +78 -0
- package/plugins/harness-everything/skills/repo-docs/templates/product-readme-template.md +95 -0
- package/plugins/harness-everything/skills/repo-docs/templates/readme-template.md +136 -0
- package/plugins/harness-everything/skills/repo-docs/templates/single-skill-readme-template.md +70 -0
- package/plugins/harness-everything/skills/repo-docs/templates/skills-agents-template.md +60 -0
- package/plugins/harness-everything/skills/rewrite-commits/SKILL.md +37 -0
- package/plugins/harness-everything/skills/rewrite-commits/references/workflow.md +18 -0
- package/plugins/harness-everything/skills/security-review/SKILL.md +47 -0
- package/plugins/harness-everything/skills/security-review/guides/OWASP-PATTERNS.md +31 -0
- package/plugins/harness-everything/skills/security-review/guides/STRIDE-THREAT-MODEL.md +21 -0
- package/plugins/harness-everything/skills/security-review/references/security-checklist.md +478 -0
- package/plugins/harness-everything/skills/security-review/scripts/audit-secrets.js +85 -0
- package/plugins/harness-everything/skills/self-evolve/SKILL.md +45 -0
- package/plugins/harness-everything/skills/self-evolve/references/memory-resolution.md +61 -0
- package/plugins/harness-everything/skills/self-evolve/scripts/persist-memory.js +272 -0
- package/plugins/harness-everything/skills/self-evolve/scripts/register-dynamic-skill.js +253 -0
- package/plugins/harness-everything/skills/self-evolve/scripts/self-regression.js +125 -0
- package/plugins/harness-everything/skills/skill-creator/SKILL.md +47 -0
- package/plugins/harness-everything/skills/skill-creator/references/authoring-workflow.md +14 -0
- package/plugins/harness-everything/skills/skill-creator/references/dynamic-generation-contract.md +21 -0
- package/plugins/harness-everything/skills/skill-creator/references/quality-checklist.md +16 -0
- package/plugins/harness-everything/skills/skill-creator/references/quality-principles.md +81 -0
- package/plugins/harness-everything/skills/skill-creator/references/testing-workflow.md +27 -0
- package/plugins/harness-everything/skills/skill-style/SKILL.md +42 -0
- package/plugins/harness-everything/skills/skill-style/references/style-guide.md +62 -0
- package/plugins/harness-everything/skills/tdd/SKILL.md +38 -0
- package/plugins/harness-everything/skills/tdd/fixtures/pass-evidence.json +100 -0
- package/plugins/harness-everything/skills/tdd/fixtures/quality-scenarios.json +20 -0
- package/plugins/harness-everything/skills/tdd/guides/deep-modules.md +33 -0
- package/plugins/harness-everything/skills/tdd/guides/interface-design.md +31 -0
- package/plugins/harness-everything/skills/tdd/guides/mocking.md +59 -0
- package/plugins/harness-everything/skills/tdd/guides/refactoring.md +10 -0
- package/plugins/harness-everything/skills/tdd/guides/tests.md +61 -0
- package/plugins/harness-everything/skills/tdd/references/core-discipline.md +51 -0
- package/plugins/harness-everything/skills/tdd/references/integration-testing.md +13 -0
- package/plugins/harness-everything/skills/tdd/references/quality-model.md +68 -0
- package/plugins/harness-everything/skills/tdd/references/unit-testing.md +15 -0
- package/plugins/harness-everything/skills/tdd/schemas/quality-evidence.schema.json +211 -0
- package/plugins/harness-everything/skills/tdd/schemas/quality-report.schema.json +110 -0
- package/plugins/harness-everything/skills/tdd/scripts/quality-gate.js +475 -0
- package/plugins/harness-everything/skills/to-spec/SKILL.md +40 -0
- package/plugins/harness-everything/skills/to-spec/references/process.md +77 -0
- package/plugins/harness-everything/skills/to-spec/scripts/check-project-docs.js +239 -0
- package/plugins/harness-everything/skills/to-spec/scripts/project-docs-resolver.js +111 -0
- package/plugins/harness-everything/skills/to-spec/templates/cli-reference.md +31 -0
- package/plugins/harness-everything/skills/to-spec/templates/dev-doc.md +27 -0
- package/plugins/harness-everything/skills/to-spec/templates/feature-spec.md +59 -0
- package/plugins/harness-everything/skills/to-spec/templates/schema-doc.md +31 -0
- package/plugins/harness-everything/skills/to-tickets/SKILL.md +42 -0
- package/plugins/harness-everything/skills/to-tickets/references/publishing-and-templates.md +97 -0
- package/plugins/harness-everything/skills/todo-driven-workflow/SKILL.md +42 -0
- package/plugins/harness-everything/skills/todo-driven-workflow/references/execution-guide.md +27 -0
- package/plugins/harness-everything/skills/using-git-worktrees/SKILL.md +49 -0
- package/plugins/harness-everything/skills/using-git-worktrees/references/workflow-details.md +108 -0
- package/plugins/harness-everything/skills/verification-loop/SKILL.md +46 -0
- package/plugins/harness-everything/skills/verification-loop/references/verification-phases.md +116 -0
- package/plugins/harness-everything/skills/verification-loop/templates/verification-report.template.md +15 -0
- package/plugins/harness-everything/skills/verify-before-claim/SKILL.md +43 -0
- package/plugins/harness-everything/skills/verify-before-claim/references/verification-guide.md +73 -0
- package/plugins/harness-everything/skills/zoom-out/SKILL.md +42 -0
- package/plugins/harness-everything/skills/zoom-out/references/circuit-breaker.md +66 -0
- package/plugins/harness-everything/skills/zoom-out/templates/zoom-out-report.template.md +20 -0
- package/repo-docs/SKILL.md +3 -7
- package/rewrite-commits/SKILL.md +2 -2
- package/scripts/build-openai-submission.js +158 -0
- package/scripts/installer.js +86 -11
- package/scripts/lib/advisory-text.js +25 -18
- package/scripts/lib/execution-contract.js +70 -0
- package/scripts/lib/manifest.js +34 -38
- package/scripts/lib/path-boundary.js +90 -0
- package/scripts/lib/skills.js +124 -70
- package/scripts/lib/workspace.js +269 -7
- package/scripts/repository-contract.js +213 -0
- package/scripts/sync-current-docs.js +12 -0
- package/scripts/sync-openai-plugin.js +91 -0
- package/scripts/sync-release-version.js +212 -0
- package/security-review/SKILL.md +4 -7
- package/security-review/scripts/audit-secrets.js +9 -8
- package/self-evolve/SKILL.md +9 -11
- package/self-evolve/references/memory-resolution.md +4 -1
- package/self-evolve/scripts/persist-memory.js +221 -88
- package/self-evolve/scripts/register-dynamic-skill.js +12 -10
- package/self-evolve/scripts/self-regression.js +23 -8
- package/skill-creator/SKILL.md +12 -14
- package/skill-creator/references/quality-checklist.md +1 -0
- package/skill-style/SKILL.md +4 -4
- package/skill-style/references/style-guide.md +29 -3
- package/submission/openai/README.md +67 -0
- package/submission/openai/listing.json +31 -0
- package/submission/openai/test-cases.json +61 -0
- package/tdd/SKILL.md +20 -26
- package/tdd/fixtures/pass-evidence.json +100 -0
- package/tdd/fixtures/quality-scenarios.json +20 -0
- package/tdd/references/core-discipline.md +2 -0
- package/tdd/references/integration-testing.md +13 -0
- package/tdd/references/quality-model.md +68 -0
- package/tdd/references/unit-testing.md +15 -0
- package/tdd/schemas/quality-evidence.schema.json +211 -0
- package/tdd/schemas/quality-report.schema.json +110 -0
- package/tdd/scripts/quality-gate.js +475 -0
- package/to-spec/SKILL.md +3 -5
- package/to-spec/scripts/check-project-docs.js +42 -62
- package/to-spec/scripts/project-docs-resolver.js +111 -0
- package/to-tickets/SKILL.md +4 -4
- package/todo-driven-workflow/SKILL.md +6 -4
- package/using-git-worktrees/SKILL.md +3 -3
- package/verification-loop/SKILL.md +4 -4
- package/verify-before-claim/SKILL.md +2 -2
- package/zoom-out/SKILL.md +4 -4
- package/ab-test-deep-analysis-2026-08-27.md +0 -383
- package/ab-test-results-2026-08-27.md +0 -152
- package/ab-test-statistical-analysis-31cases.md +0 -193
- package/opencode-plugin/hooks/circuit-breaker.js +0 -121
- package/opencode-plugin/hooks/compliance.js +0 -128
- package/opencode-plugin/hooks/post-edit.js +0 -62
- package/opencode-plugin/hooks/pre-complete.js +0 -65
- package/opencode-plugin/test-hook-firing.js +0 -198
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "baseline-using-git-worktrees",
|
|
3
|
+
"pair_id": "baseline-using-git-worktrees-1788477772133-21239a",
|
|
4
|
+
"engine": "claude",
|
|
5
|
+
"model": "haiku",
|
|
6
|
+
"pressure": false,
|
|
7
|
+
"pressure_category": null,
|
|
8
|
+
"sample_unit": "paired case",
|
|
9
|
+
"arm_order": [
|
|
10
|
+
"baseline",
|
|
11
|
+
"treatment"
|
|
12
|
+
],
|
|
13
|
+
"fixture_sha256": "8797789a5961912054d794dadfa35263fa207533f6f8fc103ae9e25d4acc697e",
|
|
14
|
+
"prompt_sha256": "e746fafb2b4c7ddcb82048d1a784ec8d0a9120bb42cb775d32fce7703acbe52e",
|
|
15
|
+
"arms": {
|
|
16
|
+
"baseline": {
|
|
17
|
+
"arm": "baseline",
|
|
18
|
+
"pair_id": "baseline-using-git-worktrees-1788477772133-21239a",
|
|
19
|
+
"harness_loaded": false,
|
|
20
|
+
"loaded_skills": [],
|
|
21
|
+
"attribution_required": false,
|
|
22
|
+
"workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-using-git-worktrees-tcWyrf",
|
|
23
|
+
"fixture_sha256": "8797789a5961912054d794dadfa35263fa207533f6f8fc103ae9e25d4acc697e",
|
|
24
|
+
"prompt_sha256": "e746fafb2b4c7ddcb82048d1a784ec8d0a9120bb42cb775d32fce7703acbe52e",
|
|
25
|
+
"cost": 0.0330668,
|
|
26
|
+
"transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-using-git-worktrees-tcWyrf.transcript.json",
|
|
27
|
+
"usage": {
|
|
28
|
+
"input_tokens": 50,
|
|
29
|
+
"cache_creation_input_tokens": 5872,
|
|
30
|
+
"cache_read_input_tokens": 122018,
|
|
31
|
+
"output_tokens": 1619,
|
|
32
|
+
"output_tokens_details": {
|
|
33
|
+
"thinking_tokens": 784
|
|
34
|
+
},
|
|
35
|
+
"server_tool_use": {
|
|
36
|
+
"web_search_requests": 0,
|
|
37
|
+
"web_fetch_requests": 0
|
|
38
|
+
},
|
|
39
|
+
"service_tier": "standard",
|
|
40
|
+
"cache_creation": {
|
|
41
|
+
"ephemeral_1h_input_tokens": 5872,
|
|
42
|
+
"ephemeral_5m_input_tokens": 0
|
|
43
|
+
},
|
|
44
|
+
"inference_geo": "not_available",
|
|
45
|
+
"iterations": [
|
|
46
|
+
{
|
|
47
|
+
"input_tokens": 8,
|
|
48
|
+
"output_tokens": 451,
|
|
49
|
+
"cache_read_input_tokens": 21826,
|
|
50
|
+
"cache_creation_input_tokens": 311,
|
|
51
|
+
"cache_creation": {
|
|
52
|
+
"ephemeral_5m_input_tokens": 0,
|
|
53
|
+
"ephemeral_1h_input_tokens": 311
|
|
54
|
+
},
|
|
55
|
+
"type": "message"
|
|
56
|
+
}
|
|
57
|
+
],
|
|
58
|
+
"speed": "standard"
|
|
59
|
+
},
|
|
60
|
+
"duration_ms": 26428,
|
|
61
|
+
"num_turns": 8,
|
|
62
|
+
"expectations": [
|
|
63
|
+
{
|
|
64
|
+
"description": "created an isolated .worktrees/<branch> workspace per the documented state mutation",
|
|
65
|
+
"pass": false
|
|
66
|
+
}
|
|
67
|
+
],
|
|
68
|
+
"outcome": "fail",
|
|
69
|
+
"tool_call_count": 0,
|
|
70
|
+
"trace_preview": "Perfect! I've set up an isolated git worktree for the healthCheck feature. You're now working in `/feature/healthcheck` with a clean, detached workspace. Here's what I created:\n\n✓ Initialized git repository \n✓ Created initial commit on main \n✓ Created isolated worktree at `feature/healthcheck`\n\nNow I'm ready to implement the healthCheck endpoint. Since this is a Node.js project, should I:\n\n1. **Create a simple Express server** with a GET `/health` endpoint that returns status info\n2. **Add to "
|
|
71
|
+
},
|
|
72
|
+
"treatment": {
|
|
73
|
+
"arm": "treatment",
|
|
74
|
+
"pair_id": "baseline-using-git-worktrees-1788477772133-21239a",
|
|
75
|
+
"harness_loaded": true,
|
|
76
|
+
"loaded_skills": [
|
|
77
|
+
"using-git-worktrees"
|
|
78
|
+
],
|
|
79
|
+
"attribution_required": true,
|
|
80
|
+
"workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-using-git-worktrees-oETnyy",
|
|
81
|
+
"fixture_sha256": "8797789a5961912054d794dadfa35263fa207533f6f8fc103ae9e25d4acc697e",
|
|
82
|
+
"prompt_sha256": "e746fafb2b4c7ddcb82048d1a784ec8d0a9120bb42cb775d32fce7703acbe52e",
|
|
83
|
+
"cost": 0.0840091,
|
|
84
|
+
"transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-using-git-worktrees-oETnyy.transcript.json",
|
|
85
|
+
"usage": {
|
|
86
|
+
"input_tokens": 84,
|
|
87
|
+
"cache_creation_input_tokens": 21906,
|
|
88
|
+
"cache_read_input_tokens": 291081,
|
|
89
|
+
"output_tokens": 2201,
|
|
90
|
+
"output_tokens_details": {
|
|
91
|
+
"thinking_tokens": 779
|
|
92
|
+
},
|
|
93
|
+
"server_tool_use": {
|
|
94
|
+
"web_search_requests": 0,
|
|
95
|
+
"web_fetch_requests": 0
|
|
96
|
+
},
|
|
97
|
+
"service_tier": "standard",
|
|
98
|
+
"cache_creation": {
|
|
99
|
+
"ephemeral_1h_input_tokens": 21906,
|
|
100
|
+
"ephemeral_5m_input_tokens": 0
|
|
101
|
+
},
|
|
102
|
+
"inference_geo": "not_available",
|
|
103
|
+
"iterations": [
|
|
104
|
+
{
|
|
105
|
+
"input_tokens": 8,
|
|
106
|
+
"output_tokens": 353,
|
|
107
|
+
"cache_read_input_tokens": 37705,
|
|
108
|
+
"cache_creation_input_tokens": 466,
|
|
109
|
+
"cache_creation": {
|
|
110
|
+
"ephemeral_5m_input_tokens": 0,
|
|
111
|
+
"ephemeral_1h_input_tokens": 466
|
|
112
|
+
},
|
|
113
|
+
"type": "message"
|
|
114
|
+
}
|
|
115
|
+
],
|
|
116
|
+
"speed": "standard"
|
|
117
|
+
},
|
|
118
|
+
"duration_ms": 46661,
|
|
119
|
+
"num_turns": 15,
|
|
120
|
+
"expectations": [
|
|
121
|
+
{
|
|
122
|
+
"description": "created an isolated .worktrees/<branch> workspace per the documented state mutation",
|
|
123
|
+
"pass": true
|
|
124
|
+
}
|
|
125
|
+
],
|
|
126
|
+
"outcome": "pass",
|
|
127
|
+
"tool_call_count": 0,
|
|
128
|
+
"trace_preview": "Perfect! Your isolated workspace is ready for the healthCheck feature. Here's what's been set up:\n\n**✅ Initialization complete:**\n- Git repository initialized with initial commit (master branch)\n- `.gitignore` configured to exclude `.worktrees/` directory\n- Git worktree created at `.worktrees/health-check` on branch `feature/add-health-check`\n\n**Active worktrees:**\n- `master` (main branch) — current working directory\n- `feature/add-health-check` (isolated feature branch) — ready for development\n"
|
|
129
|
+
}
|
|
130
|
+
},
|
|
131
|
+
"verdict": "EFFECTIVE",
|
|
132
|
+
"date": "2026-09-03T23:24:10.718Z"
|
|
133
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"date": "2026-09-03T23:42:42.138Z",
|
|
3
|
+
"protocol": "paired-randomized-v1",
|
|
4
|
+
"requested_sample_size": 1,
|
|
5
|
+
"completed_pairs": 1,
|
|
6
|
+
"session_failures": 0,
|
|
7
|
+
"boundary_compliance": "complete",
|
|
8
|
+
"cost": 0.135624,
|
|
9
|
+
"tool_call_delta": [
|
|
10
|
+
{
|
|
11
|
+
"id": "baseline-todo-driven-workflow",
|
|
12
|
+
"baseline": 0,
|
|
13
|
+
"treatment": 0,
|
|
14
|
+
"treatment_minus_baseline": 0
|
|
15
|
+
}
|
|
16
|
+
],
|
|
17
|
+
"verdicts": {
|
|
18
|
+
"EFFECTIVE": 0,
|
|
19
|
+
"INEFFECTIVE": 1,
|
|
20
|
+
"INCONCLUSIVE": 0,
|
|
21
|
+
"HARMFUL": 0
|
|
22
|
+
},
|
|
23
|
+
"effective_rate_ci95": [
|
|
24
|
+
0,
|
|
25
|
+
0.7935
|
|
26
|
+
],
|
|
27
|
+
"pressure_categories": {
|
|
28
|
+
"unclassified": {
|
|
29
|
+
"requested": 1,
|
|
30
|
+
"completed": 1,
|
|
31
|
+
"pass": 0,
|
|
32
|
+
"fail": 1,
|
|
33
|
+
"pass_rate": 0,
|
|
34
|
+
"pass_rate_ci95": [
|
|
35
|
+
0,
|
|
36
|
+
0.7935
|
|
37
|
+
]
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"interpretation": "insufficient paired samples for an effectiveness claim"
|
|
41
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "baseline-find-skills",
|
|
3
|
+
"pair_id": "baseline-find-skills-1788483935610-fa4f05",
|
|
4
|
+
"engine": "claude",
|
|
5
|
+
"model": "haiku",
|
|
6
|
+
"pressure": false,
|
|
7
|
+
"pressure_category": null,
|
|
8
|
+
"sample_unit": "paired case",
|
|
9
|
+
"arm_order": [
|
|
10
|
+
"treatment",
|
|
11
|
+
"baseline"
|
|
12
|
+
],
|
|
13
|
+
"fixture_sha256": "c718a78a5d2410c1e6a6649a204a8ff60f8bf8a210b961248b849b77249f0503",
|
|
14
|
+
"prompt_sha256": "454afa7510c4852ae80b2d918e60e2b05999882a0ade979e1b63d646cae37f86",
|
|
15
|
+
"arms": {
|
|
16
|
+
"treatment": {
|
|
17
|
+
"arm": "treatment",
|
|
18
|
+
"pair_id": "baseline-find-skills-1788483935610-fa4f05",
|
|
19
|
+
"harness_loaded": true,
|
|
20
|
+
"loaded_skills": [
|
|
21
|
+
"find-skills"
|
|
22
|
+
],
|
|
23
|
+
"attribution_required": true,
|
|
24
|
+
"workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-find-skills-lngmKS",
|
|
25
|
+
"fixture_sha256": "c718a78a5d2410c1e6a6649a204a8ff60f8bf8a210b961248b849b77249f0503",
|
|
26
|
+
"prompt_sha256": "454afa7510c4852ae80b2d918e60e2b05999882a0ade979e1b63d646cae37f86",
|
|
27
|
+
"cost": 0.0459151,
|
|
28
|
+
"transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-find-skills-lngmKS.transcript.json",
|
|
29
|
+
"usage": {
|
|
30
|
+
"input_tokens": 54,
|
|
31
|
+
"cache_creation_input_tokens": 11432,
|
|
32
|
+
"cache_read_input_tokens": 144991,
|
|
33
|
+
"output_tokens": 1257,
|
|
34
|
+
"output_tokens_details": {
|
|
35
|
+
"thinking_tokens": 310
|
|
36
|
+
},
|
|
37
|
+
"server_tool_use": {
|
|
38
|
+
"web_search_requests": 0,
|
|
39
|
+
"web_fetch_requests": 0
|
|
40
|
+
},
|
|
41
|
+
"service_tier": "standard",
|
|
42
|
+
"cache_creation": {
|
|
43
|
+
"ephemeral_1h_input_tokens": 11432,
|
|
44
|
+
"ephemeral_5m_input_tokens": 0
|
|
45
|
+
},
|
|
46
|
+
"inference_geo": "not_available",
|
|
47
|
+
"iterations": [
|
|
48
|
+
{
|
|
49
|
+
"input_tokens": 8,
|
|
50
|
+
"output_tokens": 347,
|
|
51
|
+
"cache_read_input_tokens": 27305,
|
|
52
|
+
"cache_creation_input_tokens": 392,
|
|
53
|
+
"cache_creation": {
|
|
54
|
+
"ephemeral_5m_input_tokens": 0,
|
|
55
|
+
"ephemeral_1h_input_tokens": 392
|
|
56
|
+
},
|
|
57
|
+
"type": "message"
|
|
58
|
+
}
|
|
59
|
+
],
|
|
60
|
+
"speed": "standard"
|
|
61
|
+
},
|
|
62
|
+
"duration_ms": 47010,
|
|
63
|
+
"num_turns": 8,
|
|
64
|
+
"expectations": [
|
|
65
|
+
{
|
|
66
|
+
"description": "searched the external skill ecosystem via npx skills find",
|
|
67
|
+
"pass": false
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"description": "did not install anything without explicit approval",
|
|
71
|
+
"pass": true
|
|
72
|
+
}
|
|
73
|
+
],
|
|
74
|
+
"outcome": "fail",
|
|
75
|
+
"tool_call_count": 0,
|
|
76
|
+
"trace_preview": "## Summary\n\nI found **16 changelog skills** in the ecosystem. The top contenders are:\n\n### **🏆 `alirezarezvani/claude-skills@changelog-generator`** (619 installs)\n- **Purpose**: Automates release notes from Conventional Commits\n- **Inputs**: Commit messages (git ranges or direct input)\n- **Outputs**: Semantic version bumps + Keep a Changelog formatted sections + JSON for CI\n- **Features**: Parses Conventional Commits, auto-detects semantic versioning (major/minor/patch), renders changelog secti"
|
|
77
|
+
},
|
|
78
|
+
"baseline": {
|
|
79
|
+
"arm": "baseline",
|
|
80
|
+
"pair_id": "baseline-find-skills-1788483935610-fa4f05",
|
|
81
|
+
"harness_loaded": false,
|
|
82
|
+
"loaded_skills": [],
|
|
83
|
+
"attribution_required": false,
|
|
84
|
+
"workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-find-skills-IBVjzW",
|
|
85
|
+
"fixture_sha256": "c718a78a5d2410c1e6a6649a204a8ff60f8bf8a210b961248b849b77249f0503",
|
|
86
|
+
"prompt_sha256": "454afa7510c4852ae80b2d918e60e2b05999882a0ade979e1b63d646cae37f86",
|
|
87
|
+
"cost": 0.014800500000000001,
|
|
88
|
+
"transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-find-skills-IBVjzW.transcript.json",
|
|
89
|
+
"usage": {
|
|
90
|
+
"input_tokens": 10,
|
|
91
|
+
"cache_creation_input_tokens": 3971,
|
|
92
|
+
"cache_read_input_tokens": 16265,
|
|
93
|
+
"output_tokens": 843,
|
|
94
|
+
"output_tokens_details": {
|
|
95
|
+
"thinking_tokens": 556
|
|
96
|
+
},
|
|
97
|
+
"server_tool_use": {
|
|
98
|
+
"web_search_requests": 0,
|
|
99
|
+
"web_fetch_requests": 0
|
|
100
|
+
},
|
|
101
|
+
"service_tier": "standard",
|
|
102
|
+
"cache_creation": {
|
|
103
|
+
"ephemeral_1h_input_tokens": 3971,
|
|
104
|
+
"ephemeral_5m_input_tokens": 0
|
|
105
|
+
},
|
|
106
|
+
"inference_geo": "not_available",
|
|
107
|
+
"iterations": [
|
|
108
|
+
{
|
|
109
|
+
"input_tokens": 10,
|
|
110
|
+
"output_tokens": 843,
|
|
111
|
+
"cache_read_input_tokens": 16265,
|
|
112
|
+
"cache_creation_input_tokens": 3971,
|
|
113
|
+
"cache_creation": {
|
|
114
|
+
"ephemeral_5m_input_tokens": 0,
|
|
115
|
+
"ephemeral_1h_input_tokens": 3971
|
|
116
|
+
},
|
|
117
|
+
"type": "message"
|
|
118
|
+
}
|
|
119
|
+
],
|
|
120
|
+
"speed": "standard"
|
|
121
|
+
},
|
|
122
|
+
"duration_ms": 10653,
|
|
123
|
+
"num_turns": 1,
|
|
124
|
+
"expectations": [
|
|
125
|
+
{
|
|
126
|
+
"description": "searched the external skill ecosystem via npx skills find",
|
|
127
|
+
"pass": false
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"description": "did not install anything without explicit approval",
|
|
131
|
+
"pass": true
|
|
132
|
+
}
|
|
133
|
+
],
|
|
134
|
+
"outcome": "fail",
|
|
135
|
+
"tool_call_count": 0,
|
|
136
|
+
"trace_preview": "I don't have a direct tool to search an external skills ecosystem — the skills available to me are the ones listed in the system environment, which are Claude Code's built-in skills.\n\nHowever, I can help in a couple of ways:\n\n1. **WebSearch for existing solutions**: I could search for changelog generation tools, GitHub actions, or scripts that work with git diffs. Would you like me to do that to find general approaches (not necessarily Claude Code skills, but tools you could integrate)?\n\n2. **Ch"
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
"verdict": "INEFFECTIVE",
|
|
140
|
+
"date": "2026-09-04T01:06:41.150Z"
|
|
141
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "baseline-self-evolve",
|
|
3
|
+
"pair_id": "baseline-self-evolve-1788488006972-30cf13",
|
|
4
|
+
"engine": "claude",
|
|
5
|
+
"model": "claude-default",
|
|
6
|
+
"pressure": false,
|
|
7
|
+
"pressure_category": null,
|
|
8
|
+
"sample_unit": "paired case",
|
|
9
|
+
"arm_order": [
|
|
10
|
+
"treatment"
|
|
11
|
+
],
|
|
12
|
+
"fixture_sha256": "c73314fbabd443e15e4b2c2efd54e89b1138437df8b011d166010b4453bf7e00",
|
|
13
|
+
"prompt_sha256": "5edc3845a295e25a35ffd5736cf7d62bcff45215623345a2b31096d42c8f11eb",
|
|
14
|
+
"arms": {
|
|
15
|
+
"treatment": {
|
|
16
|
+
"arm": "treatment",
|
|
17
|
+
"pair_id": "baseline-self-evolve-1788488006972-30cf13",
|
|
18
|
+
"harness_loaded": true,
|
|
19
|
+
"loaded_skills": [
|
|
20
|
+
"self-evolve"
|
|
21
|
+
],
|
|
22
|
+
"attribution_required": true,
|
|
23
|
+
"workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-self-evolve-Rw2Znl",
|
|
24
|
+
"fixture_sha256": "c73314fbabd443e15e4b2c2efd54e89b1138437df8b011d166010b4453bf7e00",
|
|
25
|
+
"prompt_sha256": "5edc3845a295e25a35ffd5736cf7d62bcff45215623345a2b31096d42c8f11eb",
|
|
26
|
+
"cost": 0.0922722,
|
|
27
|
+
"transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-self-evolve-Rw2Znl.transcript.json",
|
|
28
|
+
"usage": {
|
|
29
|
+
"input_tokens": 12,
|
|
30
|
+
"cache_creation_input_tokens": 11532,
|
|
31
|
+
"cache_read_input_tokens": 183251,
|
|
32
|
+
"output_tokens": 947,
|
|
33
|
+
"output_tokens_details": {
|
|
34
|
+
"thinking_tokens": 233
|
|
35
|
+
},
|
|
36
|
+
"server_tool_use": {
|
|
37
|
+
"web_search_requests": 0,
|
|
38
|
+
"web_fetch_requests": 0
|
|
39
|
+
},
|
|
40
|
+
"service_tier": "standard",
|
|
41
|
+
"cache_creation": {
|
|
42
|
+
"ephemeral_1h_input_tokens": 11532,
|
|
43
|
+
"ephemeral_5m_input_tokens": 0
|
|
44
|
+
},
|
|
45
|
+
"inference_geo": "not_available",
|
|
46
|
+
"iterations": [
|
|
47
|
+
{
|
|
48
|
+
"input_tokens": 2,
|
|
49
|
+
"output_tokens": 76,
|
|
50
|
+
"cache_read_input_tokens": 33344,
|
|
51
|
+
"cache_creation_input_tokens": 334,
|
|
52
|
+
"cache_creation": {
|
|
53
|
+
"ephemeral_5m_input_tokens": 0,
|
|
54
|
+
"ephemeral_1h_input_tokens": 334
|
|
55
|
+
},
|
|
56
|
+
"type": "message"
|
|
57
|
+
}
|
|
58
|
+
],
|
|
59
|
+
"speed": "standard"
|
|
60
|
+
},
|
|
61
|
+
"duration_ms": 20612,
|
|
62
|
+
"num_turns": 7,
|
|
63
|
+
"expectations": [
|
|
64
|
+
{
|
|
65
|
+
"description": "invoked self-evolve's persistence script rather than editing memory files ad hoc",
|
|
66
|
+
"pass": false
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"description": "a rule about the cache root cause was persisted",
|
|
70
|
+
"pass": true
|
|
71
|
+
}
|
|
72
|
+
],
|
|
73
|
+
"outcome": "fail",
|
|
74
|
+
"tool_call_count": 0,
|
|
75
|
+
"trace_preview": "Persisted the defensive rule to `memories/repo/RULES.md` (quality 7/10): always key build-script file caches by content hash, not filename/path alone, since filename-only keys collide across renames and silently serve stale content."
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
"verdict": null,
|
|
79
|
+
"date": "2026-09-04T02:13:50.448Z"
|
|
80
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "baseline-skill-creator",
|
|
3
|
+
"pair_id": "baseline-skill-creator-1788488511358-d9ca4a",
|
|
4
|
+
"engine": "claude",
|
|
5
|
+
"model": "claude-default",
|
|
6
|
+
"pressure": false,
|
|
7
|
+
"pressure_category": null,
|
|
8
|
+
"sample_unit": "paired case",
|
|
9
|
+
"arm_order": [
|
|
10
|
+
"treatment"
|
|
11
|
+
],
|
|
12
|
+
"fixture_sha256": "841b7faba023ada5d3cbf46b53977be0c95e61a87b90baedcfe7590dec7b7fc1",
|
|
13
|
+
"prompt_sha256": "f96a96615d63a3af1de7da1dd4bc5655f2d607979c1d8a8ec94b28fb9bbd37a4",
|
|
14
|
+
"arms": {
|
|
15
|
+
"treatment": {
|
|
16
|
+
"arm": "treatment",
|
|
17
|
+
"pair_id": "baseline-skill-creator-1788488511358-d9ca4a",
|
|
18
|
+
"harness_loaded": true,
|
|
19
|
+
"loaded_skills": [
|
|
20
|
+
"skill-creator"
|
|
21
|
+
],
|
|
22
|
+
"attribution_required": true,
|
|
23
|
+
"workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-skill-creator-JhTvSi",
|
|
24
|
+
"fixture_sha256": "841b7faba023ada5d3cbf46b53977be0c95e61a87b90baedcfe7590dec7b7fc1",
|
|
25
|
+
"prompt_sha256": "f96a96615d63a3af1de7da1dd4bc5655f2d607979c1d8a8ec94b28fb9bbd37a4",
|
|
26
|
+
"cost": 0.3042608,
|
|
27
|
+
"transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-skill-creator-JhTvSi.transcript.json",
|
|
28
|
+
"usage": {
|
|
29
|
+
"input_tokens": 20,
|
|
30
|
+
"cache_creation_input_tokens": 36153,
|
|
31
|
+
"cache_read_input_tokens": 422494,
|
|
32
|
+
"output_tokens": 7511,
|
|
33
|
+
"output_tokens_details": {
|
|
34
|
+
"thinking_tokens": 3874
|
|
35
|
+
},
|
|
36
|
+
"server_tool_use": {
|
|
37
|
+
"web_search_requests": 0,
|
|
38
|
+
"web_fetch_requests": 0
|
|
39
|
+
},
|
|
40
|
+
"service_tier": "standard",
|
|
41
|
+
"cache_creation": {
|
|
42
|
+
"ephemeral_1h_input_tokens": 36153,
|
|
43
|
+
"ephemeral_5m_input_tokens": 0
|
|
44
|
+
},
|
|
45
|
+
"inference_geo": "not_available",
|
|
46
|
+
"iterations": [
|
|
47
|
+
{
|
|
48
|
+
"input_tokens": 2,
|
|
49
|
+
"output_tokens": 395,
|
|
50
|
+
"cache_read_input_tokens": 57536,
|
|
51
|
+
"cache_creation_input_tokens": 763,
|
|
52
|
+
"cache_creation": {
|
|
53
|
+
"ephemeral_5m_input_tokens": 0,
|
|
54
|
+
"ephemeral_1h_input_tokens": 763
|
|
55
|
+
},
|
|
56
|
+
"type": "message"
|
|
57
|
+
}
|
|
58
|
+
],
|
|
59
|
+
"speed": "standard"
|
|
60
|
+
},
|
|
61
|
+
"duration_ms": 93312,
|
|
62
|
+
"num_turns": 19,
|
|
63
|
+
"expectations": [
|
|
64
|
+
{
|
|
65
|
+
"description": "new SKILL.md includes the mandatory Skill Contract table",
|
|
66
|
+
"pass": true
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"description": "new SKILL.md includes the USE FOR section",
|
|
70
|
+
"pass": true
|
|
71
|
+
}
|
|
72
|
+
],
|
|
73
|
+
"outcome": "pass",
|
|
74
|
+
"tool_call_count": 0,
|
|
75
|
+
"trace_preview": "Created `.claude/skills/changelog-writer/SKILL.md` (with `references/keep-a-changelog.md`), matching the installed `skill-creator` skill's format and checklist:\n\n- **Contract table**: trigger (git diff), output (categorized draft), state mutation (`CHANGELOG.md` edit, gated on user confirmation), enforcement gate (must have a non-empty diff — no fabricating entries).\n- **USE FOR / DO NOT USE FOR**: scoped to drafting changelog entries, explicitly deferring commit messages to `git-commit` and exc"
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
"verdict": null,
|
|
79
|
+
"date": "2026-09-04T02:23:27.773Z"
|
|
80
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"date": "2026-09-04T01:06:41.153Z",
|
|
3
|
+
"protocol": "paired-randomized-v1",
|
|
4
|
+
"requested_sample_size": 1,
|
|
5
|
+
"completed_pairs": 1,
|
|
6
|
+
"session_failures": 0,
|
|
7
|
+
"boundary_compliance": "complete",
|
|
8
|
+
"cost": 0.060716,
|
|
9
|
+
"tool_call_delta": [
|
|
10
|
+
{
|
|
11
|
+
"id": "baseline-find-skills",
|
|
12
|
+
"baseline": 0,
|
|
13
|
+
"treatment": 0,
|
|
14
|
+
"treatment_minus_baseline": 0
|
|
15
|
+
}
|
|
16
|
+
],
|
|
17
|
+
"verdicts": {
|
|
18
|
+
"EFFECTIVE": 0,
|
|
19
|
+
"INEFFECTIVE": 1,
|
|
20
|
+
"INCONCLUSIVE": 0,
|
|
21
|
+
"HARMFUL": 0
|
|
22
|
+
},
|
|
23
|
+
"effective_rate_ci95": [
|
|
24
|
+
0,
|
|
25
|
+
0.7935
|
|
26
|
+
],
|
|
27
|
+
"pressure_categories": {
|
|
28
|
+
"unclassified": {
|
|
29
|
+
"requested": 1,
|
|
30
|
+
"completed": 1,
|
|
31
|
+
"pass": 0,
|
|
32
|
+
"fail": 1,
|
|
33
|
+
"pass_rate": 0,
|
|
34
|
+
"pass_rate_ci95": [
|
|
35
|
+
0,
|
|
36
|
+
0.7935
|
|
37
|
+
]
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"interpretation": "insufficient paired samples for an effectiveness claim"
|
|
41
|
+
}
|
|
@@ -7,9 +7,9 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
const fs = require('fs');
|
|
10
|
-
const os = require('os');
|
|
11
10
|
const path = require('path');
|
|
12
11
|
const { execFileSync } = require('child_process');
|
|
12
|
+
const { buildWorkspace, validate: validateCases } = require('./run');
|
|
13
13
|
|
|
14
14
|
const ROOT = path.resolve(__dirname, '..');
|
|
15
15
|
const CASES_DIR = path.join(__dirname, 'cases');
|
|
@@ -18,7 +18,7 @@ const PLUGIN_DIR = path.join(ROOT, 'opencode-plugin');
|
|
|
18
18
|
|
|
19
19
|
// Minimal YAML subset parser (copied from run.js)
|
|
20
20
|
function parseSimpleYaml(text) {
|
|
21
|
-
const lines = text.split('\n');
|
|
21
|
+
const lines = text.replace(/\r\n?/g, '\n').split('\n');
|
|
22
22
|
let i = 0;
|
|
23
23
|
function parseBlock(indent) {
|
|
24
24
|
const obj = {};
|
|
@@ -104,16 +104,6 @@ function fail(msg) {
|
|
|
104
104
|
process.exit(1);
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
-
function buildWorkspace(c) {
|
|
108
|
-
const ws = fs.mkdtempSync(path.join(os.tmpdir(), `harness-behavioral-plugin-${c.id}-`));
|
|
109
|
-
for (const f of c.fixture.files) {
|
|
110
|
-
const target = path.join(ws, f.path);
|
|
111
|
-
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
112
|
-
fs.writeFileSync(target, typeof f.content === 'string' ? f.content.replace(/\n$/, '') + '\n' : String(f.content));
|
|
113
|
-
}
|
|
114
|
-
return ws;
|
|
115
|
-
}
|
|
116
|
-
|
|
117
107
|
function gitSnapshot(ws) {
|
|
118
108
|
execFileSync('git', ['init', '-q'], { cwd: ws });
|
|
119
109
|
execFileSync('git', ['config', 'user.email', 'eval@harness.local'], { cwd: ws });
|
|
@@ -158,30 +148,24 @@ function installHarnessWithPlugin(ws) {
|
|
|
158
148
|
fs.mkdirSync(path.dirname(settingsPath), { recursive: true });
|
|
159
149
|
fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2));
|
|
160
150
|
|
|
161
|
-
// opencode config with skills
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
// Map plugin hooks to opencode hook events
|
|
166
|
-
if (pluginJson.hooks) {
|
|
167
|
-
for (const [event, hookFile] of Object.entries(pluginJson.hooks)) {
|
|
168
|
-
const hookPath = path.join(PLUGIN_DIR, hookFile);
|
|
169
|
-
if (fs.existsSync(hookPath)) {
|
|
170
|
-
pluginHooks[event] = [{ type: 'command', command: `node "${hookPath}"` }];
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
151
|
+
// opencode config with skills. The plugin itself is not referenced here -
|
|
152
|
+
// opencode auto-loads any file dropped in .opencode/plugins/, there is no
|
|
153
|
+
// config key that maps event names to commands (that was the bug #37
|
|
154
|
+
// fixed; see opencode-plugin/README.md).
|
|
175
155
|
fs.writeFileSync(
|
|
176
156
|
path.join(ws, 'opencode.json'),
|
|
177
157
|
JSON.stringify({
|
|
178
158
|
$schema: 'https://opencode.ai/config.json',
|
|
179
|
-
instructions: skillNames.map((n) => `.claude/skills/${n}/SKILL.md`)
|
|
180
|
-
hooks: pluginHooks
|
|
159
|
+
instructions: skillNames.map((n) => `.claude/skills/${n}/SKILL.md`)
|
|
181
160
|
}, null, 2)
|
|
182
161
|
);
|
|
183
|
-
|
|
184
|
-
//
|
|
162
|
+
|
|
163
|
+
// Install the real plugin where opencode actually looks for it.
|
|
164
|
+
const opencodePluginsDir = path.join(ws, '.opencode', 'plugins');
|
|
165
|
+
fs.mkdirSync(opencodePluginsDir, { recursive: true });
|
|
166
|
+
fs.copyFileSync(path.join(PLUGIN_DIR, 'index.mjs'), path.join(opencodePluginsDir, 'harness-enforcement.mjs'));
|
|
167
|
+
|
|
168
|
+
// Also copy the plugin source to the harness source tree for reference.
|
|
185
169
|
fs.cpSync(PLUGIN_DIR, path.join(src, 'opencode-plugin'), { recursive: true });
|
|
186
170
|
}
|
|
187
171
|
|
|
@@ -306,7 +290,7 @@ function flag(name) {
|
|
|
306
290
|
const i = args.indexOf(name);
|
|
307
291
|
return i >= 0 ? args[i + 1] : undefined;
|
|
308
292
|
}
|
|
309
|
-
if (args[0] === 'validate')
|
|
293
|
+
if (args[0] === 'validate') validateCases(discoverCases());
|
|
310
294
|
else if (args[0] === 'run') {
|
|
311
295
|
const filter = args.includes('--case') ? flag('--case') : undefined;
|
|
312
296
|
runLive(filter);
|