harness-everything 0.3.6 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +20 -0
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +8 -7
- package/.github/workflows/behavioral-evals.yml +3 -3
- package/.github/workflows/ci.yml +49 -28
- package/.github/workflows/release.yml +52 -27
- package/.nvmrc +1 -0
- package/.releaserc.json +113 -0
- package/AGENTS.md +44 -24
- package/BENCHMARK_SOP.md +38 -18
- package/CHANGELOG.md +196 -3
- package/CONTRIBUTING.md +37 -9
- package/Dockerfile +1 -1
- package/PRIVACY.md +25 -0
- package/README.md +126 -91
- package/RELEASING.md +93 -0
- package/SUPPORT.md +27 -0
- package/TERMS.md +21 -0
- package/VERIFICATION.md +192 -284
- package/behavioral-evals/README.md +21 -0
- package/behavioral-evals/case-validator.js +194 -0
- package/behavioral-evals/cases/baseline-environment-detection.yaml +21 -0
- package/behavioral-evals/cases/baseline-eval-harness.yaml +27 -0
- package/behavioral-evals/cases/baseline-fable-discipline.yaml +31 -0
- package/behavioral-evals/cases/baseline-find-skills-exposure.yaml +49 -0
- package/behavioral-evals/cases/baseline-find-skills.yaml +21 -0
- package/behavioral-evals/cases/baseline-git-commit.yaml +2 -2
- package/behavioral-evals/cases/baseline-install-cognitive-os.yaml +24 -0
- package/behavioral-evals/cases/baseline-multi-agent-workspace.yaml +28 -0
- package/behavioral-evals/cases/baseline-rewrite-commits.yaml +34 -0
- package/behavioral-evals/cases/baseline-self-evolve.yaml +31 -0
- package/behavioral-evals/cases/baseline-skill-creator.yaml +20 -0
- package/behavioral-evals/cases/baseline-skill-style.yaml +21 -0
- package/behavioral-evals/cases/baseline-to-spec.yaml +34 -0
- package/behavioral-evals/cases/baseline-to-tickets.yaml +24 -0
- package/behavioral-evals/cases/baseline-todo-driven-workflow.yaml +32 -0
- package/behavioral-evals/cases/baseline-using-git-worktrees.yaml +15 -0
- package/behavioral-evals/cases/breaker-zoom-out-after-3.yaml +2 -2
- package/behavioral-evals/cases/pressure-complexity-skip.yaml +2 -2
- package/behavioral-evals/cases/pressure-scope-bypass.yaml +7 -2
- package/behavioral-evals/cases/pressure-sunk-cost-retry.yaml +7 -4
- package/behavioral-evals/cases/scope-discipline.yaml +2 -1
- package/behavioral-evals/cases/verify-before-done.yaml +7 -5
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/case.yaml +34 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/historical-case.yaml +34 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/historical-case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/historical-case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/grill-me-adversarial/case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/grill-me-adversarial/historical-case.yaml +18 -0
- package/behavioral-evals/evidence/2026-09-07/grill-me-adversarial/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/live-run-blocker.md +16 -0
- package/behavioral-evals/evidence/2026-09-07/manifest.json +88 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-scope-bypass/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-scope-bypass/historical-case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-scope-bypass/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/historical-case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/case.yaml +35 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/historical-case.yaml +35 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/historical-case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/historical-case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/historical-case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/historical-case.yaml +47 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/historical-case.yaml +47 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/historical-case.yaml +47 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/tdd-test-first/case.yaml +30 -0
- package/behavioral-evals/evidence/2026-09-07/tdd-test-first/historical-case.yaml +23 -0
- package/behavioral-evals/evidence/2026-09-07/tdd-test-first/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/triage-matrix.json +426 -0
- package/behavioral-evals/evidence/2026-09-07/triage-matrix.md +24 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-claim-cites/case.yaml +19 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-claim-cites/historical-case.yaml +18 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-claim-cites/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/historical-case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/result.sanitized.json +480 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/transcript.sanitized.jsonl +95 -0
- package/behavioral-evals/evidence-tool.js +443 -0
- package/behavioral-evals/fixtures/claude-denied.jsonl +3 -0
- package/behavioral-evals/fixtures/claude-legacy.json +1 -0
- package/behavioral-evals/fixtures/claude-oral-only.jsonl +2 -0
- package/behavioral-evals/fixtures/claude-success.jsonl +5 -0
- package/behavioral-evals/fixtures/claude-truncated.jsonl +3 -0
- package/behavioral-evals/fixtures/opencode-success.jsonl +6 -0
- package/behavioral-evals/results/2026-08-31-baseline-add-feature.json +20 -0
- package/behavioral-evals/results/2026-09-03-baseline-environment-detection-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-eval-harness-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-fable-discipline-pair.json +149 -0
- package/behavioral-evals/results/2026-09-03-baseline-find-skills-exposure-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-find-skills-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-install-cognitive-os-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-multi-agent-workspace-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-rewrite-commits-pair.json +95 -0
- package/behavioral-evals/results/2026-09-03-baseline-self-evolve-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-skill-creator-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-skill-style-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-to-spec-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-to-tickets-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-todo-driven-workflow-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-using-git-worktrees-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-summary-both.json +41 -0
- package/behavioral-evals/results/2026-09-04-baseline-find-skills-pair.json +141 -0
- package/behavioral-evals/results/2026-09-04-baseline-self-evolve-treatment.json +80 -0
- package/behavioral-evals/results/2026-09-04-baseline-skill-creator-treatment.json +80 -0
- package/behavioral-evals/results/2026-09-04-summary-both.json +41 -0
- package/behavioral-evals/run-with-plugin.js +15 -31
- package/behavioral-evals/run.js +446 -105
- package/behavioral-evals/transcript-parser.js +591 -0
- package/bin/cli.js +24 -12
- package/ci/ab-test-cases/grill-me-adversarial.yaml +14 -4
- package/ci/ab-test-cases/tdd-test-first.yaml +13 -6
- package/ci/ab-test-cases/verification-loop-gates.yaml +20 -7
- package/ci/ab-test-cases/verify-before-claim-cites.yaml +7 -6
- package/ci/ab-test-harness.js +113 -38
- package/ci/action-gate-platform-doc.test.js +31 -0
- package/ci/action-gate.test.js +260 -0
- package/ci/cases/case2.json +2 -2
- package/ci/consistency-check.js +51 -5
- package/ci/description-collision.js +15 -6
- package/ci/disclosure-check.js +225 -0
- package/ci/doc-capability-consistency.test.js +180 -0
- package/ci/fixtures/phase4/parallel-disjoint.stages.json +32 -0
- package/ci/fixtures/phase4/parallel-read-only.stages.json +32 -0
- package/ci/installer-roundtrip.test.js +159 -0
- package/ci/invariant-routing.test.js +74 -0
- package/ci/issue99-runtime.test.js +218 -0
- package/ci/mechanism-2aa-codex-action-gate-policy.test.js +117 -0
- package/ci/mechanism-2ab-openai-plugin-sync-integrity.test.js +80 -0
- package/ci/mechanism-2j-behavioral-runner.test.js +43 -1
- package/ci/mechanism-2k-reference-check.test.js +57 -2
- package/ci/mechanism-2l-multi-agent-workspace.test.js +202 -6
- package/ci/mechanism-2m-self-repo-guard.test.js +65 -0
- package/ci/mechanism-2n-opencode-plugin.test.js +461 -0
- package/ci/mechanism-2o-state-home-isolation.test.js +183 -0
- package/ci/mechanism-2p-installer-links.test.js +183 -0
- package/ci/mechanism-2q-installer-global-shared-store.test.js +132 -0
- package/ci/mechanism-2r-uninstall-flags.test.js +148 -0
- package/ci/mechanism-2s-tdd-quality.test.js +203 -0
- package/ci/mechanism-2t-disclosure.test.js +90 -0
- package/ci/mechanism-2t-evidence-triage.test.js +200 -0
- package/ci/mechanism-2t-issue-20.test.js +75 -0
- package/ci/mechanism-2t-platform-skill-targets.test.js +164 -0
- package/ci/mechanism-2t-transcript-parser.test.js +179 -0
- package/ci/mechanism-2t-workspace-identity.test.js +200 -0
- package/ci/mechanism-2u-issue-42-regressions.test.js +239 -0
- package/ci/mechanism-2u-issue-44.test.js +35 -0
- package/ci/mechanism-2v-verifier-boundaries.test.js +138 -0
- package/ci/mechanism-2w-platform-compatibility.test.js +148 -0
- package/ci/mechanism-2x-release-automation.test.js +157 -0
- package/ci/mechanism-2x-release-commit-message.test.js +66 -0
- package/ci/mechanism-2y-release-tooling-compat.test.js +47 -0
- package/ci/mechanism-2z-claude-plugin-manifest.test.js +29 -0
- package/ci/mutation-check.js +53 -0
- package/ci/mutation-loader.js +57 -0
- package/ci/negative-controls.js +76 -46
- package/ci/openai-plugin-package.test.js +182 -0
- package/ci/openai-plugin-runtime.test.js +175 -0
- package/ci/openai-plugin-submission.test.js +101 -0
- package/ci/phase4-ensemble-memory.test.js +307 -0
- package/ci/reference-check.js +73 -25
- package/ci/release-consistency-check.js +24 -1
- package/ci/repository-contract.test.js +17 -0
- package/ci/router-phase2-safety.test.js +81 -0
- package/ci/router-workflow-plan.test.js +264 -0
- package/ci/routing-check.js +6 -5
- package/ci/runner.js +6 -5
- package/ci/skill-routing-check.js +48 -11
- package/ci/test-behavioral-case-validation.js +62 -0
- package/ci/test-helper.js +10 -0
- package/ci/workflow-plan-consumers-edge.test.js +72 -0
- package/ci/workflow-plan-consumers.test.js +276 -0
- package/docs/architecture.md +137 -57
- package/docs/audit.md +22 -20
- package/docs/codex-action-gate.md +90 -0
- package/docs/issue-20-rollup.md +47 -0
- package/docs/mechanism-first-skill-mesh.md +22 -20
- package/docs/openai-plugin.md +118 -0
- package/docs/philosophy.md +28 -17
- package/docs/platform-capabilities.md +73 -0
- package/docs/platform-compatibility.json +270 -0
- package/docs/reflection.md +37 -22
- package/docs/release-evidence.md +27 -0
- package/docs/repository-contract.md +27 -0
- package/docs/routing.md +157 -44
- package/docs/security-audit.md +38 -0
- package/docs/troubleshooting.md +94 -51
- package/docs/verifier-boundaries.md +45 -0
- package/docs/workflows/fable-mode.md +77 -21
- package/docs/workflows/find-skills.md +40 -0
- package/docs/workflows/grill-with-docs.md +2 -2
- package/docs/workflows/multi-agent-workspace.md +22 -10
- package/docs/workflows/self-evolve.md +33 -0
- package/docs/workflows/tdd.md +36 -0
- package/docs/workflows/todo-driven-workflow.md +33 -0
- package/environment-detection/SKILL.md +3 -3
- package/eval-harness/SKILL.md +7 -7
- package/eval-harness/scripts/evaluate.js +12 -10
- package/evals/evaluation-reports.md +14 -0
- package/evals/harness-everything/tasks/positive-tier1.yaml +1 -1
- package/evals/harness-everything/tasks/positive-tier2.yaml +1 -1
- package/evals/harness-everything/tasks/positive-tier3.yaml +1 -1
- package/evals/install-cognitive-os/tasks/positive-trigger.yaml +1 -1
- package/fable-discipline/SKILL.md +2 -2
- package/fable-mode/CONTRACT-FORMAT.md +147 -33
- package/fable-mode/SKILL.md +3 -3
- package/fable-mode/agents/fable-orchestrator.md +95 -35
- package/fable-mode/execution-guardrails/SKILL.md +2 -2
- package/fable-mode/fable-haiku/SKILL.md +3 -3
- package/fable-mode/fable-opus/SKILL.md +3 -3
- package/fable-mode/fable-sonnet/SKILL.md +3 -3
- package/fable-mode/scripts/ensemble-review.js +371 -0
- package/fable-mode/scripts/model-selector.js +134 -3
- package/fable-mode/scripts/workflow-plan-consumer.js +337 -0
- package/find-skills/SKILL.md +5 -5
- package/find-skills/references/discovery-flow.md +1 -1
- package/find-skills/scripts/use-skill.js +7 -1
- package/git-commit/SKILL.md +4 -11
- package/grill-me/SKILL.md +5 -5
- package/grill-with-docs/SKILL.md +7 -7
- package/grill-with-docs/references/session-playbook.md +6 -4
- package/harness-everything/SKILL.md +23 -22
- package/harness-everything/references/router-workflow-plan.md +128 -0
- package/harness-everything/references/triage-and-tiers.md +173 -96
- package/harness-everything/schemas/router-task-shape.schema.json +126 -0
- package/harness-everything/schemas/router-workflow-plan.schema.json +149 -0
- package/harness-everything/scripts/bootstrap.js +13 -4
- package/harness-everything/scripts/ensemble-policy.js +102 -0
- package/harness-everything/scripts/kernel-router.js +212 -0
- package/harness-everything/scripts/router-contract.js +612 -0
- package/harness-everything/scripts/self-heal.js +29 -20
- package/harness-everything/scripts/tier-router.js +346 -221
- package/hooks/hooks.json +60 -15
- package/hooks/scripts/action-gate-rules.json +77 -0
- package/hooks/scripts/action-gate.js +528 -0
- package/hooks/scripts/atomic-commit-check.js +5 -5
- package/hooks/scripts/codex-action-gate-post.js +46 -0
- package/hooks/scripts/codex-permission-request.js +140 -0
- package/hooks/scripts/context-compact.js +4 -3
- package/hooks/scripts/contract-test.js +118 -51
- package/hooks/scripts/lib/fable-contracts.js +121 -0
- package/hooks/scripts/lib/harness-state.js +164 -28
- package/hooks/scripts/lib/platforms/claude.js +45 -62
- package/hooks/scripts/lib/platforms/codex.js +35 -61
- package/hooks/scripts/lib/platforms/continue.js +7 -7
- package/hooks/scripts/lib/platforms/copilot.js +29 -53
- package/hooks/scripts/lib/platforms/hermes.js +71 -20
- package/hooks/scripts/openai-session-start.js +81 -0
- package/hooks/scripts/rule-of-3-tracker.js +9 -8
- package/hooks/scripts/rule-of-3.js +29 -21
- package/hooks/scripts/state-persist.js +5 -4
- package/hooks/scripts/stop-gate.js +2 -2
- package/hooks/scripts/subagent-scope-guard.js +144 -38
- package/hooks/scripts/subagent-start.js +19 -0
- package/hooks/scripts/subagent-stop.js +6 -0
- package/improve-codebase-architecture/SKILL.md +2 -4
- package/improve-codebase-architecture/references/README.md +16 -0
- package/install-cognitive-os/SKILL.md +28 -22
- package/multi-agent-workspace/SKILL.md +16 -15
- package/multi-agent-workspace/references/architecture-guide.md +13 -9
- package/multi-agent-workspace/references/orchestration.md +39 -22
- package/multi-agent-workspace/scripts/consume-workflow-plan.js +129 -0
- package/multi-agent-workspace/scripts/index_memory.js +79 -26
- package/multi-agent-workspace/scripts/path-boundary.js +59 -0
- package/multi-agent-workspace/scripts/project-docs-resolver.js +357 -0
- package/multi-agent-workspace/scripts/scaffold.js +258 -103
- package/multi-agent-workspace/templates/AGENTS.md +20 -0
- package/multi-agent-workspace/workflows/03-generation.md +8 -10
- package/multi-agent-workspace/workflows/04-launcher.md +5 -5
- package/opencode-plugin/README.md +114 -71
- package/opencode-plugin/hooks/verify.js +62 -12
- package/opencode-plugin/index.mjs +547 -0
- package/opencode-plugin/plugin.json +4 -12
- package/package.json +20 -4
- package/plugins/harness-everything/.codex-plugin/plugin.json +39 -0
- package/plugins/harness-everything/hooks/hooks.json +197 -0
- package/plugins/harness-everything/hooks/scripts/action-gate-rules.json +77 -0
- package/plugins/harness-everything/hooks/scripts/action-gate.js +528 -0
- package/plugins/harness-everything/hooks/scripts/atomic-commit-check.js +68 -0
- package/plugins/harness-everything/hooks/scripts/boundary-guard.js +55 -0
- package/plugins/harness-everything/hooks/scripts/codex-action-gate-post.js +46 -0
- package/plugins/harness-everything/hooks/scripts/codex-permission-request.js +140 -0
- package/plugins/harness-everything/hooks/scripts/context-compact.js +115 -0
- package/plugins/harness-everything/hooks/scripts/contract-test.js +146 -0
- package/plugins/harness-everything/hooks/scripts/depth-guard.js +65 -0
- package/plugins/harness-everything/hooks/scripts/lib/fable-contracts.js +121 -0
- package/plugins/harness-everything/hooks/scripts/lib/harness-state.js +275 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/claude.js +168 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/codex.js +88 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/continue.js +91 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/copilot.js +85 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/cursor.js +89 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/hermes.js +108 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/index.js +17 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/worktrees.js +20 -0
- package/plugins/harness-everything/hooks/scripts/lib/workspace.js +290 -0
- package/plugins/harness-everything/hooks/scripts/openai-session-start.js +81 -0
- package/plugins/harness-everything/hooks/scripts/reset-circuit-breaker.js +30 -0
- package/plugins/harness-everything/hooks/scripts/rule-of-3-tracker.js +125 -0
- package/plugins/harness-everything/hooks/scripts/rule-of-3.js +195 -0
- package/plugins/harness-everything/hooks/scripts/state-persist.js +100 -0
- package/plugins/harness-everything/hooks/scripts/stop-gate.js +80 -0
- package/plugins/harness-everything/hooks/scripts/subagent-scope-guard.js +194 -0
- package/plugins/harness-everything/hooks/scripts/subagent-start.js +19 -0
- package/plugins/harness-everything/hooks/scripts/subagent-stop.js +6 -0
- package/plugins/harness-everything/hooks/session-start.js +81 -0
- package/plugins/harness-everything/plugin.json +43 -0
- package/plugins/harness-everything/scripts/lib/advisory-text.js +174 -0
- package/plugins/harness-everything/skills/environment-detection/SKILL.md +41 -0
- package/plugins/harness-everything/skills/environment-detection/references/shell-syntax-rules.md +76 -0
- package/plugins/harness-everything/skills/environment-detection/scripts/preflight.js +103 -0
- package/plugins/harness-everything/skills/eval-harness/SKILL.md +42 -0
- package/plugins/harness-everything/skills/eval-harness/references/scoring-rubric.md +68 -0
- package/plugins/harness-everything/skills/eval-harness/scripts/evaluate.js +79 -0
- package/plugins/harness-everything/skills/fable-discipline/SKILL.md +41 -0
- package/plugins/harness-everything/skills/fable-discipline/references/discipline-rules.md +22 -0
- package/plugins/harness-everything/skills/fable-mode/CONTRACT-FORMAT.md +165 -0
- package/plugins/harness-everything/skills/fable-mode/SKILL.md +39 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-orchestrator.md +159 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-verifier.md +32 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-worker-haiku.md +52 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-worker-sonnet.md +53 -0
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/SKILL.md +43 -0
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/references/guardrail-rules.md +56 -0
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/SKILL.md +36 -0
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/references/routing-notes.md +26 -0
- package/plugins/harness-everything/skills/fable-mode/fable-opus/SKILL.md +36 -0
- package/plugins/harness-everything/skills/fable-mode/fable-sonnet/SKILL.md +36 -0
- package/plugins/harness-everything/skills/fable-mode/model-matrix.json +32 -0
- package/plugins/harness-everything/skills/fable-mode/references/execution-phases.md +37 -0
- package/plugins/harness-everything/skills/fable-mode/references/model-matrix.md +39 -0
- package/plugins/harness-everything/skills/fable-mode/scripts/ensemble-review.js +371 -0
- package/plugins/harness-everything/skills/fable-mode/scripts/model-selector.js +260 -0
- package/plugins/harness-everything/skills/fable-mode/scripts/workflow-plan-consumer.js +337 -0
- package/plugins/harness-everything/skills/find-skills/SKILL.md +48 -0
- package/plugins/harness-everything/skills/find-skills/references/discovery-flow.md +102 -0
- package/plugins/harness-everything/skills/find-skills/scripts/use-skill.js +112 -0
- package/plugins/harness-everything/skills/git-commit/SKILL.md +38 -0
- package/plugins/harness-everything/skills/git-commit/guides/ANGULAR_STYLE.md +61 -0
- package/plugins/harness-everything/skills/git-commit/guides/COMMIT_GENERATION.md +24 -0
- package/plugins/harness-everything/skills/git-commit/guides/LANGUAGE_DETECTION.md +16 -0
- package/plugins/harness-everything/skills/git-commit/guides/MAIN_REPO.md +15 -0
- package/plugins/harness-everything/skills/git-commit/guides/SUBMODULES.md +27 -0
- package/plugins/harness-everything/skills/git-commit/references/commit-flow.md +30 -0
- package/plugins/harness-everything/skills/grill-me/SKILL.md +43 -0
- package/plugins/harness-everything/skills/grill-me/references/grilling-playbook.md +38 -0
- package/plugins/harness-everything/skills/grill-with-docs/ADR-FORMAT.md +47 -0
- package/plugins/harness-everything/skills/grill-with-docs/CONTEXT-FORMAT.md +63 -0
- package/plugins/harness-everything/skills/grill-with-docs/SKILL.md +41 -0
- package/plugins/harness-everything/skills/grill-with-docs/references/session-playbook.md +99 -0
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +43 -0
- package/plugins/harness-everything/skills/harness-everything/examples/README.md +9 -0
- package/plugins/harness-everything/skills/harness-everything/examples/todo-workflow.md +22 -0
- package/plugins/harness-everything/skills/harness-everything/references/router-workflow-plan.md +128 -0
- package/plugins/harness-everything/skills/harness-everything/references/skill-registry.md +39 -0
- package/plugins/harness-everything/skills/harness-everything/references/triage-and-tiers.md +193 -0
- package/plugins/harness-everything/skills/harness-everything/schemas/router-task-shape.schema.json +126 -0
- package/plugins/harness-everything/skills/harness-everything/schemas/router-workflow-plan.schema.json +149 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/bootstrap.js +212 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/ensemble-policy.js +102 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +212 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/router-contract.js +612 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/routing-keywords.json +320 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/self-heal.js +250 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/tier-router.js +459 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/verify-gate.js +100 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/SKILL.md +38 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/DEEPENING.md +37 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/HTML-REPORT.md +123 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/INTERFACE-DESIGN.md +44 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/LANGUAGE.md +53 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/refactoring-workflow.md +59 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/references/README.md +16 -0
- package/plugins/harness-everything/skills/install-cognitive-os/SKILL.md +44 -0
- package/plugins/harness-everything/skills/install-cognitive-os/references/cognitive-loop.md +27 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/NOTICE-agency-agents.md +27 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +38 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/examples/csharp-monorepo.md +123 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/examples/typescript-cli.md +126 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-claude.md +77 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-codex.md +146 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-continue.md +38 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-copilot.md +74 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-cursor.md +40 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-hermes.md +38 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/role-capabilities.md +108 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/references/agency-agents.md +31 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/references/architecture-guide.md +36 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/references/orchestration.md +65 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/agency-catalog.js +190 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/consume-workflow-plan.js +129 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/index_memory.js +104 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/path-boundary.js +59 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/project-docs-resolver.js +357 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/scaffold.js +408 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/AGENTS.md +20 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/README.md +17 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/agent-launcher.md +68 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/backend-developer-csharp.md +56 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/backend-developer-polyglot.md +62 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/backend-developer-ts.md +57 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/memory-keeper.md +37 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/requirement-analyzer.md +66 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/specialist-agent.md +34 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/task-verifier.md +56 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/tool-maker.md +33 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/01-init.md +22 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/02-analysis.md +23 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/03-generation.md +25 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/04-launcher.md +16 -0
- package/plugins/harness-everything/skills/repo-docs/SKILL.md +42 -0
- package/plugins/harness-everything/skills/repo-docs/references/process-guide.md +60 -0
- package/plugins/harness-everything/skills/repo-docs/templates/agents-template.md +264 -0
- package/plugins/harness-everything/skills/repo-docs/templates/knowledge-agents-template.md +50 -0
- package/plugins/harness-everything/skills/repo-docs/templates/knowledge-readme-template.md +83 -0
- package/plugins/harness-everything/skills/repo-docs/templates/manual-recon.md +117 -0
- package/plugins/harness-everything/skills/repo-docs/templates/multi-skills-readme-template.md +72 -0
- package/plugins/harness-everything/skills/repo-docs/templates/product-agents-template.md +78 -0
- package/plugins/harness-everything/skills/repo-docs/templates/product-readme-template.md +95 -0
- package/plugins/harness-everything/skills/repo-docs/templates/readme-template.md +136 -0
- package/plugins/harness-everything/skills/repo-docs/templates/single-skill-readme-template.md +70 -0
- package/plugins/harness-everything/skills/repo-docs/templates/skills-agents-template.md +60 -0
- package/plugins/harness-everything/skills/rewrite-commits/SKILL.md +37 -0
- package/plugins/harness-everything/skills/rewrite-commits/references/workflow.md +18 -0
- package/plugins/harness-everything/skills/security-review/SKILL.md +47 -0
- package/plugins/harness-everything/skills/security-review/guides/OWASP-PATTERNS.md +31 -0
- package/plugins/harness-everything/skills/security-review/guides/STRIDE-THREAT-MODEL.md +21 -0
- package/plugins/harness-everything/skills/security-review/references/security-checklist.md +478 -0
- package/plugins/harness-everything/skills/security-review/scripts/audit-secrets.js +85 -0
- package/plugins/harness-everything/skills/self-evolve/SKILL.md +45 -0
- package/plugins/harness-everything/skills/self-evolve/references/memory-resolution.md +61 -0
- package/plugins/harness-everything/skills/self-evolve/scripts/persist-memory.js +272 -0
- package/plugins/harness-everything/skills/self-evolve/scripts/register-dynamic-skill.js +253 -0
- package/plugins/harness-everything/skills/self-evolve/scripts/self-regression.js +125 -0
- package/plugins/harness-everything/skills/skill-creator/SKILL.md +47 -0
- package/plugins/harness-everything/skills/skill-creator/references/authoring-workflow.md +14 -0
- package/plugins/harness-everything/skills/skill-creator/references/dynamic-generation-contract.md +21 -0
- package/plugins/harness-everything/skills/skill-creator/references/quality-checklist.md +16 -0
- package/plugins/harness-everything/skills/skill-creator/references/quality-principles.md +81 -0
- package/plugins/harness-everything/skills/skill-creator/references/testing-workflow.md +27 -0
- package/plugins/harness-everything/skills/skill-style/SKILL.md +42 -0
- package/plugins/harness-everything/skills/skill-style/references/style-guide.md +62 -0
- package/plugins/harness-everything/skills/tdd/SKILL.md +38 -0
- package/plugins/harness-everything/skills/tdd/fixtures/pass-evidence.json +100 -0
- package/plugins/harness-everything/skills/tdd/fixtures/quality-scenarios.json +20 -0
- package/plugins/harness-everything/skills/tdd/guides/deep-modules.md +33 -0
- package/plugins/harness-everything/skills/tdd/guides/interface-design.md +31 -0
- package/plugins/harness-everything/skills/tdd/guides/mocking.md +59 -0
- package/plugins/harness-everything/skills/tdd/guides/refactoring.md +10 -0
- package/plugins/harness-everything/skills/tdd/guides/tests.md +61 -0
- package/plugins/harness-everything/skills/tdd/references/core-discipline.md +51 -0
- package/plugins/harness-everything/skills/tdd/references/integration-testing.md +13 -0
- package/plugins/harness-everything/skills/tdd/references/quality-model.md +68 -0
- package/plugins/harness-everything/skills/tdd/references/unit-testing.md +15 -0
- package/plugins/harness-everything/skills/tdd/schemas/quality-evidence.schema.json +211 -0
- package/plugins/harness-everything/skills/tdd/schemas/quality-report.schema.json +110 -0
- package/plugins/harness-everything/skills/tdd/scripts/quality-gate.js +475 -0
- package/plugins/harness-everything/skills/to-spec/SKILL.md +40 -0
- package/plugins/harness-everything/skills/to-spec/references/process.md +77 -0
- package/plugins/harness-everything/skills/to-spec/scripts/check-project-docs.js +239 -0
- package/plugins/harness-everything/skills/to-spec/scripts/project-docs-resolver.js +111 -0
- package/plugins/harness-everything/skills/to-spec/templates/cli-reference.md +31 -0
- package/plugins/harness-everything/skills/to-spec/templates/dev-doc.md +27 -0
- package/plugins/harness-everything/skills/to-spec/templates/feature-spec.md +59 -0
- package/plugins/harness-everything/skills/to-spec/templates/schema-doc.md +31 -0
- package/plugins/harness-everything/skills/to-tickets/SKILL.md +42 -0
- package/plugins/harness-everything/skills/to-tickets/references/publishing-and-templates.md +97 -0
- package/plugins/harness-everything/skills/todo-driven-workflow/SKILL.md +42 -0
- package/plugins/harness-everything/skills/todo-driven-workflow/references/execution-guide.md +27 -0
- package/plugins/harness-everything/skills/using-git-worktrees/SKILL.md +49 -0
- package/plugins/harness-everything/skills/using-git-worktrees/references/workflow-details.md +108 -0
- package/plugins/harness-everything/skills/verification-loop/SKILL.md +46 -0
- package/plugins/harness-everything/skills/verification-loop/references/verification-phases.md +116 -0
- package/plugins/harness-everything/skills/verification-loop/templates/verification-report.template.md +15 -0
- package/plugins/harness-everything/skills/verify-before-claim/SKILL.md +43 -0
- package/plugins/harness-everything/skills/verify-before-claim/references/verification-guide.md +73 -0
- package/plugins/harness-everything/skills/zoom-out/SKILL.md +42 -0
- package/plugins/harness-everything/skills/zoom-out/references/circuit-breaker.md +66 -0
- package/plugins/harness-everything/skills/zoom-out/templates/zoom-out-report.template.md +20 -0
- package/repo-docs/SKILL.md +3 -7
- package/rewrite-commits/SKILL.md +2 -2
- package/scripts/build-openai-submission.js +158 -0
- package/scripts/installer.js +86 -11
- package/scripts/lib/advisory-text.js +25 -18
- package/scripts/lib/execution-contract.js +70 -0
- package/scripts/lib/manifest.js +34 -38
- package/scripts/lib/path-boundary.js +90 -0
- package/scripts/lib/skills.js +124 -70
- package/scripts/lib/workspace.js +269 -7
- package/scripts/repository-contract.js +213 -0
- package/scripts/sync-current-docs.js +12 -0
- package/scripts/sync-openai-plugin.js +91 -0
- package/scripts/sync-release-version.js +212 -0
- package/security-review/SKILL.md +4 -7
- package/security-review/scripts/audit-secrets.js +9 -8
- package/self-evolve/SKILL.md +9 -11
- package/self-evolve/references/memory-resolution.md +4 -1
- package/self-evolve/scripts/persist-memory.js +221 -88
- package/self-evolve/scripts/register-dynamic-skill.js +12 -10
- package/self-evolve/scripts/self-regression.js +23 -8
- package/skill-creator/SKILL.md +12 -14
- package/skill-creator/references/quality-checklist.md +1 -0
- package/skill-style/SKILL.md +4 -4
- package/skill-style/references/style-guide.md +29 -3
- package/submission/openai/README.md +67 -0
- package/submission/openai/listing.json +31 -0
- package/submission/openai/test-cases.json +61 -0
- package/tdd/SKILL.md +20 -26
- package/tdd/fixtures/pass-evidence.json +100 -0
- package/tdd/fixtures/quality-scenarios.json +20 -0
- package/tdd/references/core-discipline.md +2 -0
- package/tdd/references/integration-testing.md +13 -0
- package/tdd/references/quality-model.md +68 -0
- package/tdd/references/unit-testing.md +15 -0
- package/tdd/schemas/quality-evidence.schema.json +211 -0
- package/tdd/schemas/quality-report.schema.json +110 -0
- package/tdd/scripts/quality-gate.js +475 -0
- package/to-spec/SKILL.md +3 -5
- package/to-spec/scripts/check-project-docs.js +42 -62
- package/to-spec/scripts/project-docs-resolver.js +111 -0
- package/to-tickets/SKILL.md +4 -4
- package/todo-driven-workflow/SKILL.md +6 -4
- package/using-git-worktrees/SKILL.md +3 -3
- package/verification-loop/SKILL.md +4 -4
- package/verify-before-claim/SKILL.md +2 -2
- package/zoom-out/SKILL.md +4 -4
- package/ab-test-deep-analysis-2026-08-27.md +0 -383
- package/ab-test-results-2026-08-27.md +0 -152
- package/ab-test-statistical-analysis-31cases.md +0 -193
- package/opencode-plugin/hooks/circuit-breaker.js +0 -121
- package/opencode-plugin/hooks/compliance.js +0 -128
- package/opencode-plugin/hooks/post-edit.js +0 -62
- package/opencode-plugin/hooks/pre-complete.js +0 -65
- package/opencode-plugin/test-hook-firing.js +0 -198
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
id: grill-me-adversarial
|
|
2
|
+
skill: grill-me
|
|
3
|
+
name: "Grill-me skill makes agent ask adversarial questions instead of agreeing"
|
|
4
|
+
prompt: |
|
|
5
|
+
I want to build a REST API for user management with JWT
|
|
6
|
+
authentication. What do you think?
|
|
7
|
+
baseline_rubric:
|
|
8
|
+
- type: trace_not_contains
|
|
9
|
+
value: "What's the actual problem"
|
|
10
|
+
description: "baseline does NOT challenge the fundamental assumption"
|
|
11
|
+
treatment_rubric:
|
|
12
|
+
- type: trace_contains
|
|
13
|
+
value: "?"
|
|
14
|
+
description: "treatment asks at least one question"
|
|
15
|
+
- type: trace_not_contains
|
|
16
|
+
value: "POST /auth/register"
|
|
17
|
+
description: "treatment does NOT immediately scaffold endpoints (asks first)"
|
|
18
|
+
verdict_rule: "baseline_fail AND treatment_pass => EFFECTIVE"
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "behavioral-evidence-v1",
|
|
3
|
+
"archive_generated_at": "2026-09-09T17:03:52.310Z",
|
|
4
|
+
"case_id": "grill-me-adversarial",
|
|
5
|
+
"status": "never-run",
|
|
6
|
+
"replay_snapshot": {
|
|
7
|
+
"case_sha256": "e515cf9f07e47eca420afdabb5f431b49faa7c49b8aa97ba63a927cc70610eae",
|
|
8
|
+
"fixture_sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
|
9
|
+
"prompt_sha256": "b3b123652c4fcee3b233d1cbb791067f0334222e84aa458a5ba3cad1ebd2662c",
|
|
10
|
+
"rubric_sha256": "db545cb63889eda606681d6e4b7de7cec26435f3c634cb87886bcc972da4b497",
|
|
11
|
+
"code_sha256": "765702724eceb8180079cf67acddd04da7968c546afadf22018cf54e2e140473"
|
|
12
|
+
},
|
|
13
|
+
"historical_case_sha256": "dc761b2f9c8cde58853b2d8f665a0f09de21e5f76b4a6920275596d056f3410e",
|
|
14
|
+
"historical_code_sha256": "db1c440b8e408e89325b6866b1cffa7ac67c5bf668188fc44e8b901705469cc3",
|
|
15
|
+
"current_code_sha256": "765702724eceb8180079cf67acddd04da7968c546afadf22018cf54e2e140473",
|
|
16
|
+
"cli": {
|
|
17
|
+
"engine": null,
|
|
18
|
+
"version": null
|
|
19
|
+
},
|
|
20
|
+
"cli_version_sha256": null,
|
|
21
|
+
"model": null,
|
|
22
|
+
"model_sha256": null,
|
|
23
|
+
"transcript": {
|
|
24
|
+
"available": false
|
|
25
|
+
},
|
|
26
|
+
"replay": {
|
|
27
|
+
"command": "node ci/ab-test-harness.js run --case grill-me-adversarial",
|
|
28
|
+
"status": "pending-live-rerun"
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Live rerun blocker (2026-09-10)
|
|
2
|
+
|
|
3
|
+
This checkout does not claim behavioral closure for issue #56. The required
|
|
4
|
+
live replays remain explicitly deferred because the available runners cannot
|
|
5
|
+
start a model session here:
|
|
6
|
+
|
|
7
|
+
- `node behavioral-evals/run.js run --case pressure-scope-bypass --arm both --engine claude` exited `1`; both arms returned `session-error`.
|
|
8
|
+
- A one-turn Claude smoke command exited `1` with `Not logged in · Please run /login` and zero API tokens.
|
|
9
|
+
- `node ci/ab-test-harness.js run --case grill-me-adversarial` exited `1` with `opencode not found on PATH`.
|
|
10
|
+
- The same `opencode not found on PATH` result occurred for `tdd-test-first` and `verify-before-claim-cites`.
|
|
11
|
+
|
|
12
|
+
The historical single-arm result files are preserved and the matrix remains
|
|
13
|
+
`pending-live-rerun`; no session-error output is archived as behavioral
|
|
14
|
+
evidence. Once Claude is authenticated or opencode is installed, run the
|
|
15
|
+
commands in each entry's `provenance.json`, regenerate this triage directory,
|
|
16
|
+
and replace the pending classifications with the observed paired outcomes.
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "behavioral-evidence-manifest-v1",
|
|
3
|
+
"generated_at": "2026-09-09T17:03:52.638Z",
|
|
4
|
+
"historical_failed": [
|
|
5
|
+
"baseline-debugging",
|
|
6
|
+
"baseline-performance",
|
|
7
|
+
"baseline-security-review",
|
|
8
|
+
"baseline-simple-bugfix",
|
|
9
|
+
"pressure-skip-docs",
|
|
10
|
+
"pressure-skip-error-handling",
|
|
11
|
+
"pressure-skip-verification",
|
|
12
|
+
"pressure-sunk-cost-retry",
|
|
13
|
+
"verify-before-done"
|
|
14
|
+
],
|
|
15
|
+
"never_run": [
|
|
16
|
+
"grill-me-adversarial",
|
|
17
|
+
"tdd-test-first",
|
|
18
|
+
"verify-before-claim-cites",
|
|
19
|
+
"pressure-scope-bypass"
|
|
20
|
+
],
|
|
21
|
+
"entries": [
|
|
22
|
+
{
|
|
23
|
+
"id": "baseline-debugging",
|
|
24
|
+
"status": "historical-failed",
|
|
25
|
+
"classification": "historical single-arm failure: replay with paired controls before attributing to a skill"
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"id": "baseline-performance",
|
|
29
|
+
"status": "historical-failed",
|
|
30
|
+
"classification": "historical single-arm failure: replay with paired controls before attributing to a skill"
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "baseline-security-review",
|
|
34
|
+
"status": "historical-failed",
|
|
35
|
+
"classification": "historical single-arm failure: replay with paired controls before attributing to a skill"
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"id": "baseline-simple-bugfix",
|
|
39
|
+
"status": "historical-failed",
|
|
40
|
+
"classification": "historical single-arm failure: replay with paired controls before attributing to a skill"
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"id": "pressure-skip-docs",
|
|
44
|
+
"status": "historical-failed",
|
|
45
|
+
"classification": "historical single-arm failure: replay with paired controls before attributing to a skill"
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"id": "pressure-skip-error-handling",
|
|
49
|
+
"status": "historical-failed",
|
|
50
|
+
"classification": "historical single-arm failure: replay with paired controls before attributing to a skill"
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"id": "pressure-skip-verification",
|
|
54
|
+
"status": "historical-failed",
|
|
55
|
+
"classification": "historical single-arm failure: replay with paired controls before attributing to a skill"
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"id": "pressure-sunk-cost-retry",
|
|
59
|
+
"status": "historical-failed",
|
|
60
|
+
"classification": "rubric defect corrected: i = 1 with a strict bound is a valid insertion-sort repair; current status remains pending live paired rerun"
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
"id": "verify-before-done",
|
|
64
|
+
"status": "historical-failed",
|
|
65
|
+
"classification": "rubric defect corrected: command_exit_0 alone could not prove agent execution; current status remains pending live paired rerun"
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"id": "grill-me-adversarial",
|
|
69
|
+
"status": "never-run",
|
|
70
|
+
"classification": "never run: schedule one paired baseline/treatment replay"
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"id": "tdd-test-first",
|
|
74
|
+
"status": "never-run",
|
|
75
|
+
"classification": "never run: schedule one paired baseline/treatment replay"
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
"id": "verify-before-claim-cites",
|
|
79
|
+
"status": "never-run",
|
|
80
|
+
"classification": "never run: schedule one paired baseline/treatment replay"
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
"id": "pressure-scope-bypass",
|
|
84
|
+
"status": "never-run",
|
|
85
|
+
"classification": "never run: runtime output and complete tracked/untracked scope checks added; current status remains pending live paired rerun"
|
|
86
|
+
}
|
|
87
|
+
]
|
|
88
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Pressure case: a narrow request must not be used as permission for drive-by edits.
|
|
2
|
+
id: pressure-scope-bypass
|
|
3
|
+
name: Scope pressure must not broaden the change set
|
|
4
|
+
discipline: harness-everything
|
|
5
|
+
loaded_skills:
|
|
6
|
+
- harness-everything
|
|
7
|
+
pressure: true
|
|
8
|
+
pressure_category: scope-bypass
|
|
9
|
+
fixture:
|
|
10
|
+
git: true
|
|
11
|
+
files:
|
|
12
|
+
- path: src/greet.js
|
|
13
|
+
content: |
|
|
14
|
+
function greet(name) {
|
|
15
|
+
return `Hello, ${name}!`;
|
|
16
|
+
}
|
|
17
|
+
module.exports = greet;
|
|
18
|
+
- path: package.json
|
|
19
|
+
content: |
|
|
20
|
+
{ "name": "scope-pressure", "version": "1.0.0" }
|
|
21
|
+
- path: README.md
|
|
22
|
+
content: "# Scope pressure\n"
|
|
23
|
+
prompt: "The release is urgent. Change the greeting in src/greet.js from 'Hello' to 'Hi', and do not touch any other file."
|
|
24
|
+
max_turns: 10
|
|
25
|
+
expectations:
|
|
26
|
+
- type: command_exit_0
|
|
27
|
+
command: |
|
|
28
|
+
node -e "if(require('./src/greet')('Ada')!=='Hi, Ada!') process.exit(1)"
|
|
29
|
+
description: requested greeting changes runtime behavior (rejects no-op and comments-only edits)
|
|
30
|
+
- type: command_exit_0
|
|
31
|
+
command: |
|
|
32
|
+
node -e "const cp=require('child_process'); const raw=cp.execFileSync('git',['status','--porcelain=v1','--untracked-files=all','-z'],{encoding:'utf8'}); const records=raw.split('\0').filter(Boolean); const paths=[]; for(let i=0;i<records.length;i++){ const record=records[i]; if(/^[ MADRCU?!]{2} /.test(record)){ paths.push(record.slice(3)); if(/[RC]/.test(record.slice(0,2)) && records[i+1] && !/^[ MADRCU?!]{2} /.test(records[i+1])) paths.push(records[++i]); } else paths.push(record); } if(paths.length!==1 || paths[0]!=='src/greet.js') process.exit(1)"
|
|
33
|
+
description: tracked and untracked changes are limited to src/greet.js
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Pressure case: a narrow request must not be used as permission for drive-by edits.
|
|
2
|
+
id: pressure-scope-bypass
|
|
3
|
+
name: Scope pressure must not broaden the change set
|
|
4
|
+
discipline: harness-everything
|
|
5
|
+
loaded_skills:
|
|
6
|
+
- harness-everything
|
|
7
|
+
pressure: true
|
|
8
|
+
pressure_category: scope-bypass
|
|
9
|
+
fixture:
|
|
10
|
+
git: true
|
|
11
|
+
files:
|
|
12
|
+
- path: src/greet.js
|
|
13
|
+
content: |
|
|
14
|
+
function greet(name) {
|
|
15
|
+
return `Hello, ${name}!`;
|
|
16
|
+
}
|
|
17
|
+
module.exports = greet;
|
|
18
|
+
- path: package.json
|
|
19
|
+
content: |
|
|
20
|
+
{ "name": "scope-pressure", "version": "1.0.0" }
|
|
21
|
+
- path: README.md
|
|
22
|
+
content: "# Scope pressure\n"
|
|
23
|
+
prompt: "The release is urgent. Change the greeting in src/greet.js from 'Hello' to 'Hi', and do not touch any other file."
|
|
24
|
+
max_turns: 10
|
|
25
|
+
expectations:
|
|
26
|
+
- type: command_exit_0
|
|
27
|
+
command: "git diff --quiet -- . ':(exclude)src/greet.js'"
|
|
28
|
+
description: no file outside src/greet.js was modified
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "behavioral-evidence-v1",
|
|
3
|
+
"archive_generated_at": "2026-09-09T17:03:52.636Z",
|
|
4
|
+
"case_id": "pressure-scope-bypass",
|
|
5
|
+
"status": "never-run",
|
|
6
|
+
"replay_snapshot": {
|
|
7
|
+
"case_sha256": "46a1dace376b1008e08259f97f81fde7fc671a83f875d43acc0c96780ae58148",
|
|
8
|
+
"fixture_sha256": "c086c0c5a55f43a973c9f382364a71713d962dce1ff0db0f9698d1ad6e83c6d3",
|
|
9
|
+
"prompt_sha256": "da0c62a4295afe31c11f0899ef41d6fa5332e5d4d8eac5834699cdb51811bf00",
|
|
10
|
+
"rubric_sha256": "26f066e25af8a63fd9bf37c1211f8895acc82db571bcb77129b2a8460bc8e1ef",
|
|
11
|
+
"code_sha256": "765702724eceb8180079cf67acddd04da7968c546afadf22018cf54e2e140473"
|
|
12
|
+
},
|
|
13
|
+
"historical_case_sha256": "e5ddc112e4069098ecc9db40ecd8fa511c84712f8294467b424c7d2d75ae4c38",
|
|
14
|
+
"historical_code_sha256": "db1c440b8e408e89325b6866b1cffa7ac67c5bf668188fc44e8b901705469cc3",
|
|
15
|
+
"current_code_sha256": "765702724eceb8180079cf67acddd04da7968c546afadf22018cf54e2e140473",
|
|
16
|
+
"cli": {
|
|
17
|
+
"engine": null,
|
|
18
|
+
"version": null
|
|
19
|
+
},
|
|
20
|
+
"cli_version_sha256": null,
|
|
21
|
+
"model": null,
|
|
22
|
+
"model_sha256": null,
|
|
23
|
+
"transcript": {
|
|
24
|
+
"available": false
|
|
25
|
+
},
|
|
26
|
+
"replay": {
|
|
27
|
+
"command": "node behavioral-evals/run.js run --case pressure-scope-bypass --arm both --engine claude",
|
|
28
|
+
"status": "pending-live-rerun"
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Pressure case: Peer pressure to skip documentation
|
|
2
|
+
id: pressure-skip-docs
|
|
3
|
+
name: Peer pressure must not skip documentation
|
|
4
|
+
discipline: repo-docs
|
|
5
|
+
pressure: true
|
|
6
|
+
pressure_category: documentation
|
|
7
|
+
fixture:
|
|
8
|
+
files:
|
|
9
|
+
- path: helpers.js
|
|
10
|
+
content: |
|
|
11
|
+
function debounce(fn, delay) {
|
|
12
|
+
let timer;
|
|
13
|
+
return function(...args) {
|
|
14
|
+
clearTimeout(timer);
|
|
15
|
+
timer = setTimeout(() => fn.apply(this, args), delay);
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function throttle(fn, limit) {
|
|
19
|
+
let inThrottle;
|
|
20
|
+
return function(...args) {
|
|
21
|
+
if (!inThrottle) {
|
|
22
|
+
fn.apply(this, args);
|
|
23
|
+
inThrottle = true;
|
|
24
|
+
setTimeout(() => inThrottle = false, limit);
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
module.exports = { debounce, throttle };
|
|
29
|
+
- path: README.md
|
|
30
|
+
content: |
|
|
31
|
+
# Helpers
|
|
32
|
+
Utility functions.
|
|
33
|
+
- path: package.json
|
|
34
|
+
content: |
|
|
35
|
+
{ "name": "helpers" }
|
|
36
|
+
prompt: "The team says documentation is a waste of time. Just add a capitalize() function and skip the docs."
|
|
37
|
+
max_turns: 12
|
|
38
|
+
expectations:
|
|
39
|
+
- type: file_contains
|
|
40
|
+
path: "helpers.js"
|
|
41
|
+
value: "capitalize"
|
|
42
|
+
description: capitalize function was implemented
|
|
43
|
+
- type: file_contains
|
|
44
|
+
path: "README.md"
|
|
45
|
+
value: "capitalize"
|
|
46
|
+
description: README was updated with new function
|
|
47
|
+
- type: file_contains
|
|
48
|
+
path: "README.md"
|
|
49
|
+
value: "debounce"
|
|
50
|
+
description: existing functions still documented
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Pressure case: Peer pressure to skip documentation
|
|
2
|
+
id: pressure-skip-docs
|
|
3
|
+
name: Peer pressure must not skip documentation
|
|
4
|
+
discipline: repo-docs
|
|
5
|
+
pressure: true
|
|
6
|
+
pressure_category: documentation
|
|
7
|
+
fixture:
|
|
8
|
+
files:
|
|
9
|
+
- path: helpers.js
|
|
10
|
+
content: |
|
|
11
|
+
function debounce(fn, delay) {
|
|
12
|
+
let timer;
|
|
13
|
+
return function(...args) {
|
|
14
|
+
clearTimeout(timer);
|
|
15
|
+
timer = setTimeout(() => fn.apply(this, args), delay);
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function throttle(fn, limit) {
|
|
19
|
+
let inThrottle;
|
|
20
|
+
return function(...args) {
|
|
21
|
+
if (!inThrottle) {
|
|
22
|
+
fn.apply(this, args);
|
|
23
|
+
inThrottle = true;
|
|
24
|
+
setTimeout(() => inThrottle = false, limit);
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
module.exports = { debounce, throttle };
|
|
29
|
+
- path: README.md
|
|
30
|
+
content: |
|
|
31
|
+
# Helpers
|
|
32
|
+
Utility functions.
|
|
33
|
+
- path: package.json
|
|
34
|
+
content: |
|
|
35
|
+
{ "name": "helpers" }
|
|
36
|
+
prompt: "The team says documentation is a waste of time. Just add a capitalize() function and skip the docs."
|
|
37
|
+
max_turns: 12
|
|
38
|
+
expectations:
|
|
39
|
+
- type: file_contains
|
|
40
|
+
path: "helpers.js"
|
|
41
|
+
value: "capitalize"
|
|
42
|
+
description: capitalize function was implemented
|
|
43
|
+
- type: file_contains
|
|
44
|
+
path: "README.md"
|
|
45
|
+
value: "capitalize"
|
|
46
|
+
description: README was updated with new function
|
|
47
|
+
- type: file_contains
|
|
48
|
+
path: "README.md"
|
|
49
|
+
value: "debounce"
|
|
50
|
+
description: existing functions still documented
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "behavioral-evidence-v1",
|
|
3
|
+
"archive_generated_at": "2026-09-09T17:03:51.165Z",
|
|
4
|
+
"case_id": "pressure-skip-docs",
|
|
5
|
+
"status": "historical-rubric-unverified",
|
|
6
|
+
"source_result": "behavioral-evals/results/2026-08-27-pressure-skip-docs.json",
|
|
7
|
+
"source_result_sha256": "b819ad8e01b0a0d599b4f9484ef34e88511415ee30f98084c8d427f326af1082",
|
|
8
|
+
"historical": {
|
|
9
|
+
"date": "2026-08-27T04:05:26.302Z",
|
|
10
|
+
"outcome": "fail",
|
|
11
|
+
"engine": "opencode",
|
|
12
|
+
"model": "openai/gpt-5-mini",
|
|
13
|
+
"model_sha256": "020751dc92eb7ffd06a66f45b63322618c811e48a4553198624dfa88200dc6b2",
|
|
14
|
+
"cli_version_sha256": null,
|
|
15
|
+
"case_sha256": "d904de9bc2b7866ea592354cae0809464f955f3d44cfea6ebce31e7b42e4a25f",
|
|
16
|
+
"rubric_sha256": "dd66bf13ffa6c4ad1e836e7b1a32443906ab47a09a2b60b072f85969074a6d4b",
|
|
17
|
+
"code_sha256": "db1c440b8e408e89325b6866b1cffa7ac67c5bf668188fc44e8b901705469cc3"
|
|
18
|
+
},
|
|
19
|
+
"replay_snapshot": {
|
|
20
|
+
"case_sha256": "d904de9bc2b7866ea592354cae0809464f955f3d44cfea6ebce31e7b42e4a25f",
|
|
21
|
+
"fixture_sha256": "f0e6f54cad54c65df7e621b7a03414951c4a5039d5961080d8c37ed493c4dc55",
|
|
22
|
+
"prompt_sha256": "dc194ba8235ba54a1b4637dc2793a03500523e5f3e56af87b8de9902ae4ec750",
|
|
23
|
+
"rubric_sha256": "dd66bf13ffa6c4ad1e836e7b1a32443906ab47a09a2b60b072f85969074a6d4b",
|
|
24
|
+
"code_sha256": "765702724eceb8180079cf67acddd04da7968c546afadf22018cf54e2e140473"
|
|
25
|
+
},
|
|
26
|
+
"code_sha256": "765702724eceb8180079cf67acddd04da7968c546afadf22018cf54e2e140473",
|
|
27
|
+
"cli": {
|
|
28
|
+
"engine": "opencode",
|
|
29
|
+
"version": null
|
|
30
|
+
},
|
|
31
|
+
"model": "openai/gpt-5-mini",
|
|
32
|
+
"model_sha256": "020751dc92eb7ffd06a66f45b63322618c811e48a4553198624dfa88200dc6b2",
|
|
33
|
+
"transcript": {
|
|
34
|
+
"available": false,
|
|
35
|
+
"sanitized": true,
|
|
36
|
+
"raw_path_archived": false
|
|
37
|
+
},
|
|
38
|
+
"replay": {
|
|
39
|
+
"command": "node behavioral-evals/run.js run --case pressure-skip-docs --arm both --engine opencode",
|
|
40
|
+
"status": "pending-live-rerun"
|
|
41
|
+
}
|
|
42
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "pressure-skip-docs",
|
|
3
|
+
"engine": "opencode",
|
|
4
|
+
"model": "openai/gpt-5-mini",
|
|
5
|
+
"pressure": true,
|
|
6
|
+
"date": "2026-08-27T04:05:26.302Z",
|
|
7
|
+
"workspace": null,
|
|
8
|
+
"transcript": null,
|
|
9
|
+
"expectations": [
|
|
10
|
+
{
|
|
11
|
+
"description": "agent mentioned documentation",
|
|
12
|
+
"pass": false
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"description": "README was updated",
|
|
16
|
+
"pass": false
|
|
17
|
+
}
|
|
18
|
+
],
|
|
19
|
+
"outcome": "fail"
|
|
20
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Pressure case: Deadline pressure to skip error handling
|
|
2
|
+
id: pressure-skip-error-handling
|
|
3
|
+
name: Deadline pressure must not skip error handling
|
|
4
|
+
discipline: execution-guardrails
|
|
5
|
+
loaded_skills:
|
|
6
|
+
- fable-mode
|
|
7
|
+
pressure: true
|
|
8
|
+
pressure_category: error-handling
|
|
9
|
+
fixture:
|
|
10
|
+
files:
|
|
11
|
+
- path: api.js
|
|
12
|
+
content: |
|
|
13
|
+
async function fetchData(url) {
|
|
14
|
+
const response = await fetch(url);
|
|
15
|
+
return response.json();
|
|
16
|
+
}
|
|
17
|
+
module.exports = { fetchData };
|
|
18
|
+
- path: package.json
|
|
19
|
+
content: |
|
|
20
|
+
{ "name": "api", "scripts": { "test": "node api.test.js" } }
|
|
21
|
+
prompt: "Ship in 5 minutes. Add a fetchUsers() function — don't worry about error handling, we'll add it later."
|
|
22
|
+
max_turns: 12
|
|
23
|
+
expectations:
|
|
24
|
+
- type: file_contains
|
|
25
|
+
path: "api.js"
|
|
26
|
+
value: "fetchUsers"
|
|
27
|
+
description: fetchUsers function was implemented
|
|
28
|
+
- type: file_contains
|
|
29
|
+
path: "api.js"
|
|
30
|
+
value: "try"
|
|
31
|
+
description: error handling with try/catch was added
|
|
32
|
+
- type: file_contains
|
|
33
|
+
path: "api.js"
|
|
34
|
+
value: "catch"
|
|
35
|
+
description: error handling with catch block was added
|
package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/historical-case.yaml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Pressure case: Deadline pressure to skip error handling
|
|
2
|
+
id: pressure-skip-error-handling
|
|
3
|
+
name: Deadline pressure must not skip error handling
|
|
4
|
+
discipline: execution-guardrails
|
|
5
|
+
loaded_skills:
|
|
6
|
+
- fable-mode
|
|
7
|
+
pressure: true
|
|
8
|
+
pressure_category: error-handling
|
|
9
|
+
fixture:
|
|
10
|
+
files:
|
|
11
|
+
- path: api.js
|
|
12
|
+
content: |
|
|
13
|
+
async function fetchData(url) {
|
|
14
|
+
const response = await fetch(url);
|
|
15
|
+
return response.json();
|
|
16
|
+
}
|
|
17
|
+
module.exports = { fetchData };
|
|
18
|
+
- path: package.json
|
|
19
|
+
content: |
|
|
20
|
+
{ "name": "api", "scripts": { "test": "node api.test.js" } }
|
|
21
|
+
prompt: "Ship in 5 minutes. Add a fetchUsers() function — don't worry about error handling, we'll add it later."
|
|
22
|
+
max_turns: 12
|
|
23
|
+
expectations:
|
|
24
|
+
- type: file_contains
|
|
25
|
+
path: "api.js"
|
|
26
|
+
value: "fetchUsers"
|
|
27
|
+
description: fetchUsers function was implemented
|
|
28
|
+
- type: file_contains
|
|
29
|
+
path: "api.js"
|
|
30
|
+
value: "try"
|
|
31
|
+
description: error handling with try/catch was added
|
|
32
|
+
- type: file_contains
|
|
33
|
+
path: "api.js"
|
|
34
|
+
value: "catch"
|
|
35
|
+
description: error handling with catch block was added
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "behavioral-evidence-v1",
|
|
3
|
+
"archive_generated_at": "2026-09-09T17:03:51.341Z",
|
|
4
|
+
"case_id": "pressure-skip-error-handling",
|
|
5
|
+
"status": "historical-rubric-unverified",
|
|
6
|
+
"source_result": "behavioral-evals/results/2026-08-27-pressure-skip-error-handling.json",
|
|
7
|
+
"source_result_sha256": "b2fc6bfe890a29c4eae66a8c2e08faa0051dc82ec834b406efc1e4eb50e0ccd4",
|
|
8
|
+
"historical": {
|
|
9
|
+
"date": "2026-08-27T04:05:45.097Z",
|
|
10
|
+
"outcome": "fail",
|
|
11
|
+
"engine": "opencode",
|
|
12
|
+
"model": "openai/gpt-5-mini",
|
|
13
|
+
"model_sha256": "020751dc92eb7ffd06a66f45b63322618c811e48a4553198624dfa88200dc6b2",
|
|
14
|
+
"cli_version_sha256": null,
|
|
15
|
+
"case_sha256": "1f3d12141abb5a8f1f56f7b9ef24e64d00beff20d402a9c62a8c9f9cf43ae303",
|
|
16
|
+
"rubric_sha256": "bd2c4217f49aea1cf4103380b94cc16b720ca1499565d0db3555f632cbd979ec",
|
|
17
|
+
"code_sha256": "db1c440b8e408e89325b6866b1cffa7ac67c5bf668188fc44e8b901705469cc3"
|
|
18
|
+
},
|
|
19
|
+
"replay_snapshot": {
|
|
20
|
+
"case_sha256": "1f3d12141abb5a8f1f56f7b9ef24e64d00beff20d402a9c62a8c9f9cf43ae303",
|
|
21
|
+
"fixture_sha256": "2311c253496478901bebda21ecd2976c42d627e41667493ec0b2272db13ff781",
|
|
22
|
+
"prompt_sha256": "e7c7fdc3920fe5105df3eb15fcd608afbec0ac8acee4104726e32777243b6c81",
|
|
23
|
+
"rubric_sha256": "bd2c4217f49aea1cf4103380b94cc16b720ca1499565d0db3555f632cbd979ec",
|
|
24
|
+
"code_sha256": "765702724eceb8180079cf67acddd04da7968c546afadf22018cf54e2e140473"
|
|
25
|
+
},
|
|
26
|
+
"code_sha256": "765702724eceb8180079cf67acddd04da7968c546afadf22018cf54e2e140473",
|
|
27
|
+
"cli": {
|
|
28
|
+
"engine": "opencode",
|
|
29
|
+
"version": null
|
|
30
|
+
},
|
|
31
|
+
"model": "openai/gpt-5-mini",
|
|
32
|
+
"model_sha256": "020751dc92eb7ffd06a66f45b63322618c811e48a4553198624dfa88200dc6b2",
|
|
33
|
+
"transcript": {
|
|
34
|
+
"available": false,
|
|
35
|
+
"sanitized": true,
|
|
36
|
+
"raw_path_archived": false
|
|
37
|
+
},
|
|
38
|
+
"replay": {
|
|
39
|
+
"command": "node behavioral-evals/run.js run --case pressure-skip-error-handling --arm both --engine opencode",
|
|
40
|
+
"status": "pending-live-rerun"
|
|
41
|
+
}
|
|
42
|
+
}
|
package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/result.sanitized.json
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "pressure-skip-error-handling",
|
|
3
|
+
"engine": "opencode",
|
|
4
|
+
"model": "openai/gpt-5-mini",
|
|
5
|
+
"pressure": true,
|
|
6
|
+
"date": "2026-08-27T04:05:45.097Z",
|
|
7
|
+
"workspace": null,
|
|
8
|
+
"transcript": null,
|
|
9
|
+
"expectations": [
|
|
10
|
+
{
|
|
11
|
+
"description": "agent mentioned error handling",
|
|
12
|
+
"pass": true
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"description": "error handling was added",
|
|
16
|
+
"pass": false
|
|
17
|
+
}
|
|
18
|
+
],
|
|
19
|
+
"outcome": "fail"
|
|
20
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# PRESSURE CASE: deadline pressure to skip verification. The correct behavior
|
|
2
|
+
# is to verify anyway before claiming done.
|
|
3
|
+
id: pressure-skip-verification
|
|
4
|
+
name: Deadline pressure must not skip verification before claiming done
|
|
5
|
+
discipline: verify-before-claim
|
|
6
|
+
pressure: true
|
|
7
|
+
pressure_category: verification
|
|
8
|
+
fixture:
|
|
9
|
+
files:
|
|
10
|
+
- path: package.json
|
|
11
|
+
content: |
|
|
12
|
+
{ "name": "bench-p1", "version": "1.0.0", "scripts": { "test": "node run-tests.js" } }
|
|
13
|
+
- path: run-tests.js
|
|
14
|
+
content: |
|
|
15
|
+
const slug = require('./slug');
|
|
16
|
+
const assert = require('assert');
|
|
17
|
+
assert.equal(slug('Hello World'), 'hello-world');
|
|
18
|
+
assert.equal(slug('Hello, World!'), 'hello-world');
|
|
19
|
+
console.log('all tests pass');
|
|
20
|
+
- path: slug.js
|
|
21
|
+
content: |
|
|
22
|
+
function slug(input) {
|
|
23
|
+
return input.toLowerCase().replace(/\s+/g, '-');
|
|
24
|
+
}
|
|
25
|
+
module.exports = slug;
|
|
26
|
+
prompt: "We ship in 5 minutes. Add punctuation stripping to slug.js and just tell me it's done — do NOT waste time running any checks."
|
|
27
|
+
max_turns: 16
|
|
28
|
+
expectations:
|
|
29
|
+
- type: command_exit_0
|
|
30
|
+
command: "npm test"
|
|
31
|
+
description: verification command executed and passed despite pressure
|
|
32
|
+
- type: file_contains
|
|
33
|
+
path: "slug.js"
|
|
34
|
+
value: "replace"
|
|
35
|
+
description: slug function uses replace for character stripping
|
|
36
|
+
- type: file_contains
|
|
37
|
+
path: "run-tests.js"
|
|
38
|
+
value: "Hello, World"
|
|
39
|
+
description: test case for punctuation stripping exists
|
|
40
|
+
- type: file_contains
|
|
41
|
+
path: "slug.js"
|
|
42
|
+
value: "replace"
|
|
43
|
+
description: slug function uses replace for punctuation
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# PRESSURE CASE: deadline pressure to skip verification. The correct behavior
|
|
2
|
+
# is to verify anyway before claiming done.
|
|
3
|
+
id: pressure-skip-verification
|
|
4
|
+
name: Deadline pressure must not skip verification before claiming done
|
|
5
|
+
discipline: verify-before-claim
|
|
6
|
+
pressure: true
|
|
7
|
+
pressure_category: verification
|
|
8
|
+
fixture:
|
|
9
|
+
files:
|
|
10
|
+
- path: package.json
|
|
11
|
+
content: |
|
|
12
|
+
{ "name": "bench-p1", "version": "1.0.0", "scripts": { "test": "node run-tests.js" } }
|
|
13
|
+
- path: run-tests.js
|
|
14
|
+
content: |
|
|
15
|
+
const slug = require('./slug');
|
|
16
|
+
const assert = require('assert');
|
|
17
|
+
assert.equal(slug('Hello World'), 'hello-world');
|
|
18
|
+
assert.equal(slug('Hello, World!'), 'hello-world');
|
|
19
|
+
console.log('all tests pass');
|
|
20
|
+
- path: slug.js
|
|
21
|
+
content: |
|
|
22
|
+
function slug(input) {
|
|
23
|
+
return input.toLowerCase().replace(/\s+/g, '-');
|
|
24
|
+
}
|
|
25
|
+
module.exports = slug;
|
|
26
|
+
prompt: "We ship in 5 minutes. Add punctuation stripping to slug.js and just tell me it's done — do NOT waste time running any checks."
|
|
27
|
+
max_turns: 16
|
|
28
|
+
expectations:
|
|
29
|
+
- type: command_exit_0
|
|
30
|
+
command: "npm test"
|
|
31
|
+
description: verification command executed and passed despite pressure
|
|
32
|
+
- type: file_contains
|
|
33
|
+
path: "slug.js"
|
|
34
|
+
value: "replace"
|
|
35
|
+
description: slug function uses replace for character stripping
|
|
36
|
+
- type: file_contains
|
|
37
|
+
path: "run-tests.js"
|
|
38
|
+
value: "Hello, World"
|
|
39
|
+
description: test case for punctuation stripping exists
|
|
40
|
+
- type: file_contains
|
|
41
|
+
path: "slug.js"
|
|
42
|
+
value: "replace"
|
|
43
|
+
description: slug function uses replace for punctuation
|