harness-everything 0.2.2 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +27 -0
- package/.claude-plugin/plugin.json +51 -0
- package/.gitattributes +1 -0
- package/.github/workflows/behavioral-evals.yml +36 -0
- package/.github/workflows/ci.yml +117 -0
- package/.github/workflows/release.yml +45 -0
- package/.husky/commit-msg +1 -0
- package/.waza.yaml +22 -0
- package/AGENTS.md +60 -2
- package/BENCHMARK_SOP.md +62 -62
- package/CHANGELOG.md +297 -114
- package/CODE_OF_CONDUCT.md +24 -0
- package/CONTRIBUTING.md +47 -0
- package/Dockerfile +18 -18
- package/LICENSE +201 -201
- package/README.md +279 -197
- package/SECURITY.md +33 -0
- package/VERIFICATION.md +394 -347
- package/ab-test-deep-analysis-2026-08-27.md +383 -0
- package/ab-test-results-2026-08-27.md +152 -0
- package/ab-test-statistical-analysis-31cases.md +193 -0
- package/behavioral-evals/README.md +85 -0
- package/behavioral-evals/analyze-plugin-results.js +110 -0
- package/behavioral-evals/cases/baseline-add-feature.yaml +34 -0
- package/behavioral-evals/cases/baseline-api-design.yaml +32 -0
- package/behavioral-evals/cases/baseline-debugging.yaml +31 -0
- package/behavioral-evals/cases/baseline-dependency.yaml +24 -0
- package/behavioral-evals/cases/baseline-documentation.yaml +34 -0
- package/behavioral-evals/cases/baseline-error-handling.yaml +31 -0
- package/behavioral-evals/cases/baseline-git-commit.yaml +33 -0
- package/behavioral-evals/cases/baseline-performance-monitoring.yaml +28 -0
- package/behavioral-evals/cases/baseline-performance.yaml +34 -0
- package/behavioral-evals/cases/baseline-refactor-with-tests.yaml +35 -0
- package/behavioral-evals/cases/baseline-refactor.yaml +27 -0
- package/behavioral-evals/cases/baseline-security-review.yaml +28 -0
- package/behavioral-evals/cases/baseline-simple-bugfix.yaml +33 -0
- package/behavioral-evals/cases/baseline-tdd-workflow.yaml +28 -0
- package/behavioral-evals/cases/breaker-zoom-out-after-3.yaml +50 -0
- package/behavioral-evals/cases/pressure-budget-skip-review.yaml +27 -0
- package/behavioral-evals/cases/pressure-bypass-review.yaml +27 -0
- package/behavioral-evals/cases/pressure-complexity-skip.yaml +37 -0
- package/behavioral-evals/cases/pressure-expert-skip-testing.yaml +31 -0
- package/behavioral-evals/cases/pressure-fatigue-skip.yaml +33 -0
- package/behavioral-evals/cases/pressure-management-skip-all.yaml +28 -0
- package/behavioral-evals/cases/pressure-scope-bypass.yaml +28 -0
- package/behavioral-evals/cases/pressure-skip-docs.yaml +50 -0
- package/behavioral-evals/cases/pressure-skip-error-handling.yaml +35 -0
- package/behavioral-evals/cases/pressure-skip-security.yaml +32 -0
- package/behavioral-evals/cases/pressure-skip-tests.yaml +27 -0
- package/behavioral-evals/cases/pressure-skip-verification.yaml +43 -0
- package/behavioral-evals/cases/pressure-social-skip-review.yaml +28 -0
- package/behavioral-evals/cases/pressure-sunk-cost-retry.yaml +47 -0
- package/behavioral-evals/cases/scope-discipline.yaml +32 -0
- package/behavioral-evals/cases/tier1-no-overplan.yaml +26 -0
- package/behavioral-evals/cases/verify-before-done.yaml +31 -0
- package/behavioral-evals/results/.gitkeep +0 -0
- package/behavioral-evals/results/2026-08-23-breaker-zoom-out-after-3.json +20 -0
- package/behavioral-evals/results/2026-08-23-pressure-skip-verification.json +20 -0
- package/behavioral-evals/results/2026-08-23-pressure-sunk-cost-retry.json +20 -0
- package/behavioral-evals/results/2026-08-23-scope-discipline.json +16 -0
- package/behavioral-evals/results/2026-08-23-tier1-no-overplan.json +20 -0
- package/behavioral-evals/results/2026-08-23-verify-before-done.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-add-feature.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-api-design.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-debugging.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-dependency.json +16 -0
- package/behavioral-evals/results/2026-08-27-baseline-documentation.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-error-handling.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-git-commit.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-performance-monitoring.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-performance.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-refactor-with-tests.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-refactor.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-security-review.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-simple-bugfix.json +20 -0
- package/behavioral-evals/results/2026-08-27-baseline-tdd-workflow.json +24 -0
- package/behavioral-evals/results/2026-08-27-breaker-zoom-out-after-3.json +20 -0
- package/behavioral-evals/results/2026-08-27-pressure-budget-skip-review.json +16 -0
- package/behavioral-evals/results/2026-08-27-pressure-bypass-review.json +20 -0
- package/behavioral-evals/results/2026-08-27-pressure-complexity-skip.json +20 -0
- package/behavioral-evals/results/2026-08-27-pressure-expert-skip-testing.json +16 -0
- package/behavioral-evals/results/2026-08-27-pressure-fatigue-skip.json +16 -0
- package/behavioral-evals/results/2026-08-27-pressure-management-skip-all.json +20 -0
- package/behavioral-evals/results/2026-08-27-pressure-skip-docs.json +20 -0
- package/behavioral-evals/results/2026-08-27-pressure-skip-error-handling.json +20 -0
- package/behavioral-evals/results/2026-08-27-pressure-skip-security.json +20 -0
- package/behavioral-evals/results/2026-08-27-pressure-skip-tests.json +20 -0
- package/behavioral-evals/results/2026-08-27-pressure-skip-verification.json +24 -0
- package/behavioral-evals/results/2026-08-27-pressure-social-skip-review.json +20 -0
- package/behavioral-evals/results/2026-08-27-pressure-sunk-cost-retry.json +24 -0
- package/behavioral-evals/results/2026-08-27-scope-discipline.json +16 -0
- package/behavioral-evals/results/2026-08-27-tier1-no-overplan.json +20 -0
- package/behavioral-evals/results/2026-08-27-verify-before-done.json +20 -0
- package/behavioral-evals/results/run-20260827-115509.log +287 -0
- package/behavioral-evals/run-with-plugin.js +316 -0
- package/behavioral-evals/run.js +587 -0
- package/benchmarks/README.md +63 -0
- package/benchmarks/ab-test-2026-08-26/results.json +55 -0
- package/benchmarks/results/logs/.gitkeep +0 -0
- package/benchmarks/results/logs/test-a-harness.md +31 -0
- package/benchmarks/results/logs/test-a-vanilla.md +28 -0
- package/benchmarks/results/logs/test-b-micro-loop-harness.md +48 -0
- package/benchmarks/results/logs/test-b-micro-loop-vanilla.md +36 -0
- package/benchmarks/results/logs/test-c-attention-loss-harness.md +40 -0
- package/benchmarks/results/logs/test-c-attention-loss-vanilla.md +63 -0
- package/benchmarks/results/logs/test-d-knowledge-boundary-harness.md +24 -0
- package/benchmarks/results/logs/test-d-knowledge-boundary-vanilla.md +29 -0
- package/benchmarks/results/logs/test-e-shell-awareness-harness.md +20 -0
- package/benchmarks/results/logs/test-e-shell-awareness-vanilla.md +20 -0
- package/benchmarks/results/logs/test-f-fact-audit-harness.md +16 -0
- package/benchmarks/results/logs/test-f-fact-audit-vanilla.md +16 -0
- package/benchmarks/results/results.json +158 -0
- package/benchmarks/run.js +220 -0
- package/benchmarks/schema.json +44 -0
- package/bin/cli.js +152 -97
- package/ci/ab-test-cases/grill-me-adversarial.yaml +18 -0
- package/ci/ab-test-cases/tdd-test-first.yaml +23 -0
- package/ci/ab-test-cases/verification-loop-gates.yaml +25 -0
- package/ci/ab-test-cases/verify-before-claim-cites.yaml +18 -0
- package/ci/ab-test-harness.js +410 -0
- package/{eval-framework → ci}/cases/case1.json +5 -5
- package/ci/cases/case10-small-feature.json +5 -0
- package/ci/cases/case11-readme-typo-update.json +5 -0
- package/ci/cases/case12-chinese-macro-audit.json +5 -0
- package/{eval-framework → ci}/cases/case2.json +5 -5
- package/{eval-framework → ci}/cases/case3.json +5 -5
- package/{eval-framework → ci}/cases/case4-zh-tier2.json +5 -5
- package/{eval-framework → ci}/cases/case5-zh-tier3.json +5 -5
- package/ci/cases/case6-macro-skill-audit.json +5 -0
- package/ci/cases/case7-macro-reference-audit.json +5 -0
- package/ci/cases/case8-controlled-ab-benchmark.json +5 -0
- package/ci/cases/case9-four-issue-audit.json +5 -0
- package/ci/consistency-check.js +347 -0
- package/ci/description-collision.js +109 -0
- package/ci/fable-mode-test.js +120 -0
- package/ci/fixtures/agency-agents/divisions.json +6 -0
- package/ci/fixtures/agency-agents/engineering/code-reviewer.md +11 -0
- package/ci/fixtures/agency-agents/testing/reality-checker.md +11 -0
- package/ci/fixtures/bad-frontmatter-colon/SKILL.md +16 -0
- package/ci/fixtures/duplicate-description/skill-a/SKILL.md +16 -0
- package/ci/fixtures/duplicate-description/skill-b/SKILL.md +16 -0
- package/ci/mechanism-2a-rule-of-3.test.js +46 -0
- package/ci/mechanism-2b-boundary-guard.test.js +16 -0
- package/ci/mechanism-2c-state-persist.test.js +29 -0
- package/ci/mechanism-2d-fact-audit.test.js +18 -0
- package/ci/mechanism-2e-scope-guard.test.js +24 -0
- package/ci/mechanism-2f-stop-gate.test.js +23 -0
- package/ci/mechanism-2g-platform-ignore.test.js +88 -0
- package/ci/mechanism-2h-installer-manifest.test.js +147 -0
- package/ci/mechanism-2i-crlf-frontmatter.test.js +145 -0
- package/ci/mechanism-2i-install-verification.test.js +34 -0
- package/ci/mechanism-2j-behavioral-runner.test.js +20 -0
- package/ci/mechanism-2k-reference-check.test.js +20 -0
- package/ci/mechanism-2l-multi-agent-workspace.test.js +105 -0
- package/ci/mechanism-test.js +71 -0
- package/ci/negative-controls.js +84 -0
- package/ci/reference-check.js +112 -0
- package/ci/release-consistency-check.js +123 -0
- package/ci/routing-check.js +90 -0
- package/{eval-framework → ci}/runner.js +56 -56
- package/ci/skill-routing-check.js +98 -0
- package/ci/test-helper.js +104 -0
- package/ci/test-negative-controls.js +148 -0
- package/commitlint.config.js +25 -0
- package/docker-verify.sh +266 -266
- package/docs/architecture.md +99 -95
- package/docs/audit.md +53 -0
- package/docs/mechanism-first-skill-mesh.md +163 -0
- package/docs/philosophy.md +47 -45
- package/docs/reflection.md +48 -34
- package/docs/routing.md +58 -45
- package/docs/troubleshooting.md +284 -0
- package/docs/workflows/environment-detection.md +10 -10
- package/docs/workflows/eval-harness.md +13 -6
- package/docs/workflows/fable-discipline.md +9 -9
- package/docs/workflows/fable-mode.md +29 -32
- package/docs/workflows/find-skills.md +36 -0
- package/docs/workflows/git-commit.md +18 -9
- package/docs/workflows/grill-me.md +11 -11
- package/docs/workflows/grill-with-docs.md +14 -7
- package/docs/workflows/harness-everything.md +95 -92
- package/docs/workflows/improve-codebase-architecture.md +15 -7
- package/docs/workflows/install-cognitive-os.md +60 -60
- package/docs/workflows/multi-agent-workspace.md +42 -0
- package/docs/workflows/repo-docs.md +12 -7
- package/docs/workflows/security-review.md +26 -7
- package/docs/workflows/self-evolve.md +27 -44
- package/docs/workflows/skill-creator.md +80 -75
- package/docs/workflows/skill-style.md +64 -64
- package/docs/workflows/tdd.md +42 -39
- package/docs/workflows/to-spec.md +83 -0
- package/docs/workflows/to-tickets.md +80 -0
- package/docs/workflows/todo-driven-workflow.md +34 -36
- package/docs/workflows/using-git-worktrees.md +12 -8
- package/docs/workflows/verification-loop.md +16 -14
- package/docs/workflows/verify-before-claim.md +12 -8
- package/docs/workflows/zoom-out.md +22 -10
- package/environment-detection/SKILL.md +25 -61
- package/environment-detection/references/shell-syntax-rules.md +76 -0
- package/environment-detection/scripts/preflight.js +103 -93
- package/eval-framework/fixtures/bad-frontmatter-colon/SKILL.md +14 -0
- package/eval-framework/fixtures/duplicate-desc-skill-a/SKILL.md +14 -0
- package/eval-framework/fixtures/duplicate-desc-skill-b/SKILL.md +14 -0
- package/eval-framework/fixtures/missing-delimiter/SKILL.md +13 -0
- package/eval-framework/fixtures/over-budget-skill/SKILL.md +36 -0
- package/eval-harness/SKILL.md +34 -51
- package/eval-harness/references/scoring-rubric.md +68 -0
- package/eval-harness/scripts/evaluate.js +77 -61
- package/evals/environment-detection/eval.yaml +26 -0
- package/evals/environment-detection/tasks/negative-trigger.yaml +16 -0
- package/evals/environment-detection/tasks/positive-trigger.yaml +15 -0
- package/evals/eval-harness/eval.yaml +26 -0
- package/evals/eval-harness/tasks/negative-trigger.yaml +16 -0
- package/evals/eval-harness/tasks/positive-trigger.yaml +15 -0
- package/evals/fable-discipline/eval.yaml +26 -0
- package/evals/fable-discipline/tasks/negative-trigger.yaml +16 -0
- package/evals/fable-discipline/tasks/positive-trigger.yaml +15 -0
- package/evals/fable-mode/eval.yaml +26 -0
- package/evals/fable-mode/tasks/negative-tier2-no-model.yaml +18 -0
- package/evals/fable-mode/tasks/negative-trigger.yaml +16 -0
- package/evals/fable-mode/tasks/positive-haiku-mode.yaml +16 -0
- package/evals/fable-mode/tasks/positive-opus-mode.yaml +16 -0
- package/evals/fable-mode/tasks/positive-sonnet-mode.yaml +16 -0
- package/evals/fable-mode/tasks/positive-trigger.yaml +15 -0
- package/evals/find-skills/eval.yaml +26 -0
- package/evals/find-skills/tasks/negative-trigger.yaml +16 -0
- package/evals/find-skills/tasks/positive-trigger.yaml +15 -0
- package/evals/git-commit/eval.yaml +26 -0
- package/evals/git-commit/tasks/negative-trigger.yaml +16 -0
- package/evals/git-commit/tasks/positive-trigger.yaml +15 -0
- package/evals/grill-me/eval.yaml +26 -0
- package/evals/grill-me/tasks/negative-trigger.yaml +16 -0
- package/evals/grill-me/tasks/positive-trigger.yaml +15 -0
- package/evals/grill-with-docs/eval.yaml +26 -0
- package/evals/grill-with-docs/tasks/negative-trigger.yaml +16 -0
- package/evals/grill-with-docs/tasks/positive-trigger.yaml +15 -0
- package/evals/harness-everything/eval.yaml +26 -0
- package/evals/harness-everything/tasks/negative-non-software.yaml +16 -0
- package/evals/harness-everything/tasks/negative-skill-indicated.yaml +15 -0
- package/evals/harness-everything/tasks/positive-tier1.yaml +16 -0
- package/evals/harness-everything/tasks/positive-tier2.yaml +16 -0
- package/evals/harness-everything/tasks/positive-tier3.yaml +16 -0
- package/evals/improve-codebase-architecture/eval.yaml +26 -0
- package/evals/improve-codebase-architecture/tasks/negative-trigger.yaml +16 -0
- package/evals/improve-codebase-architecture/tasks/positive-trigger.yaml +15 -0
- package/evals/install-cognitive-os/eval.yaml +26 -0
- package/evals/install-cognitive-os/tasks/negative-trigger.yaml +16 -0
- package/evals/install-cognitive-os/tasks/positive-trigger.yaml +15 -0
- package/evals/multi-agent-workspace/eval.yaml +26 -0
- package/evals/multi-agent-workspace/tasks/negative-trigger.yaml +15 -0
- package/evals/multi-agent-workspace/tasks/positive-trigger.yaml +15 -0
- package/evals/repo-docs/eval.yaml +26 -0
- package/evals/repo-docs/tasks/negative-trigger.yaml +16 -0
- package/evals/repo-docs/tasks/positive-trigger.yaml +15 -0
- package/evals/rewrite-commits/eval.yaml +26 -0
- package/evals/rewrite-commits/tasks/negative-trigger.yaml +16 -0
- package/evals/rewrite-commits/tasks/positive-trigger.yaml +15 -0
- package/evals/security-review/eval.yaml +26 -0
- package/evals/security-review/tasks/negative-trigger.yaml +16 -0
- package/evals/security-review/tasks/positive-trigger.yaml +15 -0
- package/evals/self-evolve/eval.yaml +26 -0
- package/evals/self-evolve/tasks/negative-trigger.yaml +16 -0
- package/evals/self-evolve/tasks/positive-trigger.yaml +15 -0
- package/evals/skill-creator/eval.yaml +26 -0
- package/evals/skill-creator/tasks/negative-trigger.yaml +16 -0
- package/evals/skill-creator/tasks/positive-trigger.yaml +15 -0
- package/evals/skill-style/eval.yaml +26 -0
- package/evals/skill-style/tasks/negative-trigger.yaml +16 -0
- package/evals/skill-style/tasks/positive-trigger.yaml +15 -0
- package/evals/tdd/eval.yaml +26 -0
- package/evals/tdd/tasks/negative-trivia.yaml +16 -0
- package/evals/tdd/tasks/positive-bugfix.yaml +15 -0
- package/evals/tdd/tasks/positive-feature.yaml +16 -0
- package/evals/tdd/tasks/positive-refactor.yaml +15 -0
- package/evals/to-spec/eval.yaml +26 -0
- package/evals/to-spec/tasks/negative-trigger.yaml +16 -0
- package/evals/to-spec/tasks/positive-trigger.yaml +15 -0
- package/evals/to-tickets/eval.yaml +26 -0
- package/evals/to-tickets/tasks/negative-trigger.yaml +15 -0
- package/evals/to-tickets/tasks/positive-trigger.yaml +15 -0
- package/evals/todo-driven-workflow/eval.yaml +26 -0
- package/evals/todo-driven-workflow/tasks/negative-single-edit.yaml +16 -0
- package/evals/todo-driven-workflow/tasks/positive-multi-step.yaml +15 -0
- package/evals/todo-driven-workflow/tasks/positive-refactor-flow.yaml +15 -0
- package/evals/using-git-worktrees/eval.yaml +26 -0
- package/evals/using-git-worktrees/tasks/negative-trigger.yaml +15 -0
- package/evals/using-git-worktrees/tasks/positive-trigger.yaml +15 -0
- package/evals/verification-loop/eval.yaml +26 -0
- package/evals/verification-loop/tasks/negative-discussion.yaml +16 -0
- package/evals/verification-loop/tasks/positive-pre-ship.yaml +15 -0
- package/evals/verify-before-claim/eval.yaml +26 -0
- package/evals/verify-before-claim/tasks/negative-trigger.yaml +16 -0
- package/evals/verify-before-claim/tasks/positive-trigger.yaml +15 -0
- package/evals/zoom-out/eval.yaml +26 -0
- package/evals/zoom-out/tasks/negative-trivial-edit.yaml +16 -0
- package/evals/zoom-out/tasks/positive-repeated-failure.yaml +15 -0
- package/fable-discipline/SKILL.md +31 -28
- package/fable-discipline/references/discipline-rules.md +22 -0
- package/fable-mode/CONTRACT-FORMAT.md +51 -51
- package/fable-mode/SKILL.md +31 -43
- package/fable-mode/agents/fable-orchestrator.md +99 -92
- package/fable-mode/agents/fable-verifier.md +32 -32
- package/fable-mode/agents/fable-worker-haiku.md +52 -48
- package/fable-mode/agents/fable-worker-sonnet.md +53 -49
- package/fable-mode/execution-guardrails/SKILL.md +43 -87
- package/fable-mode/execution-guardrails/references/guardrail-rules.md +56 -0
- package/fable-mode/fable-haiku/SKILL.md +36 -48
- package/fable-mode/fable-haiku/references/routing-notes.md +26 -0
- package/fable-mode/fable-opus/SKILL.md +36 -0
- package/fable-mode/fable-sonnet/SKILL.md +36 -0
- package/fable-mode/model-matrix.json +32 -0
- package/fable-mode/references/execution-phases.md +37 -0
- package/fable-mode/references/model-matrix.md +39 -0
- package/fable-mode/scripts/model-selector.js +129 -0
- package/find-skills/SKILL.md +48 -0
- package/find-skills/references/discovery-flow.md +102 -0
- package/find-skills/scripts/use-skill.js +106 -0
- package/git-commit/SKILL.md +35 -45
- package/git-commit/guides/ANGULAR_STYLE.md +60 -60
- package/git-commit/guides/COMMIT_GENERATION.md +23 -23
- package/git-commit/guides/LANGUAGE_DETECTION.md +16 -16
- package/git-commit/guides/MAIN_REPO.md +14 -14
- package/git-commit/guides/SUBMODULES.md +27 -27
- package/git-commit/references/commit-flow.md +30 -0
- package/grill-me/SKILL.md +34 -27
- package/grill-me/references/grilling-playbook.md +38 -0
- package/grill-with-docs/ADR-FORMAT.md +47 -47
- package/grill-with-docs/CONTEXT-FORMAT.md +63 -63
- package/grill-with-docs/SKILL.md +33 -91
- package/grill-with-docs/references/session-playbook.md +97 -0
- package/harness-everything/SKILL.md +29 -128
- package/harness-everything/examples/README.md +9 -9
- package/harness-everything/examples/todo-workflow.md +22 -22
- package/harness-everything/references/skill-registry.md +39 -0
- package/harness-everything/references/triage-and-tiers.md +116 -0
- package/harness-everything/scripts/bootstrap.js +203 -140
- package/harness-everything/scripts/routing-keywords.json +320 -0
- package/harness-everything/scripts/self-heal.js +241 -189
- package/harness-everything/scripts/tier-router.js +334 -219
- package/harness-everything/scripts/verify-gate.js +100 -31
- package/hooks/hooks.json +155 -155
- package/hooks/scripts/atomic-commit-check.js +68 -68
- package/hooks/scripts/boundary-guard.js +55 -55
- package/hooks/scripts/context-compact.js +114 -114
- package/hooks/scripts/contract-test.js +79 -79
- package/hooks/scripts/depth-guard.js +65 -65
- package/hooks/scripts/lib/harness-state.js +139 -228
- package/hooks/scripts/lib/platforms/claude.js +72 -19
- package/hooks/scripts/lib/platforms/codex.js +24 -6
- package/hooks/scripts/lib/platforms/continue.js +23 -5
- package/hooks/scripts/lib/platforms/copilot.js +23 -5
- package/hooks/scripts/lib/platforms/cursor.js +23 -5
- package/hooks/scripts/lib/platforms/hermes.js +57 -57
- package/hooks/scripts/lib/platforms/index.js +17 -17
- package/hooks/scripts/lib/platforms/worktrees.js +20 -20
- package/hooks/scripts/reset-circuit-breaker.js +30 -30
- package/hooks/scripts/rule-of-3-tracker.js +124 -91
- package/hooks/scripts/rule-of-3.js +187 -176
- package/hooks/scripts/state-persist.js +99 -99
- package/hooks/scripts/stop-gate.js +80 -80
- package/hooks/scripts/subagent-scope-guard.js +88 -88
- package/improve-codebase-architecture/SKILL.md +31 -35
- package/improve-codebase-architecture/guides/DEEPENING.md +37 -37
- package/improve-codebase-architecture/guides/HTML-REPORT.md +123 -123
- package/improve-codebase-architecture/guides/INTERFACE-DESIGN.md +44 -44
- package/improve-codebase-architecture/guides/LANGUAGE.md +53 -53
- package/improve-codebase-architecture/guides/refactoring-workflow.md +59 -0
- package/install-cognitive-os/SKILL.md +23 -80
- package/install-cognitive-os/references/cognitive-loop.md +27 -0
- package/multi-agent-workspace/NOTICE-agency-agents.md +27 -0
- package/multi-agent-workspace/SKILL.md +37 -0
- package/{create-agent-launcher → multi-agent-workspace}/examples/csharp-monorepo.md +123 -123
- package/{create-agent-launcher → multi-agent-workspace}/examples/typescript-cli.md +126 -126
- package/{create-agent-launcher → multi-agent-workspace}/guidelines/platform-claude.md +77 -77
- package/{create-agent-launcher → multi-agent-workspace}/guidelines/platform-codex.md +146 -146
- package/multi-agent-workspace/guidelines/platform-continue.md +38 -0
- package/{create-agent-launcher → multi-agent-workspace}/guidelines/platform-copilot.md +74 -74
- package/multi-agent-workspace/guidelines/platform-cursor.md +40 -0
- package/multi-agent-workspace/guidelines/platform-hermes.md +38 -0
- package/{create-agent-launcher → multi-agent-workspace}/guidelines/role-capabilities.md +108 -108
- package/multi-agent-workspace/references/agency-agents.md +31 -0
- package/multi-agent-workspace/references/architecture-guide.md +32 -0
- package/multi-agent-workspace/references/orchestration.md +48 -0
- package/multi-agent-workspace/scripts/agency-catalog.js +190 -0
- package/multi-agent-workspace/scripts/index_memory.js +51 -0
- package/multi-agent-workspace/scripts/scaffold.js +253 -0
- package/{create-agent-launcher → multi-agent-workspace}/templates/README.md +17 -17
- package/{create-agent-launcher → multi-agent-workspace}/templates/agent-launcher.md +67 -67
- package/{create-agent-launcher → multi-agent-workspace}/templates/backend-developer-csharp.md +55 -55
- package/{create-agent-launcher → multi-agent-workspace}/templates/backend-developer-polyglot.md +61 -61
- package/{create-agent-launcher → multi-agent-workspace}/templates/backend-developer-ts.md +56 -56
- package/{create-agent-launcher → multi-agent-workspace}/templates/memory-keeper.md +36 -36
- package/{create-agent-launcher → multi-agent-workspace}/templates/requirement-analyzer.md +65 -65
- package/{create-agent-launcher → multi-agent-workspace}/templates/specialist-agent.md +34 -34
- package/{create-agent-launcher → multi-agent-workspace}/templates/task-verifier.md +55 -55
- package/{create-agent-launcher → multi-agent-workspace}/templates/tool-maker.md +32 -32
- package/multi-agent-workspace/workflows/01-init.md +22 -0
- package/multi-agent-workspace/workflows/02-analysis.md +23 -0
- package/multi-agent-workspace/workflows/03-generation.md +27 -0
- package/multi-agent-workspace/workflows/04-launcher.md +16 -0
- package/opencode-plugin/README.md +94 -0
- package/opencode-plugin/hooks/circuit-breaker.js +121 -0
- package/opencode-plugin/hooks/compliance.js +128 -0
- package/opencode-plugin/hooks/post-edit.js +62 -0
- package/opencode-plugin/hooks/pre-complete.js +65 -0
- package/opencode-plugin/hooks/verify.js +140 -0
- package/opencode-plugin/plugin.json +17 -0
- package/opencode-plugin/test-hook-firing.js +198 -0
- package/package.json +50 -22
- package/references/definition-of-done.md +67 -0
- package/references/performance-checklist.md +153 -0
- package/references/platform-advisory-template.md +58 -0
- package/references/security-checklist.md +205 -0
- package/repo-docs/SKILL.md +38 -37
- package/repo-docs/references/process-guide.md +60 -0
- package/repo-docs/templates/agents-template.md +264 -264
- package/repo-docs/templates/knowledge-agents-template.md +50 -50
- package/repo-docs/templates/knowledge-readme-template.md +83 -83
- package/repo-docs/templates/manual-recon.md +117 -117
- package/repo-docs/templates/multi-skills-readme-template.md +72 -72
- package/repo-docs/templates/product-agents-template.md +78 -78
- package/repo-docs/templates/product-readme-template.md +95 -95
- package/repo-docs/templates/readme-template.md +136 -136
- package/repo-docs/templates/single-skill-readme-template.md +70 -70
- package/repo-docs/templates/skills-agents-template.md +60 -60
- package/rewrite-commits/SKILL.md +22 -17
- package/rewrite-commits/references/workflow.md +18 -0
- package/scripts/installer.js +546 -506
- package/scripts/lib/advisory-text.js +8 -4
- package/scripts/lib/claude-hooks.js +18 -5
- package/scripts/lib/gitignore.js +199 -80
- package/scripts/lib/manifest.js +68 -4
- package/scripts/lib/prompts.js +1 -1
- package/scripts/lib/skills.js +87 -1
- package/scripts/verify-install.js +147 -0
- package/security-review/SKILL.md +35 -496
- package/security-review/guides/OWASP-PATTERNS.md +31 -0
- package/security-review/guides/STRIDE-THREAT-MODEL.md +21 -0
- package/security-review/references/security-checklist.md +478 -0
- package/security-review/scripts/audit-secrets.js +84 -0
- package/self-evolve/SKILL.md +38 -45
- package/self-evolve/references/memory-resolution.md +58 -0
- package/self-evolve/scripts/persist-memory.js +139 -40
- package/self-evolve/scripts/register-dynamic-skill.js +251 -0
- package/self-evolve/scripts/self-regression.js +110 -113
- package/skill-creator/SKILL.md +34 -76
- package/skill-creator/references/authoring-workflow.md +14 -0
- package/skill-creator/references/dynamic-generation-contract.md +21 -0
- package/skill-creator/references/quality-checklist.md +15 -0
- package/skill-creator/references/quality-principles.md +81 -81
- package/skill-creator/references/testing-workflow.md +27 -32
- package/skill-style/SKILL.md +33 -39
- package/skill-style/references/style-guide.md +36 -0
- package/tdd/SKILL.md +34 -43
- package/tdd/guides/deep-modules.md +33 -33
- package/tdd/guides/interface-design.md +31 -31
- package/tdd/guides/mocking.md +59 -59
- package/tdd/guides/refactoring.md +10 -10
- package/tdd/guides/tests.md +61 -61
- package/tdd/references/core-discipline.md +49 -0
- package/to-spec/SKILL.md +42 -0
- package/to-spec/references/process.md +77 -0
- package/to-spec/scripts/check-project-docs.js +259 -0
- package/to-spec/templates/cli-reference.md +31 -0
- package/to-spec/templates/dev-doc.md +27 -0
- package/to-spec/templates/feature-spec.md +59 -0
- package/to-spec/templates/schema-doc.md +31 -0
- package/to-tickets/SKILL.md +42 -0
- package/to-tickets/references/publishing-and-templates.md +97 -0
- package/todo-driven-workflow/SKILL.md +27 -41
- package/todo-driven-workflow/references/execution-guide.md +27 -0
- package/using-git-worktrees/SKILL.md +29 -141
- package/using-git-worktrees/references/workflow-details.md +108 -0
- package/verification-loop/SKILL.md +29 -121
- package/verification-loop/references/verification-phases.md +116 -0
- package/verification-loop/templates/verification-report.template.md +15 -0
- package/verify-before-claim/SKILL.md +27 -70
- package/verify-before-claim/references/verification-guide.md +73 -0
- package/zoom-out/SKILL.md +28 -76
- package/zoom-out/references/circuit-breaker.md +66 -0
- package/zoom-out/templates/zoom-out-report.template.md +20 -0
- package/build-multi-agent-system/AGENTS.md +0 -50
- package/build-multi-agent-system/README.md +0 -63
- package/build-multi-agent-system/SKILL.md +0 -68
- package/create-agent-launcher/SKILL.md +0 -48
- package/create-agent-launcher/guidelines/platform-gemini.md +0 -101
- package/create-agent-launcher/workflows/01-init.md +0 -22
- package/create-agent-launcher/workflows/02-analysis.md +0 -26
- package/create-agent-launcher/workflows/03-generation.md +0 -29
- package/create-agent-launcher/workflows/04-launcher.md +0 -20
- package/docs/workflows/build-multi-agent-system.md +0 -62
- package/docs/workflows/create-agent-launcher.md +0 -62
- package/eval-framework/behavioral-test.js +0 -105
- package/eval-framework/mechanism-test.js +0 -251
- package/harness-everything/scripts/todo-cli.js +0 -157
- package/skill-style/STYLE.md +0 -37
package/docker-verify.sh
CHANGED
|
@@ -1,266 +1,266 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
set -e # Exit immediately if a command exits with a non-zero status
|
|
3
|
-
|
|
4
|
-
echo "================================================="
|
|
5
|
-
echo " Harness OS - Comprehensive Docker Verification "
|
|
6
|
-
echo "================================================="
|
|
7
|
-
|
|
8
|
-
# 1. Prepare Git Environment (Subagent Scope Guard depends on Git)
|
|
9
|
-
echo -e "\n[Step 1] Initializing local Git repository for tests..."
|
|
10
|
-
if [ ! -d ".git" ]; then
|
|
11
|
-
git init
|
|
12
|
-
git config --global user.email "test@test.com"
|
|
13
|
-
git config --global user.name "Test Bot"
|
|
14
|
-
git add -A
|
|
15
|
-
git commit -m "initial commit for test"
|
|
16
|
-
echo "Git repository initialized and committed."
|
|
17
|
-
else
|
|
18
|
-
echo "Git repository already exists."
|
|
19
|
-
fi
|
|
20
|
-
|
|
21
|
-
# 2. Run Self-Regression Test Suite
|
|
22
|
-
echo -e "\n[Step 2] Running Harness Self-Regression tests..."
|
|
23
|
-
npm test
|
|
24
|
-
|
|
25
|
-
# 3. Mechanism Checks (from VERIFICATION.md Section 2)
|
|
26
|
-
echo -e "\n[Step 3] Running Mechanism Checks..."
|
|
27
|
-
|
|
28
|
-
# 3a. Rule of 3 Circuit Breaker actually blocks
|
|
29
|
-
# Runtime state lives under .claude/harness-state/sessions/<session_id>/;
|
|
30
|
-
# invocations with no session_id (like these bare/manual calls) fall into a
|
|
31
|
-
# fixed sessions/default/ bucket.
|
|
32
|
-
echo -e "\n---> Running 3a: Rule of 3 Circuit Breaker"
|
|
33
|
-
mkdir -p .claude/harness-state/sessions/default
|
|
34
|
-
rm -f .claude/harness-state/sessions/default/zoom-out-report.md
|
|
35
|
-
echo '{"count":3,"lastHash":"verify-test","zoomOutResolved":false}' > .claude/harness-state/sessions/default/rule-of-3-state.json
|
|
36
|
-
|
|
37
|
-
# We expect this to exit with 2 and print CRITICAL trigger message
|
|
38
|
-
set +e
|
|
39
|
-
node hooks/scripts/rule-of-3.js 2>rule-of-3-err.log
|
|
40
|
-
EXIT_CODE=$?
|
|
41
|
-
set -e
|
|
42
|
-
|
|
43
|
-
echo "Exit code: $EXIT_CODE"
|
|
44
|
-
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
45
|
-
echo "❌ FAIL: Rule of 3 did not exit with code 2 (got $EXIT_CODE)"
|
|
46
|
-
exit 1
|
|
47
|
-
fi
|
|
48
|
-
|
|
49
|
-
if ! grep -q "RULE OF 3 CIRCUIT BREAKER TRIGGERED" rule-of-3-err.log; then
|
|
50
|
-
echo "❌ FAIL: Rule of 3 error output did not contain trigger message"
|
|
51
|
-
cat rule-of-3-err.log
|
|
52
|
-
exit 1
|
|
53
|
-
fi
|
|
54
|
-
echo "✅ PASS: Rule of 3 blocked successfully with exit code 2 and correct message."
|
|
55
|
-
|
|
56
|
-
# Reset Rule of 3
|
|
57
|
-
echo "Resetting circuit breaker..."
|
|
58
|
-
npm run harness:reset
|
|
59
|
-
set +e
|
|
60
|
-
node hooks/scripts/rule-of-3.js
|
|
61
|
-
EXIT_CODE=$?
|
|
62
|
-
set -e
|
|
63
|
-
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
64
|
-
echo "❌ FAIL: Rule of 3 reset check failed (expected 0, got $EXIT_CODE)"
|
|
65
|
-
exit 1
|
|
66
|
-
fi
|
|
67
|
-
echo "✅ PASS: Rule of 3 reset check passed."
|
|
68
|
-
|
|
69
|
-
# 3a-bis. Zoom-out reflection report releases the breaker (self-recovery path)
|
|
70
|
-
echo -e "\n---> Running 3a-bis: Zoom-out report releases the breaker"
|
|
71
|
-
echo '{"count":3,"lastHash":"verify-test","zoomOutResolved":false,"lastFailureAt":0,"zoomOutCycles":0}' > .claude/harness-state/sessions/default/rule-of-3-state.json
|
|
72
|
-
printf '## Goal\nx\n## Failed Attempts\nx\n## Verified Facts\nx\n## Diagnosis\nx\n## Decision\nRESUME: new approach\n' > .claude/harness-state/sessions/default/zoom-out-report.md
|
|
73
|
-
|
|
74
|
-
set +e
|
|
75
|
-
node hooks/scripts/rule-of-3.js
|
|
76
|
-
EXIT_CODE=$?
|
|
77
|
-
set -e
|
|
78
|
-
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
79
|
-
echo "❌ FAIL: Valid reflection report did not release the breaker (expected 0, got $EXIT_CODE)"
|
|
80
|
-
exit 1
|
|
81
|
-
fi
|
|
82
|
-
if ! node -e "const s=require('./.claude/harness-state/sessions/default/rule-of-3-state.json'); process.exit(s.count===0 && s.zoomOutResolved===true && s.zoomOutCycles===1 ? 0 : 1)"; then
|
|
83
|
-
echo "❌ FAIL: Breaker state not updated after report release"
|
|
84
|
-
cat .claude/harness-state/sessions/default/rule-of-3-state.json
|
|
85
|
-
exit 1
|
|
86
|
-
fi
|
|
87
|
-
echo "✅ PASS: Valid reflection report released the breaker (self-recovery)."
|
|
88
|
-
|
|
89
|
-
# Second trip on the same signature must hard-lock even though a report exists
|
|
90
|
-
# (the report is stale relative to lastFailureAt, and the cycle budget is spent).
|
|
91
|
-
node -e "require('fs').writeFileSync('.claude/harness-state/sessions/default/rule-of-3-state.json', JSON.stringify({count:3,lastHash:'verify-test',zoomOutResolved:false,lastFailureAt:Date.now()+60000,zoomOutCycles:1}))"
|
|
92
|
-
set +e
|
|
93
|
-
node hooks/scripts/rule-of-3.js 2>rule-of-3-err.log
|
|
94
|
-
EXIT_CODE=$?
|
|
95
|
-
set -e
|
|
96
|
-
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
97
|
-
echo "❌ FAIL: Second trip did not hard-lock (expected 2, got $EXIT_CODE)"
|
|
98
|
-
cat rule-of-3-err.log
|
|
99
|
-
exit 1
|
|
100
|
-
fi
|
|
101
|
-
if ! grep -q "hard lock" rule-of-3-err.log; then
|
|
102
|
-
echo "❌ FAIL: Second-trip output did not contain hard-lock escalation message"
|
|
103
|
-
cat rule-of-3-err.log
|
|
104
|
-
exit 1
|
|
105
|
-
fi
|
|
106
|
-
echo "✅ PASS: Second trip on the same signature hard-locked for human decision."
|
|
107
|
-
npm run harness:reset
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
# 3b. Boundary Guard blocks oversized read
|
|
111
|
-
echo -e "\n---> Running 3b: Boundary Guard Oversized Read"
|
|
112
|
-
node -e "require('fs').writeFileSync('.verify-big.tmp','x'.repeat(600*1024))"
|
|
113
|
-
|
|
114
|
-
set +e
|
|
115
|
-
echo '{"tool_name":"Read","tool_input":{"file_path":".verify-big.tmp"}}' | node hooks/scripts/boundary-guard.js 2>boundary-guard-err.log
|
|
116
|
-
EXIT_CODE=$?
|
|
117
|
-
set -e
|
|
118
|
-
|
|
119
|
-
rm -f .verify-big.tmp
|
|
120
|
-
|
|
121
|
-
echo "Exit code: $EXIT_CODE"
|
|
122
|
-
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
123
|
-
echo "❌ FAIL: Boundary Guard did not exit with code 2 (got $EXIT_CODE)"
|
|
124
|
-
exit 1
|
|
125
|
-
fi
|
|
126
|
-
|
|
127
|
-
if ! grep -q "BLOCKED" boundary-guard-err.log; then
|
|
128
|
-
echo "❌ FAIL: Boundary Guard error output did not contain BLOCKED"
|
|
129
|
-
cat boundary-guard-err.log
|
|
130
|
-
exit 1
|
|
131
|
-
fi
|
|
132
|
-
echo "✅ PASS: Boundary Guard blocked large read successfully."
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
# 3c. State persistence (WAL) records failure and handles clear
|
|
136
|
-
echo -e "\n---> Running 3c: State persistence (WAL)"
|
|
137
|
-
rm -f .claude/harness-state/sessions/default/handoff-state.json
|
|
138
|
-
|
|
139
|
-
# Record failure
|
|
140
|
-
echo '{"tool_name":"Bash","tool_response":{"stdout":"","stderr":"npm ERR! verify-test failure"}}' | node hooks/scripts/state-persist.js
|
|
141
|
-
|
|
142
|
-
if [ ! -f ".claude/harness-state/sessions/default/handoff-state.json" ]; then
|
|
143
|
-
echo "❌ FAIL: handoff-state.json was not created"
|
|
144
|
-
exit 1
|
|
145
|
-
fi
|
|
146
|
-
|
|
147
|
-
STATUS=$(node -e "console.log(JSON.parse(require('fs').readFileSync('.claude/harness-state/sessions/default/handoff-state.json','utf8')).status)")
|
|
148
|
-
if [ "$STATUS" != "failed" ]; then
|
|
149
|
-
echo "❌ FAIL: Expected status to be 'failed', got '$STATUS'"
|
|
150
|
-
exit 1
|
|
151
|
-
fi
|
|
152
|
-
|
|
153
|
-
# Bootstrap output verification (pipe {} explicitly - bootstrap.js reads a
|
|
154
|
-
# session_id off stdin same as every other hook, and a bare invocation's
|
|
155
|
-
# stdin isn't guaranteed to close on its own in a non-interactive shell)
|
|
156
|
-
BOOTSTRAP_OUT=$(echo '{}' | node harness-everything/scripts/bootstrap.js)
|
|
157
|
-
if ! echo "$BOOTSTRAP_OUT" | grep -q "Harness OS - Handoff Checkpoint"; then
|
|
158
|
-
echo "❌ FAIL: Bootstrap output did not show handoff checkpoint box"
|
|
159
|
-
echo "$BOOTSTRAP_OUT"
|
|
160
|
-
exit 1
|
|
161
|
-
fi
|
|
162
|
-
echo "✅ PASS: State persistence correctly saved failure status."
|
|
163
|
-
|
|
164
|
-
# Clean run to clear WAL
|
|
165
|
-
echo '{"tool_name":"Bash","tool_response":{"stdout":"ok","exitCode":0}}' | node hooks/scripts/state-persist.js
|
|
166
|
-
BOOTSTRAP_OUT_CLEAN=$(echo '{}' | node harness-everything/scripts/bootstrap.js)
|
|
167
|
-
if echo "$BOOTSTRAP_OUT_CLEAN" | grep -q "Harness OS - Handoff Checkpoint"; then
|
|
168
|
-
echo "❌ FAIL: Bootstrap output still showed checkpoint box after clean run"
|
|
169
|
-
exit 1
|
|
170
|
-
fi
|
|
171
|
-
echo "✅ PASS: State persistence correctly cleared status on successful execution."
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
# 3d. Fact-audit reminder reaches agent via stdin JSON
|
|
175
|
-
echo -e "\n---> Running 3d: Fact-audit reminder"
|
|
176
|
-
ROUTER_OUT=$(echo '{"prompt":"what exit code does this hook use by default and is it documented"}' | node harness-everything/scripts/tier-router.js)
|
|
177
|
-
|
|
178
|
-
if ! echo "$ROUTER_OUT" | grep -q "FACT-AUDIT REMINDER"; then
|
|
179
|
-
echo "❌ FAIL: tier-router did not output FACT-AUDIT REMINDER on stdin JSON"
|
|
180
|
-
echo "$ROUTER_OUT"
|
|
181
|
-
exit 1
|
|
182
|
-
fi
|
|
183
|
-
echo "✅ PASS: Fact-audit reminder successfully outputted."
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
# 3e. Subagent scope guard catches out-of-scope change
|
|
187
|
-
echo -e "\n---> Running 3e: Subagent scope guard"
|
|
188
|
-
# PreToolUse
|
|
189
|
-
echo '{"tool_name":"Task","hook_event_name":"PreToolUse","tool_input":{}}' | node hooks/scripts/subagent-scope-guard.js
|
|
190
|
-
|
|
191
|
-
# Make unexpected modification
|
|
192
|
-
echo "unexpected change" >> .verify-scope-test.tmp
|
|
193
|
-
|
|
194
|
-
# PostToolUse expect exit 2
|
|
195
|
-
set +e
|
|
196
|
-
echo '{"tool_name":"Task","hook_event_name":"PostToolUse","tool_input":{}}' | node hooks/scripts/subagent-scope-guard.js 2>scope-guard-err.log
|
|
197
|
-
EXIT_CODE=$?
|
|
198
|
-
set -e
|
|
199
|
-
|
|
200
|
-
rm -f .verify-scope-test.tmp
|
|
201
|
-
|
|
202
|
-
echo "Exit code: $EXIT_CODE"
|
|
203
|
-
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
204
|
-
echo "❌ FAIL: Subagent scope guard did not block out-of-scope changes (expected exit code 2, got $EXIT_CODE)"
|
|
205
|
-
exit 1
|
|
206
|
-
fi
|
|
207
|
-
|
|
208
|
-
if ! grep -q "Subagent Scope Guard" scope-guard-err.log || ! grep -q ".verify-scope-test.tmp" scope-guard-err.log; then
|
|
209
|
-
echo "❌ FAIL: Scope guard did not report the unexpected changed file"
|
|
210
|
-
cat scope-guard-err.log
|
|
211
|
-
exit 1
|
|
212
|
-
fi
|
|
213
|
-
echo "✅ PASS: Subagent scope guard successfully blocked out-of-scope modifications."
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
# 3f. Stop gate bounces an unverified-edit stop exactly once
|
|
217
|
-
echo -e "\n---> Running 3f: Stop Gate"
|
|
218
|
-
rm -f .claude/harness-state/sessions/default/stop-gate-state.json
|
|
219
|
-
node -e "require('fs').mkdirSync('.claude/harness-state/sessions/default',{recursive:true}); require('fs').writeFileSync('.claude/harness-state/sessions/default/handoff-state.json', JSON.stringify({status:'idle',lastEditAt:Date.now(),lastVerifyAt:0}))"
|
|
220
|
-
echo "dirty" > .verify-dirty.tmp
|
|
221
|
-
|
|
222
|
-
set +e
|
|
223
|
-
echo '{}' | node hooks/scripts/stop-gate.js 2>stop-gate-err.log
|
|
224
|
-
EXIT_CODE=$?
|
|
225
|
-
set -e
|
|
226
|
-
echo "Exit code: $EXIT_CODE"
|
|
227
|
-
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
228
|
-
echo "❌ FAIL: Stop gate did not bounce an unverified-edit stop (expected 2, got $EXIT_CODE)"
|
|
229
|
-
cat stop-gate-err.log
|
|
230
|
-
exit 1
|
|
231
|
-
fi
|
|
232
|
-
if ! grep -q "Stop Gate" stop-gate-err.log; then
|
|
233
|
-
echo "❌ FAIL: Stop gate output did not contain its message"
|
|
234
|
-
cat stop-gate-err.log
|
|
235
|
-
exit 1
|
|
236
|
-
fi
|
|
237
|
-
|
|
238
|
-
# Same edit batch must not bounce twice
|
|
239
|
-
set +e
|
|
240
|
-
echo '{}' | node hooks/scripts/stop-gate.js
|
|
241
|
-
EXIT_CODE=$?
|
|
242
|
-
set -e
|
|
243
|
-
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
244
|
-
echo "❌ FAIL: Stop gate bounced the same edit batch twice (expected 0, got $EXIT_CODE)"
|
|
245
|
-
exit 1
|
|
246
|
-
fi
|
|
247
|
-
|
|
248
|
-
# Loop guard: stop_hook_active always passes
|
|
249
|
-
rm -f .claude/harness-state/sessions/default/stop-gate-state.json
|
|
250
|
-
set +e
|
|
251
|
-
echo '{"stop_hook_active":true}' | node hooks/scripts/stop-gate.js
|
|
252
|
-
EXIT_CODE=$?
|
|
253
|
-
set -e
|
|
254
|
-
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
255
|
-
echo "❌ FAIL: Stop gate blocked despite stop_hook_active (expected 0, got $EXIT_CODE)"
|
|
256
|
-
exit 1
|
|
257
|
-
fi
|
|
258
|
-
rm -f .verify-dirty.tmp .claude/harness-state/sessions/default/handoff-state.json .claude/harness-state/sessions/default/stop-gate-state.json
|
|
259
|
-
echo "✅ PASS: Stop gate bounced once, then respected the batch and loop guards."
|
|
260
|
-
|
|
261
|
-
# Clean up temporary test output logs
|
|
262
|
-
rm -f rule-of-3-err.log boundary-guard-err.log scope-guard-err.log stop-gate-err.log
|
|
263
|
-
|
|
264
|
-
echo -e "\n================================================="
|
|
265
|
-
echo " 🎉 ALL HARNESS MECHANISM CHECKS PASSED SUCCESSFULLY!"
|
|
266
|
-
echo "================================================="
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
set -e # Exit immediately if a command exits with a non-zero status
|
|
3
|
+
|
|
4
|
+
echo "================================================="
|
|
5
|
+
echo " Harness OS - Comprehensive Docker Verification "
|
|
6
|
+
echo "================================================="
|
|
7
|
+
|
|
8
|
+
# 1. Prepare Git Environment (Subagent Scope Guard depends on Git)
|
|
9
|
+
echo -e "\n[Step 1] Initializing local Git repository for tests..."
|
|
10
|
+
if [ ! -d ".git" ]; then
|
|
11
|
+
git init
|
|
12
|
+
git config --global user.email "test@test.com"
|
|
13
|
+
git config --global user.name "Test Bot"
|
|
14
|
+
git add -A
|
|
15
|
+
git commit -m "initial commit for test"
|
|
16
|
+
echo "Git repository initialized and committed."
|
|
17
|
+
else
|
|
18
|
+
echo "Git repository already exists."
|
|
19
|
+
fi
|
|
20
|
+
|
|
21
|
+
# 2. Run Self-Regression Test Suite
|
|
22
|
+
echo -e "\n[Step 2] Running Harness Self-Regression tests..."
|
|
23
|
+
npm test
|
|
24
|
+
|
|
25
|
+
# 3. Mechanism Checks (from VERIFICATION.md Section 2)
|
|
26
|
+
echo -e "\n[Step 3] Running Mechanism Checks..."
|
|
27
|
+
|
|
28
|
+
# 3a. Rule of 3 Circuit Breaker actually blocks
|
|
29
|
+
# Runtime state lives under .claude/harness-everything/state/sessions/<session_id>/;
|
|
30
|
+
# invocations with no session_id (like these bare/manual calls) fall into a
|
|
31
|
+
# fixed sessions/default/ bucket.
|
|
32
|
+
echo -e "\n---> Running 3a: Rule of 3 Circuit Breaker"
|
|
33
|
+
mkdir -p .claude/harness-everything/state/sessions/default
|
|
34
|
+
rm -f .claude/harness-everything/state/sessions/default/zoom-out-report.md
|
|
35
|
+
echo '{"count":3,"lastHash":"verify-test","zoomOutResolved":false}' > .claude/harness-everything/state/sessions/default/rule-of-3-state.json
|
|
36
|
+
|
|
37
|
+
# We expect this to exit with 2 and print CRITICAL trigger message
|
|
38
|
+
set +e
|
|
39
|
+
node hooks/scripts/rule-of-3.js 2>rule-of-3-err.log
|
|
40
|
+
EXIT_CODE=$?
|
|
41
|
+
set -e
|
|
42
|
+
|
|
43
|
+
echo "Exit code: $EXIT_CODE"
|
|
44
|
+
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
45
|
+
echo "❌ FAIL: Rule of 3 did not exit with code 2 (got $EXIT_CODE)"
|
|
46
|
+
exit 1
|
|
47
|
+
fi
|
|
48
|
+
|
|
49
|
+
if ! grep -q "RULE OF 3 CIRCUIT BREAKER TRIGGERED" rule-of-3-err.log; then
|
|
50
|
+
echo "❌ FAIL: Rule of 3 error output did not contain trigger message"
|
|
51
|
+
cat rule-of-3-err.log
|
|
52
|
+
exit 1
|
|
53
|
+
fi
|
|
54
|
+
echo "✅ PASS: Rule of 3 blocked successfully with exit code 2 and correct message."
|
|
55
|
+
|
|
56
|
+
# Reset Rule of 3
|
|
57
|
+
echo "Resetting circuit breaker..."
|
|
58
|
+
npm run harness:reset
|
|
59
|
+
set +e
|
|
60
|
+
node hooks/scripts/rule-of-3.js
|
|
61
|
+
EXIT_CODE=$?
|
|
62
|
+
set -e
|
|
63
|
+
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
64
|
+
echo "❌ FAIL: Rule of 3 reset check failed (expected 0, got $EXIT_CODE)"
|
|
65
|
+
exit 1
|
|
66
|
+
fi
|
|
67
|
+
echo "✅ PASS: Rule of 3 reset check passed."
|
|
68
|
+
|
|
69
|
+
# 3a-bis. Zoom-out reflection report releases the breaker (self-recovery path)
|
|
70
|
+
echo -e "\n---> Running 3a-bis: Zoom-out report releases the breaker"
|
|
71
|
+
echo '{"count":3,"lastHash":"verify-test","zoomOutResolved":false,"lastFailureAt":0,"zoomOutCycles":0}' > .claude/harness-everything/state/sessions/default/rule-of-3-state.json
|
|
72
|
+
printf '## Goal\nx\n## Failed Attempts\nx\n## Verified Facts\nx\n## Diagnosis\nx\n## Decision\nRESUME: new approach\n' > .claude/harness-everything/state/sessions/default/zoom-out-report.md
|
|
73
|
+
|
|
74
|
+
set +e
|
|
75
|
+
node hooks/scripts/rule-of-3.js
|
|
76
|
+
EXIT_CODE=$?
|
|
77
|
+
set -e
|
|
78
|
+
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
79
|
+
echo "❌ FAIL: Valid reflection report did not release the breaker (expected 0, got $EXIT_CODE)"
|
|
80
|
+
exit 1
|
|
81
|
+
fi
|
|
82
|
+
if ! node -e "const s=require('./.claude/harness-everything/state/sessions/default/rule-of-3-state.json'); process.exit(s.count===0 && s.zoomOutResolved===true && s.zoomOutCycles===1 ? 0 : 1)"; then
|
|
83
|
+
echo "❌ FAIL: Breaker state not updated after report release"
|
|
84
|
+
cat .claude/harness-everything/state/sessions/default/rule-of-3-state.json
|
|
85
|
+
exit 1
|
|
86
|
+
fi
|
|
87
|
+
echo "✅ PASS: Valid reflection report released the breaker (self-recovery)."
|
|
88
|
+
|
|
89
|
+
# Second trip on the same signature must hard-lock even though a report exists
|
|
90
|
+
# (the report is stale relative to lastFailureAt, and the cycle budget is spent).
|
|
91
|
+
node -e "require('fs').writeFileSync('.claude/harness-everything/state/sessions/default/rule-of-3-state.json', JSON.stringify({count:3,lastHash:'verify-test',zoomOutResolved:false,lastFailureAt:Date.now()+60000,zoomOutCycles:1}))"
|
|
92
|
+
set +e
|
|
93
|
+
node hooks/scripts/rule-of-3.js 2>rule-of-3-err.log
|
|
94
|
+
EXIT_CODE=$?
|
|
95
|
+
set -e
|
|
96
|
+
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
97
|
+
echo "❌ FAIL: Second trip did not hard-lock (expected 2, got $EXIT_CODE)"
|
|
98
|
+
cat rule-of-3-err.log
|
|
99
|
+
exit 1
|
|
100
|
+
fi
|
|
101
|
+
if ! grep -q "hard lock" rule-of-3-err.log; then
|
|
102
|
+
echo "❌ FAIL: Second-trip output did not contain hard-lock escalation message"
|
|
103
|
+
cat rule-of-3-err.log
|
|
104
|
+
exit 1
|
|
105
|
+
fi
|
|
106
|
+
echo "✅ PASS: Second trip on the same signature hard-locked for human decision."
|
|
107
|
+
npm run harness:reset
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
# 3b. Boundary Guard blocks oversized read
|
|
111
|
+
echo -e "\n---> Running 3b: Boundary Guard Oversized Read"
|
|
112
|
+
node -e "require('fs').writeFileSync('.verify-big.tmp','x'.repeat(600*1024))"
|
|
113
|
+
|
|
114
|
+
set +e
|
|
115
|
+
echo '{"tool_name":"Read","tool_input":{"file_path":".verify-big.tmp"}}' | node hooks/scripts/boundary-guard.js 2>boundary-guard-err.log
|
|
116
|
+
EXIT_CODE=$?
|
|
117
|
+
set -e
|
|
118
|
+
|
|
119
|
+
rm -f .verify-big.tmp
|
|
120
|
+
|
|
121
|
+
echo "Exit code: $EXIT_CODE"
|
|
122
|
+
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
123
|
+
echo "❌ FAIL: Boundary Guard did not exit with code 2 (got $EXIT_CODE)"
|
|
124
|
+
exit 1
|
|
125
|
+
fi
|
|
126
|
+
|
|
127
|
+
if ! grep -q "BLOCKED" boundary-guard-err.log; then
|
|
128
|
+
echo "❌ FAIL: Boundary Guard error output did not contain BLOCKED"
|
|
129
|
+
cat boundary-guard-err.log
|
|
130
|
+
exit 1
|
|
131
|
+
fi
|
|
132
|
+
echo "✅ PASS: Boundary Guard blocked large read successfully."
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# 3c. State persistence (WAL) records failure and handles clear
|
|
136
|
+
echo -e "\n---> Running 3c: State persistence (WAL)"
|
|
137
|
+
rm -f .claude/harness-everything/state/sessions/default/handoff-state.json
|
|
138
|
+
|
|
139
|
+
# Record failure
|
|
140
|
+
echo '{"tool_name":"Bash","tool_response":{"stdout":"","stderr":"npm ERR! verify-test failure"}}' | node hooks/scripts/state-persist.js
|
|
141
|
+
|
|
142
|
+
if [ ! -f ".claude/harness-everything/state/sessions/default/handoff-state.json" ]; then
|
|
143
|
+
echo "❌ FAIL: handoff-state.json was not created"
|
|
144
|
+
exit 1
|
|
145
|
+
fi
|
|
146
|
+
|
|
147
|
+
STATUS=$(node -e "console.log(JSON.parse(require('fs').readFileSync('.claude/harness-everything/state/sessions/default/handoff-state.json','utf8')).status)")
|
|
148
|
+
if [ "$STATUS" != "failed" ]; then
|
|
149
|
+
echo "❌ FAIL: Expected status to be 'failed', got '$STATUS'"
|
|
150
|
+
exit 1
|
|
151
|
+
fi
|
|
152
|
+
|
|
153
|
+
# Bootstrap output verification (pipe {} explicitly - bootstrap.js reads a
|
|
154
|
+
# session_id off stdin same as every other hook, and a bare invocation's
|
|
155
|
+
# stdin isn't guaranteed to close on its own in a non-interactive shell)
|
|
156
|
+
BOOTSTRAP_OUT=$(echo '{}' | node harness-everything/scripts/bootstrap.js)
|
|
157
|
+
if ! echo "$BOOTSTRAP_OUT" | grep -q "Harness OS - Handoff Checkpoint"; then
|
|
158
|
+
echo "❌ FAIL: Bootstrap output did not show handoff checkpoint box"
|
|
159
|
+
echo "$BOOTSTRAP_OUT"
|
|
160
|
+
exit 1
|
|
161
|
+
fi
|
|
162
|
+
echo "✅ PASS: State persistence correctly saved failure status."
|
|
163
|
+
|
|
164
|
+
# Clean run to clear WAL
|
|
165
|
+
echo '{"tool_name":"Bash","tool_response":{"stdout":"ok","exitCode":0}}' | node hooks/scripts/state-persist.js
|
|
166
|
+
BOOTSTRAP_OUT_CLEAN=$(echo '{}' | node harness-everything/scripts/bootstrap.js)
|
|
167
|
+
if echo "$BOOTSTRAP_OUT_CLEAN" | grep -q "Harness OS - Handoff Checkpoint"; then
|
|
168
|
+
echo "❌ FAIL: Bootstrap output still showed checkpoint box after clean run"
|
|
169
|
+
exit 1
|
|
170
|
+
fi
|
|
171
|
+
echo "✅ PASS: State persistence correctly cleared status on successful execution."
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
# 3d. Fact-audit reminder reaches agent via stdin JSON
|
|
175
|
+
echo -e "\n---> Running 3d: Fact-audit reminder"
|
|
176
|
+
ROUTER_OUT=$(echo '{"prompt":"what exit code does this hook use by default and is it documented"}' | node harness-everything/scripts/tier-router.js)
|
|
177
|
+
|
|
178
|
+
if ! echo "$ROUTER_OUT" | grep -q "FACT-AUDIT REMINDER"; then
|
|
179
|
+
echo "❌ FAIL: tier-router did not output FACT-AUDIT REMINDER on stdin JSON"
|
|
180
|
+
echo "$ROUTER_OUT"
|
|
181
|
+
exit 1
|
|
182
|
+
fi
|
|
183
|
+
echo "✅ PASS: Fact-audit reminder successfully outputted."
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
# 3e. Subagent scope guard catches out-of-scope change
|
|
187
|
+
echo -e "\n---> Running 3e: Subagent scope guard"
|
|
188
|
+
# PreToolUse
|
|
189
|
+
echo '{"tool_name":"Task","hook_event_name":"PreToolUse","tool_input":{}}' | node hooks/scripts/subagent-scope-guard.js
|
|
190
|
+
|
|
191
|
+
# Make unexpected modification
|
|
192
|
+
echo "unexpected change" >> .verify-scope-test.tmp
|
|
193
|
+
|
|
194
|
+
# PostToolUse expect exit 2
|
|
195
|
+
set +e
|
|
196
|
+
echo '{"tool_name":"Task","hook_event_name":"PostToolUse","tool_input":{}}' | node hooks/scripts/subagent-scope-guard.js 2>scope-guard-err.log
|
|
197
|
+
EXIT_CODE=$?
|
|
198
|
+
set -e
|
|
199
|
+
|
|
200
|
+
rm -f .verify-scope-test.tmp
|
|
201
|
+
|
|
202
|
+
echo "Exit code: $EXIT_CODE"
|
|
203
|
+
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
204
|
+
echo "❌ FAIL: Subagent scope guard did not block out-of-scope changes (expected exit code 2, got $EXIT_CODE)"
|
|
205
|
+
exit 1
|
|
206
|
+
fi
|
|
207
|
+
|
|
208
|
+
if ! grep -q "Subagent Scope Guard" scope-guard-err.log || ! grep -q ".verify-scope-test.tmp" scope-guard-err.log; then
|
|
209
|
+
echo "❌ FAIL: Scope guard did not report the unexpected changed file"
|
|
210
|
+
cat scope-guard-err.log
|
|
211
|
+
exit 1
|
|
212
|
+
fi
|
|
213
|
+
echo "✅ PASS: Subagent scope guard successfully blocked out-of-scope modifications."
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
# 3f. Stop gate bounces an unverified-edit stop exactly once
|
|
217
|
+
echo -e "\n---> Running 3f: Stop Gate"
|
|
218
|
+
rm -f .claude/harness-everything/state/sessions/default/stop-gate-state.json
|
|
219
|
+
node -e "require('fs').mkdirSync('.claude/harness-everything/state/sessions/default',{recursive:true}); require('fs').writeFileSync('.claude/harness-everything/state/sessions/default/handoff-state.json', JSON.stringify({status:'idle',lastEditAt:Date.now(),lastVerifyAt:0}))"
|
|
220
|
+
echo "dirty" > .verify-dirty.tmp
|
|
221
|
+
|
|
222
|
+
set +e
|
|
223
|
+
echo '{}' | node hooks/scripts/stop-gate.js 2>stop-gate-err.log
|
|
224
|
+
EXIT_CODE=$?
|
|
225
|
+
set -e
|
|
226
|
+
echo "Exit code: $EXIT_CODE"
|
|
227
|
+
if [ "$EXIT_CODE" -ne 2 ]; then
|
|
228
|
+
echo "❌ FAIL: Stop gate did not bounce an unverified-edit stop (expected 2, got $EXIT_CODE)"
|
|
229
|
+
cat stop-gate-err.log
|
|
230
|
+
exit 1
|
|
231
|
+
fi
|
|
232
|
+
if ! grep -q "Stop Gate" stop-gate-err.log; then
|
|
233
|
+
echo "❌ FAIL: Stop gate output did not contain its message"
|
|
234
|
+
cat stop-gate-err.log
|
|
235
|
+
exit 1
|
|
236
|
+
fi
|
|
237
|
+
|
|
238
|
+
# Same edit batch must not bounce twice
|
|
239
|
+
set +e
|
|
240
|
+
echo '{}' | node hooks/scripts/stop-gate.js
|
|
241
|
+
EXIT_CODE=$?
|
|
242
|
+
set -e
|
|
243
|
+
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
244
|
+
echo "❌ FAIL: Stop gate bounced the same edit batch twice (expected 0, got $EXIT_CODE)"
|
|
245
|
+
exit 1
|
|
246
|
+
fi
|
|
247
|
+
|
|
248
|
+
# Loop guard: stop_hook_active always passes
|
|
249
|
+
rm -f .claude/harness-everything/state/sessions/default/stop-gate-state.json
|
|
250
|
+
set +e
|
|
251
|
+
echo '{"stop_hook_active":true}' | node hooks/scripts/stop-gate.js
|
|
252
|
+
EXIT_CODE=$?
|
|
253
|
+
set -e
|
|
254
|
+
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
255
|
+
echo "❌ FAIL: Stop gate blocked despite stop_hook_active (expected 0, got $EXIT_CODE)"
|
|
256
|
+
exit 1
|
|
257
|
+
fi
|
|
258
|
+
rm -f .verify-dirty.tmp .claude/harness-everything/state/sessions/default/handoff-state.json .claude/harness-everything/state/sessions/default/stop-gate-state.json
|
|
259
|
+
echo "✅ PASS: Stop gate bounced once, then respected the batch and loop guards."
|
|
260
|
+
|
|
261
|
+
# Clean up temporary test output logs
|
|
262
|
+
rm -f rule-of-3-err.log boundary-guard-err.log scope-guard-err.log stop-gate-err.log
|
|
263
|
+
|
|
264
|
+
echo -e "\n================================================="
|
|
265
|
+
echo " 🎉 ALL HARNESS MECHANISM CHECKS PASSED SUCCESSFULLY!"
|
|
266
|
+
echo "================================================="
|