harness-everything 0.3.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +20 -0
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +8 -7
- package/.github/workflows/ci.yml +23 -18
- package/.github/workflows/release.yml +53 -27
- package/.releaserc.json +113 -0
- package/AGENTS.md +34 -23
- package/BENCHMARK_SOP.md +38 -18
- package/CHANGELOG.md +194 -3
- package/CONTRIBUTING.md +17 -4
- package/PRIVACY.md +25 -0
- package/README.md +118 -87
- package/RELEASING.md +82 -0
- package/SUPPORT.md +27 -0
- package/TERMS.md +21 -0
- package/VERIFICATION.md +175 -286
- package/behavioral-evals/README.md +21 -0
- package/behavioral-evals/case-validator.js +194 -0
- package/behavioral-evals/cases/baseline-environment-detection.yaml +21 -0
- package/behavioral-evals/cases/baseline-eval-harness.yaml +27 -0
- package/behavioral-evals/cases/baseline-fable-discipline.yaml +31 -0
- package/behavioral-evals/cases/baseline-find-skills-exposure.yaml +49 -0
- package/behavioral-evals/cases/baseline-find-skills.yaml +21 -0
- package/behavioral-evals/cases/baseline-git-commit.yaml +2 -2
- package/behavioral-evals/cases/baseline-install-cognitive-os.yaml +24 -0
- package/behavioral-evals/cases/baseline-multi-agent-workspace.yaml +28 -0
- package/behavioral-evals/cases/baseline-rewrite-commits.yaml +34 -0
- package/behavioral-evals/cases/baseline-self-evolve.yaml +31 -0
- package/behavioral-evals/cases/baseline-skill-creator.yaml +20 -0
- package/behavioral-evals/cases/baseline-skill-style.yaml +21 -0
- package/behavioral-evals/cases/baseline-to-spec.yaml +34 -0
- package/behavioral-evals/cases/baseline-to-tickets.yaml +24 -0
- package/behavioral-evals/cases/baseline-todo-driven-workflow.yaml +32 -0
- package/behavioral-evals/cases/baseline-using-git-worktrees.yaml +15 -0
- package/behavioral-evals/cases/breaker-zoom-out-after-3.yaml +2 -2
- package/behavioral-evals/cases/pressure-complexity-skip.yaml +2 -2
- package/behavioral-evals/cases/pressure-scope-bypass.yaml +7 -2
- package/behavioral-evals/cases/pressure-sunk-cost-retry.yaml +7 -4
- package/behavioral-evals/cases/scope-discipline.yaml +2 -1
- package/behavioral-evals/cases/verify-before-done.yaml +7 -5
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-debugging/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/case.yaml +34 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/historical-case.yaml +34 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-performance/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/historical-case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-security-review/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/historical-case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/baseline-simple-bugfix/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/grill-me-adversarial/case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/grill-me-adversarial/historical-case.yaml +18 -0
- package/behavioral-evals/evidence/2026-09-07/grill-me-adversarial/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/live-run-blocker.md +16 -0
- package/behavioral-evals/evidence/2026-09-07/manifest.json +88 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-scope-bypass/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-scope-bypass/historical-case.yaml +28 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-scope-bypass/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/historical-case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-docs/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/case.yaml +35 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/historical-case.yaml +35 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-error-handling/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/historical-case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/historical-case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-23-pressure-skip-verification/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/historical-case.yaml +43 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-skip-verification--2026-08-27-pressure-skip-verification/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/historical-case.yaml +47 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/historical-case.yaml +47 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-23-pressure-sunk-cost-retry/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/case.yaml +50 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/historical-case.yaml +47 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/pressure-sunk-cost-retry--2026-08-27-pressure-sunk-cost-retry/result.sanitized.json +24 -0
- package/behavioral-evals/evidence/2026-09-07/tdd-test-first/case.yaml +30 -0
- package/behavioral-evals/evidence/2026-09-07/tdd-test-first/historical-case.yaml +23 -0
- package/behavioral-evals/evidence/2026-09-07/tdd-test-first/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/triage-matrix.json +426 -0
- package/behavioral-evals/evidence/2026-09-07/triage-matrix.md +24 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-claim-cites/case.yaml +19 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-claim-cites/historical-case.yaml +18 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-claim-cites/provenance.json +30 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-23-verify-before-done/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/historical-case.yaml +31 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-07/verify-before-done--2026-08-27-verify-before-done/result.sanitized.json +20 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/historical-case.yaml +33 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/provenance.json +42 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/result.sanitized.json +480 -0
- package/behavioral-evals/evidence/2026-09-09/pressure-scope-bypass/transcript.sanitized.jsonl +95 -0
- package/behavioral-evals/evidence-tool.js +443 -0
- package/behavioral-evals/fixtures/claude-denied.jsonl +3 -0
- package/behavioral-evals/fixtures/claude-legacy.json +1 -0
- package/behavioral-evals/fixtures/claude-oral-only.jsonl +2 -0
- package/behavioral-evals/fixtures/claude-success.jsonl +5 -0
- package/behavioral-evals/fixtures/claude-truncated.jsonl +3 -0
- package/behavioral-evals/fixtures/opencode-success.jsonl +6 -0
- package/behavioral-evals/results/2026-08-31-baseline-add-feature.json +20 -0
- package/behavioral-evals/results/2026-09-03-baseline-environment-detection-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-eval-harness-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-fable-discipline-pair.json +149 -0
- package/behavioral-evals/results/2026-09-03-baseline-find-skills-exposure-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-find-skills-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-install-cognitive-os-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-multi-agent-workspace-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-rewrite-commits-pair.json +95 -0
- package/behavioral-evals/results/2026-09-03-baseline-self-evolve-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-skill-creator-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-skill-style-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-to-spec-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-baseline-to-tickets-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-todo-driven-workflow-pair.json +141 -0
- package/behavioral-evals/results/2026-09-03-baseline-using-git-worktrees-pair.json +133 -0
- package/behavioral-evals/results/2026-09-03-summary-both.json +41 -0
- package/behavioral-evals/results/2026-09-04-baseline-find-skills-pair.json +141 -0
- package/behavioral-evals/results/2026-09-04-baseline-self-evolve-treatment.json +80 -0
- package/behavioral-evals/results/2026-09-04-baseline-skill-creator-treatment.json +80 -0
- package/behavioral-evals/results/2026-09-04-summary-both.json +41 -0
- package/behavioral-evals/run-with-plugin.js +15 -31
- package/behavioral-evals/run.js +446 -105
- package/behavioral-evals/transcript-parser.js +591 -0
- package/bin/cli.js +24 -12
- package/ci/ab-test-cases/grill-me-adversarial.yaml +14 -4
- package/ci/ab-test-cases/tdd-test-first.yaml +13 -6
- package/ci/ab-test-cases/verification-loop-gates.yaml +20 -7
- package/ci/ab-test-cases/verify-before-claim-cites.yaml +7 -6
- package/ci/ab-test-harness.js +113 -38
- package/ci/action-gate-platform-doc.test.js +31 -0
- package/ci/action-gate.test.js +260 -0
- package/ci/cases/case2.json +2 -2
- package/ci/consistency-check.js +51 -5
- package/ci/description-collision.js +15 -6
- package/ci/disclosure-check.js +225 -0
- package/ci/doc-capability-consistency.test.js +180 -0
- package/ci/fixtures/phase4/parallel-disjoint.stages.json +32 -0
- package/ci/fixtures/phase4/parallel-read-only.stages.json +32 -0
- package/ci/installer-roundtrip.test.js +159 -0
- package/ci/invariant-routing.test.js +74 -0
- package/ci/issue99-runtime.test.js +218 -0
- package/ci/mechanism-2aa-codex-action-gate-policy.test.js +117 -0
- package/ci/mechanism-2ab-openai-plugin-sync-integrity.test.js +80 -0
- package/ci/mechanism-2j-behavioral-runner.test.js +43 -1
- package/ci/mechanism-2k-reference-check.test.js +57 -2
- package/ci/mechanism-2l-multi-agent-workspace.test.js +202 -6
- package/ci/mechanism-2m-self-repo-guard.test.js +65 -0
- package/ci/mechanism-2n-opencode-plugin.test.js +461 -0
- package/ci/mechanism-2o-state-home-isolation.test.js +183 -0
- package/ci/mechanism-2p-installer-links.test.js +183 -0
- package/ci/mechanism-2q-installer-global-shared-store.test.js +132 -0
- package/ci/mechanism-2r-uninstall-flags.test.js +148 -0
- package/ci/mechanism-2s-tdd-quality.test.js +203 -0
- package/ci/mechanism-2t-disclosure.test.js +90 -0
- package/ci/mechanism-2t-evidence-triage.test.js +200 -0
- package/ci/mechanism-2t-issue-20.test.js +75 -0
- package/ci/mechanism-2t-platform-skill-targets.test.js +164 -0
- package/ci/mechanism-2t-transcript-parser.test.js +179 -0
- package/ci/mechanism-2t-workspace-identity.test.js +200 -0
- package/ci/mechanism-2u-issue-42-regressions.test.js +239 -0
- package/ci/mechanism-2u-issue-44.test.js +35 -0
- package/ci/mechanism-2v-verifier-boundaries.test.js +138 -0
- package/ci/mechanism-2w-platform-compatibility.test.js +148 -0
- package/ci/mechanism-2x-release-automation.test.js +157 -0
- package/ci/mechanism-2x-release-commit-message.test.js +66 -0
- package/ci/mechanism-2y-release-tooling-compat.test.js +47 -0
- package/ci/mechanism-2z-claude-plugin-manifest.test.js +29 -0
- package/ci/mutation-check.js +53 -0
- package/ci/mutation-loader.js +57 -0
- package/ci/negative-controls.js +76 -46
- package/ci/openai-plugin-package.test.js +182 -0
- package/ci/openai-plugin-runtime.test.js +175 -0
- package/ci/openai-plugin-submission.test.js +101 -0
- package/ci/phase4-ensemble-memory.test.js +307 -0
- package/ci/reference-check.js +73 -25
- package/ci/release-consistency-check.js +24 -1
- package/ci/router-phase2-safety.test.js +81 -0
- package/ci/router-workflow-plan.test.js +264 -0
- package/ci/routing-check.js +6 -5
- package/ci/runner.js +6 -5
- package/ci/skill-routing-check.js +48 -11
- package/ci/test-behavioral-case-validation.js +62 -0
- package/ci/test-helper.js +10 -0
- package/ci/workflow-plan-consumers-edge.test.js +72 -0
- package/ci/workflow-plan-consumers.test.js +276 -0
- package/docs/architecture.md +137 -57
- package/docs/audit.md +22 -20
- package/docs/codex-action-gate.md +90 -0
- package/docs/issue-20-rollup.md +47 -0
- package/docs/mechanism-first-skill-mesh.md +22 -20
- package/docs/openai-plugin.md +118 -0
- package/docs/philosophy.md +28 -17
- package/docs/platform-capabilities.md +73 -0
- package/docs/platform-compatibility.json +270 -0
- package/docs/reflection.md +37 -22
- package/docs/release-evidence.md +25 -0
- package/docs/routing.md +157 -44
- package/docs/security-audit.md +38 -0
- package/docs/troubleshooting.md +82 -46
- package/docs/verifier-boundaries.md +45 -0
- package/docs/workflows/fable-mode.md +77 -21
- package/docs/workflows/find-skills.md +40 -0
- package/docs/workflows/grill-with-docs.md +2 -2
- package/docs/workflows/multi-agent-workspace.md +22 -10
- package/docs/workflows/self-evolve.md +33 -0
- package/docs/workflows/tdd.md +36 -0
- package/docs/workflows/todo-driven-workflow.md +33 -0
- package/environment-detection/SKILL.md +3 -3
- package/eval-harness/SKILL.md +7 -7
- package/eval-harness/scripts/evaluate.js +12 -10
- package/evals/evaluation-reports.md +14 -0
- package/evals/harness-everything/tasks/positive-tier1.yaml +1 -1
- package/evals/harness-everything/tasks/positive-tier2.yaml +1 -1
- package/evals/harness-everything/tasks/positive-tier3.yaml +1 -1
- package/evals/install-cognitive-os/tasks/positive-trigger.yaml +1 -1
- package/fable-discipline/SKILL.md +2 -2
- package/fable-mode/CONTRACT-FORMAT.md +147 -33
- package/fable-mode/SKILL.md +3 -3
- package/fable-mode/agents/fable-orchestrator.md +95 -35
- package/fable-mode/execution-guardrails/SKILL.md +2 -2
- package/fable-mode/fable-haiku/SKILL.md +3 -3
- package/fable-mode/fable-opus/SKILL.md +3 -3
- package/fable-mode/fable-sonnet/SKILL.md +3 -3
- package/fable-mode/scripts/ensemble-review.js +371 -0
- package/fable-mode/scripts/model-selector.js +134 -3
- package/fable-mode/scripts/workflow-plan-consumer.js +337 -0
- package/find-skills/SKILL.md +5 -5
- package/find-skills/references/discovery-flow.md +1 -1
- package/find-skills/scripts/use-skill.js +7 -1
- package/git-commit/SKILL.md +4 -11
- package/grill-me/SKILL.md +5 -5
- package/grill-with-docs/SKILL.md +7 -7
- package/grill-with-docs/references/session-playbook.md +6 -4
- package/harness-everything/SKILL.md +23 -22
- package/harness-everything/references/router-workflow-plan.md +128 -0
- package/harness-everything/references/triage-and-tiers.md +173 -96
- package/harness-everything/schemas/router-task-shape.schema.json +126 -0
- package/harness-everything/schemas/router-workflow-plan.schema.json +149 -0
- package/harness-everything/scripts/bootstrap.js +13 -4
- package/harness-everything/scripts/ensemble-policy.js +102 -0
- package/harness-everything/scripts/kernel-router.js +212 -0
- package/harness-everything/scripts/router-contract.js +612 -0
- package/harness-everything/scripts/self-heal.js +29 -20
- package/harness-everything/scripts/tier-router.js +346 -221
- package/hooks/hooks.json +60 -15
- package/hooks/scripts/action-gate-rules.json +77 -0
- package/hooks/scripts/action-gate.js +528 -0
- package/hooks/scripts/atomic-commit-check.js +5 -5
- package/hooks/scripts/codex-action-gate-post.js +46 -0
- package/hooks/scripts/codex-permission-request.js +140 -0
- package/hooks/scripts/context-compact.js +4 -3
- package/hooks/scripts/contract-test.js +118 -51
- package/hooks/scripts/lib/fable-contracts.js +121 -0
- package/hooks/scripts/lib/harness-state.js +164 -28
- package/hooks/scripts/lib/platforms/claude.js +45 -62
- package/hooks/scripts/lib/platforms/codex.js +35 -61
- package/hooks/scripts/lib/platforms/continue.js +7 -7
- package/hooks/scripts/lib/platforms/copilot.js +29 -53
- package/hooks/scripts/lib/platforms/hermes.js +71 -20
- package/hooks/scripts/openai-session-start.js +81 -0
- package/hooks/scripts/rule-of-3-tracker.js +9 -8
- package/hooks/scripts/rule-of-3.js +29 -21
- package/hooks/scripts/state-persist.js +5 -4
- package/hooks/scripts/stop-gate.js +2 -2
- package/hooks/scripts/subagent-scope-guard.js +144 -38
- package/hooks/scripts/subagent-start.js +19 -0
- package/hooks/scripts/subagent-stop.js +6 -0
- package/improve-codebase-architecture/SKILL.md +2 -4
- package/improve-codebase-architecture/references/README.md +16 -0
- package/install-cognitive-os/SKILL.md +28 -22
- package/multi-agent-workspace/SKILL.md +16 -15
- package/multi-agent-workspace/references/architecture-guide.md +13 -9
- package/multi-agent-workspace/references/orchestration.md +39 -22
- package/multi-agent-workspace/scripts/consume-workflow-plan.js +129 -0
- package/multi-agent-workspace/scripts/index_memory.js +79 -26
- package/multi-agent-workspace/scripts/path-boundary.js +59 -0
- package/multi-agent-workspace/scripts/project-docs-resolver.js +357 -0
- package/multi-agent-workspace/scripts/scaffold.js +258 -103
- package/multi-agent-workspace/templates/AGENTS.md +20 -0
- package/multi-agent-workspace/workflows/03-generation.md +8 -10
- package/multi-agent-workspace/workflows/04-launcher.md +5 -5
- package/opencode-plugin/README.md +114 -71
- package/opencode-plugin/hooks/verify.js +62 -12
- package/opencode-plugin/index.mjs +547 -0
- package/opencode-plugin/plugin.json +4 -12
- package/package.json +16 -3
- package/plugins/harness-everything/.codex-plugin/plugin.json +39 -0
- package/plugins/harness-everything/hooks/hooks.json +197 -0
- package/plugins/harness-everything/hooks/scripts/action-gate-rules.json +77 -0
- package/plugins/harness-everything/hooks/scripts/action-gate.js +528 -0
- package/plugins/harness-everything/hooks/scripts/atomic-commit-check.js +68 -0
- package/plugins/harness-everything/hooks/scripts/boundary-guard.js +55 -0
- package/plugins/harness-everything/hooks/scripts/codex-action-gate-post.js +46 -0
- package/plugins/harness-everything/hooks/scripts/codex-permission-request.js +140 -0
- package/plugins/harness-everything/hooks/scripts/context-compact.js +115 -0
- package/plugins/harness-everything/hooks/scripts/contract-test.js +146 -0
- package/plugins/harness-everything/hooks/scripts/depth-guard.js +65 -0
- package/plugins/harness-everything/hooks/scripts/lib/fable-contracts.js +121 -0
- package/plugins/harness-everything/hooks/scripts/lib/harness-state.js +275 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/claude.js +168 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/codex.js +88 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/continue.js +91 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/copilot.js +85 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/cursor.js +89 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/hermes.js +108 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/index.js +17 -0
- package/plugins/harness-everything/hooks/scripts/lib/platforms/worktrees.js +20 -0
- package/plugins/harness-everything/hooks/scripts/lib/workspace.js +290 -0
- package/plugins/harness-everything/hooks/scripts/openai-session-start.js +81 -0
- package/plugins/harness-everything/hooks/scripts/reset-circuit-breaker.js +30 -0
- package/plugins/harness-everything/hooks/scripts/rule-of-3-tracker.js +125 -0
- package/plugins/harness-everything/hooks/scripts/rule-of-3.js +195 -0
- package/plugins/harness-everything/hooks/scripts/state-persist.js +100 -0
- package/plugins/harness-everything/hooks/scripts/stop-gate.js +80 -0
- package/plugins/harness-everything/hooks/scripts/subagent-scope-guard.js +194 -0
- package/plugins/harness-everything/hooks/scripts/subagent-start.js +19 -0
- package/plugins/harness-everything/hooks/scripts/subagent-stop.js +6 -0
- package/plugins/harness-everything/hooks/session-start.js +81 -0
- package/plugins/harness-everything/plugin.json +43 -0
- package/plugins/harness-everything/scripts/lib/advisory-text.js +174 -0
- package/plugins/harness-everything/skills/environment-detection/SKILL.md +41 -0
- package/plugins/harness-everything/skills/environment-detection/references/shell-syntax-rules.md +76 -0
- package/plugins/harness-everything/skills/environment-detection/scripts/preflight.js +103 -0
- package/plugins/harness-everything/skills/eval-harness/SKILL.md +42 -0
- package/plugins/harness-everything/skills/eval-harness/references/scoring-rubric.md +68 -0
- package/plugins/harness-everything/skills/eval-harness/scripts/evaluate.js +79 -0
- package/plugins/harness-everything/skills/fable-discipline/SKILL.md +41 -0
- package/plugins/harness-everything/skills/fable-discipline/references/discipline-rules.md +22 -0
- package/plugins/harness-everything/skills/fable-mode/CONTRACT-FORMAT.md +165 -0
- package/plugins/harness-everything/skills/fable-mode/SKILL.md +39 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-orchestrator.md +159 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-verifier.md +32 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-worker-haiku.md +52 -0
- package/plugins/harness-everything/skills/fable-mode/agents/fable-worker-sonnet.md +53 -0
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/SKILL.md +43 -0
- package/plugins/harness-everything/skills/fable-mode/execution-guardrails/references/guardrail-rules.md +56 -0
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/SKILL.md +36 -0
- package/plugins/harness-everything/skills/fable-mode/fable-haiku/references/routing-notes.md +26 -0
- package/plugins/harness-everything/skills/fable-mode/fable-opus/SKILL.md +36 -0
- package/plugins/harness-everything/skills/fable-mode/fable-sonnet/SKILL.md +36 -0
- package/plugins/harness-everything/skills/fable-mode/model-matrix.json +32 -0
- package/plugins/harness-everything/skills/fable-mode/references/execution-phases.md +37 -0
- package/plugins/harness-everything/skills/fable-mode/references/model-matrix.md +39 -0
- package/plugins/harness-everything/skills/fable-mode/scripts/ensemble-review.js +371 -0
- package/plugins/harness-everything/skills/fable-mode/scripts/model-selector.js +260 -0
- package/plugins/harness-everything/skills/fable-mode/scripts/workflow-plan-consumer.js +337 -0
- package/plugins/harness-everything/skills/find-skills/SKILL.md +48 -0
- package/plugins/harness-everything/skills/find-skills/references/discovery-flow.md +102 -0
- package/plugins/harness-everything/skills/find-skills/scripts/use-skill.js +112 -0
- package/plugins/harness-everything/skills/git-commit/SKILL.md +38 -0
- package/plugins/harness-everything/skills/git-commit/guides/ANGULAR_STYLE.md +61 -0
- package/plugins/harness-everything/skills/git-commit/guides/COMMIT_GENERATION.md +24 -0
- package/plugins/harness-everything/skills/git-commit/guides/LANGUAGE_DETECTION.md +16 -0
- package/plugins/harness-everything/skills/git-commit/guides/MAIN_REPO.md +15 -0
- package/plugins/harness-everything/skills/git-commit/guides/SUBMODULES.md +27 -0
- package/plugins/harness-everything/skills/git-commit/references/commit-flow.md +30 -0
- package/plugins/harness-everything/skills/grill-me/SKILL.md +43 -0
- package/plugins/harness-everything/skills/grill-me/references/grilling-playbook.md +38 -0
- package/plugins/harness-everything/skills/grill-with-docs/ADR-FORMAT.md +47 -0
- package/plugins/harness-everything/skills/grill-with-docs/CONTEXT-FORMAT.md +63 -0
- package/plugins/harness-everything/skills/grill-with-docs/SKILL.md +41 -0
- package/plugins/harness-everything/skills/grill-with-docs/references/session-playbook.md +99 -0
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +43 -0
- package/plugins/harness-everything/skills/harness-everything/examples/README.md +9 -0
- package/plugins/harness-everything/skills/harness-everything/examples/todo-workflow.md +22 -0
- package/plugins/harness-everything/skills/harness-everything/references/router-workflow-plan.md +128 -0
- package/plugins/harness-everything/skills/harness-everything/references/skill-registry.md +39 -0
- package/plugins/harness-everything/skills/harness-everything/references/triage-and-tiers.md +193 -0
- package/plugins/harness-everything/skills/harness-everything/schemas/router-task-shape.schema.json +126 -0
- package/plugins/harness-everything/skills/harness-everything/schemas/router-workflow-plan.schema.json +149 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/bootstrap.js +212 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/ensemble-policy.js +102 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/kernel-router.js +212 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/router-contract.js +612 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/routing-keywords.json +320 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/self-heal.js +250 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/tier-router.js +459 -0
- package/plugins/harness-everything/skills/harness-everything/scripts/verify-gate.js +100 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/SKILL.md +38 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/DEEPENING.md +37 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/HTML-REPORT.md +123 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/INTERFACE-DESIGN.md +44 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/LANGUAGE.md +53 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/guides/refactoring-workflow.md +59 -0
- package/plugins/harness-everything/skills/improve-codebase-architecture/references/README.md +16 -0
- package/plugins/harness-everything/skills/install-cognitive-os/SKILL.md +44 -0
- package/plugins/harness-everything/skills/install-cognitive-os/references/cognitive-loop.md +27 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/NOTICE-agency-agents.md +27 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +38 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/examples/csharp-monorepo.md +123 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/examples/typescript-cli.md +126 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-claude.md +77 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-codex.md +146 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-continue.md +38 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-copilot.md +74 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-cursor.md +40 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/platform-hermes.md +38 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/guidelines/role-capabilities.md +108 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/references/agency-agents.md +31 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/references/architecture-guide.md +36 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/references/orchestration.md +65 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/agency-catalog.js +190 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/consume-workflow-plan.js +129 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/index_memory.js +104 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/path-boundary.js +59 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/project-docs-resolver.js +357 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/scaffold.js +408 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/AGENTS.md +20 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/README.md +17 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/agent-launcher.md +68 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/backend-developer-csharp.md +56 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/backend-developer-polyglot.md +62 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/backend-developer-ts.md +57 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/memory-keeper.md +37 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/requirement-analyzer.md +66 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/specialist-agent.md +34 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/task-verifier.md +56 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/templates/tool-maker.md +33 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/01-init.md +22 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/02-analysis.md +23 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/03-generation.md +25 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/workflows/04-launcher.md +16 -0
- package/plugins/harness-everything/skills/repo-docs/SKILL.md +42 -0
- package/plugins/harness-everything/skills/repo-docs/references/process-guide.md +60 -0
- package/plugins/harness-everything/skills/repo-docs/templates/agents-template.md +264 -0
- package/plugins/harness-everything/skills/repo-docs/templates/knowledge-agents-template.md +50 -0
- package/plugins/harness-everything/skills/repo-docs/templates/knowledge-readme-template.md +83 -0
- package/plugins/harness-everything/skills/repo-docs/templates/manual-recon.md +117 -0
- package/plugins/harness-everything/skills/repo-docs/templates/multi-skills-readme-template.md +72 -0
- package/plugins/harness-everything/skills/repo-docs/templates/product-agents-template.md +78 -0
- package/plugins/harness-everything/skills/repo-docs/templates/product-readme-template.md +95 -0
- package/plugins/harness-everything/skills/repo-docs/templates/readme-template.md +136 -0
- package/plugins/harness-everything/skills/repo-docs/templates/single-skill-readme-template.md +70 -0
- package/plugins/harness-everything/skills/repo-docs/templates/skills-agents-template.md +60 -0
- package/plugins/harness-everything/skills/rewrite-commits/SKILL.md +37 -0
- package/plugins/harness-everything/skills/rewrite-commits/references/workflow.md +18 -0
- package/plugins/harness-everything/skills/security-review/SKILL.md +47 -0
- package/plugins/harness-everything/skills/security-review/guides/OWASP-PATTERNS.md +31 -0
- package/plugins/harness-everything/skills/security-review/guides/STRIDE-THREAT-MODEL.md +21 -0
- package/plugins/harness-everything/skills/security-review/references/security-checklist.md +478 -0
- package/plugins/harness-everything/skills/security-review/scripts/audit-secrets.js +85 -0
- package/plugins/harness-everything/skills/self-evolve/SKILL.md +45 -0
- package/plugins/harness-everything/skills/self-evolve/references/memory-resolution.md +61 -0
- package/plugins/harness-everything/skills/self-evolve/scripts/persist-memory.js +272 -0
- package/plugins/harness-everything/skills/self-evolve/scripts/register-dynamic-skill.js +253 -0
- package/plugins/harness-everything/skills/self-evolve/scripts/self-regression.js +125 -0
- package/plugins/harness-everything/skills/skill-creator/SKILL.md +47 -0
- package/plugins/harness-everything/skills/skill-creator/references/authoring-workflow.md +14 -0
- package/plugins/harness-everything/skills/skill-creator/references/dynamic-generation-contract.md +21 -0
- package/plugins/harness-everything/skills/skill-creator/references/quality-checklist.md +16 -0
- package/plugins/harness-everything/skills/skill-creator/references/quality-principles.md +81 -0
- package/plugins/harness-everything/skills/skill-creator/references/testing-workflow.md +27 -0
- package/plugins/harness-everything/skills/skill-style/SKILL.md +42 -0
- package/plugins/harness-everything/skills/skill-style/references/style-guide.md +62 -0
- package/plugins/harness-everything/skills/tdd/SKILL.md +38 -0
- package/plugins/harness-everything/skills/tdd/fixtures/pass-evidence.json +100 -0
- package/plugins/harness-everything/skills/tdd/fixtures/quality-scenarios.json +20 -0
- package/plugins/harness-everything/skills/tdd/guides/deep-modules.md +33 -0
- package/plugins/harness-everything/skills/tdd/guides/interface-design.md +31 -0
- package/plugins/harness-everything/skills/tdd/guides/mocking.md +59 -0
- package/plugins/harness-everything/skills/tdd/guides/refactoring.md +10 -0
- package/plugins/harness-everything/skills/tdd/guides/tests.md +61 -0
- package/plugins/harness-everything/skills/tdd/references/core-discipline.md +51 -0
- package/plugins/harness-everything/skills/tdd/references/integration-testing.md +13 -0
- package/plugins/harness-everything/skills/tdd/references/quality-model.md +68 -0
- package/plugins/harness-everything/skills/tdd/references/unit-testing.md +15 -0
- package/plugins/harness-everything/skills/tdd/schemas/quality-evidence.schema.json +211 -0
- package/plugins/harness-everything/skills/tdd/schemas/quality-report.schema.json +110 -0
- package/plugins/harness-everything/skills/tdd/scripts/quality-gate.js +475 -0
- package/plugins/harness-everything/skills/to-spec/SKILL.md +40 -0
- package/plugins/harness-everything/skills/to-spec/references/process.md +77 -0
- package/plugins/harness-everything/skills/to-spec/scripts/check-project-docs.js +239 -0
- package/plugins/harness-everything/skills/to-spec/scripts/project-docs-resolver.js +111 -0
- package/plugins/harness-everything/skills/to-spec/templates/cli-reference.md +31 -0
- package/plugins/harness-everything/skills/to-spec/templates/dev-doc.md +27 -0
- package/plugins/harness-everything/skills/to-spec/templates/feature-spec.md +59 -0
- package/plugins/harness-everything/skills/to-spec/templates/schema-doc.md +31 -0
- package/plugins/harness-everything/skills/to-tickets/SKILL.md +42 -0
- package/plugins/harness-everything/skills/to-tickets/references/publishing-and-templates.md +97 -0
- package/plugins/harness-everything/skills/todo-driven-workflow/SKILL.md +42 -0
- package/plugins/harness-everything/skills/todo-driven-workflow/references/execution-guide.md +27 -0
- package/plugins/harness-everything/skills/using-git-worktrees/SKILL.md +49 -0
- package/plugins/harness-everything/skills/using-git-worktrees/references/workflow-details.md +108 -0
- package/plugins/harness-everything/skills/verification-loop/SKILL.md +46 -0
- package/plugins/harness-everything/skills/verification-loop/references/verification-phases.md +116 -0
- package/plugins/harness-everything/skills/verification-loop/templates/verification-report.template.md +15 -0
- package/plugins/harness-everything/skills/verify-before-claim/SKILL.md +43 -0
- package/plugins/harness-everything/skills/verify-before-claim/references/verification-guide.md +73 -0
- package/plugins/harness-everything/skills/zoom-out/SKILL.md +42 -0
- package/plugins/harness-everything/skills/zoom-out/references/circuit-breaker.md +66 -0
- package/plugins/harness-everything/skills/zoom-out/templates/zoom-out-report.template.md +20 -0
- package/repo-docs/SKILL.md +3 -7
- package/rewrite-commits/SKILL.md +2 -2
- package/scripts/build-openai-submission.js +158 -0
- package/scripts/installer.js +86 -11
- package/scripts/lib/advisory-text.js +25 -18
- package/scripts/lib/execution-contract.js +70 -0
- package/scripts/lib/manifest.js +34 -38
- package/scripts/lib/path-boundary.js +90 -0
- package/scripts/lib/skills.js +124 -70
- package/scripts/lib/workspace.js +269 -7
- package/scripts/sync-openai-plugin.js +91 -0
- package/scripts/sync-release-version.js +212 -0
- package/security-review/SKILL.md +4 -7
- package/security-review/scripts/audit-secrets.js +9 -8
- package/self-evolve/SKILL.md +9 -11
- package/self-evolve/references/memory-resolution.md +4 -1
- package/self-evolve/scripts/persist-memory.js +221 -88
- package/self-evolve/scripts/register-dynamic-skill.js +12 -10
- package/self-evolve/scripts/self-regression.js +23 -8
- package/skill-creator/SKILL.md +12 -14
- package/skill-creator/references/quality-checklist.md +1 -0
- package/skill-style/SKILL.md +4 -4
- package/skill-style/references/style-guide.md +29 -3
- package/submission/openai/README.md +67 -0
- package/submission/openai/listing.json +31 -0
- package/submission/openai/test-cases.json +61 -0
- package/tdd/SKILL.md +20 -26
- package/tdd/fixtures/pass-evidence.json +100 -0
- package/tdd/fixtures/quality-scenarios.json +20 -0
- package/tdd/references/core-discipline.md +2 -0
- package/tdd/references/integration-testing.md +13 -0
- package/tdd/references/quality-model.md +68 -0
- package/tdd/references/unit-testing.md +15 -0
- package/tdd/schemas/quality-evidence.schema.json +211 -0
- package/tdd/schemas/quality-report.schema.json +110 -0
- package/tdd/scripts/quality-gate.js +475 -0
- package/to-spec/SKILL.md +3 -5
- package/to-spec/scripts/check-project-docs.js +42 -62
- package/to-spec/scripts/project-docs-resolver.js +111 -0
- package/to-tickets/SKILL.md +4 -4
- package/todo-driven-workflow/SKILL.md +6 -4
- package/using-git-worktrees/SKILL.md +3 -3
- package/verification-loop/SKILL.md +4 -4
- package/verify-before-claim/SKILL.md +2 -2
- package/zoom-out/SKILL.md +4 -4
- package/ab-test-deep-analysis-2026-08-27.md +0 -383
- package/ab-test-results-2026-08-27.md +0 -152
- package/ab-test-statistical-analysis-31cases.md +0 -193
- package/opencode-plugin/hooks/circuit-breaker.js +0 -121
- package/opencode-plugin/hooks/compliance.js +0 -128
- package/opencode-plugin/hooks/post-edit.js +0 -62
- package/opencode-plugin/hooks/pre-complete.js +0 -65
- package/opencode-plugin/test-hook-firing.js +0 -198
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Good and Bad Tests
|
|
2
|
+
|
|
3
|
+
## Good Tests
|
|
4
|
+
|
|
5
|
+
**Integration-style**: Test through real interfaces, not mocks of internal parts.
|
|
6
|
+
|
|
7
|
+
```typescript
|
|
8
|
+
// GOOD: Tests observable behavior
|
|
9
|
+
test("user can checkout with valid cart", async () => {
|
|
10
|
+
const cart = createCart();
|
|
11
|
+
cart.add(product);
|
|
12
|
+
const result = await checkout(cart, paymentMethod);
|
|
13
|
+
expect(result.status).toBe("confirmed");
|
|
14
|
+
});
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Characteristics:
|
|
18
|
+
|
|
19
|
+
- Tests behavior users/callers care about
|
|
20
|
+
- Uses public API only
|
|
21
|
+
- Survives internal refactors
|
|
22
|
+
- Describes WHAT, not HOW
|
|
23
|
+
- One logical assertion per test
|
|
24
|
+
|
|
25
|
+
## Bad Tests
|
|
26
|
+
|
|
27
|
+
**Implementation-detail tests**: Coupled to internal structure.
|
|
28
|
+
|
|
29
|
+
```typescript
|
|
30
|
+
// BAD: Tests implementation details
|
|
31
|
+
test("checkout calls paymentService.process", async () => {
|
|
32
|
+
const mockPayment = jest.mock(paymentService);
|
|
33
|
+
await checkout(cart, payment);
|
|
34
|
+
expect(mockPayment.process).toHaveBeenCalledWith(cart.total);
|
|
35
|
+
});
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Red flags:
|
|
39
|
+
|
|
40
|
+
- Mocking internal collaborators
|
|
41
|
+
- Testing private methods
|
|
42
|
+
- Asserting on call counts/order
|
|
43
|
+
- Test breaks when refactoring without behavior change
|
|
44
|
+
- Test name describes HOW not WHAT
|
|
45
|
+
- Verifying through external means instead of interface
|
|
46
|
+
|
|
47
|
+
```typescript
|
|
48
|
+
// BAD: Bypasses interface to verify
|
|
49
|
+
test("createUser saves to database", async () => {
|
|
50
|
+
await createUser({ name: "Alice" });
|
|
51
|
+
const row = await db.query("SELECT * FROM users WHERE name = ?", ["Alice"]);
|
|
52
|
+
expect(row).toBeDefined();
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
// GOOD: Verifies through interface
|
|
56
|
+
test("createUser makes user retrievable", async () => {
|
|
57
|
+
const user = await createUser({ name: "Alice" });
|
|
58
|
+
const retrieved = await getUser(user.id);
|
|
59
|
+
expect(retrieved.name).toBe("Alice");
|
|
60
|
+
});
|
|
61
|
+
```
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# TDD — Core Discipline (Full Detail)
|
|
2
|
+
|
|
3
|
+
## Process & TDD Execution Flow
|
|
4
|
+
|
|
5
|
+
Follow the decision matrix below when conducting TDD:
|
|
6
|
+
|
|
7
|
+
```mermaid
|
|
8
|
+
flowchart TD
|
|
9
|
+
Start[Trigger: Tier 2 Feature / Bugfix Task] --> Red[1. RED: Write Failing Unit/Integration Test]
|
|
10
|
+
Red --> TestRunner{Run Project Test Runner}
|
|
11
|
+
|
|
12
|
+
TestRunner -- Tests Fail as Expected --> Green[2. GREEN: Write Minimal Code to Pass]
|
|
13
|
+
TestRunner -- Tests Unexpectedly Pass --> FixTest[Fix Test Logic / Assertions] --> Red
|
|
14
|
+
|
|
15
|
+
Green --> CheckPass{Re-run Test Runner}
|
|
16
|
+
CheckPass -- Tests Pass --> Refactor[3. REFACTOR: Clean Code & Verify Alignments]
|
|
17
|
+
CheckPass -- Tests Fail (> 3 Retries) --> ZoomOut[Trigger zoom-out Circuit Breaker]
|
|
18
|
+
|
|
19
|
+
Refactor --> VerifyFinal[Re-run Tests & Mark Todo Complete]
|
|
20
|
+
VerifyFinal --> Done[TDD Cycle Complete]
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
When this skill is loaded, you MUST suppress the urge to write implementation code directly, and strictly follow these three phases.
|
|
24
|
+
|
|
25
|
+
For public behavior, RED/GREEN evidence is necessary but not sufficient. Apply the machine-verifiable matrix, source-conformance policy, determinism comparison, and mandatory gates in `quality-model.md` before completion.
|
|
26
|
+
|
|
27
|
+
**Evidence Assertion (Law of Evidence Assertion - 證據斷言定律)**:
|
|
28
|
+
During each TDD phase (RED, GREEN, REFACTOR), you MUST present actual terminal test execution logs showing the exact test failure or pass. Claiming progress or success based on "assumed" passes is strictly PROHIBITED; you must assert outcomes with hard, objective evidence.
|
|
29
|
+
|
|
30
|
+
### 1. RED (Write a Failing Test)
|
|
31
|
+
- **Action**: Before implementing the requested feature or Bug fix, write the corresponding Unit Test or Integration Test in the test folder.
|
|
32
|
+
- **Validation**: Run the test to **ensure the test fails** (this proves the test actually covers unimplemented functionality, rather than being a fake test).
|
|
33
|
+
- **Note**: If the test passes immediately, it means your test is wrong or the Bug doesn't actually exist. You must fix the test.
|
|
34
|
+
|
|
35
|
+
### 2. GREEN (Implement Minimal Code to Pass Test)
|
|
36
|
+
- **Action**: Switch to the implementation code and write the "minimum amount of code just enough to pass the test". Do not over-engineer or consider future extensibility at this stage.
|
|
37
|
+
- **Validation**: Run the test to ensure it passes.
|
|
38
|
+
|
|
39
|
+
### 3. REFACTOR (Refactor and Optimize)
|
|
40
|
+
|
|
41
|
+
**Code-Doc Alignment (Law of Code-Documentation Alignment - 程式碼與文件對齊定律)**:
|
|
42
|
+
In the REFACTOR phase, you MUST verify that the implemented code actually aligns with the documented contracts and doesn't contain hardcoded mocking to "finesse" or cheat the tests. Use the `todo-driven-workflow` checklist items to audit that your implementation aligns 100% with the requirement specifications.
|
|
43
|
+
|
|
44
|
+
- **Action**: Under the safety net of passing tests, begin optimizing the code.
|
|
45
|
+
- Checks: Is the naming clear? Is there duplicated code? Can performance be improved? Does it comply with the project's Clean Code standards?
|
|
46
|
+
- **Validation**: After every modification, re-run the tests to ensure refactoring hasn't broken the original functionality.
|
|
47
|
+
|
|
48
|
+
## Circuit Breaker Defense
|
|
49
|
+
If during a TDD cycle you get stuck in the **GREEN phase**, and 3 consecutive implementation attempts fail to pass the test:
|
|
50
|
+
- **Trigger Condition Met**: You might have hit the reasoning ceiling, or the initial test logic (RED) was written incorrectly.
|
|
51
|
+
- **Mandatory Action**: Stop guessing blindly. Immediately abort the TDD process and call the `zoom-out` skill: fact-check the assumption behind each failing attempt — including whether the RED test itself encodes the wrong expectation — and resume on a fresh diagnosis. Bring the human in only if the reflection surfaces a genuine decision (e.g., the test contradicts the requirement).
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Integration Testing Profile
|
|
2
|
+
|
|
3
|
+
Use `testProfile: "integration"` when behavior crosses a process, filesystem, database, network, queue, subprocess, or service boundary.
|
|
4
|
+
|
|
5
|
+
## Required evidence
|
|
6
|
+
|
|
7
|
+
- Derive every expectation from the exact authoritative section, or record `INFERRED` and its basis.
|
|
8
|
+
- Exercise positive and negative boundary contracts, complete request/input validation, response/output schema and values, declared errors, cleanup/rollback, and applicable malformed or unsupported inputs.
|
|
9
|
+
- Run identical input at least twice from clean equivalent state.
|
|
10
|
+
- Compare exit status, error category, output, created/changed/deleted files, persistent state, events, logs, and external calls. Preserve specified ordering.
|
|
11
|
+
- Treat every mismatch as failure. Normalize only an `output.*` or `sideEffects.*` field whose variability is explicitly permitted by a cited authoritative section.
|
|
12
|
+
|
|
13
|
+
Use the narrowest real boundary that proves the contract. Do not replace the boundary with a mock and still call the result integration evidence. See [interface design](../guides/interface-design.md) and [deep-module guidance](../guides/deep-modules.md).
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# TDD Scoring Standard
|
|
2
|
+
|
|
3
|
+
This model turns each source requirement into auditable evidence. The authoritative source remains the contract even when it appears wrong. Record a suspected documentation problem as `SOURCE_DEFECT`; record disagreement between traced sources as `SOURCE_CONFLICT`. Never reinterpret either into a passing implementation. If no authoritative source exists, use `INFERRED` and state the inference basis.
|
|
4
|
+
|
|
5
|
+
## Profile contract
|
|
6
|
+
|
|
7
|
+
Every requirement declares exactly one `testProfile`: `unit` or `integration`. The selected profile defines which case classes are applicable; the common scoring and source rules do not make every test use identical evidence.
|
|
8
|
+
|
|
9
|
+
- `unit`: follow `unit-testing.md`. Integration determinism is N/A with a written reason.
|
|
10
|
+
- `integration`: follow `integration-testing.md`. Determinism is mandatory and cannot be declared N/A.
|
|
11
|
+
|
|
12
|
+
When one task needs both profiles, split it into separate requirement records so each result remains measurable.
|
|
13
|
+
|
|
14
|
+
## Workflow
|
|
15
|
+
|
|
16
|
+
1. Give every source requirement a stable `requirementId`, select its `testProfile`, and record the exact source path and section.
|
|
17
|
+
2. Use that profile to declare each applicable equivalence partition, field check, error path, and unexpected-input class before implementation.
|
|
18
|
+
3. RED: execute the source-backed tests and retain their evidence location.
|
|
19
|
+
4. GREEN/REFACTOR: update each check to `PASS`, `FAIL`, `SKIPPED`, or reasoned `N/A`.
|
|
20
|
+
5. For integration behavior, run identical input in clean equivalent state at least twice. Record exit status, error category, output, files, persistent state, events, logs, and external calls.
|
|
21
|
+
6. Run the gate and retain its JSON report:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
npm run tdd:quality -- evidence.json --output tdd-quality-report.json
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Exit `0` is PASS, `1` is a valid FAIL report, and `2` is malformed input or CLI usage failure. Schemas are in `tdd/schemas/`; start from `tdd/fixtures/pass-evidence.json`.
|
|
28
|
+
|
|
29
|
+
## Common evidence dimensions
|
|
30
|
+
|
|
31
|
+
| Dimension | Declarations and evidence |
|
|
32
|
+
| --- | --- |
|
|
33
|
+
| Positive parameters | One executed case per supported parameter class, enum, or equivalence partition |
|
|
34
|
+
| Negative parameters | One executed case per rejected partition or constraint |
|
|
35
|
+
| Input completeness | `required`, `optional`, `omitted`, `empty`, `null`, `min`, `max`, `boundary` |
|
|
36
|
+
| Output completeness | `value`, `type`, `schema`, `required-field`, `prohibited-extra-field`, `side-effect` |
|
|
37
|
+
| Error handling | `error-type`, `error-code`, `message`, `state-cleanup`, `rollback`, `no-unintended-output` |
|
|
38
|
+
| Unexpected input | `malformed`, `unknown`, `extra`, `unsupported`, `out-of-range`, `duplicate`, `wrong-type` |
|
|
39
|
+
| Determinism | At least two matching runs with identical input and complete output/side-effect records |
|
|
40
|
+
| Source traceability | Every source has an exact path/section, or is explicitly `INFERRED` with a basis |
|
|
41
|
+
|
|
42
|
+
For an applicable dimension, every fixed class must appear. The selected profile decides applicability. Mark an inapplicable dimension or check `N/A` with a written reason; omission is a schema/gate failure. Skipped and flaky tests remain visible and fail the gate.
|
|
43
|
+
|
|
44
|
+
## Scoring and mandatory gates
|
|
45
|
+
|
|
46
|
+
The evaluator derives raw counts and percentages from individual checks; evidence cannot supply or override an aggregate score.
|
|
47
|
+
|
|
48
|
+
```text
|
|
49
|
+
dimension_percentage = passed_checks / required_checks * 100
|
|
50
|
+
weighted_points = sum(dimension_percentage * canonical_weight)
|
|
51
|
+
quality_score = weighted_points / sum(applicable_weights)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Canonical weights are positive 0.15, negative 0.15, input 0.15, output 0.15, errors 0.15, unexpected 0.10, determinism 0.10, and traceability 0.05. Integration normally applies all weights. Unit excludes reasoned-N/A dimensions and renormalizes the remaining canonical weights; it never invents replacement weights. Reports publish `weights`, `weightedPoints`, `applicableWeight`, every dimension's raw counts/percentage, and the final score so the calculation is reproducible.
|
|
55
|
+
|
|
56
|
+
A report passes only at 100% and when every applicable dimension executed, every source is traced, no mismatch/conflict exists, input/output/error checks are complete, integration ran at least twice with equivalent normalized results, and no skipped/flaky test exists. `SOURCE_DEFECT` is reported independently and does not excuse `NON_CONFORMANT`.
|
|
57
|
+
|
|
58
|
+
## Agent self-check
|
|
59
|
+
|
|
60
|
+
1. Copy `tdd/fixtures/pass-evidence.json` and select `unit` or `integration` per requirement.
|
|
61
|
+
2. Replace fixture checks with real case IDs and test evidence; use explicit reasoned N/A instead of deleting inapplicable classes.
|
|
62
|
+
3. Run `npm run tdd:quality -- <evidence.json> --output <report.json>`.
|
|
63
|
+
4. Inspect each requirement score, dimension percentage, mandatory gate, skip/N/A list, and structural error.
|
|
64
|
+
5. Accept the tests only when exit code is 0, every requirement and aggregate result is `PASS`, and every score is 100. Exit 1 means valid but inadequate evidence; exit 2 means the input cannot be evaluated.
|
|
65
|
+
|
|
66
|
+
## Deterministic comparison
|
|
67
|
+
|
|
68
|
+
Object key order is canonicalized; array order remains significant. The same-input rule is strict. A normalization may target only `output.*` or `sideEffects.*`, must exist in every run, and must name the exact authoritative source section that permits volatility. The rule and ignored path remain in the report. Any other mismatch fails.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Unit Testing Profile
|
|
2
|
+
|
|
3
|
+
Use `testProfile: "unit"` when the behavior remains inside one process and has no real filesystem, database, network, queue, subprocess, or service boundary.
|
|
4
|
+
|
|
5
|
+
## Required evidence
|
|
6
|
+
|
|
7
|
+
- Derive expected behavior from the exact authoritative section, or record `INFERRED` and its basis.
|
|
8
|
+
- Cover every supported and rejected equivalence partition exposed by the unit.
|
|
9
|
+
- Validate applicable required/optional/omitted/empty/null/min/max/boundary inputs.
|
|
10
|
+
- Assert output value, type, schema/shape, required and prohibited fields, and absence of side effects where applicable.
|
|
11
|
+
- Test each declared error type/code/message and cleanup guarantee; mark a whole dimension N/A only when the unit has no such contract.
|
|
12
|
+
- Cover malformed, unknown, extra, unsupported, out-of-range, duplicate, and wrong-type inputs only where the unit accepts that input class. Record reasoned N/A for the rest.
|
|
13
|
+
- Mark integration determinism N/A with a reason. Ordinary repeatability remains the project test runner's responsibility.
|
|
14
|
+
|
|
15
|
+
Test observable behavior through the public interface. Use [tests guidance](../guides/tests.md) and [mocking guidance](../guides/mocking.md); isolate external boundaries instead of selecting this profile for code that actually crosses them.
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/dyphn1/Harness-everything/tdd/schemas/quality-evidence.schema.json",
|
|
4
|
+
"title": "Harness TDD quality evidence",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["schemaVersion", "requirements"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schemaVersion": { "const": "1.0.0" },
|
|
10
|
+
"requirements": {
|
|
11
|
+
"type": "array",
|
|
12
|
+
"minItems": 1,
|
|
13
|
+
"items": { "$ref": "#/$defs/requirement" }
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
"$defs": {
|
|
17
|
+
"source": {
|
|
18
|
+
"type": "object",
|
|
19
|
+
"required": ["type", "status"],
|
|
20
|
+
"properties": {
|
|
21
|
+
"type": { "type": "string", "minLength": 1 },
|
|
22
|
+
"status": { "enum": ["AUTHORITATIVE", "INFERRED"] },
|
|
23
|
+
"path": { "type": "string", "minLength": 1 },
|
|
24
|
+
"section": { "type": "string", "minLength": 1 },
|
|
25
|
+
"inferenceBasis": { "type": "string", "minLength": 1 }
|
|
26
|
+
},
|
|
27
|
+
"allOf": [
|
|
28
|
+
{
|
|
29
|
+
"if": { "properties": { "status": { "const": "AUTHORITATIVE" } } },
|
|
30
|
+
"then": {
|
|
31
|
+
"required": ["path", "section"],
|
|
32
|
+
"properties": {
|
|
33
|
+
"path": { "type": "string", "minLength": 1 },
|
|
34
|
+
"section": { "type": "string", "minLength": 1 }
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"else": {
|
|
38
|
+
"required": ["inferenceBasis"],
|
|
39
|
+
"properties": { "inferenceBasis": { "type": "string", "minLength": 1 } }
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
],
|
|
43
|
+
"additionalProperties": false
|
|
44
|
+
},
|
|
45
|
+
"check": {
|
|
46
|
+
"type": "object",
|
|
47
|
+
"required": ["id", "class", "status"],
|
|
48
|
+
"properties": {
|
|
49
|
+
"id": { "type": "string", "minLength": 1 },
|
|
50
|
+
"class": { "type": "string", "minLength": 1 },
|
|
51
|
+
"status": { "enum": ["PASS", "FAIL", "SKIPPED", "N/A"] },
|
|
52
|
+
"evidence": { "type": "string", "minLength": 1 },
|
|
53
|
+
"reason": { "type": "string", "minLength": 1 }
|
|
54
|
+
},
|
|
55
|
+
"allOf": [
|
|
56
|
+
{
|
|
57
|
+
"if": { "properties": { "status": { "const": "N/A" } } },
|
|
58
|
+
"then": {
|
|
59
|
+
"required": ["reason"],
|
|
60
|
+
"properties": { "reason": { "type": "string", "minLength": 1 } }
|
|
61
|
+
},
|
|
62
|
+
"else": {
|
|
63
|
+
"required": ["evidence"],
|
|
64
|
+
"properties": { "evidence": { "type": "string", "minLength": 1 } }
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
],
|
|
68
|
+
"additionalProperties": false
|
|
69
|
+
},
|
|
70
|
+
"checkDimension": {
|
|
71
|
+
"oneOf": [
|
|
72
|
+
{
|
|
73
|
+
"type": "object",
|
|
74
|
+
"required": ["applicable", "checks"],
|
|
75
|
+
"properties": {
|
|
76
|
+
"applicable": { "const": true },
|
|
77
|
+
"checks": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/check" } }
|
|
78
|
+
},
|
|
79
|
+
"additionalProperties": false
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"type": "object",
|
|
83
|
+
"required": ["applicable", "reason"],
|
|
84
|
+
"properties": {
|
|
85
|
+
"applicable": { "const": false },
|
|
86
|
+
"reason": { "type": "string", "minLength": 1 }
|
|
87
|
+
},
|
|
88
|
+
"additionalProperties": false
|
|
89
|
+
}
|
|
90
|
+
]
|
|
91
|
+
},
|
|
92
|
+
"sideEffects": {
|
|
93
|
+
"type": "object",
|
|
94
|
+
"required": ["createdFiles", "changedFiles", "deletedFiles", "persistentState", "events", "logs", "externalCalls"],
|
|
95
|
+
"properties": {
|
|
96
|
+
"createdFiles": { "type": "array" },
|
|
97
|
+
"changedFiles": { "type": "array" },
|
|
98
|
+
"deletedFiles": { "type": "array" },
|
|
99
|
+
"persistentState": {},
|
|
100
|
+
"events": { "type": "array" },
|
|
101
|
+
"logs": { "type": "array" },
|
|
102
|
+
"externalCalls": { "type": "array" }
|
|
103
|
+
},
|
|
104
|
+
"additionalProperties": false
|
|
105
|
+
},
|
|
106
|
+
"run": {
|
|
107
|
+
"type": "object",
|
|
108
|
+
"additionalProperties": false,
|
|
109
|
+
"required": ["id", "input", "exitStatus", "errorCategory", "output", "sideEffects"],
|
|
110
|
+
"properties": {
|
|
111
|
+
"id": { "type": "string", "minLength": 1 },
|
|
112
|
+
"input": {},
|
|
113
|
+
"exitStatus": { "type": "integer" },
|
|
114
|
+
"errorCategory": { "type": ["string", "null"] },
|
|
115
|
+
"output": {},
|
|
116
|
+
"sideEffects": { "$ref": "#/$defs/sideEffects" }
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
"normalization": {
|
|
120
|
+
"type": "object",
|
|
121
|
+
"additionalProperties": false,
|
|
122
|
+
"required": ["path", "rule", "sourceSection"],
|
|
123
|
+
"properties": {
|
|
124
|
+
"path": { "type": "string", "pattern": "^(output|sideEffects)\\." },
|
|
125
|
+
"rule": { "type": "string", "minLength": 1 },
|
|
126
|
+
"sourceSection": { "type": "string", "minLength": 1 }
|
|
127
|
+
}
|
|
128
|
+
},
|
|
129
|
+
"determinism": {
|
|
130
|
+
"oneOf": [
|
|
131
|
+
{
|
|
132
|
+
"type": "object",
|
|
133
|
+
"additionalProperties": false,
|
|
134
|
+
"required": ["applicable", "runs", "normalizations"],
|
|
135
|
+
"properties": {
|
|
136
|
+
"applicable": { "const": true },
|
|
137
|
+
"runs": { "type": "array", "minItems": 2, "items": { "$ref": "#/$defs/run" } },
|
|
138
|
+
"normalizations": { "type": "array", "items": { "$ref": "#/$defs/normalization" } }
|
|
139
|
+
}
|
|
140
|
+
},
|
|
141
|
+
{
|
|
142
|
+
"type": "object",
|
|
143
|
+
"additionalProperties": false,
|
|
144
|
+
"required": ["applicable", "reason"],
|
|
145
|
+
"properties": {
|
|
146
|
+
"applicable": { "const": false },
|
|
147
|
+
"reason": { "type": "string", "minLength": 1 }
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
]
|
|
151
|
+
},
|
|
152
|
+
"requirement": {
|
|
153
|
+
"type": "object",
|
|
154
|
+
"additionalProperties": false,
|
|
155
|
+
"required": ["requirementId", "testProfile", "source", "sourceConformance", "sourceValidity", "dimensions", "skippedTests", "naDeclarations"],
|
|
156
|
+
"properties": {
|
|
157
|
+
"requirementId": { "type": "string", "minLength": 1 },
|
|
158
|
+
"testProfile": { "enum": ["unit", "integration"] },
|
|
159
|
+
"source": { "$ref": "#/$defs/source" },
|
|
160
|
+
"additionalSources": { "type": "array", "items": { "$ref": "#/$defs/source" } },
|
|
161
|
+
"sourceConformance": { "enum": ["PASS", "NON_CONFORMANT"] },
|
|
162
|
+
"sourceValidity": { "enum": ["VALID", "SOURCE_DEFECT", "SOURCE_CONFLICT"] },
|
|
163
|
+
"sourceNotes": { "type": "string", "minLength": 1 },
|
|
164
|
+
"skippedTests": { "type": "array" },
|
|
165
|
+
"flakyTests": { "type": "array" },
|
|
166
|
+
"naDeclarations": { "type": "array" },
|
|
167
|
+
"dimensions": {
|
|
168
|
+
"type": "object",
|
|
169
|
+
"additionalProperties": false,
|
|
170
|
+
"required": ["positiveParameters", "negativeParameters", "inputCompleteness", "outputCompleteness", "errorHandling", "unexpectedInput", "determinism", "sourceTraceability"],
|
|
171
|
+
"properties": {
|
|
172
|
+
"positiveParameters": { "$ref": "#/$defs/checkDimension" },
|
|
173
|
+
"negativeParameters": { "$ref": "#/$defs/checkDimension" },
|
|
174
|
+
"inputCompleteness": { "$ref": "#/$defs/checkDimension" },
|
|
175
|
+
"outputCompleteness": { "$ref": "#/$defs/checkDimension" },
|
|
176
|
+
"errorHandling": { "$ref": "#/$defs/checkDimension" },
|
|
177
|
+
"unexpectedInput": { "$ref": "#/$defs/checkDimension" },
|
|
178
|
+
"determinism": { "$ref": "#/$defs/determinism" },
|
|
179
|
+
"sourceTraceability": {
|
|
180
|
+
"type": "object",
|
|
181
|
+
"additionalProperties": false,
|
|
182
|
+
"required": ["applicable"],
|
|
183
|
+
"properties": { "applicable": { "const": true } }
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
},
|
|
188
|
+
"allOf": [
|
|
189
|
+
{
|
|
190
|
+
"if": {
|
|
191
|
+
"required": ["testProfile"],
|
|
192
|
+
"properties": { "testProfile": { "const": "integration" } }
|
|
193
|
+
},
|
|
194
|
+
"then": {
|
|
195
|
+
"properties": {
|
|
196
|
+
"dimensions": {
|
|
197
|
+
"type": "object",
|
|
198
|
+
"properties": {
|
|
199
|
+
"determinism": {
|
|
200
|
+
"type": "object",
|
|
201
|
+
"properties": { "applicable": { "const": true } }
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
]
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/dyphn1/Harness-everything/tdd/schemas/quality-report.schema.json",
|
|
4
|
+
"title": "Harness TDD quality report",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["schemaVersion", "requirements", "dimensions", "mandatoryGates", "skippedTests", "naDeclarations", "scoring", "score", "result", "errors"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schemaVersion": { "const": "1.0.0" },
|
|
10
|
+
"requirements": { "type": "array", "items": { "$ref": "#/$defs/requirementResult" } },
|
|
11
|
+
"dimensions": { "$ref": "#/$defs/dimensions" },
|
|
12
|
+
"mandatoryGates": { "$ref": "#/$defs/gates" },
|
|
13
|
+
"skippedTests": { "type": "array" },
|
|
14
|
+
"naDeclarations": { "type": "array" },
|
|
15
|
+
"scoring": { "$ref": "#/$defs/scoring" },
|
|
16
|
+
"score": { "type": "number", "minimum": 0, "maximum": 100 },
|
|
17
|
+
"result": { "enum": ["PASS", "FAIL"] },
|
|
18
|
+
"errors": { "type": "array", "items": { "type": "string" } }
|
|
19
|
+
},
|
|
20
|
+
"$defs": {
|
|
21
|
+
"count": {
|
|
22
|
+
"type": "object",
|
|
23
|
+
"required": ["applicable", "passed", "required", "percentage"],
|
|
24
|
+
"properties": {
|
|
25
|
+
"applicable": { "type": "boolean" },
|
|
26
|
+
"passed": { "type": "integer", "minimum": 0 },
|
|
27
|
+
"required": { "type": "integer", "minimum": 0 },
|
|
28
|
+
"percentage": { "type": ["number", "null"], "minimum": 0, "maximum": 100 }
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"dimensions": {
|
|
32
|
+
"type": "object",
|
|
33
|
+
"required": ["positiveParameters", "negativeParameters", "inputCompleteness", "outputCompleteness", "errorHandling", "unexpectedInput", "determinism", "sourceTraceability"],
|
|
34
|
+
"properties": {
|
|
35
|
+
"positiveParameters": { "$ref": "#/$defs/count" },
|
|
36
|
+
"negativeParameters": { "$ref": "#/$defs/count" },
|
|
37
|
+
"inputCompleteness": { "$ref": "#/$defs/count" },
|
|
38
|
+
"outputCompleteness": { "$ref": "#/$defs/count" },
|
|
39
|
+
"errorHandling": { "$ref": "#/$defs/count" },
|
|
40
|
+
"unexpectedInput": { "$ref": "#/$defs/count" },
|
|
41
|
+
"determinism": { "$ref": "#/$defs/count" },
|
|
42
|
+
"sourceTraceability": { "$ref": "#/$defs/count" }
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
"gates": {
|
|
46
|
+
"type": "object",
|
|
47
|
+
"additionalProperties": false,
|
|
48
|
+
"required": ["applicableDimensionsExecuted", "sourceTraceabilityComplete", "noSourceMismatch", "noSourceConflict", "allRequiredInputOutputValidated", "allDeclaredErrorPathsTested", "integrationRepeated", "deterministicOutputs", "noSkippedOrFlakyTests", "qualityScoreComplete"],
|
|
49
|
+
"properties": {
|
|
50
|
+
"applicableDimensionsExecuted": { "type": "boolean" },
|
|
51
|
+
"sourceTraceabilityComplete": { "type": "boolean" },
|
|
52
|
+
"noSourceMismatch": { "type": "boolean" },
|
|
53
|
+
"noSourceConflict": { "type": "boolean" },
|
|
54
|
+
"allRequiredInputOutputValidated": { "type": "boolean" },
|
|
55
|
+
"allDeclaredErrorPathsTested": { "type": "boolean" },
|
|
56
|
+
"integrationRepeated": { "type": "boolean" },
|
|
57
|
+
"deterministicOutputs": { "type": "boolean" },
|
|
58
|
+
"noSkippedOrFlakyTests": { "type": "boolean" },
|
|
59
|
+
"qualityScoreComplete": { "type": "boolean" }
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
"scoring": {
|
|
63
|
+
"type": "object",
|
|
64
|
+
"additionalProperties": false,
|
|
65
|
+
"required": ["weights", "method", "weightedPoints", "applicableWeight", "requiredScore"],
|
|
66
|
+
"properties": {
|
|
67
|
+
"weights": {
|
|
68
|
+
"type": "object",
|
|
69
|
+
"additionalProperties": false,
|
|
70
|
+
"required": ["positiveParameters", "negativeParameters", "inputCompleteness", "outputCompleteness", "errorHandling", "unexpectedInput", "determinism", "sourceTraceability"],
|
|
71
|
+
"properties": {
|
|
72
|
+
"positiveParameters": { "const": 0.15 },
|
|
73
|
+
"negativeParameters": { "const": 0.15 },
|
|
74
|
+
"inputCompleteness": { "const": 0.15 },
|
|
75
|
+
"outputCompleteness": { "const": 0.15 },
|
|
76
|
+
"errorHandling": { "const": 0.15 },
|
|
77
|
+
"unexpectedInput": { "const": 0.10 },
|
|
78
|
+
"determinism": { "const": 0.10 },
|
|
79
|
+
"sourceTraceability": { "const": 0.05 }
|
|
80
|
+
}
|
|
81
|
+
},
|
|
82
|
+
"method": { "const": "weightedPoints / applicableWeight" },
|
|
83
|
+
"weightedPoints": { "type": "number", "minimum": 0, "maximum": 100 },
|
|
84
|
+
"applicableWeight": { "type": "number", "exclusiveMinimum": 0, "maximum": 1 },
|
|
85
|
+
"requiredScore": { "const": 100 }
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
"requirementResult": {
|
|
89
|
+
"type": "object",
|
|
90
|
+
"required": ["requirementId", "testProfile", "source", "additionalSources", "sourceConformance", "sourceValidity", "dimensions", "skippedTests", "flakyTests", "naDeclarations", "mandatoryGates", "scoring", "score", "result"],
|
|
91
|
+
"properties": {
|
|
92
|
+
"requirementId": { "type": ["string", "null"] },
|
|
93
|
+
"testProfile": { "enum": ["unit", "integration", null] },
|
|
94
|
+
"source": { "type": "object" },
|
|
95
|
+
"additionalSources": { "type": "array" },
|
|
96
|
+
"sourceConformance": { "enum": ["PASS", "NON_CONFORMANT", null] },
|
|
97
|
+
"sourceValidity": { "enum": ["VALID", "SOURCE_DEFECT", "SOURCE_CONFLICT", null] },
|
|
98
|
+
"sourceNotes": { "type": "string" },
|
|
99
|
+
"dimensions": { "$ref": "#/$defs/dimensions" },
|
|
100
|
+
"skippedTests": { "type": "array" },
|
|
101
|
+
"flakyTests": { "type": "array" },
|
|
102
|
+
"naDeclarations": { "type": "array" },
|
|
103
|
+
"mandatoryGates": { "$ref": "#/$defs/gates" },
|
|
104
|
+
"scoring": { "$ref": "#/$defs/scoring" },
|
|
105
|
+
"score": { "type": "number", "minimum": 0, "maximum": 100 },
|
|
106
|
+
"result": { "enum": ["PASS", "FAIL"] }
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|