@harness-engineering/cli 2.8.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/dist/agents/skills/claude-code/harness-audit-harness-strength/SKILL.md +188 -0
  2. package/dist/agents/skills/claude-code/harness-audit-harness-strength/skill.yaml +54 -0
  3. package/dist/agents/skills/claude-code/harness-autopilot/SKILL.md +2 -0
  4. package/dist/agents/skills/claude-code/harness-brainstorming/SKILL.md +28 -14
  5. package/dist/agents/skills/claude-code/harness-execution/SKILL.md +1 -1
  6. package/dist/agents/skills/claude-code/harness-git-workflow/SKILL.md +2 -0
  7. package/dist/agents/skills/claude-code/harness-ideate/SKILL.md +2 -0
  8. package/dist/agents/skills/claude-code/harness-knowledge-pipeline/SKILL.md +2 -0
  9. package/dist/agents/skills/claude-code/harness-roadmap/SKILL.md +62 -4
  10. package/dist/agents/skills/claude-code/harness-roadmap-pilot/SKILL.md +2 -0
  11. package/dist/agents/skills/claude-code/harness-test-advisor/SKILL.md +80 -2
  12. package/dist/agents/skills/claude-code/harness-test-advisor/skill.yaml +4 -1
  13. package/dist/agents/skills/claude-code/initialize-harness-project/SKILL.md +27 -25
  14. package/dist/agents/skills/claude-code/initialize-harness-project/skill.yaml +1 -0
  15. package/dist/agents/skills/claude-code/outcome-eval/SKILL.md +97 -0
  16. package/dist/agents/skills/claude-code/outcome-eval/skill.yaml +51 -0
  17. package/dist/agents/skills/codex/harness-audit-harness-strength/SKILL.md +188 -0
  18. package/dist/agents/skills/codex/harness-audit-harness-strength/skill.yaml +54 -0
  19. package/dist/agents/skills/codex/harness-autopilot/SKILL.md +2 -0
  20. package/dist/agents/skills/codex/harness-brainstorming/SKILL.md +28 -14
  21. package/dist/agents/skills/codex/harness-execution/SKILL.md +1 -1
  22. package/dist/agents/skills/codex/harness-git-workflow/SKILL.md +2 -0
  23. package/dist/agents/skills/codex/harness-ideate/SKILL.md +2 -0
  24. package/dist/agents/skills/codex/harness-knowledge-pipeline/SKILL.md +2 -0
  25. package/dist/agents/skills/codex/harness-roadmap/SKILL.md +62 -4
  26. package/dist/agents/skills/codex/harness-roadmap-pilot/SKILL.md +2 -0
  27. package/dist/agents/skills/codex/harness-test-advisor/SKILL.md +80 -2
  28. package/dist/agents/skills/codex/harness-test-advisor/skill.yaml +4 -1
  29. package/dist/agents/skills/codex/initialize-harness-project/SKILL.md +27 -25
  30. package/dist/agents/skills/codex/initialize-harness-project/skill.yaml +1 -0
  31. package/dist/agents/skills/codex/outcome-eval/SKILL.md +97 -0
  32. package/dist/agents/skills/codex/outcome-eval/skill.yaml +51 -0
  33. package/dist/agents/skills/cursor/harness-audit-harness-strength/SKILL.md +188 -0
  34. package/dist/agents/skills/cursor/harness-audit-harness-strength/skill.yaml +54 -0
  35. package/dist/agents/skills/cursor/harness-autopilot/SKILL.md +2 -0
  36. package/dist/agents/skills/cursor/harness-brainstorming/SKILL.md +28 -14
  37. package/dist/agents/skills/cursor/harness-execution/SKILL.md +1 -1
  38. package/dist/agents/skills/cursor/harness-git-workflow/SKILL.md +2 -0
  39. package/dist/agents/skills/cursor/harness-ideate/SKILL.md +2 -0
  40. package/dist/agents/skills/cursor/harness-knowledge-pipeline/SKILL.md +2 -0
  41. package/dist/agents/skills/cursor/harness-roadmap/SKILL.md +62 -4
  42. package/dist/agents/skills/cursor/harness-roadmap-pilot/SKILL.md +2 -0
  43. package/dist/agents/skills/cursor/harness-test-advisor/SKILL.md +80 -2
  44. package/dist/agents/skills/cursor/harness-test-advisor/skill.yaml +4 -1
  45. package/dist/agents/skills/cursor/initialize-harness-project/SKILL.md +27 -25
  46. package/dist/agents/skills/cursor/initialize-harness-project/skill.yaml +1 -0
  47. package/dist/agents/skills/cursor/outcome-eval/SKILL.md +97 -0
  48. package/dist/agents/skills/cursor/outcome-eval/skill.yaml +51 -0
  49. package/dist/agents/skills/gemini-cli/harness-audit-harness-strength/SKILL.md +188 -0
  50. package/dist/agents/skills/gemini-cli/harness-audit-harness-strength/skill.yaml +54 -0
  51. package/dist/agents/skills/gemini-cli/harness-autopilot/SKILL.md +2 -0
  52. package/dist/agents/skills/gemini-cli/harness-brainstorming/SKILL.md +28 -14
  53. package/dist/agents/skills/gemini-cli/harness-execution/SKILL.md +1 -1
  54. package/dist/agents/skills/gemini-cli/harness-git-workflow/SKILL.md +2 -0
  55. package/dist/agents/skills/gemini-cli/harness-ideate/SKILL.md +2 -0
  56. package/dist/agents/skills/gemini-cli/harness-knowledge-pipeline/SKILL.md +2 -0
  57. package/dist/agents/skills/gemini-cli/harness-roadmap/SKILL.md +62 -4
  58. package/dist/agents/skills/gemini-cli/harness-roadmap-pilot/SKILL.md +2 -0
  59. package/dist/agents/skills/gemini-cli/harness-test-advisor/SKILL.md +80 -2
  60. package/dist/agents/skills/gemini-cli/harness-test-advisor/skill.yaml +4 -1
  61. package/dist/agents/skills/gemini-cli/initialize-harness-project/SKILL.md +27 -25
  62. package/dist/agents/skills/gemini-cli/initialize-harness-project/skill.yaml +1 -0
  63. package/dist/agents/skills/gemini-cli/outcome-eval/SKILL.md +97 -0
  64. package/dist/agents/skills/gemini-cli/outcome-eval/skill.yaml +51 -0
  65. package/dist/agents/skills/tests/harness-test-advisor.test.ts +94 -0
  66. package/dist/{agents-md-WMJO5HSM.js → agents-md-XOJE5ETA.js} +2 -2
  67. package/dist/{architecture-EVYVGPRZ.js → architecture-JEZPJ725.js} +3 -3
  68. package/dist/{assess-project-KZG563SO.js → assess-project-RHXOQNMM.js} +1 -1
  69. package/dist/bin/harness-mcp.js +13 -13
  70. package/dist/bin/harness.js +22 -22
  71. package/dist/{check-phase-gate-MK364VXS.js → check-phase-gate-GNDOMS35.js} +3 -3
  72. package/dist/{chunk-VBNJAKUH.js → chunk-3ISHDWO7.js} +1 -1
  73. package/dist/{chunk-UJFNIWC7.js → chunk-527G27VI.js} +2 -2
  74. package/dist/{chunk-5XL5QJ5B.js → chunk-6WQCLCBO.js} +1 -1
  75. package/dist/{chunk-UXPWSV3B.js → chunk-BYVT5LVO.js} +3 -3
  76. package/dist/{chunk-HRSE6IQD.js → chunk-DE5U6KOL.js} +1 -1
  77. package/dist/{chunk-B7KMTHN3.js → chunk-GGKRA7A7.js} +1 -1
  78. package/dist/{chunk-STESM73J.js → chunk-GSP2XIVS.js} +1 -1
  79. package/dist/{chunk-KBCPELBC.js → chunk-HEIMJJI4.js} +2 -2
  80. package/dist/{chunk-KI6PHH6Q.js → chunk-HWTUUQOW.js} +3 -3
  81. package/dist/{chunk-2SNRCAC3.js → chunk-KAJOS3BA.js} +2 -2
  82. package/dist/{chunk-C7J55C6E.js → chunk-MGUPSE6D.js} +5 -5
  83. package/dist/{chunk-GOHWCHCS.js → chunk-N24HCQA4.js} +2243 -776
  84. package/dist/{chunk-EEFRPEWT.js → chunk-NWLGL3IR.js} +1 -1
  85. package/dist/{chunk-UUO4K7NX.js → chunk-ONCPJGHY.js} +6 -6
  86. package/dist/{chunk-X4EFYVCZ.js → chunk-PC2R5RUJ.js} +1 -1
  87. package/dist/{chunk-7EQAZKJ5.js → chunk-QBC7DI4Q.js} +1 -1
  88. package/dist/{chunk-UBZQE3JN.js → chunk-UX3PQKVZ.js} +7 -2
  89. package/dist/{chunk-CAQPSFPT.js → chunk-WWEVS7RO.js} +453 -91
  90. package/dist/{chunk-4BDOSCNY.js → chunk-XFU6SB2Q.js} +1015 -648
  91. package/dist/{chunk-LADPYELQ.js → chunk-XX4OLNWQ.js} +2 -2
  92. package/dist/{chunk-FZ5MBXEG.js → chunk-YQSYCBAH.js} +6 -6
  93. package/dist/{chunk-ACOOG3UW.js → chunk-Z4AWEIBY.js} +3 -3
  94. package/dist/{ci-workflow-IW2H5DC4.js → ci-workflow-ORZ4F45F.js} +2 -2
  95. package/dist/{dist-JCFK263L.js → dist-LHINSVK4.js} +85 -1
  96. package/dist/{docs-7CP6VV42.js → docs-AU3DXFFO.js} +3 -3
  97. package/dist/{engine-MKJEBWU7.js → engine-6GPWOYQH.js} +2 -2
  98. package/dist/{entropy-6JOZJYG7.js → entropy-VLKDRMRT.js} +2 -2
  99. package/dist/{feedback-T65AEJEG.js → feedback-YXYY44ZC.js} +1 -1
  100. package/dist/{generate-agent-definitions-4VI4YTO2.js → generate-agent-definitions-XIHQPKG3.js} +3 -3
  101. package/dist/hooks/cost-tracker.js +29 -6
  102. package/dist/index.js +22 -22
  103. package/dist/{loader-AWTHHE3A.js → loader-YAN56MT3.js} +2 -2
  104. package/dist/{mcp-JBIBINO2.js → mcp-RK3SFVYB.js} +13 -13
  105. package/dist/{performance-NCOUDOMW.js → performance-64DPTLLQ.js} +3 -3
  106. package/dist/{review-pipeline-YIC6JPMY.js → review-pipeline-NENG6LW7.js} +2 -2
  107. package/dist/{runtime-5MHV4UEE.js → runtime-FFIIRT2T.js} +2 -2
  108. package/dist/{security-VZPJNIDP.js → security-3PIBN35B.js} +1 -1
  109. package/dist/templates/basic/harness.config.json.hbs +34 -0
  110. package/dist/templates/ci/README.md +86 -0
  111. package/dist/templates/ci/required-review.ruleset.json +20 -0
  112. package/dist/templates/ci/required-review.yml.hbs +57 -0
  113. package/dist/templates/ci/template.json +5 -0
  114. package/dist/templates/intermediate/harness.config.json.hbs +36 -0
  115. package/dist/templates/orchestrator/harness.orchestrator.md +9 -0
  116. package/dist/{tool-tiers-B7JC2XC3.js → tool-tiers-OKK5FE57.js} +2 -0
  117. package/dist/{validate-7BE4VTFI.js → validate-7IAF2ES3.js} +3 -3
  118. package/dist/{validate-cross-check-F7GEBRDH.js → validate-cross-check-QPSAOZKD.js} +2 -2
  119. package/package.json +7 -7
@@ -0,0 +1,51 @@
1
+ name: outcome-eval
2
+ version: '0.1.0'
3
+ description: >-
4
+ LLM-judgment skill that produces a structured, confidence-rated verdict on
5
+ whether an implementation satisfied its spec. Reads the spec's acceptance
6
+ section, the change diff, and test output; emits an OutcomeVerdict
7
+ (SATISFIED | NOT_SATISFIED | INCONCLUSIVE) with confidence, rationale, and
8
+ unmet criteria. Authority is derived in TypeScript, never from the LLM: a
9
+ high-confidence NOT_SATISFIED blocks ship; every other verdict is advisory.
10
+ The verdict persists as an execution_outcome node and feeds skill-effectiveness
11
+ baselines. The harness's first blocking post-execution spec-satisfaction gate.
12
+ stability: draft
13
+ cognitive_mode: constructive-architect
14
+ triggers:
15
+ - manual
16
+ - on_pr
17
+ platforms:
18
+ - claude-code
19
+ tools:
20
+ - Bash
21
+ - Read
22
+ - Glob
23
+ - Grep
24
+ mcp:
25
+ tool: outcome_eval
26
+ # specPath, diff, and testOutput are ALL required. diff/testOutput are the
27
+ # evidence the judge reasons over: omitting them degrades the verdict to
28
+ # INCONCLUSIVE/advisory (never blocking), so the agent MUST supply them.
29
+ input:
30
+ specPath: 'string (required) — absolute or repo-relative path to the spec markdown to judge against'
31
+ diff: 'string (required) — unified diff of the change under judgment (git diff); omission degrades the verdict to INCONCLUSIVE/advisory'
32
+ testOutput: 'string (required) — captured test-runner stdout+stderr; omission degrades the verdict to INCONCLUSIVE/advisory'
33
+ model: 'string (optional) — model override for the outcome-eval LLM call'
34
+ path: 'string (optional) — project root used to resolve the knowledge graph (default cwd)'
35
+ type: rigid
36
+ tier: 2
37
+ phases:
38
+ - name: gather
39
+ description: Collect the unified diff (git diff), capture test output, resolve the spec path
40
+ required: true
41
+ - name: resolve
42
+ description: Resolve the judgment section (Success Criteria -> User-Visible Behavior -> Overview)
43
+ required: true
44
+ - name: judge
45
+ description: Invoke OutcomeEvaluator.evaluate(); LLM returns verdict/confidence/rationale/unmetCriteria; authority derived in TS
46
+ required: true
47
+ - name: gate
48
+ description: Render the verdict; on a blocking verdict (NOT_SATISFIED + high confidence) halt before Ship
49
+ required: true
50
+ state:
51
+ persistent: false
@@ -0,0 +1,188 @@
1
+ # Harness Audit: Harness Strength
2
+
3
+ > Mechanically audit whether a project's harness is load-bearing. Orchestrates the deterministic check-harness-strength engine; interprets results. Not a deep/AI review.
4
+
5
+ ## When to Use
6
+
7
+ - On a milestone gate, to confirm the harness still constrains rather than merely decorates the project
8
+ - When adding the strength audit as a required CI check, so the seven STRENGTH patterns block the merge instead of being prose advice
9
+ - When validating a harness-distribution (toolkit) repo before shipping templates downstream — to catch a default that would weaken every adopter
10
+ - When a reviewer suspects a gate "passes" without doing anything (a snapshot that asserts `passed: true` for a check it never ran)
11
+ - NOT for deep security review — use `harness-security-review` (semantic, AI-assisted)
12
+ - NOT for design-system drift — use `detect-design-drift`
13
+ - NOT for reimplementing pattern detection by hand — the engine (`HarnessStrengthAuditor`) owns detection. This skill runs it and interprets the output.
14
+
15
+ ## Process
16
+
17
+ This skill ORCHESTRATES `harness check-harness-strength`. It never re-greps configs, never re-parses hooks, and never re-derives findings. The engine reads the project once and returns a structured `AuditResult`; the skill's job is to run it with the right mode/severity and turn the JSON into an actionable report.
18
+
19
+ ### Phase 1: SCAN — Run the engine
20
+
21
+ 1. **Resolve the project root.** Use the provided `path` argument, else the current working directory.
22
+
23
+ 2. **Resolve the mode.**
24
+ - If `--toolkit` or `--adopter` (or `--mode toolkit|adopter`) is supplied, honor it. Explicit mode wins.
25
+ - Otherwise auto-detect: treat the repo as **toolkit** when it is a harness distribution — both `templates/` and `agents/skills/` exist at the root. Otherwise **adopter**.
26
+ - Do not inspect config files to "double-check" the mode. The engine resolves and reports the mode it used.
27
+
28
+ 3. **Invoke the command** via Bash, requesting JSON:
29
+
30
+ ```bash
31
+ harness check-harness-strength [--mode <adopter|toolkit>] [--severity <error|warning|info>] [--report-only] --json
32
+ ```
33
+
34
+ The `--json` payload is the raw structured `AuditResult` (score, tier, mode, findings with `id`, `gearPiece`, `severity`, `message`, `remediation`, and file:line evidence). Parse this — do not re-derive it.
35
+
36
+ 4. **Do NOT inspect config files by hand.** The engine reads `harness.config.json`, hook profiles, baselines, snapshots, and tier defaults exactly once. Hand-grepping `.husky/pre-commit` or `harness.config.json` to "confirm" a finding reimplements detection and violates the core decision behind this skill (ADR 0039 / spec D1).
37
+
38
+ ### Phase 2: DETECT — Interpret findings
39
+
40
+ 1. **Map each finding to its STRENGTH pattern** using the JSON `id` and `gearPiece` fields. Every finding the engine emits corresponds to exactly one of the seven patterns below. Do not invent findings the engine did not emit, and do not silence findings it did.
41
+
42
+ 2. **The seven STRENGTH patterns** (the engine's rule registry — for interpretation only; the engine, not the skill, decides which fire):
43
+
44
+ | ID | Gear piece | Pattern | Default |
45
+ | ------------ | ----------------------- | -------------------------------------------------------------------- | ------- |
46
+ | STRENGTH-001 | blocking-gate | Hook documented "never blocks"/"always exits 0" in an active profile | error |
47
+ | STRENGTH-002 | regression-baseline | Pre-commit auto-updates baselines/thresholds on regression | error |
48
+ | STRENGTH-003 | skip-discipline | `--skip` list > 2 categories without inline justification | warning |
49
+ | STRENGTH-004 | architecture-thresholds | `layers` defined but `architecture.thresholds` empty/absent | error |
50
+ | STRENGTH-005 | tier-default | Init/config defaults to lowest tier (`basic`) | warning |
51
+ | STRENGTH-006 | review-gate | Baseline-update PR auto-approved without independent review | error |
52
+ | STRENGTH-007 | snapshot-honesty | `passed:true` in health snapshot whose `signals[]` names that check | error |
53
+
54
+ 3. **Attach evidence.** Each finding carries a `file:line` (or config-key) reference and a remediation string from the engine. Surface them verbatim — do not paraphrase the remediation.
55
+
56
+ ### Phase 3: SCORE/REPORT
57
+
58
+ 1. **Surface the score and tier.** The engine returns a 0-100 strength score and a tier label:
59
+ - `solid` — score >= 85
60
+ - `at-risk` — score 50-84
61
+ - `theatre` — score < 50
62
+
63
+ 2. **Report format** (mirror of the security-scan report block):
64
+
65
+ ```
66
+ Harness Strength: [PASS/FAIL] — tier: <solid|at-risk|theatre> (score N/100, mode <adopter|toolkit>)
67
+ Findings: <count> (Errors: N | Warnings: N | Info: N)
68
+
69
+ [STRENGTH-00N] <gearPiece> <file:line> (severity)
70
+ <message>
71
+ Remediation: <engine remediation string>
72
+ ```
73
+
74
+ 3. **PASS/FAIL.** FAIL if any error-severity finding survives the severity threshold, unless `--report-only` was passed (then exit softens to PASS but findings are still listed). Otherwise PASS.
75
+
76
+ ## Gates
77
+
78
+ - **Error-severity findings are blocking.** The report is FAIL and the exit is non-zero when any error-severity finding survives filtering — unless `--report-only` was explicitly requested. Do not report PASS over an unsuppressed error finding.
79
+ - **No reimplementation.** The skill must run `harness check-harness-strength` and interpret its JSON. It must never hand-grep hooks, hand-parse `harness.config.json`, or re-derive a score. Reimplementing detection violates ADR 0039.
80
+ - **"Not evaluable" is not a pass.** When an input the engine needs is absent (e.g., no hook profile, no snapshot), the engine reports that state. Surface it as-is. Do not convert "could not evaluate" into "passed."
81
+ - **Mechanical only.** No AI judgment about whether a weakness "really matters." The patterns are deterministic; the engine decides what fires.
82
+
83
+ ## Escalation
84
+
85
+ - **Disputed finding / false positive:** Do not suppress by editing files or skipping the rule in the skill. Adjust severity via config — `audit.harnessStrength.severities` in `harness.config.json` (e.g., downgrade STRENGTH-003 to `info` for a justified wide `--skip`). Document the rationale alongside the override.
86
+ - **Engine misses a known weakness:** This skill cannot add detection. A missing pattern is engine work — file a new `StrengthRule` (`{ id, gearPiece, defaultSeverity, appliesIn(mode), evaluable?(ctx), detect(ctx) }`) in `packages/core/src/harness-strength/`. Out of scope for the skill.
87
+ - **Mode mis-detected:** If auto-detection picks the wrong mode (e.g., a repo with `templates/` that is not a distribution), pass `--mode adopter` (or `--toolkit`) explicitly. Do not work around it by hand-editing detection.
88
+ - **Engine throws or the build is stale:** If `harness check-harness-strength` errors (command not found / not in the built CLI), the dist may be stale. Rebuild the CLI and re-run. Do not substitute a manual config read for the engine.
89
+
90
+ ## Rationalizations to Reject
91
+
92
+ ### Universal
93
+
94
+ These reasoning patterns sound plausible but lead to bad outcomes. Reject them.
95
+
96
+ - **"It's probably fine"** — "Probably" is not evidence. Run the engine and cite the result.
97
+ - **"This is best practice"** — Best practice in what context? Cite the source and confirm it applies to this codebase.
98
+ - **"We can fix it later"** — If it is worth flagging, it is worth documenting now with a concrete follow-up plan.
99
+
100
+ ### Domain-Specific
101
+
102
+ | Rationalization | Reality |
103
+ | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
104
+ | "The config looks fine when I read it" | The engine exists because manual reading misses these patterns — that is the whole point of D1. Reading is not running. Run `harness check-harness-strength`. |
105
+ | "It only warns, not errors, so it's fine" | A gate that warns but does not stop IS STRENGTH-001 — this is the recursion item the audit was built to catch. Do not normalize "warns but doesn't block." |
106
+ | "I'll re-grep the hooks myself to double-check the engine" | Reimplementing detection violates ADR 0039 / spec D1. The engine is the single source of truth — trust and interpret its output, do not shadow it. |
107
+ | "Score is 70, that's a passing grade" | 70 is `at-risk`, not `solid`. Tier labels are thresholds, not letter grades. Below 85 means the harness has load-bearing gaps. |
108
+ | "No error findings, so I can skip reading the warnings" | Warnings (STRENGTH-003, -005) are erosion signals — a `basic` tier default or an unjustified wide `--skip` weakens every future run. Surface them. |
109
+
110
+ ## Success Criteria
111
+
112
+ - The engine (`harness check-harness-strength`) ran and produced a score, tier label, and findings (or a clean result).
113
+ - Every finding is interpreted against the seven-pattern table, with the engine's file:line evidence and remediation surfaced verbatim.
114
+ - The exit code / PASS-FAIL reflects the gate (error-severity findings fail unless `--report-only`).
115
+ - No manual config inspection was substituted for the engine — the report is derived entirely from the command's `--json` output.
116
+ - "Not evaluable" states are surfaced as-is, never converted to passes.
117
+
118
+ ## Evidence Requirements
119
+
120
+ When this skill makes claims about existing code, architecture, or behavior, it MUST cite evidence using one of:
121
+
122
+ 1. **File reference:** `file:line` format (e.g., `.husky/pre-commit:12`) — taken from the engine's finding, not re-derived.
123
+ 2. **Code pattern reference:** `file` with description (e.g., `harness.config.json` — "architecture.layers defined, thresholds absent").
124
+ 3. **Test/command output:** Inline or referenced output from the `harness check-harness-strength --json` run.
125
+ 4. **Session evidence:** Write to the `evidence` session section via `manage_state`.
126
+
127
+ **Uncited claims:** Technical assertions without citations MUST be prefixed with `[UNVERIFIED]`. Example: `[UNVERIFIED] The pre-commit hook auto-updates baselines`.
128
+
129
+ ## Harness Integration
130
+
131
+ - **`harness check-harness-strength`** — The CLI command this skill runs and interprets. Options: `--mode <adopter|toolkit>`, `--toolkit`/`--adopter` shortcuts, `--severity <error|warning|info>` (default `warning`), `--report-only`, `--json`.
132
+ - **`HarnessStrengthAuditor`** — Core class from `@harness-engineering/core` that builds a `ProjectContext` once, runs the applicable `StrengthRule`s, and aggregates the score/tier/findings. The skill never instantiates it directly; it goes through the command.
133
+ - **`harness.config.json` → `audit.harnessStrength.severities`** — Per-pattern severity overrides. The supported escalation path for false positives — never suppress by editing source.
134
+ - **`docs/standard/article-failure-patterns.md`** — Narrative companion documenting the seven patterns (forthcoming — downstream item; may not exist yet).
135
+
136
+ ## Examples
137
+
138
+ ### Example: Clean toolkit-mode run
139
+
140
+ **Input:** A harness-distribution repo (`templates/` and `agents/skills/` present), run at a milestone gate. `harness check-harness-strength --toolkit --json`.
141
+
142
+ **Output:**
143
+
144
+ ```
145
+ Harness Strength: PASS — tier: solid (score 100/100, mode toolkit)
146
+ Findings: 0 (Errors: 0 | Warnings: 0 | Info: 0)
147
+ ```
148
+
149
+ Every gear piece is load-bearing: no "never blocks" hook, baselines are not auto-updated on regression, architecture thresholds are set, the init default is not `basic`, baseline-update PRs require independent review, and no snapshot claims a check passed that it never ran.
150
+
151
+ ### Example: Findings detected (adopter mode)
152
+
153
+ **Input:** An adopter repo, run with default severity. `harness check-harness-strength --json`.
154
+
155
+ **Output:**
156
+
157
+ ```
158
+ Harness Strength: FAIL — tier: at-risk (score 62/100, mode adopter)
159
+ Findings: 2 (Errors: 2 | Warnings: 0 | Info: 0)
160
+
161
+ [STRENGTH-001] blocking-gate .husky/pre-commit:12 (error)
162
+ Active pre-commit profile documents "always exits 0" — the gate reports but never blocks.
163
+ Remediation: Remove the unconditional `exit 0`; let failing checks return a non-zero status.
164
+
165
+ [STRENGTH-004] architecture-thresholds harness.config.json (error)
166
+ architecture.layers is defined but architecture.thresholds is empty — layer rules cannot be enforced.
167
+ Remediation: Set architecture.thresholds (e.g. maxModuleLines, maxDependencyDepth) so the layer
168
+ definitions become enforceable, or remove the unused layers block.
169
+ ```
170
+
171
+ Two error-severity findings → FAIL. STRENGTH-001 is the recursion item: a hook that runs but cannot block. Run with `--report-only` only when you explicitly want to surface findings without failing the build (e.g., a first baseline-capture run) — and never to normalize the weakness.
172
+
173
+ ## Skill Test Scenarios
174
+
175
+ ### Scenario 1: Red Flag — "I'll re-grep the hooks myself to double-check the engine"
176
+
177
+ Input: The agent has the `harness check-harness-strength --json` output in hand but is about to open `.husky/pre-commit` and grep it by hand to "confirm" the STRENGTH-001 finding.
178
+ Expected: Agent stops, cites the no-reimplementation gate (ADR 0039 / spec D1), and interprets the engine's JSON instead of hand-grepping the hook.
179
+
180
+ ### Scenario 2: Rationalization — "it only warns, not errors, so it's fine"
181
+
182
+ Input: The engine reports a documented "never blocks" pre-commit profile, and the agent is tempted to treat it as acceptable because nothing currently fails the build.
183
+ Expected: Agent rejects the rationalization, recognizes that "warns but doesn't stop" IS STRENGTH-001 (the recursion item), and surfaces it as a finding rather than normalizing it.
184
+
185
+ ### Scenario 3: Gate — error-severity STRENGTH-001 finding present without `--report-only`
186
+
187
+ Input: The `--json` output contains an error-severity STRENGTH-001 finding, `--report-only` was not passed, and the agent is about to summarize the run as PASS.
188
+ Expected: Agent halts, reports FAIL with a non-zero exit, lists the finding with its file:line evidence and remediation, and does not proceed past the gate.
@@ -0,0 +1,54 @@
1
+ name: harness-audit-harness-strength
2
+ version: '0.1.0'
3
+ description: Mechanically audit a project's own harness setup against the seven STRENGTH failure patterns; reports per-pattern findings, a 0-100 strength score, and a tier label (solid/at-risk/theatre). Orchestrates harness check-harness-strength; never reimplements detection.
4
+ stability: draft
5
+ cognitive_mode: constructive-architect
6
+ triggers:
7
+ - manual
8
+ - on_milestone
9
+ platforms:
10
+ - claude-code
11
+ - gemini-cli
12
+ - cursor
13
+ - codex
14
+ tools:
15
+ - Bash
16
+ - Read
17
+ - Grep
18
+ - Glob
19
+ cli:
20
+ command: harness skill run harness-audit-harness-strength
21
+ args:
22
+ - name: path
23
+ description: Project root path
24
+ required: false
25
+ - name: mode
26
+ description: 'adopter (default) or toolkit (auto-detected in a harness-distribution repo)'
27
+ required: false
28
+ - name: severity
29
+ description: Minimum severity threshold (error, warning, info)
30
+ required: false
31
+ - name: report-only
32
+ description: Soften exit to 0 even when error-severity findings exist
33
+ required: false
34
+ mcp:
35
+ tool: run_skill
36
+ input:
37
+ skill: harness-audit-harness-strength
38
+ path: string
39
+ type: rigid
40
+ tier: 2
41
+ phases:
42
+ - name: scan
43
+ description: Resolve mode and run harness check-harness-strength against the target repo
44
+ required: true
45
+ - name: detect
46
+ description: Interpret the 7 STRENGTH pattern findings with file-line evidence
47
+ required: true
48
+ - name: score_report
49
+ description: Surface the 0-100 score, tier label, and per-pattern remediation
50
+ required: true
51
+ state:
52
+ persistent: false
53
+ files: []
54
+ depends_on: []
@@ -213,6 +213,8 @@ Persist findings to `{sessionDir}/phase-{N}-review.json`. No blocking → PHASE_
213
213
 
214
214
  ## Process
215
215
 
216
+ **Prompt the human in plain text** — every phase-continue and human-decision interaction in this skill is plain text only. Do not elevate to `AskUserQuestion`: natural headers like "Continue phase" exceed its 12-char cap, rendering the call as ERR.
217
+
216
218
  1. **INIT** — Resolve spec, derive session slug, check for existing state, parse phases.
217
219
  2. **ASSESS** — Route by complexity: low/medium auto-plans, high pauses for interactive planning.
218
220
  3. **PLAN → APPROVE** — Dispatch harness-planner, check approval signals, auto-approve or pause.
@@ -136,7 +136,7 @@ These flow into `handoff.json` `contextKeywords` field. Select keywords that hel
136
136
  - **Entry Points** -- Which system entry points does this feature touch or create? (e.g., new CLI command, new MCP tool, new skill, new API route, new barrel export)
137
137
  - **Registrations Required** -- What registrations are needed for the feature to be discoverable? (e.g., barrel export regeneration, skill tier assignment, route registration)
138
138
  - **Documentation Updates** -- What docs need updating to reflect the new capability? (e.g., AGENTS.md section, API docs, README, guides)
139
- - **Architectural Decisions** -- What decisions warrant ADRs? List the decision and a one-line rationale. Only for medium/large tier changes -- omit for small changes.
139
+ - **Architectural Decisions** -- Which decisions from the **Decisions made** section rise to a standalone ADR? Reference each by name with a one-line note on _why_ it warrants an ADR. Do **not** restate the decisions here -- this subsection points back to the canonical **Decisions made** section; duplicating them creates two sources that drift (spec-craft flags this as SPEC-R004). Only for medium/large tier changes -- "None" for small changes.
140
140
  - **Knowledge Impact** -- What domain concepts, patterns, or relationships should enter the knowledge graph?
141
141
 
142
142
  If the feature is a small change (bug fix, config tweak, < 3 files), the section may contain only Entry Points and Registrations Required with "None" for the others. The section must still be present.
@@ -186,21 +186,35 @@ These flow into `handoff.json` `contextKeywords` field. Select keywords that hel
186
186
 
187
187
  The human must explicitly approve before this skill is complete.
188
188
 
189
- 7. **Commit spec artifacts.** After approval, commit the spec and skill recommendations so the paper trail enters git history at sign-off rather than retroactively:
189
+ 7. **Promote the roadmap row.** If `docs/roadmap.md` exists, transition the named row to `planned` and link the spec in a single structured call (steps 7 and 8 are intentionally ordered so the commit captures the roadmap mutation). Skip silently only when no roadmap exists.
190
+ - Derive the lookup key from the slash-command `ARGUMENTS` string (D1). Derive the summary from the spec title (the H1 heading).
191
+ - Call `manage_roadmap` with action `promote`, `feature: "<ARGUMENTS>"`, `spec: "docs/changes/<feature>/proposal.md"`, and `summary: "<H1>"`.
192
+ - Branch on the returned `PromoteResult` envelope:
193
+
194
+ | Envelope | Skill behavior |
195
+ | ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
196
+ | `ok: true, transitioned: 'backlog→planned'` | Log "Promoted `<feature>`: backlog → planned". Continue to step 8. |
197
+ | `ok: true, transitioned: 'spec-updated'` | Log "Updated spec link for `<feature>` (status preserved)". Continue to step 8. |
198
+ | `ok: true, transitioned: 'noop'` | Log "No change — `<feature>` already promoted with this spec". Continue to step 8. |
199
+ | `ok: false, reason: 'in-progress'` | **STOP.** Surface: "Refused to promote `<feature>`: an agent is currently dispatched against this row. Stop the agent or use a different feature name." |
200
+ | `ok: false, reason: 'done'` | **STOP.** Surface: "Refused to promote `<feature>`: row is already 'done'. To revise a shipped feature, use a new name." |
201
+ | `ok: false, reason: 'not-found'` | If no row was expected to exist, fall through to a `created` outcome. Otherwise surface `closestMatches` as a typo hint and **STOP**. |
202
+ | `ok: false, reason: 'ambiguous'` | **STOP.** Surface: "Refused to promote `<feature>`: matches multiple rows across milestones. Re-invoke with one of: `<matches>`." Matches are milestone-qualified. |
203
+ | `ok: false, reason: 'write-failed'` | **STOP.** Surface `detail` verbatim. The brainstorm is not considered complete. |
204
+
205
+ - The `not-found` create path: when the row genuinely does not exist yet, call `manage_roadmap` action `add` with `status: "planned"`, `milestone: "Current Work"`, the spec path, and the H1 summary (unchanged from the legacy behavior), then continue to step 8.
206
+ - "STOP" cases skip step 8 (no commit) and step 9 (no handoff/transition). The spec file written in step 3 stays on disk; the user re-runs the skill after resolving the conflict.
207
+ - If `manage_roadmap` is unavailable, fall back to `parseRoadmap`/`promoteFeature`/`serializeRoadmap` from core. Warn: "External sync skipped (MCP unavailable). Run `manage_roadmap sync` when MCP is restored."
208
+ - If no roadmap exists, skip silently (no envelope) and continue to step 8; the commit then contains only the spec files.
209
+
210
+ 8. **Commit spec artifacts and roadmap promotion.** After a successful promote (or a silent skip when no roadmap exists), commit the spec, skill recommendations, and the roadmap mutation together so the promotion ships atomically with the spec:
190
211
 
191
212
  ```bash
192
- git add docs/changes/<feature>/proposal.md docs/changes/<feature>/SKILLS.md
193
- git commit -m "docs(<feature>): add spec"
213
+ git add docs/changes/<feature>/proposal.md docs/changes/<feature>/SKILLS.md docs/roadmap.md
214
+ git commit -m "docs(<feature>): add spec and promote to planned"
194
215
  ```
195
216
 
196
- Do not skip this step. `harness-execution` commits only implementation files, so a spec uncommitted here stays untracked until someone notices. If a pre-commit hook reformats either file, re-add and re-commit.
197
-
198
- 8. **Add feature to roadmap.** If `docs/roadmap.md` exists:
199
- - Derive feature name from the spec title (the H1 heading).
200
- - Call `manage_roadmap` with action `add`, `status: "planned"`, `milestone: "Current Work"`, and the spec path.
201
- - If the feature already exists, skip silently.
202
- - If `manage_roadmap` is unavailable, fall back to `parseRoadmap`/`serializeRoadmap` from core. Warn: "External sync skipped (MCP unavailable). Run `manage_roadmap sync` when MCP is restored."
203
- - If no roadmap exists, skip silently.
217
+ Include `docs/roadmap.md` in `git add` only when it exists; omit it when the project has no roadmap. Do not skip this step `harness-execution` commits only implementation files, so a spec uncommitted here stays untracked until someone notices. If a pre-commit hook reformats a file, re-add and re-commit.
204
218
 
205
219
  9. **Write handoff and suggest transition.** After approval:
206
220
 
@@ -336,8 +350,8 @@ Technical claims about existing code, architecture, or tradeoffs MUST cite evide
336
350
  - **Spec location** -- `docs/changes/<feature>/proposal.md`.
337
351
  - **Handoff** -- Once approved, invoke harness-planning to create the implementation plan.
338
352
  - **Session directory** — When session slug is known, handoff goes to `.harness/sessions/<slug>/handoff.json`. The session directory structure is: `handoff.json`, `state.json`, `artifacts.json` (registry of spec/plan paths and file lists). Do not write to `.harness/handoff.json` in session context.
339
- - **Spec commit** -- After sign-off (Phase 4 Step 7), commit `docs/changes/<feature>/proposal.md` and `docs/changes/<feature>/SKILLS.md` so the spec enters git history at approval, not retroactively. `harness-execution` does not backfill these — see issue #487.
340
- - **Roadmap sync** -- After approval, call `manage_roadmap` action `add` to register as `planned`. Skip silently if no roadmap. Duplicates ignored.
353
+ - **Spec commit** -- After sign-off, the Phase 4 step 8 commit captures `docs/changes/<feature>/proposal.md`, `docs/changes/<feature>/SKILLS.md`, and the promoted `docs/roadmap.md` together so the spec enters git history at approval, not retroactively. `harness-execution` does not backfill these — see issue #487.
354
+ - **Roadmap promotion** -- After approval (Phase 4 step 7), call `manage_roadmap` action `promote` to transition the named row to `planned` and link the spec atomically with the spec commit. Falls back to `add` (create new row) when the row does not exist; refuses on `in-progress`/`done`/`ambiguous`. Skip silently if no roadmap.
341
355
  - **`emit_interaction`** -- End of Phase 4 to suggest transition to harness-planning (confirmed transition).
342
356
 
343
357
  #### Requirement Phrasing
@@ -56,7 +56,7 @@ When no arguments are provided (standalone invocation), discover plan from `docs
56
56
  4. **Load session summary for cold start.** If resuming (session slug known):
57
57
  - Call `listActiveSessions()` to read the session index.
58
58
  - Call `loadSessionSummary()` for the target session.
59
- - If ambiguous, present the index and ask which session to resume.
59
+ - If ambiguous, present the index and ask which session to resume. Ask in plain text — do not elevate this to `AskUserQuestion` (the session index can exceed its 4-option cap and a natural header like "Pick session" can exceed its 12-char cap, rendering the call as ERR).
60
60
 
61
61
  5. **Check for known dead ends.** Review `learnings` tagged `[outcome:failure]`. Warn if any match current plan approaches.
62
62
 
@@ -13,6 +13,8 @@
13
13
 
14
14
  ## Process
15
15
 
16
+ **Prompt the human in plain text** — every choice and destructive confirmation in this skill (worktree location, discard-experiment, etc.) is plain text only. Do not elevate to `AskUserQuestion`: option labels like the A/B/C worktree-location choices and natural headers like "Discard commits" routinely exceed its 4-option / 12-char caps and render as ERR.
17
+
16
18
  ### Part A: Worktree Creation
17
19
 
18
20
  #### Step 1: Choose Worktree Location
@@ -15,6 +15,8 @@
15
15
 
16
16
  ## Process
17
17
 
18
+ **Prompt the human in plain text** — every confirmation and selection in this skill (`Proceed? (y/n)`, post-critique selection, etc.) is plain text only. Do not elevate to `AskUserQuestion`: natural headers like "Confirm inputs" exceed its 12-char cap and option lists routinely exceed its 4-option cap, rendering the call as ERR.
19
+
18
20
  ### Iron Law
19
21
 
20
22
  **The skill produces exactly ONE ranked artifact per run, at `docs/ideation/<slug>-YYYY-MM-DD.md`, and NEVER produces a spec, plan, ADR, or code.** If the user wants a spec from a ranked idea, they invoke `harness-brainstorming` next — that is the contract.
@@ -14,6 +14,8 @@
14
14
 
15
15
  ## Process
16
16
 
17
+ **Prompt the human in plain text** — every confirmation, drift acknowledgement, and contradiction resolution in this skill is plain text only. Do not elevate to `AskUserQuestion`: source labels and finding descriptions routinely exceed its 4-option / 12-char caps, rendering the call as ERR.
18
+
17
19
  1. **EXTRACT:** Run code signal extractors, diagram parsers, BusinessKnowledgeIngestor, and KnowledgeLinker against the project
18
20
  2. **RECONCILE:** Build pre-extraction snapshot (current graph state) and post-extraction snapshot, then run StructuralDriftDetector
19
21
  3. **DETECT:** Partition findings by classification (new/stale/drifted/contradicting), generate gap report
@@ -9,6 +9,7 @@
9
9
  - When adding a new feature to an existing roadmap (`--add <feature-name>`)
10
10
  - When roadmap statuses may be stale and need updating from plan execution state (`--sync`)
11
11
  - When features need reordering, moving between milestones, or blocker updates (`--edit`)
12
+ - When the roadmap needs tidying -- completed work archived, dead `planned` rows demoted (`--groom`)
12
13
  - When user asks about project status and no roadmap exists -- suggest `--create`
13
14
  - NOT for programmatic CRUD (use `manage_roadmap` MCP tool directly)
14
15
 
@@ -20,6 +21,8 @@
20
21
 
21
22
  If the human has not seen and approved the milestone groupings and feature list, do not write the file. Present. Wait. Confirm. Then write.
22
23
 
24
+ **Prompt the human in plain text — every `(y/n)`, "which feature?", and similar prompt in this skill is plain text only.** Do not elevate them to `AskUserQuestion`: feature lists routinely exceed its 4-option cap, and natural header choices ("Pick feature", "Remove feature") exceed its 12-char cap, causing the call to render as ERR.
25
+
23
26
  ---
24
27
 
25
28
  ### Command: `--create` -- Bootstrap Roadmap
@@ -428,12 +431,67 @@ Choice?
428
431
 
429
432
  ---
430
433
 
434
+ ### Command: `--groom` -- Tidy the Roadmap
435
+
436
+ Keeps the roadmap manageable over time. **Milestones are themes; statuses are lifecycle stages** -- grooming enforces that separation so the backlog never decays back into an undifferentiated dump.
437
+
438
+ #### Phase 1: SCAN -- Detect Untidiness
439
+
440
+ 1. Check if `docs/roadmap.md` exists. If missing: error and direct the user to `--create`.
441
+ 2. Run `manage_roadmap` (`action: "groom"`) in a dry-run frame, or call `checkRoadmapHealth` from `@harness-engineering/core`, to surface the four health signals:
442
+ - **RMH001** -- completed (`done`) features still sitting in an active milestone.
443
+ - **RMH002** -- `planned` rows with neither a spec nor a plan (the orchestrator cannot auto-execute these; it escalates them to a human).
444
+ - **RMH003** -- lifecycle catch-all milestones (`Backlog`, `Current Work`) that should not exist.
445
+ - **RMH004** -- active milestones that have grown past the size cap (a mini-dump).
446
+
447
+ #### Phase 2: PROPOSE -- Present the Plan
448
+
449
+ Show the human exactly what grooming will do, in plain text:
450
+
451
+ ```
452
+ GROOM PLAN
453
+
454
+ Demote to backlog (planned with no spec/plan):
455
+ - Feature A (Theme X)
456
+ - Feature B (Theme Y)
457
+
458
+ Archive to docs/roadmap-archive.md (completed):
459
+ - Feature C (Theme X)
460
+
461
+ Flagged for manual routing (not auto-changed):
462
+ - Intake lane has 3 items awaiting a theme
463
+ - "Theme Z" has 28 features (cap 25) -- consider splitting
464
+
465
+ Apply? (y/n)
466
+ ```
467
+
468
+ Wait for confirmation. The mechanical changes (demote, archive) are safe and automated; **draining the Intake lane into themed milestones and splitting oversized milestones are human decisions** -- propose, do not auto-apply.
469
+
470
+ #### Phase 3: WRITE -- Apply
471
+
472
+ 1. Run `manage_roadmap` (`action: "groom"`). It demotes unactionable `planned` rows to `backlog` and moves `done` features into `docs/roadmap-archive.md` under a `Shipped` milestone, returning the list of changes.
473
+ 2. For Intake-draining or milestone-splitting the human approved, follow up with `--edit` (move features between milestones).
474
+
475
+ #### Phase 4: VALIDATE -- Verify
476
+
477
+ 1. Run `harness validate` and confirm the `roadmapHealth` check passes (no RMH003 errors; RMH001/002/004 warnings cleared or acknowledged).
478
+ 2. Summarize:
479
+
480
+ ```
481
+ Groom complete.
482
+ Demoted: N | Archived: N -> docs/roadmap-archive.md | Flagged for manual routing: N
483
+ harness validate (roadmapHealth): passed
484
+ ```
485
+
486
+ ---
487
+
431
488
  ## Harness Integration
432
489
 
433
- - **`manage_roadmap` MCP tool** -- Primary read/write interface for roadmap operations. Supports `show`, `add`, `update`, `remove`, and `query` actions. Use this when MCP is available for structured CRUD.
434
- - **`harness validate`** -- Run after any roadmap modification to verify project health. Mandatory in the VALIDATE phase of both `--create` and `--add`.
435
- - **Core `parseRoadmap`/`serializeRoadmap`** -- Fallback when MCP is unavailable. These functions in `packages/core/src/roadmap/` handle parsing and serializing the roadmap markdown format directly.
436
- - **Roadmap file** -- Always at `docs/roadmap.md`. This is the single source of truth for the project roadmap.
490
+ - **`manage_roadmap` MCP tool** -- Primary read/write interface for roadmap operations. Supports `show`, `add`, `update`, `remove`, `query`, `sync`, `promote`, and `groom` actions. Use this when MCP is available for structured CRUD.
491
+ - **`harness validate`** -- Run after any roadmap modification to verify project health. Mandatory in the VALIDATE phase of `--create`, `--add`, and `--groom`. The `roadmapHealth` check enforces the maintenance rules (RMH001-RMH004) as a regression guard.
492
+ - **Core `checkRoadmapHealth`/`groomRoadmap`** -- Maintenance engine in `packages/core/src/roadmap/health.ts`. `checkRoadmapHealth` is read-only diagnostics; `groomRoadmap` is the pure transform (demote unactionable planned, archive done). Both are surfaced via `manage_roadmap` and `harness validate`.
493
+ - **Core `parseRoadmap`/`serializeRoadmap`** -- Fallback when MCP is unavailable. These functions in `packages/core/src/roadmap/` handle parsing and serializing the roadmap markdown format directly. Note: the serializer only preserves frontmatter, milestones, features, and the Assignment History table -- never add convention prose or comments to `docs/roadmap.md`, they are dropped on the next write.
494
+ - **Roadmap files** -- Live work in `docs/roadmap.md` (the orchestrator's source of truth); completed work archived to `docs/roadmap-archive.md` by `--groom`. Milestones are themes, not lifecycle stages -- promoted items land in the `Intake` lane and are groomed into themes.
437
495
 
438
496
  ## Success Criteria
439
497
 
@@ -126,6 +126,8 @@ Proceed with Feature A? (y/n/pick another)
126
126
 
127
127
  ### Phase 3: CONFIRM -- Human Decision
128
128
 
129
+ Ask the human in plain text (matching the `y/n/pick another` example above). Do not elevate this confirmation to an `AskUserQuestion` tool call — candidate labels and natural header choices ("Pick candidate", etc.) exceed its 12-char `header` cap and the prompt is rejected as ERR.
130
+
129
131
  1. Wait for human confirmation.
130
132
  - If **yes**: proceed to Phase 4.
131
133
  - If **pick another**: ask which candidate number, then proceed with that pick.
@@ -8,8 +8,9 @@
8
8
  - In CI — optimize test suite execution order
9
9
  - When a test fails — understand which changes could have caused it
10
10
  - When `on_pr` triggers fire
11
- - NOT for writing tests (use harness-tdd)
12
- - NOT for test quality analysis (out of scope)
11
+ - **Coverage Audit mode** — when no diff is available and the user asks for "coverage gaps", "deep dive", "coverage plan", or "what's untested", run project-wide gap analysis instead of test selection
12
+ - NOT for writing tests (use harness-tdd, or `canary:canary-write-test` for uncovered files surfaced by Coverage Audit)
13
+ - NOT for test quality analysis at the test-selection level (Coverage Audit mode does include a capped quality review via `canary:canary-review-test`)
13
14
 
14
15
  ## Prerequisites
15
16
 
@@ -112,6 +113,83 @@ npx vitest run tests/services/auth.test.ts tests/types/user.test.ts tests/routes
112
113
  npx vitest run tests/services/auth.test.ts tests/types/user.test.ts tests/routes/login.test.ts tests/middleware/verify.test.ts tests/integration/auth-flow.test.ts
113
114
  ```
114
115
 
116
+ ## Coverage Audit Mode
117
+
118
+ Activates when `--audit` is passed, OR when no diff is available AND the user's
119
+ language matches audit intent ("coverage gaps", "deep dive", "coverage plan",
120
+ "what's untested"). When this mode is selected, skip Phases 1–3 above and run
121
+ the audit phases below instead.
122
+
123
+ ### Audit Phase 0: PROBE — Detect canary CLI availability
124
+
125
+ Call the `canary_probe` MCP tool once at the start of the audit. It returns
126
+ `{ status: "available" | "degraded", version?, reason? }` and never errors.
127
+
128
+ - **`available`** — the deterministic canary CLI is usable; use `canary_recommend_framework`
129
+ for framework selection in Phase 3 (GAP REPORT).
130
+ - **`degraded`** — the CLI is not usable (reason: `not-installed`, `binary-missing`,
131
+ `exec-failed`, or `bad-output`). Print one line:
132
+
133
+ > canary CLI unavailable (`<reason>`) — install `canary-test-cli` for deterministic
134
+ > framework recommendations. Proceeding; framework picks fall back to the plugin.
135
+
136
+ Then skip `canary_recommend_framework` calls for the rest of the run. This does **not**
137
+ affect the plugin-based Quality Review in Phase 2 (`canary:canary-review-test` is a
138
+ separate Claude Code plugin, installed independently of the CLI).
139
+
140
+ ### Audit Phase 1: INVENTORY — Build the Source-to-Test Map
141
+
142
+ 1. **Enumerate source files**: glob for `.ts`, `.tsx`, `.js`, `.jsx` under the
143
+ project's source roots (e.g., `src/`, `packages/*/src/`). Skip `node_modules`,
144
+ `dist`, build outputs, and fixtures.
145
+ 2. **Enumerate test files**: glob for `*.test.*`, `*.spec.*`, and files under
146
+ `__tests__/` and parallel `tests/` directories.
147
+ 3. **Map source → test**: for each source file, find its test file using the
148
+ Tier 1 / Tier 2 / Tier 3 strategies described in `Phase 2: DISCOVER`. With a
149
+ graph, prefer `get_impact`; without one, use naming conventions and import
150
+ parsing.
151
+ 4. **Split**: produce two lists — `covered` (source files with at least one
152
+ matching test) and `uncovered` (source files with no matching test).
153
+
154
+ ### Audit Phase 2: QUALITY REVIEW — Critique Covered Test Files
155
+
156
+ 1. **Cap at 10 files per run**: a deep quality review is expensive. Pick the
157
+ top 10 covered files prioritized by size (lines of code) and criticality
158
+ (depth in the graph / co-change frequency). The remaining covered files are
159
+ accepted as-is for this run.
160
+ 2. **For each of the 10**: dispatch `canary:canary-review-test` against the
161
+ test file. Collect its findings (missing edge cases, anti-patterns,
162
+ brittleness, oracle gaps).
163
+ 3. **Aggregate**: bin findings by severity (high / medium / low) and by file.
164
+
165
+ ### Audit Phase 3: GAP REPORT — Synthesize a Unified Coverage Plan
166
+
167
+ Emit a single report with three sections:
168
+
169
+ ```
170
+ ## Coverage Audit Report
171
+
172
+ ### Uncovered Files (no test found)
173
+ | File | Lines | Priority | Suggested Action |
174
+ |------|-------|----------|------------------|
175
+ | src/services/billing.ts | 412 | high | canary:canary-write-test |
176
+ | src/utils/format.ts | 38 | low | canary:canary-write-test |
177
+
178
+ ### Quality Gaps (from Quality Review, capped at 10 reviewed)
179
+ | Test File | Severity | Top Gap |
180
+ |-----------|----------|---------|
181
+ | tests/services/auth.test.ts | high | no failure-path assertions |
182
+ | tests/utils/date.test.ts | med | mock leaks across cases |
183
+
184
+ ### Recommended Next Steps
185
+ - Generate tests for high-priority uncovered files via `canary:canary-write-test`.
186
+ - Resolve high-severity quality gaps via `canary:canary-review-test` follow-ups.
187
+ - For uncovered files, pick a framework first: when Phase 0 reported `available`, call
188
+ the `canary_recommend_framework` MCP tool with a prompt describing the file's purpose
189
+ (deterministic, no LLM round-trip) and put its `framework` in the Suggested Action.
190
+ When `degraded`, fall back to the `canary:canary-pick-framework` plugin.
191
+ ```
192
+
115
193
  ## Harness Integration
116
194
 
117
195
  - **`harness scan`** — Recommended before this skill for full graph-enhanced analysis. If graph is missing, skill uses naming convention and import parsing fallbacks.
@@ -1,6 +1,6 @@
1
1
  name: harness-test-advisor
2
2
  version: "1.0.0"
3
- description: Graph-based test selection — answers "what tests should I run?"
3
+ description: Graph-based test selection and project-wide coverage audit — answers "what tests should I run?" or "what's untested?"
4
4
  stability: static
5
5
  cognitive_mode: advisory-guide
6
6
  triggers:
@@ -25,6 +25,9 @@ cli:
25
25
  - name: files
26
26
  description: Comma-separated list of changed files
27
27
  required: false
28
+ - name: audit
29
+ description: Run Coverage Audit mode (project-wide gap analysis) instead of test selection
30
+ required: false
28
31
  mcp:
29
32
  tool: run_skill
30
33
  input: