@harness-engineering/cli 2.8.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/skills/claude-code/harness-audit-harness-strength/SKILL.md +188 -0
- package/dist/agents/skills/claude-code/harness-audit-harness-strength/skill.yaml +54 -0
- package/dist/agents/skills/claude-code/harness-autopilot/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-brainstorming/SKILL.md +28 -14
- package/dist/agents/skills/claude-code/harness-execution/SKILL.md +1 -1
- package/dist/agents/skills/claude-code/harness-git-workflow/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-ideate/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-knowledge-pipeline/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-roadmap/SKILL.md +62 -4
- package/dist/agents/skills/claude-code/harness-roadmap-pilot/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-test-advisor/SKILL.md +80 -2
- package/dist/agents/skills/claude-code/harness-test-advisor/skill.yaml +4 -1
- package/dist/agents/skills/claude-code/initialize-harness-project/SKILL.md +27 -25
- package/dist/agents/skills/claude-code/initialize-harness-project/skill.yaml +1 -0
- package/dist/agents/skills/claude-code/outcome-eval/SKILL.md +97 -0
- package/dist/agents/skills/claude-code/outcome-eval/skill.yaml +51 -0
- package/dist/agents/skills/codex/harness-audit-harness-strength/SKILL.md +188 -0
- package/dist/agents/skills/codex/harness-audit-harness-strength/skill.yaml +54 -0
- package/dist/agents/skills/codex/harness-autopilot/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-brainstorming/SKILL.md +28 -14
- package/dist/agents/skills/codex/harness-execution/SKILL.md +1 -1
- package/dist/agents/skills/codex/harness-git-workflow/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-ideate/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-knowledge-pipeline/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-roadmap/SKILL.md +62 -4
- package/dist/agents/skills/codex/harness-roadmap-pilot/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-test-advisor/SKILL.md +80 -2
- package/dist/agents/skills/codex/harness-test-advisor/skill.yaml +4 -1
- package/dist/agents/skills/codex/initialize-harness-project/SKILL.md +27 -25
- package/dist/agents/skills/codex/initialize-harness-project/skill.yaml +1 -0
- package/dist/agents/skills/codex/outcome-eval/SKILL.md +97 -0
- package/dist/agents/skills/codex/outcome-eval/skill.yaml +51 -0
- package/dist/agents/skills/cursor/harness-audit-harness-strength/SKILL.md +188 -0
- package/dist/agents/skills/cursor/harness-audit-harness-strength/skill.yaml +54 -0
- package/dist/agents/skills/cursor/harness-autopilot/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-brainstorming/SKILL.md +28 -14
- package/dist/agents/skills/cursor/harness-execution/SKILL.md +1 -1
- package/dist/agents/skills/cursor/harness-git-workflow/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-ideate/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-knowledge-pipeline/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-roadmap/SKILL.md +62 -4
- package/dist/agents/skills/cursor/harness-roadmap-pilot/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-test-advisor/SKILL.md +80 -2
- package/dist/agents/skills/cursor/harness-test-advisor/skill.yaml +4 -1
- package/dist/agents/skills/cursor/initialize-harness-project/SKILL.md +27 -25
- package/dist/agents/skills/cursor/initialize-harness-project/skill.yaml +1 -0
- package/dist/agents/skills/cursor/outcome-eval/SKILL.md +97 -0
- package/dist/agents/skills/cursor/outcome-eval/skill.yaml +51 -0
- package/dist/agents/skills/gemini-cli/harness-audit-harness-strength/SKILL.md +188 -0
- package/dist/agents/skills/gemini-cli/harness-audit-harness-strength/skill.yaml +54 -0
- package/dist/agents/skills/gemini-cli/harness-autopilot/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-brainstorming/SKILL.md +28 -14
- package/dist/agents/skills/gemini-cli/harness-execution/SKILL.md +1 -1
- package/dist/agents/skills/gemini-cli/harness-git-workflow/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-ideate/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-knowledge-pipeline/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-roadmap/SKILL.md +62 -4
- package/dist/agents/skills/gemini-cli/harness-roadmap-pilot/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-test-advisor/SKILL.md +80 -2
- package/dist/agents/skills/gemini-cli/harness-test-advisor/skill.yaml +4 -1
- package/dist/agents/skills/gemini-cli/initialize-harness-project/SKILL.md +27 -25
- package/dist/agents/skills/gemini-cli/initialize-harness-project/skill.yaml +1 -0
- package/dist/agents/skills/gemini-cli/outcome-eval/SKILL.md +97 -0
- package/dist/agents/skills/gemini-cli/outcome-eval/skill.yaml +51 -0
- package/dist/agents/skills/tests/harness-test-advisor.test.ts +94 -0
- package/dist/{agents-md-WMJO5HSM.js → agents-md-XOJE5ETA.js} +2 -2
- package/dist/{architecture-EVYVGPRZ.js → architecture-JEZPJ725.js} +3 -3
- package/dist/{assess-project-KZG563SO.js → assess-project-RHXOQNMM.js} +1 -1
- package/dist/bin/harness-mcp.js +13 -13
- package/dist/bin/harness.js +22 -22
- package/dist/{check-phase-gate-MK364VXS.js → check-phase-gate-GNDOMS35.js} +3 -3
- package/dist/{chunk-VBNJAKUH.js → chunk-3ISHDWO7.js} +1 -1
- package/dist/{chunk-UJFNIWC7.js → chunk-527G27VI.js} +2 -2
- package/dist/{chunk-5XL5QJ5B.js → chunk-6WQCLCBO.js} +1 -1
- package/dist/{chunk-UXPWSV3B.js → chunk-BYVT5LVO.js} +3 -3
- package/dist/{chunk-HRSE6IQD.js → chunk-DE5U6KOL.js} +1 -1
- package/dist/{chunk-B7KMTHN3.js → chunk-GGKRA7A7.js} +1 -1
- package/dist/{chunk-STESM73J.js → chunk-GSP2XIVS.js} +1 -1
- package/dist/{chunk-KBCPELBC.js → chunk-HEIMJJI4.js} +2 -2
- package/dist/{chunk-KI6PHH6Q.js → chunk-HWTUUQOW.js} +3 -3
- package/dist/{chunk-2SNRCAC3.js → chunk-KAJOS3BA.js} +2 -2
- package/dist/{chunk-C7J55C6E.js → chunk-MGUPSE6D.js} +5 -5
- package/dist/{chunk-GOHWCHCS.js → chunk-N24HCQA4.js} +2243 -776
- package/dist/{chunk-EEFRPEWT.js → chunk-NWLGL3IR.js} +1 -1
- package/dist/{chunk-UUO4K7NX.js → chunk-ONCPJGHY.js} +6 -6
- package/dist/{chunk-X4EFYVCZ.js → chunk-PC2R5RUJ.js} +1 -1
- package/dist/{chunk-7EQAZKJ5.js → chunk-QBC7DI4Q.js} +1 -1
- package/dist/{chunk-UBZQE3JN.js → chunk-UX3PQKVZ.js} +7 -2
- package/dist/{chunk-CAQPSFPT.js → chunk-WWEVS7RO.js} +453 -91
- package/dist/{chunk-4BDOSCNY.js → chunk-XFU6SB2Q.js} +1015 -648
- package/dist/{chunk-LADPYELQ.js → chunk-XX4OLNWQ.js} +2 -2
- package/dist/{chunk-FZ5MBXEG.js → chunk-YQSYCBAH.js} +6 -6
- package/dist/{chunk-ACOOG3UW.js → chunk-Z4AWEIBY.js} +3 -3
- package/dist/{ci-workflow-IW2H5DC4.js → ci-workflow-ORZ4F45F.js} +2 -2
- package/dist/{dist-JCFK263L.js → dist-LHINSVK4.js} +85 -1
- package/dist/{docs-7CP6VV42.js → docs-AU3DXFFO.js} +3 -3
- package/dist/{engine-MKJEBWU7.js → engine-6GPWOYQH.js} +2 -2
- package/dist/{entropy-6JOZJYG7.js → entropy-VLKDRMRT.js} +2 -2
- package/dist/{feedback-T65AEJEG.js → feedback-YXYY44ZC.js} +1 -1
- package/dist/{generate-agent-definitions-4VI4YTO2.js → generate-agent-definitions-XIHQPKG3.js} +3 -3
- package/dist/hooks/cost-tracker.js +29 -6
- package/dist/index.js +22 -22
- package/dist/{loader-AWTHHE3A.js → loader-YAN56MT3.js} +2 -2
- package/dist/{mcp-JBIBINO2.js → mcp-RK3SFVYB.js} +13 -13
- package/dist/{performance-NCOUDOMW.js → performance-64DPTLLQ.js} +3 -3
- package/dist/{review-pipeline-YIC6JPMY.js → review-pipeline-NENG6LW7.js} +2 -2
- package/dist/{runtime-5MHV4UEE.js → runtime-FFIIRT2T.js} +2 -2
- package/dist/{security-VZPJNIDP.js → security-3PIBN35B.js} +1 -1
- package/dist/templates/basic/harness.config.json.hbs +34 -0
- package/dist/templates/ci/README.md +86 -0
- package/dist/templates/ci/required-review.ruleset.json +20 -0
- package/dist/templates/ci/required-review.yml.hbs +57 -0
- package/dist/templates/ci/template.json +5 -0
- package/dist/templates/intermediate/harness.config.json.hbs +36 -0
- package/dist/templates/orchestrator/harness.orchestrator.md +9 -0
- package/dist/{tool-tiers-B7JC2XC3.js → tool-tiers-OKK5FE57.js} +2 -0
- package/dist/{validate-7BE4VTFI.js → validate-7IAF2ES3.js} +3 -3
- package/dist/{validate-cross-check-F7GEBRDH.js → validate-cross-check-QPSAOZKD.js} +2 -2
- package/package.json +7 -7
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# Harness Audit: Harness Strength
|
|
2
|
+
|
|
3
|
+
> Mechanically audit whether a project's harness is load-bearing. Orchestrates the deterministic check-harness-strength engine; interprets results. Not a deep/AI review.
|
|
4
|
+
|
|
5
|
+
## When to Use
|
|
6
|
+
|
|
7
|
+
- On a milestone gate, to confirm the harness still constrains rather than merely decorates the project
|
|
8
|
+
- When adding the strength audit as a required CI check, so the seven STRENGTH patterns block the merge instead of being prose advice
|
|
9
|
+
- When validating a harness-distribution (toolkit) repo before shipping templates downstream — to catch a default that would weaken every adopter
|
|
10
|
+
- When a reviewer suspects a gate "passes" without doing anything (a snapshot that asserts `passed: true` for a check it never ran)
|
|
11
|
+
- NOT for deep security review — use `harness-security-review` (semantic, AI-assisted)
|
|
12
|
+
- NOT for design-system drift — use `detect-design-drift`
|
|
13
|
+
- NOT for reimplementing pattern detection by hand — the engine (`HarnessStrengthAuditor`) owns detection. This skill runs it and interprets the output.
|
|
14
|
+
|
|
15
|
+
## Process
|
|
16
|
+
|
|
17
|
+
This skill ORCHESTRATES `harness check-harness-strength`. It never re-greps configs, never re-parses hooks, and never re-derives findings. The engine reads the project once and returns a structured `AuditResult`; the skill's job is to run it with the right mode/severity and turn the JSON into an actionable report.
|
|
18
|
+
|
|
19
|
+
### Phase 1: SCAN — Run the engine
|
|
20
|
+
|
|
21
|
+
1. **Resolve the project root.** Use the provided `path` argument, else the current working directory.
|
|
22
|
+
|
|
23
|
+
2. **Resolve the mode.**
|
|
24
|
+
- If `--toolkit` or `--adopter` (or `--mode toolkit|adopter`) is supplied, honor it. Explicit mode wins.
|
|
25
|
+
- Otherwise auto-detect: treat the repo as **toolkit** when it is a harness distribution — both `templates/` and `agents/skills/` exist at the root. Otherwise **adopter**.
|
|
26
|
+
- Do not inspect config files to "double-check" the mode. The engine resolves and reports the mode it used.
|
|
27
|
+
|
|
28
|
+
3. **Invoke the command** via Bash, requesting JSON:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
harness check-harness-strength [--mode <adopter|toolkit>] [--severity <error|warning|info>] [--report-only] --json
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The `--json` payload is the raw structured `AuditResult` (score, tier, mode, findings with `id`, `gearPiece`, `severity`, `message`, `remediation`, and file:line evidence). Parse this — do not re-derive it.
|
|
35
|
+
|
|
36
|
+
4. **Do NOT inspect config files by hand.** The engine reads `harness.config.json`, hook profiles, baselines, snapshots, and tier defaults exactly once. Hand-grepping `.husky/pre-commit` or `harness.config.json` to "confirm" a finding reimplements detection and violates the core decision behind this skill (ADR 0039 / spec D1).
|
|
37
|
+
|
|
38
|
+
### Phase 2: DETECT — Interpret findings
|
|
39
|
+
|
|
40
|
+
1. **Map each finding to its STRENGTH pattern** using the JSON `id` and `gearPiece` fields. Every finding the engine emits corresponds to exactly one of the seven patterns below. Do not invent findings the engine did not emit, and do not silence findings it did.
|
|
41
|
+
|
|
42
|
+
2. **The seven STRENGTH patterns** (the engine's rule registry — for interpretation only; the engine, not the skill, decides which fire):
|
|
43
|
+
|
|
44
|
+
| ID | Gear piece | Pattern | Default |
|
|
45
|
+
| ------------ | ----------------------- | -------------------------------------------------------------------- | ------- |
|
|
46
|
+
| STRENGTH-001 | blocking-gate | Hook documented "never blocks"/"always exits 0" in an active profile | error |
|
|
47
|
+
| STRENGTH-002 | regression-baseline | Pre-commit auto-updates baselines/thresholds on regression | error |
|
|
48
|
+
| STRENGTH-003 | skip-discipline | `--skip` list > 2 categories without inline justification | warning |
|
|
49
|
+
| STRENGTH-004 | architecture-thresholds | `layers` defined but `architecture.thresholds` empty/absent | error |
|
|
50
|
+
| STRENGTH-005 | tier-default | Init/config defaults to lowest tier (`basic`) | warning |
|
|
51
|
+
| STRENGTH-006 | review-gate | Baseline-update PR auto-approved without independent review | error |
|
|
52
|
+
| STRENGTH-007 | snapshot-honesty | `passed:true` in health snapshot whose `signals[]` names that check | error |
|
|
53
|
+
|
|
54
|
+
3. **Attach evidence.** Each finding carries a `file:line` (or config-key) reference and a remediation string from the engine. Surface them verbatim — do not paraphrase the remediation.
|
|
55
|
+
|
|
56
|
+
### Phase 3: SCORE/REPORT
|
|
57
|
+
|
|
58
|
+
1. **Surface the score and tier.** The engine returns a 0-100 strength score and a tier label:
|
|
59
|
+
- `solid` — score >= 85
|
|
60
|
+
- `at-risk` — score 50-84
|
|
61
|
+
- `theatre` — score < 50
|
|
62
|
+
|
|
63
|
+
2. **Report format** (mirror of the security-scan report block):
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
Harness Strength: [PASS/FAIL] — tier: <solid|at-risk|theatre> (score N/100, mode <adopter|toolkit>)
|
|
67
|
+
Findings: <count> (Errors: N | Warnings: N | Info: N)
|
|
68
|
+
|
|
69
|
+
[STRENGTH-00N] <gearPiece> <file:line> (severity)
|
|
70
|
+
<message>
|
|
71
|
+
Remediation: <engine remediation string>
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
3. **PASS/FAIL.** FAIL if any error-severity finding survives the severity threshold, unless `--report-only` was passed (then exit softens to PASS but findings are still listed). Otherwise PASS.
|
|
75
|
+
|
|
76
|
+
## Gates
|
|
77
|
+
|
|
78
|
+
- **Error-severity findings are blocking.** The report is FAIL and the exit is non-zero when any error-severity finding survives filtering — unless `--report-only` was explicitly requested. Do not report PASS over an unsuppressed error finding.
|
|
79
|
+
- **No reimplementation.** The skill must run `harness check-harness-strength` and interpret its JSON. It must never hand-grep hooks, hand-parse `harness.config.json`, or re-derive a score. Reimplementing detection violates ADR 0039.
|
|
80
|
+
- **"Not evaluable" is not a pass.** When an input the engine needs is absent (e.g., no hook profile, no snapshot), the engine reports that state. Surface it as-is. Do not convert "could not evaluate" into "passed."
|
|
81
|
+
- **Mechanical only.** No AI judgment about whether a weakness "really matters." The patterns are deterministic; the engine decides what fires.
|
|
82
|
+
|
|
83
|
+
## Escalation
|
|
84
|
+
|
|
85
|
+
- **Disputed finding / false positive:** Do not suppress by editing files or skipping the rule in the skill. Adjust severity via config — `audit.harnessStrength.severities` in `harness.config.json` (e.g., downgrade STRENGTH-003 to `info` for a justified wide `--skip`). Document the rationale alongside the override.
|
|
86
|
+
- **Engine misses a known weakness:** This skill cannot add detection. A missing pattern is engine work — file a new `StrengthRule` (`{ id, gearPiece, defaultSeverity, appliesIn(mode), evaluable?(ctx), detect(ctx) }`) in `packages/core/src/harness-strength/`. Out of scope for the skill.
|
|
87
|
+
- **Mode mis-detected:** If auto-detection picks the wrong mode (e.g., a repo with `templates/` that is not a distribution), pass `--mode adopter` (or `--toolkit`) explicitly. Do not work around it by hand-editing detection.
|
|
88
|
+
- **Engine throws or the build is stale:** If `harness check-harness-strength` errors (command not found / not in the built CLI), the dist may be stale. Rebuild the CLI and re-run. Do not substitute a manual config read for the engine.
|
|
89
|
+
|
|
90
|
+
## Rationalizations to Reject
|
|
91
|
+
|
|
92
|
+
### Universal
|
|
93
|
+
|
|
94
|
+
These reasoning patterns sound plausible but lead to bad outcomes. Reject them.
|
|
95
|
+
|
|
96
|
+
- **"It's probably fine"** — "Probably" is not evidence. Run the engine and cite the result.
|
|
97
|
+
- **"This is best practice"** — Best practice in what context? Cite the source and confirm it applies to this codebase.
|
|
98
|
+
- **"We can fix it later"** — If it is worth flagging, it is worth documenting now with a concrete follow-up plan.
|
|
99
|
+
|
|
100
|
+
### Domain-Specific
|
|
101
|
+
|
|
102
|
+
| Rationalization | Reality |
|
|
103
|
+
| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
104
|
+
| "The config looks fine when I read it" | The engine exists because manual reading misses these patterns — that is the whole point of D1. Reading is not running. Run `harness check-harness-strength`. |
|
|
105
|
+
| "It only warns, not errors, so it's fine" | A gate that warns but does not stop IS STRENGTH-001 — this is the recursion item the audit was built to catch. Do not normalize "warns but doesn't block." |
|
|
106
|
+
| "I'll re-grep the hooks myself to double-check the engine" | Reimplementing detection violates ADR 0039 / spec D1. The engine is the single source of truth — trust and interpret its output, do not shadow it. |
|
|
107
|
+
| "Score is 70, that's a passing grade" | 70 is `at-risk`, not `solid`. Tier labels are thresholds, not letter grades. Below 85 means the harness has load-bearing gaps. |
|
|
108
|
+
| "No error findings, so I can skip reading the warnings" | Warnings (STRENGTH-003, -005) are erosion signals — a `basic` tier default or an unjustified wide `--skip` weakens every future run. Surface them. |
|
|
109
|
+
|
|
110
|
+
## Success Criteria
|
|
111
|
+
|
|
112
|
+
- The engine (`harness check-harness-strength`) ran and produced a score, tier label, and findings (or a clean result).
|
|
113
|
+
- Every finding is interpreted against the seven-pattern table, with the engine's file:line evidence and remediation surfaced verbatim.
|
|
114
|
+
- The exit code / PASS-FAIL reflects the gate (error-severity findings fail unless `--report-only`).
|
|
115
|
+
- No manual config inspection was substituted for the engine — the report is derived entirely from the command's `--json` output.
|
|
116
|
+
- "Not evaluable" states are surfaced as-is, never converted to passes.
|
|
117
|
+
|
|
118
|
+
## Evidence Requirements
|
|
119
|
+
|
|
120
|
+
When this skill makes claims about existing code, architecture, or behavior, it MUST cite evidence using one of:
|
|
121
|
+
|
|
122
|
+
1. **File reference:** `file:line` format (e.g., `.husky/pre-commit:12`) — taken from the engine's finding, not re-derived.
|
|
123
|
+
2. **Code pattern reference:** `file` with description (e.g., `harness.config.json` — "architecture.layers defined, thresholds absent").
|
|
124
|
+
3. **Test/command output:** Inline or referenced output from the `harness check-harness-strength --json` run.
|
|
125
|
+
4. **Session evidence:** Write to the `evidence` session section via `manage_state`.
|
|
126
|
+
|
|
127
|
+
**Uncited claims:** Technical assertions without citations MUST be prefixed with `[UNVERIFIED]`. Example: `[UNVERIFIED] The pre-commit hook auto-updates baselines`.
|
|
128
|
+
|
|
129
|
+
## Harness Integration
|
|
130
|
+
|
|
131
|
+
- **`harness check-harness-strength`** — The CLI command this skill runs and interprets. Options: `--mode <adopter|toolkit>`, `--toolkit`/`--adopter` shortcuts, `--severity <error|warning|info>` (default `warning`), `--report-only`, `--json`.
|
|
132
|
+
- **`HarnessStrengthAuditor`** — Core class from `@harness-engineering/core` that builds a `ProjectContext` once, runs the applicable `StrengthRule`s, and aggregates the score/tier/findings. The skill never instantiates it directly; it goes through the command.
|
|
133
|
+
- **`harness.config.json` → `audit.harnessStrength.severities`** — Per-pattern severity overrides. The supported escalation path for false positives — never suppress by editing source.
|
|
134
|
+
- **`docs/standard/article-failure-patterns.md`** — Narrative companion documenting the seven patterns (forthcoming — downstream item; may not exist yet).
|
|
135
|
+
|
|
136
|
+
## Examples
|
|
137
|
+
|
|
138
|
+
### Example: Clean toolkit-mode run
|
|
139
|
+
|
|
140
|
+
**Input:** A harness-distribution repo (`templates/` and `agents/skills/` present), run at a milestone gate. `harness check-harness-strength --toolkit --json`.
|
|
141
|
+
|
|
142
|
+
**Output:**
|
|
143
|
+
|
|
144
|
+
```
|
|
145
|
+
Harness Strength: PASS — tier: solid (score 100/100, mode toolkit)
|
|
146
|
+
Findings: 0 (Errors: 0 | Warnings: 0 | Info: 0)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Every gear piece is load-bearing: no "never blocks" hook, baselines are not auto-updated on regression, architecture thresholds are set, the init default is not `basic`, baseline-update PRs require independent review, and no snapshot claims a check passed that it never ran.
|
|
150
|
+
|
|
151
|
+
### Example: Findings detected (adopter mode)
|
|
152
|
+
|
|
153
|
+
**Input:** An adopter repo, run with default severity. `harness check-harness-strength --json`.
|
|
154
|
+
|
|
155
|
+
**Output:**
|
|
156
|
+
|
|
157
|
+
```
|
|
158
|
+
Harness Strength: FAIL — tier: at-risk (score 62/100, mode adopter)
|
|
159
|
+
Findings: 2 (Errors: 2 | Warnings: 0 | Info: 0)
|
|
160
|
+
|
|
161
|
+
[STRENGTH-001] blocking-gate .husky/pre-commit:12 (error)
|
|
162
|
+
Active pre-commit profile documents "always exits 0" — the gate reports but never blocks.
|
|
163
|
+
Remediation: Remove the unconditional `exit 0`; let failing checks return a non-zero status.
|
|
164
|
+
|
|
165
|
+
[STRENGTH-004] architecture-thresholds harness.config.json (error)
|
|
166
|
+
architecture.layers is defined but architecture.thresholds is empty — layer rules cannot be enforced.
|
|
167
|
+
Remediation: Set architecture.thresholds (e.g. maxModuleLines, maxDependencyDepth) so the layer
|
|
168
|
+
definitions become enforceable, or remove the unused layers block.
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Two error-severity findings → FAIL. STRENGTH-001 is the recursion item: a hook that runs but cannot block. Run with `--report-only` only when you explicitly want to surface findings without failing the build (e.g., a first baseline-capture run) — and never to normalize the weakness.
|
|
172
|
+
|
|
173
|
+
## Skill Test Scenarios
|
|
174
|
+
|
|
175
|
+
### Scenario 1: Red Flag — "I'll re-grep the hooks myself to double-check the engine"
|
|
176
|
+
|
|
177
|
+
Input: The agent has the `harness check-harness-strength --json` output in hand but is about to open `.husky/pre-commit` and grep it by hand to "confirm" the STRENGTH-001 finding.
|
|
178
|
+
Expected: Agent stops, cites the no-reimplementation gate (ADR 0039 / spec D1), and interprets the engine's JSON instead of hand-grepping the hook.
|
|
179
|
+
|
|
180
|
+
### Scenario 2: Rationalization — "it only warns, not errors, so it's fine"
|
|
181
|
+
|
|
182
|
+
Input: The engine reports a documented "never blocks" pre-commit profile, and the agent is tempted to treat it as acceptable because nothing currently fails the build.
|
|
183
|
+
Expected: Agent rejects the rationalization, recognizes that "warns but doesn't stop" IS STRENGTH-001 (the recursion item), and surfaces it as a finding rather than normalizing it.
|
|
184
|
+
|
|
185
|
+
### Scenario 3: Gate — error-severity STRENGTH-001 finding present without `--report-only`
|
|
186
|
+
|
|
187
|
+
Input: The `--json` output contains an error-severity STRENGTH-001 finding, `--report-only` was not passed, and the agent is about to summarize the run as PASS.
|
|
188
|
+
Expected: Agent halts, reports FAIL with a non-zero exit, lists the finding with its file:line evidence and remediation, and does not proceed past the gate.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
name: harness-audit-harness-strength
|
|
2
|
+
version: '0.1.0'
|
|
3
|
+
description: Mechanically audit a project's own harness setup against the seven STRENGTH failure patterns; reports per-pattern findings, a 0-100 strength score, and a tier label (solid/at-risk/theatre). Orchestrates harness check-harness-strength; never reimplements detection.
|
|
4
|
+
stability: draft
|
|
5
|
+
cognitive_mode: constructive-architect
|
|
6
|
+
triggers:
|
|
7
|
+
- manual
|
|
8
|
+
- on_milestone
|
|
9
|
+
platforms:
|
|
10
|
+
- claude-code
|
|
11
|
+
- gemini-cli
|
|
12
|
+
- cursor
|
|
13
|
+
- codex
|
|
14
|
+
tools:
|
|
15
|
+
- Bash
|
|
16
|
+
- Read
|
|
17
|
+
- Grep
|
|
18
|
+
- Glob
|
|
19
|
+
cli:
|
|
20
|
+
command: harness skill run harness-audit-harness-strength
|
|
21
|
+
args:
|
|
22
|
+
- name: path
|
|
23
|
+
description: Project root path
|
|
24
|
+
required: false
|
|
25
|
+
- name: mode
|
|
26
|
+
description: 'adopter (default) or toolkit (auto-detected in a harness-distribution repo)'
|
|
27
|
+
required: false
|
|
28
|
+
- name: severity
|
|
29
|
+
description: Minimum severity threshold (error, warning, info)
|
|
30
|
+
required: false
|
|
31
|
+
- name: report-only
|
|
32
|
+
description: Soften exit to 0 even when error-severity findings exist
|
|
33
|
+
required: false
|
|
34
|
+
mcp:
|
|
35
|
+
tool: run_skill
|
|
36
|
+
input:
|
|
37
|
+
skill: harness-audit-harness-strength
|
|
38
|
+
path: string
|
|
39
|
+
type: rigid
|
|
40
|
+
tier: 2
|
|
41
|
+
phases:
|
|
42
|
+
- name: scan
|
|
43
|
+
description: Resolve mode and run harness check-harness-strength against the target repo
|
|
44
|
+
required: true
|
|
45
|
+
- name: detect
|
|
46
|
+
description: Interpret the 7 STRENGTH pattern findings with file-line evidence
|
|
47
|
+
required: true
|
|
48
|
+
- name: score_report
|
|
49
|
+
description: Surface the 0-100 score, tier label, and per-pattern remediation
|
|
50
|
+
required: true
|
|
51
|
+
state:
|
|
52
|
+
persistent: false
|
|
53
|
+
files: []
|
|
54
|
+
depends_on: []
|
|
@@ -213,6 +213,8 @@ Persist findings to `{sessionDir}/phase-{N}-review.json`. No blocking → PHASE_
|
|
|
213
213
|
|
|
214
214
|
## Process
|
|
215
215
|
|
|
216
|
+
**Prompt the human in plain text** — every phase-continue and human-decision interaction in this skill is plain text only. Do not elevate to `AskUserQuestion`: natural headers like "Continue phase" exceed its 12-char cap, rendering the call as ERR.
|
|
217
|
+
|
|
216
218
|
1. **INIT** — Resolve spec, derive session slug, check for existing state, parse phases.
|
|
217
219
|
2. **ASSESS** — Route by complexity: low/medium auto-plans, high pauses for interactive planning.
|
|
218
220
|
3. **PLAN → APPROVE** — Dispatch harness-planner, check approval signals, auto-approve or pause.
|
|
@@ -136,7 +136,7 @@ These flow into `handoff.json` `contextKeywords` field. Select keywords that hel
|
|
|
136
136
|
- **Entry Points** -- Which system entry points does this feature touch or create? (e.g., new CLI command, new MCP tool, new skill, new API route, new barrel export)
|
|
137
137
|
- **Registrations Required** -- What registrations are needed for the feature to be discoverable? (e.g., barrel export regeneration, skill tier assignment, route registration)
|
|
138
138
|
- **Documentation Updates** -- What docs need updating to reflect the new capability? (e.g., AGENTS.md section, API docs, README, guides)
|
|
139
|
-
- **Architectural Decisions** --
|
|
139
|
+
- **Architectural Decisions** -- Which decisions from the **Decisions made** section rise to a standalone ADR? Reference each by name with a one-line note on _why_ it warrants an ADR. Do **not** restate the decisions here -- this subsection points back to the canonical **Decisions made** section; duplicating them creates two sources that drift (spec-craft flags this as SPEC-R004). Only for medium/large tier changes -- "None" for small changes.
|
|
140
140
|
- **Knowledge Impact** -- What domain concepts, patterns, or relationships should enter the knowledge graph?
|
|
141
141
|
|
|
142
142
|
If the feature is a small change (bug fix, config tweak, < 3 files), the section may contain only Entry Points and Registrations Required with "None" for the others. The section must still be present.
|
|
@@ -186,21 +186,35 @@ These flow into `handoff.json` `contextKeywords` field. Select keywords that hel
|
|
|
186
186
|
|
|
187
187
|
The human must explicitly approve before this skill is complete.
|
|
188
188
|
|
|
189
|
-
7. **
|
|
189
|
+
7. **Promote the roadmap row.** If `docs/roadmap.md` exists, transition the named row to `planned` and link the spec in a single structured call (steps 7 and 8 are intentionally ordered so the commit captures the roadmap mutation). Skip silently only when no roadmap exists.
|
|
190
|
+
- Derive the lookup key from the slash-command `ARGUMENTS` string (D1). Derive the summary from the spec title (the H1 heading).
|
|
191
|
+
- Call `manage_roadmap` with action `promote`, `feature: "<ARGUMENTS>"`, `spec: "docs/changes/<feature>/proposal.md"`, and `summary: "<H1>"`.
|
|
192
|
+
- Branch on the returned `PromoteResult` envelope:
|
|
193
|
+
|
|
194
|
+
| Envelope | Skill behavior |
|
|
195
|
+
| ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
196
|
+
| `ok: true, transitioned: 'backlog→planned'` | Log "Promoted `<feature>`: backlog → planned". Continue to step 8. |
|
|
197
|
+
| `ok: true, transitioned: 'spec-updated'` | Log "Updated spec link for `<feature>` (status preserved)". Continue to step 8. |
|
|
198
|
+
| `ok: true, transitioned: 'noop'` | Log "No change — `<feature>` already promoted with this spec". Continue to step 8. |
|
|
199
|
+
| `ok: false, reason: 'in-progress'` | **STOP.** Surface: "Refused to promote `<feature>`: an agent is currently dispatched against this row. Stop the agent or use a different feature name." |
|
|
200
|
+
| `ok: false, reason: 'done'` | **STOP.** Surface: "Refused to promote `<feature>`: row is already 'done'. To revise a shipped feature, use a new name." |
|
|
201
|
+
| `ok: false, reason: 'not-found'` | If no row was expected to exist, fall through to a `created` outcome. Otherwise surface `closestMatches` as a typo hint and **STOP**. |
|
|
202
|
+
| `ok: false, reason: 'ambiguous'` | **STOP.** Surface: "Refused to promote `<feature>`: matches multiple rows across milestones. Re-invoke with one of: `<matches>`." Matches are milestone-qualified. |
|
|
203
|
+
| `ok: false, reason: 'write-failed'` | **STOP.** Surface `detail` verbatim. The brainstorm is not considered complete. |
|
|
204
|
+
|
|
205
|
+
- The `not-found` create path: when the row genuinely does not exist yet, call `manage_roadmap` action `add` with `status: "planned"`, `milestone: "Current Work"`, the spec path, and the H1 summary (unchanged from the legacy behavior), then continue to step 8.
|
|
206
|
+
- "STOP" cases skip step 8 (no commit) and step 9 (no handoff/transition). The spec file written in step 3 stays on disk; the user re-runs the skill after resolving the conflict.
|
|
207
|
+
- If `manage_roadmap` is unavailable, fall back to `parseRoadmap`/`promoteFeature`/`serializeRoadmap` from core. Warn: "External sync skipped (MCP unavailable). Run `manage_roadmap sync` when MCP is restored."
|
|
208
|
+
- If no roadmap exists, skip silently (no envelope) and continue to step 8; the commit then contains only the spec files.
|
|
209
|
+
|
|
210
|
+
8. **Commit spec artifacts and roadmap promotion.** After a successful promote (or a silent skip when no roadmap exists), commit the spec, skill recommendations, and the roadmap mutation together so the promotion ships atomically with the spec:
|
|
190
211
|
|
|
191
212
|
```bash
|
|
192
|
-
git add docs/changes/<feature>/proposal.md docs/changes/<feature>/SKILLS.md
|
|
193
|
-
git commit -m "docs(<feature>): add spec"
|
|
213
|
+
git add docs/changes/<feature>/proposal.md docs/changes/<feature>/SKILLS.md docs/roadmap.md
|
|
214
|
+
git commit -m "docs(<feature>): add spec and promote to planned"
|
|
194
215
|
```
|
|
195
216
|
|
|
196
|
-
Do not skip this step
|
|
197
|
-
|
|
198
|
-
8. **Add feature to roadmap.** If `docs/roadmap.md` exists:
|
|
199
|
-
- Derive feature name from the spec title (the H1 heading).
|
|
200
|
-
- Call `manage_roadmap` with action `add`, `status: "planned"`, `milestone: "Current Work"`, and the spec path.
|
|
201
|
-
- If the feature already exists, skip silently.
|
|
202
|
-
- If `manage_roadmap` is unavailable, fall back to `parseRoadmap`/`serializeRoadmap` from core. Warn: "External sync skipped (MCP unavailable). Run `manage_roadmap sync` when MCP is restored."
|
|
203
|
-
- If no roadmap exists, skip silently.
|
|
217
|
+
Include `docs/roadmap.md` in `git add` only when it exists; omit it when the project has no roadmap. Do not skip this step — `harness-execution` commits only implementation files, so a spec uncommitted here stays untracked until someone notices. If a pre-commit hook reformats a file, re-add and re-commit.
|
|
204
218
|
|
|
205
219
|
9. **Write handoff and suggest transition.** After approval:
|
|
206
220
|
|
|
@@ -336,8 +350,8 @@ Technical claims about existing code, architecture, or tradeoffs MUST cite evide
|
|
|
336
350
|
- **Spec location** -- `docs/changes/<feature>/proposal.md`.
|
|
337
351
|
- **Handoff** -- Once approved, invoke harness-planning to create the implementation plan.
|
|
338
352
|
- **Session directory** — When session slug is known, handoff goes to `.harness/sessions/<slug>/handoff.json`. The session directory structure is: `handoff.json`, `state.json`, `artifacts.json` (registry of spec/plan paths and file lists). Do not write to `.harness/handoff.json` in session context.
|
|
339
|
-
- **Spec commit** -- After sign-off
|
|
340
|
-
- **Roadmap
|
|
353
|
+
- **Spec commit** -- After sign-off, the Phase 4 step 8 commit captures `docs/changes/<feature>/proposal.md`, `docs/changes/<feature>/SKILLS.md`, and the promoted `docs/roadmap.md` together so the spec enters git history at approval, not retroactively. `harness-execution` does not backfill these — see issue #487.
|
|
354
|
+
- **Roadmap promotion** -- After approval (Phase 4 step 7), call `manage_roadmap` action `promote` to transition the named row to `planned` and link the spec atomically with the spec commit. Falls back to `add` (create new row) when the row does not exist; refuses on `in-progress`/`done`/`ambiguous`. Skip silently if no roadmap.
|
|
341
355
|
- **`emit_interaction`** -- End of Phase 4 to suggest transition to harness-planning (confirmed transition).
|
|
342
356
|
|
|
343
357
|
#### Requirement Phrasing
|
|
@@ -56,7 +56,7 @@ When no arguments are provided (standalone invocation), discover plan from `docs
|
|
|
56
56
|
4. **Load session summary for cold start.** If resuming (session slug known):
|
|
57
57
|
- Call `listActiveSessions()` to read the session index.
|
|
58
58
|
- Call `loadSessionSummary()` for the target session.
|
|
59
|
-
- If ambiguous, present the index and ask which session to resume.
|
|
59
|
+
- If ambiguous, present the index and ask which session to resume. Ask in plain text — do not elevate this to `AskUserQuestion` (the session index can exceed its 4-option cap and a natural header like "Pick session" can exceed its 12-char cap, rendering the call as ERR).
|
|
60
60
|
|
|
61
61
|
5. **Check for known dead ends.** Review `learnings` tagged `[outcome:failure]`. Warn if any match current plan approaches.
|
|
62
62
|
|
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
|
|
14
14
|
## Process
|
|
15
15
|
|
|
16
|
+
**Prompt the human in plain text** — every choice and destructive confirmation in this skill (worktree location, discard-experiment, etc.) is plain text only. Do not elevate to `AskUserQuestion`: option labels like the A/B/C worktree-location choices and natural headers like "Discard commits" routinely exceed its 4-option / 12-char caps and render as ERR.
|
|
17
|
+
|
|
16
18
|
### Part A: Worktree Creation
|
|
17
19
|
|
|
18
20
|
#### Step 1: Choose Worktree Location
|
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
|
|
16
16
|
## Process
|
|
17
17
|
|
|
18
|
+
**Prompt the human in plain text** — every confirmation and selection in this skill (`Proceed? (y/n)`, post-critique selection, etc.) is plain text only. Do not elevate to `AskUserQuestion`: natural headers like "Confirm inputs" exceed its 12-char cap and option lists routinely exceed its 4-option cap, rendering the call as ERR.
|
|
19
|
+
|
|
18
20
|
### Iron Law
|
|
19
21
|
|
|
20
22
|
**The skill produces exactly ONE ranked artifact per run, at `docs/ideation/<slug>-YYYY-MM-DD.md`, and NEVER produces a spec, plan, ADR, or code.** If the user wants a spec from a ranked idea, they invoke `harness-brainstorming` next — that is the contract.
|
|
@@ -14,6 +14,8 @@
|
|
|
14
14
|
|
|
15
15
|
## Process
|
|
16
16
|
|
|
17
|
+
**Prompt the human in plain text** — every confirmation, drift acknowledgement, and contradiction resolution in this skill is plain text only. Do not elevate to `AskUserQuestion`: source labels and finding descriptions routinely exceed its 4-option / 12-char caps, rendering the call as ERR.
|
|
18
|
+
|
|
17
19
|
1. **EXTRACT:** Run code signal extractors, diagram parsers, BusinessKnowledgeIngestor, and KnowledgeLinker against the project
|
|
18
20
|
2. **RECONCILE:** Build pre-extraction snapshot (current graph state) and post-extraction snapshot, then run StructuralDriftDetector
|
|
19
21
|
3. **DETECT:** Partition findings by classification (new/stale/drifted/contradicting), generate gap report
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
- When adding a new feature to an existing roadmap (`--add <feature-name>`)
|
|
10
10
|
- When roadmap statuses may be stale and need updating from plan execution state (`--sync`)
|
|
11
11
|
- When features need reordering, moving between milestones, or blocker updates (`--edit`)
|
|
12
|
+
- When the roadmap needs tidying -- completed work archived, dead `planned` rows demoted (`--groom`)
|
|
12
13
|
- When user asks about project status and no roadmap exists -- suggest `--create`
|
|
13
14
|
- NOT for programmatic CRUD (use `manage_roadmap` MCP tool directly)
|
|
14
15
|
|
|
@@ -20,6 +21,8 @@
|
|
|
20
21
|
|
|
21
22
|
If the human has not seen and approved the milestone groupings and feature list, do not write the file. Present. Wait. Confirm. Then write.
|
|
22
23
|
|
|
24
|
+
**Prompt the human in plain text — every `(y/n)`, "which feature?", and similar prompt in this skill is plain text only.** Do not elevate them to `AskUserQuestion`: feature lists routinely exceed its 4-option cap, and natural header choices ("Pick feature", "Remove feature") exceed its 12-char cap, causing the call to render as ERR.
|
|
25
|
+
|
|
23
26
|
---
|
|
24
27
|
|
|
25
28
|
### Command: `--create` -- Bootstrap Roadmap
|
|
@@ -428,12 +431,67 @@ Choice?
|
|
|
428
431
|
|
|
429
432
|
---
|
|
430
433
|
|
|
434
|
+
### Command: `--groom` -- Tidy the Roadmap
|
|
435
|
+
|
|
436
|
+
Keeps the roadmap manageable over time. **Milestones are themes; statuses are lifecycle stages** -- grooming enforces that separation so the backlog never decays back into an undifferentiated dump.
|
|
437
|
+
|
|
438
|
+
#### Phase 1: SCAN -- Detect Untidiness
|
|
439
|
+
|
|
440
|
+
1. Check if `docs/roadmap.md` exists. If missing: error and direct the user to `--create`.
|
|
441
|
+
2. Run `manage_roadmap` (`action: "groom"`) in a dry-run frame, or call `checkRoadmapHealth` from `@harness-engineering/core`, to surface the four health signals:
|
|
442
|
+
- **RMH001** -- completed (`done`) features still sitting in an active milestone.
|
|
443
|
+
- **RMH002** -- `planned` rows with neither a spec nor a plan (the orchestrator cannot auto-execute these; it escalates them to a human).
|
|
444
|
+
- **RMH003** -- lifecycle catch-all milestones (`Backlog`, `Current Work`) that should not exist.
|
|
445
|
+
- **RMH004** -- active milestones that have grown past the size cap (a mini-dump).
|
|
446
|
+
|
|
447
|
+
#### Phase 2: PROPOSE -- Present the Plan
|
|
448
|
+
|
|
449
|
+
Show the human exactly what grooming will do, in plain text:
|
|
450
|
+
|
|
451
|
+
```
|
|
452
|
+
GROOM PLAN
|
|
453
|
+
|
|
454
|
+
Demote to backlog (planned with no spec/plan):
|
|
455
|
+
- Feature A (Theme X)
|
|
456
|
+
- Feature B (Theme Y)
|
|
457
|
+
|
|
458
|
+
Archive to docs/roadmap-archive.md (completed):
|
|
459
|
+
- Feature C (Theme X)
|
|
460
|
+
|
|
461
|
+
Flagged for manual routing (not auto-changed):
|
|
462
|
+
- Intake lane has 3 items awaiting a theme
|
|
463
|
+
- "Theme Z" has 28 features (cap 25) -- consider splitting
|
|
464
|
+
|
|
465
|
+
Apply? (y/n)
|
|
466
|
+
```
|
|
467
|
+
|
|
468
|
+
Wait for confirmation. The mechanical changes (demote, archive) are safe and automated; **draining the Intake lane into themed milestones and splitting oversized milestones are human decisions** -- propose, do not auto-apply.
|
|
469
|
+
|
|
470
|
+
#### Phase 3: WRITE -- Apply
|
|
471
|
+
|
|
472
|
+
1. Run `manage_roadmap` (`action: "groom"`). It demotes unactionable `planned` rows to `backlog` and moves `done` features into `docs/roadmap-archive.md` under a `Shipped` milestone, returning the list of changes.
|
|
473
|
+
2. For Intake-draining or milestone-splitting the human approved, follow up with `--edit` (move features between milestones).
|
|
474
|
+
|
|
475
|
+
#### Phase 4: VALIDATE -- Verify
|
|
476
|
+
|
|
477
|
+
1. Run `harness validate` and confirm the `roadmapHealth` check passes (no RMH003 errors; RMH001/002/004 warnings cleared or acknowledged).
|
|
478
|
+
2. Summarize:
|
|
479
|
+
|
|
480
|
+
```
|
|
481
|
+
Groom complete.
|
|
482
|
+
Demoted: N | Archived: N -> docs/roadmap-archive.md | Flagged for manual routing: N
|
|
483
|
+
harness validate (roadmapHealth): passed
|
|
484
|
+
```
|
|
485
|
+
|
|
486
|
+
---
|
|
487
|
+
|
|
431
488
|
## Harness Integration
|
|
432
489
|
|
|
433
|
-
- **`manage_roadmap` MCP tool** -- Primary read/write interface for roadmap operations. Supports `show`, `add`, `update`, `remove`, and `
|
|
434
|
-
- **`harness validate`** -- Run after any roadmap modification to verify project health. Mandatory in the VALIDATE phase of
|
|
435
|
-
- **Core `
|
|
436
|
-
- **
|
|
490
|
+
- **`manage_roadmap` MCP tool** -- Primary read/write interface for roadmap operations. Supports `show`, `add`, `update`, `remove`, `query`, `sync`, `promote`, and `groom` actions. Use this when MCP is available for structured CRUD.
|
|
491
|
+
- **`harness validate`** -- Run after any roadmap modification to verify project health. Mandatory in the VALIDATE phase of `--create`, `--add`, and `--groom`. The `roadmapHealth` check enforces the maintenance rules (RMH001-RMH004) as a regression guard.
|
|
492
|
+
- **Core `checkRoadmapHealth`/`groomRoadmap`** -- Maintenance engine in `packages/core/src/roadmap/health.ts`. `checkRoadmapHealth` is read-only diagnostics; `groomRoadmap` is the pure transform (demote unactionable planned, archive done). Both are surfaced via `manage_roadmap` and `harness validate`.
|
|
493
|
+
- **Core `parseRoadmap`/`serializeRoadmap`** -- Fallback when MCP is unavailable. These functions in `packages/core/src/roadmap/` handle parsing and serializing the roadmap markdown format directly. Note: the serializer only preserves frontmatter, milestones, features, and the Assignment History table -- never add convention prose or comments to `docs/roadmap.md`, they are dropped on the next write.
|
|
494
|
+
- **Roadmap files** -- Live work in `docs/roadmap.md` (the orchestrator's source of truth); completed work archived to `docs/roadmap-archive.md` by `--groom`. Milestones are themes, not lifecycle stages -- promoted items land in the `Intake` lane and are groomed into themes.
|
|
437
495
|
|
|
438
496
|
## Success Criteria
|
|
439
497
|
|
|
@@ -126,6 +126,8 @@ Proceed with Feature A? (y/n/pick another)
|
|
|
126
126
|
|
|
127
127
|
### Phase 3: CONFIRM -- Human Decision
|
|
128
128
|
|
|
129
|
+
Ask the human in plain text (matching the `y/n/pick another` example above). Do not elevate this confirmation to an `AskUserQuestion` tool call — candidate labels and natural header choices ("Pick candidate", etc.) exceed its 12-char `header` cap and the prompt is rejected as ERR.
|
|
130
|
+
|
|
129
131
|
1. Wait for human confirmation.
|
|
130
132
|
- If **yes**: proceed to Phase 4.
|
|
131
133
|
- If **pick another**: ask which candidate number, then proceed with that pick.
|
|
@@ -8,8 +8,9 @@
|
|
|
8
8
|
- In CI — optimize test suite execution order
|
|
9
9
|
- When a test fails — understand which changes could have caused it
|
|
10
10
|
- When `on_pr` triggers fire
|
|
11
|
-
-
|
|
12
|
-
- NOT for test
|
|
11
|
+
- **Coverage Audit mode** — when no diff is available and the user asks for "coverage gaps", "deep dive", "coverage plan", or "what's untested", run project-wide gap analysis instead of test selection
|
|
12
|
+
- NOT for writing tests (use harness-tdd, or `canary:canary-write-test` for uncovered files surfaced by Coverage Audit)
|
|
13
|
+
- NOT for test quality analysis at the test-selection level (Coverage Audit mode does include a capped quality review via `canary:canary-review-test`)
|
|
13
14
|
|
|
14
15
|
## Prerequisites
|
|
15
16
|
|
|
@@ -112,6 +113,83 @@ npx vitest run tests/services/auth.test.ts tests/types/user.test.ts tests/routes
|
|
|
112
113
|
npx vitest run tests/services/auth.test.ts tests/types/user.test.ts tests/routes/login.test.ts tests/middleware/verify.test.ts tests/integration/auth-flow.test.ts
|
|
113
114
|
```
|
|
114
115
|
|
|
116
|
+
## Coverage Audit Mode
|
|
117
|
+
|
|
118
|
+
Activates when `--audit` is passed, OR when no diff is available AND the user's
|
|
119
|
+
language matches audit intent ("coverage gaps", "deep dive", "coverage plan",
|
|
120
|
+
"what's untested"). When this mode is selected, skip Phases 1–3 above and run
|
|
121
|
+
the audit phases below instead.
|
|
122
|
+
|
|
123
|
+
### Audit Phase 0: PROBE — Detect canary CLI availability
|
|
124
|
+
|
|
125
|
+
Call the `canary_probe` MCP tool once at the start of the audit. It returns
|
|
126
|
+
`{ status: "available" | "degraded", version?, reason? }` and never errors.
|
|
127
|
+
|
|
128
|
+
- **`available`** — the deterministic canary CLI is usable; use `canary_recommend_framework`
|
|
129
|
+
for framework selection in Phase 3 (GAP REPORT).
|
|
130
|
+
- **`degraded`** — the CLI is not usable (reason: `not-installed`, `binary-missing`,
|
|
131
|
+
`exec-failed`, or `bad-output`). Print one line:
|
|
132
|
+
|
|
133
|
+
> canary CLI unavailable (`<reason>`) — install `canary-test-cli` for deterministic
|
|
134
|
+
> framework recommendations. Proceeding; framework picks fall back to the plugin.
|
|
135
|
+
|
|
136
|
+
Then skip `canary_recommend_framework` calls for the rest of the run. This does **not**
|
|
137
|
+
affect the plugin-based Quality Review in Phase 2 (`canary:canary-review-test` is a
|
|
138
|
+
separate Claude Code plugin, installed independently of the CLI).
|
|
139
|
+
|
|
140
|
+
### Audit Phase 1: INVENTORY — Build the Source-to-Test Map
|
|
141
|
+
|
|
142
|
+
1. **Enumerate source files**: glob for `.ts`, `.tsx`, `.js`, `.jsx` under the
|
|
143
|
+
project's source roots (e.g., `src/`, `packages/*/src/`). Skip `node_modules`,
|
|
144
|
+
`dist`, build outputs, and fixtures.
|
|
145
|
+
2. **Enumerate test files**: glob for `*.test.*`, `*.spec.*`, and files under
|
|
146
|
+
`__tests__/` and parallel `tests/` directories.
|
|
147
|
+
3. **Map source → test**: for each source file, find its test file using the
|
|
148
|
+
Tier 1 / Tier 2 / Tier 3 strategies described in `Phase 2: DISCOVER`. With a
|
|
149
|
+
graph, prefer `get_impact`; without one, use naming conventions and import
|
|
150
|
+
parsing.
|
|
151
|
+
4. **Split**: produce two lists — `covered` (source files with at least one
|
|
152
|
+
matching test) and `uncovered` (source files with no matching test).
|
|
153
|
+
|
|
154
|
+
### Audit Phase 2: QUALITY REVIEW — Critique Covered Test Files
|
|
155
|
+
|
|
156
|
+
1. **Cap at 10 files per run**: a deep quality review is expensive. Pick the
|
|
157
|
+
top 10 covered files prioritized by size (lines of code) and criticality
|
|
158
|
+
(depth in the graph / co-change frequency). The remaining covered files are
|
|
159
|
+
accepted as-is for this run.
|
|
160
|
+
2. **For each of the 10**: dispatch `canary:canary-review-test` against the
|
|
161
|
+
test file. Collect its findings (missing edge cases, anti-patterns,
|
|
162
|
+
brittleness, oracle gaps).
|
|
163
|
+
3. **Aggregate**: bin findings by severity (high / medium / low) and by file.
|
|
164
|
+
|
|
165
|
+
### Audit Phase 3: GAP REPORT — Synthesize a Unified Coverage Plan
|
|
166
|
+
|
|
167
|
+
Emit a single report with three sections:
|
|
168
|
+
|
|
169
|
+
```
|
|
170
|
+
## Coverage Audit Report
|
|
171
|
+
|
|
172
|
+
### Uncovered Files (no test found)
|
|
173
|
+
| File | Lines | Priority | Suggested Action |
|
|
174
|
+
|------|-------|----------|------------------|
|
|
175
|
+
| src/services/billing.ts | 412 | high | canary:canary-write-test |
|
|
176
|
+
| src/utils/format.ts | 38 | low | canary:canary-write-test |
|
|
177
|
+
|
|
178
|
+
### Quality Gaps (from Quality Review, capped at 10 reviewed)
|
|
179
|
+
| Test File | Severity | Top Gap |
|
|
180
|
+
|-----------|----------|---------|
|
|
181
|
+
| tests/services/auth.test.ts | high | no failure-path assertions |
|
|
182
|
+
| tests/utils/date.test.ts | med | mock leaks across cases |
|
|
183
|
+
|
|
184
|
+
### Recommended Next Steps
|
|
185
|
+
- Generate tests for high-priority uncovered files via `canary:canary-write-test`.
|
|
186
|
+
- Resolve high-severity quality gaps via `canary:canary-review-test` follow-ups.
|
|
187
|
+
- For uncovered files, pick a framework first: when Phase 0 reported `available`, call
|
|
188
|
+
the `canary_recommend_framework` MCP tool with a prompt describing the file's purpose
|
|
189
|
+
(deterministic, no LLM round-trip) and put its `framework` in the Suggested Action.
|
|
190
|
+
When `degraded`, fall back to the `canary:canary-pick-framework` plugin.
|
|
191
|
+
```
|
|
192
|
+
|
|
115
193
|
## Harness Integration
|
|
116
194
|
|
|
117
195
|
- **`harness scan`** — Recommended before this skill for full graph-enhanced analysis. If graph is missing, skill uses naming convention and import parsing fallbacks.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
name: harness-test-advisor
|
|
2
2
|
version: "1.0.0"
|
|
3
|
-
description: Graph-based test selection — answers "what tests should I run?"
|
|
3
|
+
description: Graph-based test selection and project-wide coverage audit — answers "what tests should I run?" or "what's untested?"
|
|
4
4
|
stability: static
|
|
5
5
|
cognitive_mode: advisory-guide
|
|
6
6
|
triggers:
|
|
@@ -25,6 +25,9 @@ cli:
|
|
|
25
25
|
- name: files
|
|
26
26
|
description: Comma-separated list of changed files
|
|
27
27
|
required: false
|
|
28
|
+
- name: audit
|
|
29
|
+
description: Run Coverage Audit mode (project-wide gap analysis) instead of test selection
|
|
30
|
+
required: false
|
|
28
31
|
mcp:
|
|
29
32
|
tool: run_skill
|
|
30
33
|
input:
|