@harness-engineering/cli 2.8.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/skills/claude-code/harness-audit-harness-strength/SKILL.md +188 -0
- package/dist/agents/skills/claude-code/harness-audit-harness-strength/skill.yaml +54 -0
- package/dist/agents/skills/claude-code/harness-autopilot/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-brainstorming/SKILL.md +28 -14
- package/dist/agents/skills/claude-code/harness-execution/SKILL.md +1 -1
- package/dist/agents/skills/claude-code/harness-git-workflow/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-ideate/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-knowledge-pipeline/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-roadmap/SKILL.md +62 -4
- package/dist/agents/skills/claude-code/harness-roadmap-pilot/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-test-advisor/SKILL.md +80 -2
- package/dist/agents/skills/claude-code/harness-test-advisor/skill.yaml +4 -1
- package/dist/agents/skills/claude-code/initialize-harness-project/SKILL.md +27 -25
- package/dist/agents/skills/claude-code/initialize-harness-project/skill.yaml +1 -0
- package/dist/agents/skills/claude-code/outcome-eval/SKILL.md +97 -0
- package/dist/agents/skills/claude-code/outcome-eval/skill.yaml +51 -0
- package/dist/agents/skills/codex/harness-audit-harness-strength/SKILL.md +188 -0
- package/dist/agents/skills/codex/harness-audit-harness-strength/skill.yaml +54 -0
- package/dist/agents/skills/codex/harness-autopilot/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-brainstorming/SKILL.md +28 -14
- package/dist/agents/skills/codex/harness-execution/SKILL.md +1 -1
- package/dist/agents/skills/codex/harness-git-workflow/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-ideate/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-knowledge-pipeline/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-roadmap/SKILL.md +62 -4
- package/dist/agents/skills/codex/harness-roadmap-pilot/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-test-advisor/SKILL.md +80 -2
- package/dist/agents/skills/codex/harness-test-advisor/skill.yaml +4 -1
- package/dist/agents/skills/codex/initialize-harness-project/SKILL.md +27 -25
- package/dist/agents/skills/codex/initialize-harness-project/skill.yaml +1 -0
- package/dist/agents/skills/codex/outcome-eval/SKILL.md +97 -0
- package/dist/agents/skills/codex/outcome-eval/skill.yaml +51 -0
- package/dist/agents/skills/cursor/harness-audit-harness-strength/SKILL.md +188 -0
- package/dist/agents/skills/cursor/harness-audit-harness-strength/skill.yaml +54 -0
- package/dist/agents/skills/cursor/harness-autopilot/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-brainstorming/SKILL.md +28 -14
- package/dist/agents/skills/cursor/harness-execution/SKILL.md +1 -1
- package/dist/agents/skills/cursor/harness-git-workflow/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-ideate/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-knowledge-pipeline/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-roadmap/SKILL.md +62 -4
- package/dist/agents/skills/cursor/harness-roadmap-pilot/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-test-advisor/SKILL.md +80 -2
- package/dist/agents/skills/cursor/harness-test-advisor/skill.yaml +4 -1
- package/dist/agents/skills/cursor/initialize-harness-project/SKILL.md +27 -25
- package/dist/agents/skills/cursor/initialize-harness-project/skill.yaml +1 -0
- package/dist/agents/skills/cursor/outcome-eval/SKILL.md +97 -0
- package/dist/agents/skills/cursor/outcome-eval/skill.yaml +51 -0
- package/dist/agents/skills/gemini-cli/harness-audit-harness-strength/SKILL.md +188 -0
- package/dist/agents/skills/gemini-cli/harness-audit-harness-strength/skill.yaml +54 -0
- package/dist/agents/skills/gemini-cli/harness-autopilot/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-brainstorming/SKILL.md +28 -14
- package/dist/agents/skills/gemini-cli/harness-execution/SKILL.md +1 -1
- package/dist/agents/skills/gemini-cli/harness-git-workflow/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-ideate/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-knowledge-pipeline/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-roadmap/SKILL.md +62 -4
- package/dist/agents/skills/gemini-cli/harness-roadmap-pilot/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-test-advisor/SKILL.md +80 -2
- package/dist/agents/skills/gemini-cli/harness-test-advisor/skill.yaml +4 -1
- package/dist/agents/skills/gemini-cli/initialize-harness-project/SKILL.md +27 -25
- package/dist/agents/skills/gemini-cli/initialize-harness-project/skill.yaml +1 -0
- package/dist/agents/skills/gemini-cli/outcome-eval/SKILL.md +97 -0
- package/dist/agents/skills/gemini-cli/outcome-eval/skill.yaml +51 -0
- package/dist/agents/skills/tests/harness-test-advisor.test.ts +94 -0
- package/dist/{agents-md-WMJO5HSM.js → agents-md-XOJE5ETA.js} +2 -2
- package/dist/{architecture-EVYVGPRZ.js → architecture-JEZPJ725.js} +3 -3
- package/dist/{assess-project-KZG563SO.js → assess-project-RHXOQNMM.js} +1 -1
- package/dist/bin/harness-mcp.js +13 -13
- package/dist/bin/harness.js +22 -22
- package/dist/{check-phase-gate-MK364VXS.js → check-phase-gate-GNDOMS35.js} +3 -3
- package/dist/{chunk-VBNJAKUH.js → chunk-3ISHDWO7.js} +1 -1
- package/dist/{chunk-UJFNIWC7.js → chunk-527G27VI.js} +2 -2
- package/dist/{chunk-5XL5QJ5B.js → chunk-6WQCLCBO.js} +1 -1
- package/dist/{chunk-UXPWSV3B.js → chunk-BYVT5LVO.js} +3 -3
- package/dist/{chunk-HRSE6IQD.js → chunk-DE5U6KOL.js} +1 -1
- package/dist/{chunk-B7KMTHN3.js → chunk-GGKRA7A7.js} +1 -1
- package/dist/{chunk-STESM73J.js → chunk-GSP2XIVS.js} +1 -1
- package/dist/{chunk-KBCPELBC.js → chunk-HEIMJJI4.js} +2 -2
- package/dist/{chunk-KI6PHH6Q.js → chunk-HWTUUQOW.js} +3 -3
- package/dist/{chunk-2SNRCAC3.js → chunk-KAJOS3BA.js} +2 -2
- package/dist/{chunk-C7J55C6E.js → chunk-MGUPSE6D.js} +5 -5
- package/dist/{chunk-GOHWCHCS.js → chunk-N24HCQA4.js} +2243 -776
- package/dist/{chunk-EEFRPEWT.js → chunk-NWLGL3IR.js} +1 -1
- package/dist/{chunk-UUO4K7NX.js → chunk-ONCPJGHY.js} +6 -6
- package/dist/{chunk-X4EFYVCZ.js → chunk-PC2R5RUJ.js} +1 -1
- package/dist/{chunk-7EQAZKJ5.js → chunk-QBC7DI4Q.js} +1 -1
- package/dist/{chunk-UBZQE3JN.js → chunk-UX3PQKVZ.js} +7 -2
- package/dist/{chunk-CAQPSFPT.js → chunk-WWEVS7RO.js} +453 -91
- package/dist/{chunk-4BDOSCNY.js → chunk-XFU6SB2Q.js} +1015 -648
- package/dist/{chunk-LADPYELQ.js → chunk-XX4OLNWQ.js} +2 -2
- package/dist/{chunk-FZ5MBXEG.js → chunk-YQSYCBAH.js} +6 -6
- package/dist/{chunk-ACOOG3UW.js → chunk-Z4AWEIBY.js} +3 -3
- package/dist/{ci-workflow-IW2H5DC4.js → ci-workflow-ORZ4F45F.js} +2 -2
- package/dist/{dist-JCFK263L.js → dist-LHINSVK4.js} +85 -1
- package/dist/{docs-7CP6VV42.js → docs-AU3DXFFO.js} +3 -3
- package/dist/{engine-MKJEBWU7.js → engine-6GPWOYQH.js} +2 -2
- package/dist/{entropy-6JOZJYG7.js → entropy-VLKDRMRT.js} +2 -2
- package/dist/{feedback-T65AEJEG.js → feedback-YXYY44ZC.js} +1 -1
- package/dist/{generate-agent-definitions-4VI4YTO2.js → generate-agent-definitions-XIHQPKG3.js} +3 -3
- package/dist/hooks/cost-tracker.js +29 -6
- package/dist/index.d.ts +8 -8
- package/dist/index.js +22 -22
- package/dist/{loader-AWTHHE3A.js → loader-YAN56MT3.js} +2 -2
- package/dist/{mcp-JBIBINO2.js → mcp-RK3SFVYB.js} +13 -13
- package/dist/{performance-NCOUDOMW.js → performance-64DPTLLQ.js} +3 -3
- package/dist/{review-pipeline-YIC6JPMY.js → review-pipeline-NENG6LW7.js} +2 -2
- package/dist/{runtime-5MHV4UEE.js → runtime-FFIIRT2T.js} +2 -2
- package/dist/{security-VZPJNIDP.js → security-3PIBN35B.js} +1 -1
- package/dist/templates/basic/harness.config.json.hbs +34 -0
- package/dist/templates/ci/README.md +86 -0
- package/dist/templates/ci/required-review.ruleset.json +20 -0
- package/dist/templates/ci/required-review.yml.hbs +57 -0
- package/dist/templates/ci/template.json +5 -0
- package/dist/templates/intermediate/harness.config.json.hbs +36 -0
- package/dist/templates/orchestrator/harness.orchestrator.md +9 -0
- package/dist/{tool-tiers-B7JC2XC3.js → tool-tiers-OKK5FE57.js} +2 -0
- package/dist/{validate-7BE4VTFI.js → validate-7IAF2ES3.js} +3 -3
- package/dist/{validate-cross-check-F7GEBRDH.js → validate-cross-check-QPSAOZKD.js} +2 -2
- package/package.json +7 -7
|
@@ -29,6 +29,8 @@ Detect plugin-only state by checking whether `harness` resolves on PATH (`comman
|
|
|
29
29
|
|
|
30
30
|
## Process
|
|
31
31
|
|
|
32
|
+
**Prompt the human in plain text** — every framework confirmation, migration check, and telemetry-identity question in this skill is plain text only. Do not elevate to `AskUserQuestion`: the framework list (~10 options) exceeds its 4-option cap and natural headers like "Confirm framework" exceed its 12-char cap, rendering the call as ERR.
|
|
33
|
+
|
|
32
34
|
### Phase 1: ASSESS — Determine Current State
|
|
33
35
|
|
|
34
36
|
1. **Check for existing harness configuration.** Look for `.harness/` directory, `AGENTS.md`, `harness.config.json`, and any skill definitions. Their presence determines whether this is a new project or a migration.
|
|
@@ -96,7 +98,7 @@ Detect plugin-only state by checking whether `harness` resolves on PATH (`comman
|
|
|
96
98
|
5. **Configure i18n (all levels).** Ask: "Will this project support multiple languages?" Based on the answer:
|
|
97
99
|
- **Yes:** Invoke `harness-i18n-workflow` configure phase to set up i18n config in `harness.config.json` (source locale, target locales, framework, strictness). Then invoke `harness-i18n-workflow` scaffold phase to create translation file structure and extraction config. Set `i18n.enabled: true`.
|
|
98
100
|
- **No:** Set `i18n.enabled: false` in `harness.config.json`. The `harness-i18n-process` skill will still fire gentle prompts for unconfigured projects when features touch user-facing strings.
|
|
99
|
-
- **Not sure
|
|
101
|
+
- **Not sure:** Skip i18n configuration entirely. Do not set `i18n.enabled`. The project can enable i18n later by running `harness-i18n-workflow` directly.
|
|
100
102
|
|
|
101
103
|
5b. **Configure design system (non-test-suite projects).** Ask: "Will this project have a UI requiring a design system?" Mirror the i18n step's three-way response shape. Use `emit_interaction`:
|
|
102
104
|
|
|
@@ -136,7 +138,7 @@ Detect plugin-only state by checking whether `harness` resolves on PATH (`comman
|
|
|
136
138
|
Based on the answer:
|
|
137
139
|
- **Yes:** Ask a follow-up: "Which platforms? `web`, `mobile`, or both?" Write `design.enabled: true` and `design.platforms: [...]` (a non-empty array of `web` and/or `mobile`) to `harness.config.json`. Inform the user: "Design tokens will be generated when you start your first design-touching feature — `harness-design-system` fires automatically via `on_new_feature`."
|
|
138
140
|
- **No:** Write `design.enabled: false` to `harness.config.json`. Do not write `design.platforms`. The `on_new_feature` trigger respects this flag and will not fire `harness-design-system`.
|
|
139
|
-
- **Not sure
|
|
141
|
+
- **Not sure:** Do not write `design.enabled` or `design.platforms`. The project can enable design later by running `harness-design-system` directly; `on_new_feature` will prompt gently when a feature touches user-facing UI.
|
|
140
142
|
|
|
141
143
|
**Skip this step entirely if Phase 1 step 5 classified the project as a test suite.** Test-suite projects will be dispatched at step 6 below to `initialize-test-suite-project` and have no UI to govern.
|
|
142
144
|
|
|
@@ -183,7 +185,7 @@ Detect plugin-only state by checking whether `harness` resolves on PATH (`comman
|
|
|
183
185
|
- **Absent (most common on init).** Present the prompt above. Apply the answer:
|
|
184
186
|
- **Yes:** delegate to `harness-strategy` (which routes via its own Phase 0 to the first-run interview). When `harness-strategy` completes, continue with step 6.
|
|
185
187
|
- **No:** write `init.strategy.declined: true` to `.harness/state.json` (merge into existing JSON; do not clobber). This is the explicit decline location; re-running init detects the flag and skips the prompt.
|
|
186
|
-
- **Not sure
|
|
188
|
+
- **Not sure:** no state write. `/harness:strategy` remains available standalone, and a future re-run of init will re-offer.
|
|
187
189
|
|
|
188
190
|
- **Present and valid.** Skip the prompt silently. Surface a one-line note: `STRATEGY.md detected — downstream skills will pick it up as grounding`. No `init.strategy.declined` write.
|
|
189
191
|
|
|
@@ -303,7 +305,7 @@ This phase closes the parity gap that the marketplace plugin install does not co
|
|
|
303
305
|
|
|
304
306
|
Based on the answer:
|
|
305
307
|
- **Yes:** Invoke `harness-roadmap` (skill) or run `/harness:roadmap --create` to create `docs/roadmap.md`. Verify the file exists. The `manage_roadmap` MCP tool is for managing entries in an existing roadmap, not for creating one.
|
|
306
|
-
- **If `design.enabled === true` in `harness.config.json`** (set by Phase 3 step 5b), call `manage_roadmap` with `action: add`, `feature: "Set up design system"`, `status: "planned"`, `milestone: "
|
|
308
|
+
- **If `design.enabled === true` in `harness.config.json`** (set by Phase 3 step 5b), call `manage_roadmap` with `action: add`, `feature: "Set up design system"`, `status: "planned"`, `milestone: "Intake"`, `summary: "Run harness-design-system to define palette, typography, and generate W3C DTCG tokens. Deferred from project init — fires on first design-touching feature via on_new_feature."`. Skip silently if `manage_roadmap show` reports a duplicate `(feature, milestone)` pair. This closes the loop between deferred design intent and visible planning work.
|
|
307
309
|
- **No:** Skip silently. The user can still run `/harness:roadmap --create` later — that informational fallback remains valid.
|
|
308
310
|
|
|
309
311
|
2. **Commit the initialization.** All generated, configured, and instrumentation files in a single commit. Include `harness.config.json`, `AGENTS.md`, `.harness/arch/baselines.json`, `.harness/telemetry.json` (if created), updates to project `.mcp.json` (if Tier-0 integrations were wired), and any roadmap files.
|
|
@@ -323,11 +325,11 @@ This phase closes the parity gap that the marketplace plugin install does not co
|
|
|
323
325
|
- **`harness integrations list` / `harness integrations add <name>`** — Phase 5 step 6. Lists Tier-0 (zero-config) and Tier-1 (API-key) MCP integrations and adds them to project `.mcp.json`.
|
|
324
326
|
- **`harness-i18n-workflow configure` + `harness-i18n-workflow scaffold`** — Invoked during Phase 3 if the project will support multiple languages. Sets up i18n configuration and translation file structure.
|
|
325
327
|
- **`harness-design-system` (deferred via `on_new_feature`)** — Phase 3 step 5b records `design.enabled` + `design.platforms` in `harness.config.json` but does NOT run the full design-system skill. Token generation defers to the first design-touching feature, where `harness-design-system` fires via `on_new_feature` and reads `design.enabled` to decide whether to proceed.
|
|
326
|
-
- **`harness-strategy`** — Phase 3 step 5c delegates to this skill on "Yes". The skill conducts a first-run interview (pushback rules with a 2-round cap per section) and writes a valid `STRATEGY.md` at repo root via `writeStrategyDoc`. On "No" init writes `init.strategy.declined: true` to `.harness/state.json`. On "
|
|
328
|
+
- **`harness-strategy`** — Phase 3 step 5c delegates to this skill on "Yes". The skill conducts a first-run interview (pushback rules with a 2-round cap per section) and writes a valid `STRATEGY.md` at repo root via `writeStrategyDoc`. On "No" init writes `init.strategy.declined: true` to `.harness/state.json`. On "not sure" no state is written; the user can run `/harness:strategy` standalone. When `STRATEGY.md` already exists and is valid the prompt is skipped; when present-but-invalid the user gets the three-path repair offer.
|
|
327
329
|
- **`validateStrategy`** — `@harness-engineering/core` helper used by Phase 3 step 5c to detect present-but-invalid `STRATEGY.md`. Invoked through a Node one-liner so init does not require importing core directly.
|
|
328
330
|
- **`initialize-test-suite-project`** — Sub-skill. Invoked during Phase 3 step 6 when Phase 1 step 5 classified the project as a test suite. Owns archetype selection, shared-library vs in-repo decision, layer variants, tag taxonomy, reporter stack, custom report, and "prove the guards fire" verification.
|
|
329
331
|
- **`harness-roadmap` skill** — Phase 6 step 1 invokes this skill (or `/harness:roadmap --create`) when the user opts in to creating `docs/roadmap.md`. The `manage_roadmap` MCP tool does not create roadmaps; it manages entries in an existing one.
|
|
330
|
-
- **`manage_roadmap` MCP tool** — Phase 6 step 1, when `design.enabled === true`, calls `manage_roadmap` with `action: add` to insert a `planned` "Set up design system" item under milestone `
|
|
332
|
+
- **`manage_roadmap` MCP tool** — Phase 6 step 1, when `design.enabled === true`, calls `manage_roadmap` with `action: add` to insert a `planned` "Set up design system" item under milestone `Intake` with a summary describing the deferred work.
|
|
331
333
|
|
|
332
334
|
## Success Criteria
|
|
333
335
|
|
|
@@ -340,29 +342,29 @@ This phase closes the parity gap that the marketplace plugin install does not co
|
|
|
340
342
|
- The adoption level matches what was agreed upon with the human
|
|
341
343
|
- All generated files are committed in a single atomic commit
|
|
342
344
|
- i18n configuration is set if the human chose to enable it during init
|
|
343
|
-
- For non-test-suite projects, the design-system question was asked and `harness.config.json` reflects the answer: `design.enabled: true` (with `design.platforms` populated) for yes, `design.enabled: false` for no, or absent for not
|
|
344
|
-
- The strategy question (Phase 3 step 5c) was asked unless `STRATEGY.md` already existed and was valid (in which case the prompt was skipped silently with a one-line detection note). The answer was recorded according to the documented semantics: Yes → `STRATEGY.md` exists and `harness validate` passes against `StrategyDocSchema`; No → `.harness/state.json` contains `init.strategy.declined: true`; Not sure
|
|
345
|
+
- For non-test-suite projects, the design-system question was asked and `harness.config.json` reflects the answer: `design.enabled: true` (with `design.platforms` populated) for yes, `design.enabled: false` for no, or absent for not sure.
|
|
346
|
+
- The strategy question (Phase 3 step 5c) was asked unless `STRATEGY.md` already existed and was valid (in which case the prompt was skipped silently with a one-line detection note). The answer was recorded according to the documented semantics: Yes → `STRATEGY.md` exists and `harness validate` passes against `StrategyDocSchema`; No → `.harness/state.json` contains `init.strategy.declined: true`; Not sure → no `STRATEGY.md` and no `init.strategy.declined` flag.
|
|
345
347
|
- **Phase 5 (INSTRUMENT) outputs:** `.harness/graph/` is populated; `.harness/arch/baselines.json` exists; `harness check-perf` ran without errors (intermediate and above); `.harness/telemetry.json` exists if the human opted into identity tagging; legacy layout warnings were surfaced via `harness migrate --dry-run` and either resolved or explicitly deferred; Tier-0 MCP integrations (context7, sequential-thinking, playwright) are present in the project's `.mcp.json` (or the human declined and that decision is recorded).
|
|
346
348
|
- The roadmap question was asked. If the user answered yes, `docs/roadmap.md` exists and was created via `harness-roadmap` (or the documented `/harness:roadmap --create` fallback).
|
|
347
|
-
- When `design.enabled === true` AND the user answered yes to the roadmap question, `docs/roadmap.md` contains a `planned` entry titled "Set up design system" under milestone `
|
|
349
|
+
- When `design.enabled === true` AND the user answered yes to the roadmap question, `docs/roadmap.md` contains a `planned` entry titled "Set up design system" under milestone `Intake` with a summary describing the deferred work. The entry is absent in all other answer combinations.
|
|
348
350
|
- For test suites: `initialize-test-suite-project` ran to completion and its Success Criteria are also met
|
|
349
351
|
- For plugin-only invocations: every CLI invocation in this skill ran successfully via `npx @harness-engineering/cli <cmd>` without requiring a global install. If any invocation failed because the npx download failed, the human was prompted to retry or `npm install -g @harness-engineering/cli` instead.
|
|
350
352
|
|
|
351
353
|
## Rationalizations to Reject
|
|
352
354
|
|
|
353
|
-
| Rationalization | Why It Is Wrong
|
|
354
|
-
| ---------------------------------------------------------------------------- |
|
|
355
|
-
| "The generated AGENTS.md template looks fine -- no need to customize it" | Phase 3 says do not blindly accept generated content. Without project-specific descriptions, agents receive generic instructions.
|
|
356
|
-
| "We should start at the advanced level since we want full coverage" | The skill recommends basic for new projects. Each level builds on the previous. Jumping to advanced creates misconfigured rules.
|
|
357
|
-
| "I will skip the i18n question to keep setup fast" | Phase 3 requires asking about i18n and recording the decision. Skipping creates ambiguity about whether the omission was intentional.
|
|
358
|
-
| "I will skip the design-system question to keep setup fast" | Phase 3 step 5b requires asking about design and recording the answer in `design.enabled`. Skipping creates ambiguity about whether the omission was intentional and bypasses the linkage between init and the deferred `harness-design-system` invocation on `on_new_feature`.
|
|
359
|
-
| "I will skip the strategy question — it's just paperwork" | Phase 3 step 5c is the only point in the workflow where init asks the user to capture the strategic anchor. Skipping bypasses the grounding signal that brainstorming, ideate, and roadmap-pilot read; downstream skill output degrades silently. Even a "
|
|
360
|
-
| "STRATEGY.md exists already, so I should re-run the interview to refresh it" | When `STRATEGY.md` is present and valid, Phase 3 step 5c skips the prompt silently. Refreshing strategy mid-init is out of scope — that is what the standalone `/harness:strategy` update flow is for. Surface a one-line detection note and continue.
|
|
361
|
-
| "STRATEGY.md is present but invalid, so I should block init" | Phase 3 step 5c explicitly does NOT block. It surfaces the validation error, offers three repair paths (fix now / move-to-bak / ignore), and continues based on the user's choice. Init is the wrong place to gate on strategy correctness.
|
|
362
|
-
| "Validation passed, so the project is ready" | Phase 5 captures baselines, configures telemetry identity, surfaces legacy warnings, and wires Tier-0 integrations. Validation alone is not sufficient.
|
|
363
|
-
| "Plugin install means setup is done" | The marketplace plugin ships skills, slash commands, subagents, hooks, and MCP — but it cannot mutate the user's project state. `harness.config.json`, baselines, telemetry identity, and Tier-0 integrations require running this skill once per project.
|
|
364
|
-
| "Skip Phase 5 if we already ran `harness setup`" | Phase 5 is idempotent. It safely no-ops where setup already wired things and fills gaps where it didn't (common case: setup ran once, then a new Tier-0 integration was added or a layout was migrated). Re-running is the right behavior.
|
|
365
|
-
| "This is a test suite, we'll configure layers in this skill" | Phase 3 step 6 dispatches to `initialize-test-suite-project` for archetype selection, layer variants, and the rest. Do not inline test-suite-specific configuration here — the sub-skill owns it and carries the gotchas.
|
|
355
|
+
| Rationalization | Why It Is Wrong |
|
|
356
|
+
| ---------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
357
|
+
| "The generated AGENTS.md template looks fine -- no need to customize it" | Phase 3 says do not blindly accept generated content. Without project-specific descriptions, agents receive generic instructions. |
|
|
358
|
+
| "We should start at the advanced level since we want full coverage" | The skill recommends basic for new projects. Each level builds on the previous. Jumping to advanced creates misconfigured rules. |
|
|
359
|
+
| "I will skip the i18n question to keep setup fast" | Phase 3 requires asking about i18n and recording the decision. Skipping creates ambiguity about whether the omission was intentional. |
|
|
360
|
+
| "I will skip the design-system question to keep setup fast" | Phase 3 step 5b requires asking about design and recording the answer in `design.enabled`. Skipping creates ambiguity about whether the omission was intentional and bypasses the linkage between init and the deferred `harness-design-system` invocation on `on_new_feature`. |
|
|
361
|
+
| "I will skip the strategy question — it's just paperwork" | Phase 3 step 5c is the only point in the workflow where init asks the user to capture the strategic anchor. Skipping bypasses the grounding signal that brainstorming, ideate, and roadmap-pilot read; downstream skill output degrades silently. Even a "no" or "not sure" answer is better than no answer because it locks in the decision (or absence of one). |
|
|
362
|
+
| "STRATEGY.md exists already, so I should re-run the interview to refresh it" | When `STRATEGY.md` is present and valid, Phase 3 step 5c skips the prompt silently. Refreshing strategy mid-init is out of scope — that is what the standalone `/harness:strategy` update flow is for. Surface a one-line detection note and continue. |
|
|
363
|
+
| "STRATEGY.md is present but invalid, so I should block init" | Phase 3 step 5c explicitly does NOT block. It surfaces the validation error, offers three repair paths (fix now / move-to-bak / ignore), and continues based on the user's choice. Init is the wrong place to gate on strategy correctness. |
|
|
364
|
+
| "Validation passed, so the project is ready" | Phase 5 captures baselines, configures telemetry identity, surfaces legacy warnings, and wires Tier-0 integrations. Validation alone is not sufficient. |
|
|
365
|
+
| "Plugin install means setup is done" | The marketplace plugin ships skills, slash commands, subagents, hooks, and MCP — but it cannot mutate the user's project state. `harness.config.json`, baselines, telemetry identity, and Tier-0 integrations require running this skill once per project. |
|
|
366
|
+
| "Skip Phase 5 if we already ran `harness setup`" | Phase 5 is idempotent. It safely no-ops where setup already wired things and fills gaps where it didn't (common case: setup ran once, then a new Tier-0 integration was added or a layout was migrated). Re-running is the right behavior. |
|
|
367
|
+
| "This is a test suite, we'll configure layers in this skill" | Phase 3 step 6 dispatches to `initialize-test-suite-project` for archetype selection, layer variants, and the rest. Do not inline test-suite-specific configuration here — the sub-skill owns it and carries the gotchas. |
|
|
366
368
|
|
|
367
369
|
## Examples
|
|
368
370
|
|
|
@@ -458,7 +460,7 @@ Step 4 (roadmap): "Set up a project roadmap now?"
|
|
|
458
460
|
design.enabled === true detected → manage_roadmap action: add
|
|
459
461
|
feature: "Set up design system"
|
|
460
462
|
status: planned
|
|
461
|
-
milestone:
|
|
463
|
+
milestone: Intake
|
|
462
464
|
summary: "Run harness-design-system to define palette, typography, and generate W3C DTCG tokens..."
|
|
463
465
|
Result: docs/roadmap.md contains the planned design item.
|
|
464
466
|
|
|
@@ -470,7 +472,7 @@ git add harness.config.json .harness/ AGENTS.md docs/roadmap.md
|
|
|
470
472
|
git commit -m "feat: initialize harness project with design and roadmap"
|
|
471
473
|
```
|
|
472
474
|
|
|
473
|
-
**Final state:** `harness.config.json` has `design.enabled: true` + `design.platforms: ["web"]`; `docs/roadmap.md` lists "Set up design system" as a `planned` item under `
|
|
475
|
+
**Final state:** `harness.config.json` has `design.enabled: true` + `design.platforms: ["web"]`; `docs/roadmap.md` lists "Set up design system" as a `planned` item under `Intake`; on the first feature touching UI, `on_new_feature` fires `harness-design-system` which reads `design.enabled` and runs the full discover/define/generate/validate flow.
|
|
474
476
|
|
|
475
477
|
### Example: Migrating Existing Project from Basic to Intermediate
|
|
476
478
|
|
|
@@ -547,7 +549,7 @@ npx @harness-engineering/cli init # auto-detects framework
|
|
|
547
549
|
Customize AGENTS.md with project-specific content.
|
|
548
550
|
Phase 3 step 5 (i18n): "No, English only." → i18n.enabled = false.
|
|
549
551
|
Phase 3 step 5b (design): "No, this is a backend service." → design.enabled = false.
|
|
550
|
-
Phase 3 step 5c (strategy): "
|
|
552
|
+
Phase 3 step 5c (strategy): "not sure." → no STRATEGY.md, no init.strategy.declined flag; user can run /harness:strategy later.
|
|
551
553
|
Phase 3 step 6 (test-suite dispatch): not a test suite, skipped.
|
|
552
554
|
```
|
|
553
555
|
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# Outcome Eval
|
|
2
|
+
|
|
3
|
+
> Post-execution LLM-judgment: did the implementation actually satisfy its spec? Reads the spec's acceptance section, the change diff, and test output, and emits a confidence-rated `OutcomeVerdict` (`SATISFIED | NOT_SATISFIED | INCONCLUSIVE`) with a rationale and unmet criteria. Ship authority is derived in TypeScript, never trusted from the LLM: a high-confidence `NOT_SATISFIED` blocks ship; every other verdict is advisory. The harness's first blocking post-execution spec-satisfaction gate (the roadmap's named #1 gap). Each verdict persists as an `execution_outcome` node, compounding into skill-effectiveness baselines.
|
|
4
|
+
|
|
5
|
+
## When to Use
|
|
6
|
+
|
|
7
|
+
- At orchestrator step 6.5 — after Code Review, before Ship — on every change with a spec.
|
|
8
|
+
- When you need a durable, structured answer to "did this code do what the spec said?"
|
|
9
|
+
- NOT for pre-execution risk simulation (use PESL).
|
|
10
|
+
- NOT for rule-based floors (lint/architecture/entropy) or craft ceilings (naming/spec/security) — those run elsewhere.
|
|
11
|
+
- NOT for auto-remediation. outcome-eval judges; it does not fix.
|
|
12
|
+
- NOT when no judgable spec section exists — the verdict degrades to INCONCLUSIVE/advisory and never blocks.
|
|
13
|
+
|
|
14
|
+
## Process
|
|
15
|
+
|
|
16
|
+
### Phase 1: GATHER — Collect inputs
|
|
17
|
+
|
|
18
|
+
1. Capture the change under judgment as a unified diff: `git diff` (or `git diff <base>...HEAD` for a branch). Record it as `diff`.
|
|
19
|
+
2. Capture test-runner output. If a test command is known, run it and capture stdout+stderr as `testOutput`; otherwise pass the most recent captured output. Empty/unparseable test output is tolerated (degrades to advisory).
|
|
20
|
+
3. Resolve the spec path. Prefer the spec under `docs/changes/<feature>/proposal.md` for the current change. Record as `specPath`.
|
|
21
|
+
|
|
22
|
+
### Phase 2: RESOLVE — Find the judgment section
|
|
23
|
+
|
|
24
|
+
The evaluator resolves the section internally via the fallback chain `## Success Criteria` -> `## User-Visible Behavior` -> `## Overview`, recording the match in `judgedAgainst`. No manual action — pass `specPath` and let `OutcomeEvaluator` resolve. If no section is judgable, the verdict is INCONCLUSIVE/advisory.
|
|
25
|
+
|
|
26
|
+
### Phase 3: JUDGE — Invoke the evaluator
|
|
27
|
+
|
|
28
|
+
1. Invoke the MCP tool `mcp__harness__outcome_eval` with `{ specPath, diff, testOutput }` (optional `model`). The tool constructs `OutcomeEvaluator` cli-side and calls `evaluate({ specPath, diff, testOutput })`; the supported v1 provider is the anthropic analysis provider (`ANTHROPIC_API_KEY`).
|
|
29
|
+
2. **`diff` and `testOutput` are required inputs and the agent MUST supply them** from the session (`git diff` + captured test-runner output). They are the evidence the judge reasons over — passing an empty `diff` or empty `testOutput` is the degradation path, not the normal path: the verdict degrades to INCONCLUSIVE/advisory (never blocking), which defeats the gate. Do not invoke the tool without real diff/test content.
|
|
30
|
+
3. The LLM returns ONLY `verdict / confidence / rationale / unmetCriteria`. `authority` is computed in TypeScript from `(verdict, confidence)` and is never read from the LLM — do not attempt to override it. The tool returns the verdict exactly as the evaluator derives it.
|
|
31
|
+
4. The call is degrade-safe: provider failure (incl. no `ANTHROPIC_API_KEY`), empty diff, empty test output, or missing judgable section yields INCONCLUSIVE/low/advisory. It never throws and never blocks.
|
|
32
|
+
|
|
33
|
+
### Phase 4: GATE — Render and (conditionally) halt
|
|
34
|
+
|
|
35
|
+
1. Render the verdict: `verdict`, `confidence`, `judgedAgainst`, `rationale`, and `unmetCriteria`.
|
|
36
|
+
2. Authority rule (must match `deriveAuthority`): authority is `blocking` **iff** `verdict === 'NOT_SATISFIED' && confidence === 'high'`; every other combination — including all `INCONCLUSIVE` and `SATISFIED` cases, and all `medium`/`low` `NOT_SATISFIED` — is `advisory`.
|
|
37
|
+
3. **On a blocking verdict: HALT before the Ship step.** Report the unmet criteria and stop; do not proceed to step 7. Resolution requires fixing the implementation (or the spec) and re-running outcome-eval.
|
|
38
|
+
4. On an advisory verdict: report it and proceed. Advisory `NOT_SATISFIED` is surfaced for human attention but does not stop the workflow.
|
|
39
|
+
|
|
40
|
+
## Harness Integration
|
|
41
|
+
|
|
42
|
+
- **`mcp__harness__outcome_eval`** — MCP tool (the invocation surface). Inputs: `specPath` (required), `diff` (required), `testOutput` (required), `model` (optional), `path` (optional project root for graph persistence). The agent supplies `diff` and `testOutput` from the session; omitting them degrades the verdict to INCONCLUSIVE/advisory (never blocking). The handler builds the cli `AnalysisProvider` + a `GraphStore`, constructs `OutcomeEvaluator`, and returns the `OutcomeVerdict` with authority exactly as derived in TypeScript.
|
|
43
|
+
- **Evaluator surface:** `OutcomeEvaluator`, `deriveAuthority`, `verdictSchema`, `OutcomeVerdict` are exported from `@harness-engineering/intelligence`.
|
|
44
|
+
- **Provider path (v1 supported):** the anthropic analysis provider (`ANTHROPIC_API_KEY`). When no provider is configured the call degrades to INCONCLUSIVE/advisory. The openai-compatible _strict_ structured-output path is a known follow-up (see Known Limitations).
|
|
45
|
+
- **Orchestrator:** runs as step 6.5 between Code Review and Ship in `harness.orchestrator.md`.
|
|
46
|
+
- **Persistence:** each `evaluate()` writes one `execution_outcome` node via `ExecutionOutcomeConnector`, consumable by `effectiveness/scorer.ts`.
|
|
47
|
+
|
|
48
|
+
## Known Limitations
|
|
49
|
+
|
|
50
|
+
- **INCONCLUSIVE persistence:** the persisted node maps `INCONCLUSIVE -> result: 'failure'` for type-validity, but it OMITS `agentPersona` and writes `affectedSystemNodeIds: []`. The effectiveness scorer (`gatherOutcomes`) ignores any node missing `agentPersona` or `outcome_of` edges, so outcome-eval nodes are **scorer-non-counting** in v1 — the INCONCLUSIVE-as-failure mapping is therefore harmless and does not punish any persona. If a future change attaches persona/affected-system attribution, it MUST first change INCONCLUSIVE modeling (do not persist INCONCLUSIVE, or use a distinct result value the scorer excludes) before the node becomes scorer-counted.
|
|
51
|
+
- **openai-compatible strict mode:** `zodToJsonSchema` does not emit `additionalProperties: false`, which OpenAI strict structured output requires. The v1 supported path is claude-cli / anthropic. Follow-up tracked.
|
|
52
|
+
- **CI required-check wiring:** deferred to roadmap #540 (unbuilt CI workflow template).
|
|
53
|
+
|
|
54
|
+
## Success Criteria
|
|
55
|
+
|
|
56
|
+
See `docs/changes/outcome-eval/proposal.md` for the full 9 criteria. This skill satisfies SC8 (orchestrator step 6.5 + blocking halt) and SC9 (introduces no new `harness validate` findings; layer rules respected).
|
|
57
|
+
|
|
58
|
+
## Examples
|
|
59
|
+
|
|
60
|
+
### Example: NOT_SATISFIED with high confidence (blocks)
|
|
61
|
+
|
|
62
|
+
**Input:** spec Success Criteria require `GET /api/users/:id` to return 404 with `{ error: 'User not found' }`; the diff implements the happy path only, no 404 branch; test output shows the 404 test failing.
|
|
63
|
+
|
|
64
|
+
**Verdict:**
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
verdict: NOT_SATISFIED
|
|
68
|
+
confidence: high
|
|
69
|
+
judgedAgainst: success-criteria
|
|
70
|
+
authority: blocking
|
|
71
|
+
unmetCriteria:
|
|
72
|
+
- "404 path for nonexistent user is unimplemented; the failing test asserts { error: 'User not found' }."
|
|
73
|
+
rationale: "The diff adds the lookup but returns 200 with an empty body when the user is missing."
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
**Action:** HALT before Ship. Report unmet criteria; do not open the PR.
|
|
77
|
+
|
|
78
|
+
### Example: partial implementation (advisory)
|
|
79
|
+
|
|
80
|
+
**Input:** the diff meets most criteria; one acceptance item is ambiguous in the diff.
|
|
81
|
+
|
|
82
|
+
**Verdict:** `NOT_SATISFIED confidence: medium authority: advisory` — surfaced for review, workflow proceeds.
|
|
83
|
+
|
|
84
|
+
## Gates
|
|
85
|
+
|
|
86
|
+
- **Authority is never read from the LLM.** The verdict's `authority` is always `deriveAuthority(verdict, confidence)` computed in TypeScript. If you find yourself letting the model assert blocking/advisory, STOP — that defeats the entire purpose of this gate.
|
|
87
|
+
- **Block only on high-confidence NOT_SATISFIED.** `authority === 'blocking'` iff `verdict === 'NOT_SATISFIED' && confidence === 'high'`. Every other combination — all `SATISFIED`, all `INCONCLUSIVE`, and every `medium`/`low` `NOT_SATISFIED` — is advisory. Do not halt the workflow on an advisory verdict.
|
|
88
|
+
- **Always supply `diff` and `testOutput`.** Omitting them degrades the verdict to INCONCLUSIVE/advisory (a silent false-negative at the ship gate). Gather them from the session before invoking the tool.
|
|
89
|
+
- **Never block on infrastructure noise.** A provider failure, an unparseable response, or a missing spec must resolve to INCONCLUSIVE/advisory, never a thrown error or a block. The evaluator enforces this; do not reintroduce a hard failure in the wrapper.
|
|
90
|
+
- **Do not skip step 6.5.** The gate runs after Code Review and before Ship. A blocking verdict halts before step 7; it is not optional.
|
|
91
|
+
|
|
92
|
+
## Escalation
|
|
93
|
+
|
|
94
|
+
- **Blocking verdict the author disputes:** the resolution is to fix the implementation (or amend the spec's Success Criteria if they were wrong) and re-run outcome-eval — not to override the gate. If the spec itself is wrong, that is a spec change, reviewed on its own merits.
|
|
95
|
+
- **Repeated INCONCLUSIVE on a real change:** usually means `diff`/`testOutput` were not supplied, or no judgable section exists in the spec. Confirm inputs and that the spec has a Success Criteria / User-Visible Behavior / Overview section.
|
|
96
|
+
- **No `ANTHROPIC_API_KEY` configured:** every verdict degrades to INCONCLUSIVE/advisory and nothing blocks. Surface this to the human — the gate is effectively disabled until a provider is configured.
|
|
97
|
+
- **Verdict seems wrong (false positive/negative):** capture the spec section, diff, and verdict, and route to the maintainers; do not loosen the conservative-confidence prompt ad hoc.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
name: outcome-eval
|
|
2
|
+
version: '0.1.0'
|
|
3
|
+
description: >-
|
|
4
|
+
LLM-judgment skill that produces a structured, confidence-rated verdict on
|
|
5
|
+
whether an implementation satisfied its spec. Reads the spec's acceptance
|
|
6
|
+
section, the change diff, and test output; emits an OutcomeVerdict
|
|
7
|
+
(SATISFIED | NOT_SATISFIED | INCONCLUSIVE) with confidence, rationale, and
|
|
8
|
+
unmet criteria. Authority is derived in TypeScript, never from the LLM: a
|
|
9
|
+
high-confidence NOT_SATISFIED blocks ship; every other verdict is advisory.
|
|
10
|
+
The verdict persists as an execution_outcome node and feeds skill-effectiveness
|
|
11
|
+
baselines. The harness's first blocking post-execution spec-satisfaction gate.
|
|
12
|
+
stability: draft
|
|
13
|
+
cognitive_mode: constructive-architect
|
|
14
|
+
triggers:
|
|
15
|
+
- manual
|
|
16
|
+
- on_pr
|
|
17
|
+
platforms:
|
|
18
|
+
- claude-code
|
|
19
|
+
tools:
|
|
20
|
+
- Bash
|
|
21
|
+
- Read
|
|
22
|
+
- Glob
|
|
23
|
+
- Grep
|
|
24
|
+
mcp:
|
|
25
|
+
tool: outcome_eval
|
|
26
|
+
# specPath, diff, and testOutput are ALL required. diff/testOutput are the
|
|
27
|
+
# evidence the judge reasons over: omitting them degrades the verdict to
|
|
28
|
+
# INCONCLUSIVE/advisory (never blocking), so the agent MUST supply them.
|
|
29
|
+
input:
|
|
30
|
+
specPath: 'string (required) — absolute or repo-relative path to the spec markdown to judge against'
|
|
31
|
+
diff: 'string (required) — unified diff of the change under judgment (git diff); omission degrades the verdict to INCONCLUSIVE/advisory'
|
|
32
|
+
testOutput: 'string (required) — captured test-runner stdout+stderr; omission degrades the verdict to INCONCLUSIVE/advisory'
|
|
33
|
+
model: 'string (optional) — model override for the outcome-eval LLM call'
|
|
34
|
+
path: 'string (optional) — project root used to resolve the knowledge graph (default cwd)'
|
|
35
|
+
type: rigid
|
|
36
|
+
tier: 2
|
|
37
|
+
phases:
|
|
38
|
+
- name: gather
|
|
39
|
+
description: Collect the unified diff (git diff), capture test output, resolve the spec path
|
|
40
|
+
required: true
|
|
41
|
+
- name: resolve
|
|
42
|
+
description: Resolve the judgment section (Success Criteria -> User-Visible Behavior -> Overview)
|
|
43
|
+
required: true
|
|
44
|
+
- name: judge
|
|
45
|
+
description: Invoke OutcomeEvaluator.evaluate(); LLM returns verdict/confidence/rationale/unmetCriteria; authority derived in TS
|
|
46
|
+
required: true
|
|
47
|
+
- name: gate
|
|
48
|
+
description: Render the verdict; on a blocking verdict (NOT_SATISFIED + high confidence) halt before Ship
|
|
49
|
+
required: true
|
|
50
|
+
state:
|
|
51
|
+
persistent: false
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# Harness Audit: Harness Strength
|
|
2
|
+
|
|
3
|
+
> Mechanically audit whether a project's harness is load-bearing. Orchestrates the deterministic check-harness-strength engine; interprets results. Not a deep/AI review.
|
|
4
|
+
|
|
5
|
+
## When to Use
|
|
6
|
+
|
|
7
|
+
- On a milestone gate, to confirm the harness still constrains rather than merely decorates the project
|
|
8
|
+
- When adding the strength audit as a required CI check, so the seven STRENGTH patterns block the merge instead of being prose advice
|
|
9
|
+
- When validating a harness-distribution (toolkit) repo before shipping templates downstream — to catch a default that would weaken every adopter
|
|
10
|
+
- When a reviewer suspects a gate "passes" without doing anything (a snapshot that asserts `passed: true` for a check it never ran)
|
|
11
|
+
- NOT for deep security review — use `harness-security-review` (semantic, AI-assisted)
|
|
12
|
+
- NOT for design-system drift — use `detect-design-drift`
|
|
13
|
+
- NOT for reimplementing pattern detection by hand — the engine (`HarnessStrengthAuditor`) owns detection. This skill runs it and interprets the output.
|
|
14
|
+
|
|
15
|
+
## Process
|
|
16
|
+
|
|
17
|
+
This skill ORCHESTRATES `harness check-harness-strength`. It never re-greps configs, never re-parses hooks, and never re-derives findings. The engine reads the project once and returns a structured `AuditResult`; the skill's job is to run it with the right mode/severity and turn the JSON into an actionable report.
|
|
18
|
+
|
|
19
|
+
### Phase 1: SCAN — Run the engine
|
|
20
|
+
|
|
21
|
+
1. **Resolve the project root.** Use the provided `path` argument, else the current working directory.
|
|
22
|
+
|
|
23
|
+
2. **Resolve the mode.**
|
|
24
|
+
- If `--toolkit` or `--adopter` (or `--mode toolkit|adopter`) is supplied, honor it. Explicit mode wins.
|
|
25
|
+
- Otherwise auto-detect: treat the repo as **toolkit** when it is a harness distribution — both `templates/` and `agents/skills/` exist at the root. Otherwise **adopter**.
|
|
26
|
+
- Do not inspect config files to "double-check" the mode. The engine resolves and reports the mode it used.
|
|
27
|
+
|
|
28
|
+
3. **Invoke the command** via Bash, requesting JSON:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
harness check-harness-strength [--mode <adopter|toolkit>] [--severity <error|warning|info>] [--report-only] --json
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The `--json` payload is the raw structured `AuditResult` (score, tier, mode, findings with `id`, `gearPiece`, `severity`, `message`, `remediation`, and file:line evidence). Parse this — do not re-derive it.
|
|
35
|
+
|
|
36
|
+
4. **Do NOT inspect config files by hand.** The engine reads `harness.config.json`, hook profiles, baselines, snapshots, and tier defaults exactly once. Hand-grepping `.husky/pre-commit` or `harness.config.json` to "confirm" a finding reimplements detection and violates the core decision behind this skill (ADR 0039 / spec D1).
|
|
37
|
+
|
|
38
|
+
### Phase 2: DETECT — Interpret findings
|
|
39
|
+
|
|
40
|
+
1. **Map each finding to its STRENGTH pattern** using the JSON `id` and `gearPiece` fields. Every finding the engine emits corresponds to exactly one of the seven patterns below. Do not invent findings the engine did not emit, and do not silence findings it did.
|
|
41
|
+
|
|
42
|
+
2. **The seven STRENGTH patterns** (the engine's rule registry — for interpretation only; the engine, not the skill, decides which fire):
|
|
43
|
+
|
|
44
|
+
| ID | Gear piece | Pattern | Default |
|
|
45
|
+
| ------------ | ----------------------- | -------------------------------------------------------------------- | ------- |
|
|
46
|
+
| STRENGTH-001 | blocking-gate | Hook documented "never blocks"/"always exits 0" in an active profile | error |
|
|
47
|
+
| STRENGTH-002 | regression-baseline | Pre-commit auto-updates baselines/thresholds on regression | error |
|
|
48
|
+
| STRENGTH-003 | skip-discipline | `--skip` list > 2 categories without inline justification | warning |
|
|
49
|
+
| STRENGTH-004 | architecture-thresholds | `layers` defined but `architecture.thresholds` empty/absent | error |
|
|
50
|
+
| STRENGTH-005 | tier-default | Init/config defaults to lowest tier (`basic`) | warning |
|
|
51
|
+
| STRENGTH-006 | review-gate | Baseline-update PR auto-approved without independent review | error |
|
|
52
|
+
| STRENGTH-007 | snapshot-honesty | `passed:true` in health snapshot whose `signals[]` names that check | error |
|
|
53
|
+
|
|
54
|
+
3. **Attach evidence.** Each finding carries a `file:line` (or config-key) reference and a remediation string from the engine. Surface them verbatim — do not paraphrase the remediation.
|
|
55
|
+
|
|
56
|
+
### Phase 3: SCORE/REPORT
|
|
57
|
+
|
|
58
|
+
1. **Surface the score and tier.** The engine returns a 0-100 strength score and a tier label:
|
|
59
|
+
- `solid` — score >= 85
|
|
60
|
+
- `at-risk` — score 50-84
|
|
61
|
+
- `theatre` — score < 50
|
|
62
|
+
|
|
63
|
+
2. **Report format** (mirror of the security-scan report block):
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
Harness Strength: [PASS/FAIL] — tier: <solid|at-risk|theatre> (score N/100, mode <adopter|toolkit>)
|
|
67
|
+
Findings: <count> (Errors: N | Warnings: N | Info: N)
|
|
68
|
+
|
|
69
|
+
[STRENGTH-00N] <gearPiece> <file:line> (severity)
|
|
70
|
+
<message>
|
|
71
|
+
Remediation: <engine remediation string>
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
3. **PASS/FAIL.** FAIL if any error-severity finding survives the severity threshold, unless `--report-only` was passed (then exit softens to PASS but findings are still listed). Otherwise PASS.
|
|
75
|
+
|
|
76
|
+
## Gates
|
|
77
|
+
|
|
78
|
+
- **Error-severity findings are blocking.** The report is FAIL and the exit is non-zero when any error-severity finding survives filtering — unless `--report-only` was explicitly requested. Do not report PASS over an unsuppressed error finding.
|
|
79
|
+
- **No reimplementation.** The skill must run `harness check-harness-strength` and interpret its JSON. It must never hand-grep hooks, hand-parse `harness.config.json`, or re-derive a score. Reimplementing detection violates ADR 0039.
|
|
80
|
+
- **"Not evaluable" is not a pass.** When an input the engine needs is absent (e.g., no hook profile, no snapshot), the engine reports that state. Surface it as-is. Do not convert "could not evaluate" into "passed."
|
|
81
|
+
- **Mechanical only.** No AI judgment about whether a weakness "really matters." The patterns are deterministic; the engine decides what fires.
|
|
82
|
+
|
|
83
|
+
## Escalation
|
|
84
|
+
|
|
85
|
+
- **Disputed finding / false positive:** Do not suppress by editing files or skipping the rule in the skill. Adjust severity via config — `audit.harnessStrength.severities` in `harness.config.json` (e.g., downgrade STRENGTH-003 to `info` for a justified wide `--skip`). Document the rationale alongside the override.
|
|
86
|
+
- **Engine misses a known weakness:** This skill cannot add detection. A missing pattern is engine work — file a new `StrengthRule` (`{ id, gearPiece, defaultSeverity, appliesIn(mode), evaluable?(ctx), detect(ctx) }`) in `packages/core/src/harness-strength/`. Out of scope for the skill.
|
|
87
|
+
- **Mode mis-detected:** If auto-detection picks the wrong mode (e.g., a repo with `templates/` that is not a distribution), pass `--mode adopter` (or `--toolkit`) explicitly. Do not work around it by hand-editing detection.
|
|
88
|
+
- **Engine throws or the build is stale:** If `harness check-harness-strength` errors (command not found / not in the built CLI), the dist may be stale. Rebuild the CLI and re-run. Do not substitute a manual config read for the engine.
|
|
89
|
+
|
|
90
|
+
## Rationalizations to Reject
|
|
91
|
+
|
|
92
|
+
### Universal
|
|
93
|
+
|
|
94
|
+
These reasoning patterns sound plausible but lead to bad outcomes. Reject them.
|
|
95
|
+
|
|
96
|
+
- **"It's probably fine"** — "Probably" is not evidence. Run the engine and cite the result.
|
|
97
|
+
- **"This is best practice"** — Best practice in what context? Cite the source and confirm it applies to this codebase.
|
|
98
|
+
- **"We can fix it later"** — If it is worth flagging, it is worth documenting now with a concrete follow-up plan.
|
|
99
|
+
|
|
100
|
+
### Domain-Specific
|
|
101
|
+
|
|
102
|
+
| Rationalization | Reality |
|
|
103
|
+
| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
104
|
+
| "The config looks fine when I read it" | The engine exists because manual reading misses these patterns — that is the whole point of D1. Reading is not running. Run `harness check-harness-strength`. |
|
|
105
|
+
| "It only warns, not errors, so it's fine" | A gate that warns but does not stop IS STRENGTH-001 — this is the recursion item the audit was built to catch. Do not normalize "warns but doesn't block." |
|
|
106
|
+
| "I'll re-grep the hooks myself to double-check the engine" | Reimplementing detection violates ADR 0039 / spec D1. The engine is the single source of truth — trust and interpret its output, do not shadow it. |
|
|
107
|
+
| "Score is 70, that's a passing grade" | 70 is `at-risk`, not `solid`. Tier labels are thresholds, not letter grades. Below 85 means the harness has load-bearing gaps. |
|
|
108
|
+
| "No error findings, so I can skip reading the warnings" | Warnings (STRENGTH-003, -005) are erosion signals — a `basic` tier default or an unjustified wide `--skip` weakens every future run. Surface them. |
|
|
109
|
+
|
|
110
|
+
## Success Criteria
|
|
111
|
+
|
|
112
|
+
- The engine (`harness check-harness-strength`) ran and produced a score, tier label, and findings (or a clean result).
|
|
113
|
+
- Every finding is interpreted against the seven-pattern table, with the engine's file:line evidence and remediation surfaced verbatim.
|
|
114
|
+
- The exit code / PASS-FAIL reflects the gate (error-severity findings fail unless `--report-only`).
|
|
115
|
+
- No manual config inspection was substituted for the engine — the report is derived entirely from the command's `--json` output.
|
|
116
|
+
- "Not evaluable" states are surfaced as-is, never converted to passes.
|
|
117
|
+
|
|
118
|
+
## Evidence Requirements
|
|
119
|
+
|
|
120
|
+
When this skill makes claims about existing code, architecture, or behavior, it MUST cite evidence using one of:
|
|
121
|
+
|
|
122
|
+
1. **File reference:** `file:line` format (e.g., `.husky/pre-commit:12`) — taken from the engine's finding, not re-derived.
|
|
123
|
+
2. **Code pattern reference:** `file` with description (e.g., `harness.config.json` — "architecture.layers defined, thresholds absent").
|
|
124
|
+
3. **Test/command output:** Inline or referenced output from the `harness check-harness-strength --json` run.
|
|
125
|
+
4. **Session evidence:** Write to the `evidence` session section via `manage_state`.
|
|
126
|
+
|
|
127
|
+
**Uncited claims:** Technical assertions without citations MUST be prefixed with `[UNVERIFIED]`. Example: `[UNVERIFIED] The pre-commit hook auto-updates baselines`.
|
|
128
|
+
|
|
129
|
+
## Harness Integration
|
|
130
|
+
|
|
131
|
+
- **`harness check-harness-strength`** — The CLI command this skill runs and interprets. Options: `--mode <adopter|toolkit>`, `--toolkit`/`--adopter` shortcuts, `--severity <error|warning|info>` (default `warning`), `--report-only`, `--json`.
|
|
132
|
+
- **`HarnessStrengthAuditor`** — Core class from `@harness-engineering/core` that builds a `ProjectContext` once, runs the applicable `StrengthRule`s, and aggregates the score/tier/findings. The skill never instantiates it directly; it goes through the command.
|
|
133
|
+
- **`harness.config.json` → `audit.harnessStrength.severities`** — Per-pattern severity overrides. The supported escalation path for false positives — never suppress by editing source.
|
|
134
|
+
- **`docs/standard/article-failure-patterns.md`** — Narrative companion documenting the seven patterns (forthcoming — downstream item; may not exist yet).
|
|
135
|
+
|
|
136
|
+
## Examples
|
|
137
|
+
|
|
138
|
+
### Example: Clean toolkit-mode run
|
|
139
|
+
|
|
140
|
+
**Input:** A harness-distribution repo (`templates/` and `agents/skills/` present), run at a milestone gate. `harness check-harness-strength --toolkit --json`.
|
|
141
|
+
|
|
142
|
+
**Output:**
|
|
143
|
+
|
|
144
|
+
```
|
|
145
|
+
Harness Strength: PASS — tier: solid (score 100/100, mode toolkit)
|
|
146
|
+
Findings: 0 (Errors: 0 | Warnings: 0 | Info: 0)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Every gear piece is load-bearing: no "never blocks" hook, baselines are not auto-updated on regression, architecture thresholds are set, the init default is not `basic`, baseline-update PRs require independent review, and no snapshot claims a check passed that it never ran.
|
|
150
|
+
|
|
151
|
+
### Example: Findings detected (adopter mode)
|
|
152
|
+
|
|
153
|
+
**Input:** An adopter repo, run with default severity. `harness check-harness-strength --json`.
|
|
154
|
+
|
|
155
|
+
**Output:**
|
|
156
|
+
|
|
157
|
+
```
|
|
158
|
+
Harness Strength: FAIL — tier: at-risk (score 62/100, mode adopter)
|
|
159
|
+
Findings: 2 (Errors: 2 | Warnings: 0 | Info: 0)
|
|
160
|
+
|
|
161
|
+
[STRENGTH-001] blocking-gate .husky/pre-commit:12 (error)
|
|
162
|
+
Active pre-commit profile documents "always exits 0" — the gate reports but never blocks.
|
|
163
|
+
Remediation: Remove the unconditional `exit 0`; let failing checks return a non-zero status.
|
|
164
|
+
|
|
165
|
+
[STRENGTH-004] architecture-thresholds harness.config.json (error)
|
|
166
|
+
architecture.layers is defined but architecture.thresholds is empty — layer rules cannot be enforced.
|
|
167
|
+
Remediation: Set architecture.thresholds (e.g. maxModuleLines, maxDependencyDepth) so the layer
|
|
168
|
+
definitions become enforceable, or remove the unused layers block.
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Two error-severity findings → FAIL. STRENGTH-001 is the recursion item: a hook that runs but cannot block. Run with `--report-only` only when you explicitly want to surface findings without failing the build (e.g., a first baseline-capture run) — and never to normalize the weakness.
|
|
172
|
+
|
|
173
|
+
## Skill Test Scenarios
|
|
174
|
+
|
|
175
|
+
### Scenario 1: Red Flag — "I'll re-grep the hooks myself to double-check the engine"
|
|
176
|
+
|
|
177
|
+
Input: The agent has the `harness check-harness-strength --json` output in hand but is about to open `.husky/pre-commit` and grep it by hand to "confirm" the STRENGTH-001 finding.
|
|
178
|
+
Expected: Agent stops, cites the no-reimplementation gate (ADR 0039 / spec D1), and interprets the engine's JSON instead of hand-grepping the hook.
|
|
179
|
+
|
|
180
|
+
### Scenario 2: Rationalization — "it only warns, not errors, so it's fine"
|
|
181
|
+
|
|
182
|
+
Input: The engine reports a documented "never blocks" pre-commit profile, and the agent is tempted to treat it as acceptable because nothing currently fails the build.
|
|
183
|
+
Expected: Agent rejects the rationalization, recognizes that "warns but doesn't stop" IS STRENGTH-001 (the recursion item), and surfaces it as a finding rather than normalizing it.
|
|
184
|
+
|
|
185
|
+
### Scenario 3: Gate — error-severity STRENGTH-001 finding present without `--report-only`
|
|
186
|
+
|
|
187
|
+
Input: The `--json` output contains an error-severity STRENGTH-001 finding, `--report-only` was not passed, and the agent is about to summarize the run as PASS.
|
|
188
|
+
Expected: Agent halts, reports FAIL with a non-zero exit, lists the finding with its file:line evidence and remediation, and does not proceed past the gate.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
name: harness-audit-harness-strength
|
|
2
|
+
version: '0.1.0'
|
|
3
|
+
description: Mechanically audit a project's own harness setup against the seven STRENGTH failure patterns; reports per-pattern findings, a 0-100 strength score, and a tier label (solid/at-risk/theatre). Orchestrates harness check-harness-strength; never reimplements detection.
|
|
4
|
+
stability: draft
|
|
5
|
+
cognitive_mode: constructive-architect
|
|
6
|
+
triggers:
|
|
7
|
+
- manual
|
|
8
|
+
- on_milestone
|
|
9
|
+
platforms:
|
|
10
|
+
- claude-code
|
|
11
|
+
- gemini-cli
|
|
12
|
+
- cursor
|
|
13
|
+
- codex
|
|
14
|
+
tools:
|
|
15
|
+
- Bash
|
|
16
|
+
- Read
|
|
17
|
+
- Grep
|
|
18
|
+
- Glob
|
|
19
|
+
cli:
|
|
20
|
+
command: harness skill run harness-audit-harness-strength
|
|
21
|
+
args:
|
|
22
|
+
- name: path
|
|
23
|
+
description: Project root path
|
|
24
|
+
required: false
|
|
25
|
+
- name: mode
|
|
26
|
+
description: 'adopter (default) or toolkit (auto-detected in a harness-distribution repo)'
|
|
27
|
+
required: false
|
|
28
|
+
- name: severity
|
|
29
|
+
description: Minimum severity threshold (error, warning, info)
|
|
30
|
+
required: false
|
|
31
|
+
- name: report-only
|
|
32
|
+
description: Soften exit to 0 even when error-severity findings exist
|
|
33
|
+
required: false
|
|
34
|
+
mcp:
|
|
35
|
+
tool: run_skill
|
|
36
|
+
input:
|
|
37
|
+
skill: harness-audit-harness-strength
|
|
38
|
+
path: string
|
|
39
|
+
type: rigid
|
|
40
|
+
tier: 2
|
|
41
|
+
phases:
|
|
42
|
+
- name: scan
|
|
43
|
+
description: Resolve mode and run harness check-harness-strength against the target repo
|
|
44
|
+
required: true
|
|
45
|
+
- name: detect
|
|
46
|
+
description: Interpret the 7 STRENGTH pattern findings with file-line evidence
|
|
47
|
+
required: true
|
|
48
|
+
- name: score_report
|
|
49
|
+
description: Surface the 0-100 score, tier label, and per-pattern remediation
|
|
50
|
+
required: true
|
|
51
|
+
state:
|
|
52
|
+
persistent: false
|
|
53
|
+
files: []
|
|
54
|
+
depends_on: []
|
|
@@ -213,6 +213,8 @@ Persist findings to `{sessionDir}/phase-{N}-review.json`. No blocking → PHASE_
|
|
|
213
213
|
|
|
214
214
|
## Process
|
|
215
215
|
|
|
216
|
+
**Prompt the human in plain text** — every phase-continue and human-decision interaction in this skill is plain text only. Do not elevate to `AskUserQuestion`: natural headers like "Continue phase" exceed its 12-char cap, rendering the call as ERR.
|
|
217
|
+
|
|
216
218
|
1. **INIT** — Resolve spec, derive session slug, check for existing state, parse phases.
|
|
217
219
|
2. **ASSESS** — Route by complexity: low/medium auto-plans, high pauses for interactive planning.
|
|
218
220
|
3. **PLAN → APPROVE** — Dispatch harness-planner, check approval signals, auto-approve or pause.
|