@harness-engineering/cli 3.1.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/personas/harness-pm.yaml +33 -0
- package/dist/agents/skills/claude-code/acceptance-eval/SKILL.md +92 -0
- package/dist/agents/skills/claude-code/acceptance-eval/skill.yaml +54 -0
- package/dist/agents/skills/claude-code/harness-brainstorming/SKILL.md +46 -38
- package/dist/agents/skills/claude-code/harness-brainstorming/skill.yaml +1 -0
- package/dist/agents/skills/claude-code/harness-code-review/SKILL.md +10 -11
- package/dist/agents/skills/claude-code/harness-execution/SKILL.md +24 -26
- package/dist/agents/skills/claude-code/harness-integration/SKILL.md +21 -19
- package/dist/agents/skills/claude-code/harness-maintenance-pipeline/SKILL.md +173 -0
- package/dist/agents/skills/claude-code/harness-maintenance-pipeline/skill.yaml +43 -0
- package/dist/agents/skills/claude-code/harness-onboarding/SKILL.md +2 -0
- package/dist/agents/skills/claude-code/harness-planning/SKILL.md +17 -40
- package/dist/agents/skills/claude-code/harness-pulse/SKILL.md +1 -1
- package/dist/agents/skills/claude-code/harness-roadmap/SKILL.md +17 -0
- package/dist/agents/skills/claude-code/harness-roadmap-pilot/SKILL.md +8 -0
- package/dist/agents/skills/claude-code/harness-verification/SKILL.md +9 -12
- package/dist/agents/skills/claude-code/initialize-harness-project/SKILL.md +112 -126
- package/dist/agents/skills/claude-code/outcome-eval/SKILL.md +1 -0
- package/dist/agents/skills/codex/acceptance-eval/SKILL.md +92 -0
- package/dist/agents/skills/codex/acceptance-eval/skill.yaml +54 -0
- package/dist/agents/skills/codex/harness-brainstorming/SKILL.md +46 -38
- package/dist/agents/skills/codex/harness-brainstorming/skill.yaml +1 -0
- package/dist/agents/skills/codex/harness-code-review/SKILL.md +10 -11
- package/dist/agents/skills/codex/harness-execution/SKILL.md +24 -26
- package/dist/agents/skills/codex/harness-integration/SKILL.md +21 -19
- package/dist/agents/skills/codex/harness-maintenance-pipeline/SKILL.md +173 -0
- package/dist/agents/skills/codex/harness-maintenance-pipeline/skill.yaml +43 -0
- package/dist/agents/skills/codex/harness-onboarding/SKILL.md +2 -0
- package/dist/agents/skills/codex/harness-planning/SKILL.md +17 -40
- package/dist/agents/skills/codex/harness-pulse/SKILL.md +1 -1
- package/dist/agents/skills/codex/harness-roadmap/SKILL.md +17 -0
- package/dist/agents/skills/codex/harness-roadmap-pilot/SKILL.md +8 -0
- package/dist/agents/skills/codex/harness-verification/SKILL.md +9 -12
- package/dist/agents/skills/codex/initialize-harness-project/SKILL.md +112 -126
- package/dist/agents/skills/codex/outcome-eval/SKILL.md +1 -0
- package/dist/agents/skills/cursor/acceptance-eval/SKILL.md +92 -0
- package/dist/agents/skills/cursor/acceptance-eval/skill.yaml +54 -0
- package/dist/agents/skills/cursor/harness-brainstorming/SKILL.md +46 -38
- package/dist/agents/skills/cursor/harness-brainstorming/skill.yaml +1 -0
- package/dist/agents/skills/cursor/harness-code-review/SKILL.md +10 -11
- package/dist/agents/skills/cursor/harness-execution/SKILL.md +24 -26
- package/dist/agents/skills/cursor/harness-integration/SKILL.md +21 -19
- package/dist/agents/skills/cursor/harness-maintenance-pipeline/SKILL.md +173 -0
- package/dist/agents/skills/cursor/harness-maintenance-pipeline/skill.yaml +43 -0
- package/dist/agents/skills/cursor/harness-onboarding/SKILL.md +2 -0
- package/dist/agents/skills/cursor/harness-planning/SKILL.md +17 -40
- package/dist/agents/skills/cursor/harness-pulse/SKILL.md +1 -1
- package/dist/agents/skills/cursor/harness-roadmap/SKILL.md +17 -0
- package/dist/agents/skills/cursor/harness-roadmap-pilot/SKILL.md +8 -0
- package/dist/agents/skills/cursor/harness-verification/SKILL.md +9 -12
- package/dist/agents/skills/cursor/initialize-harness-project/SKILL.md +112 -126
- package/dist/agents/skills/cursor/outcome-eval/SKILL.md +1 -0
- package/dist/agents/skills/gemini-cli/acceptance-eval/SKILL.md +92 -0
- package/dist/agents/skills/gemini-cli/acceptance-eval/skill.yaml +54 -0
- package/dist/agents/skills/gemini-cli/harness-brainstorming/SKILL.md +46 -38
- package/dist/agents/skills/gemini-cli/harness-brainstorming/skill.yaml +1 -0
- package/dist/agents/skills/gemini-cli/harness-code-review/SKILL.md +10 -11
- package/dist/agents/skills/gemini-cli/harness-execution/SKILL.md +24 -26
- package/dist/agents/skills/gemini-cli/harness-integration/SKILL.md +21 -19
- package/dist/agents/skills/gemini-cli/harness-maintenance-pipeline/SKILL.md +173 -0
- package/dist/agents/skills/gemini-cli/harness-maintenance-pipeline/skill.yaml +43 -0
- package/dist/agents/skills/gemini-cli/harness-onboarding/SKILL.md +2 -0
- package/dist/agents/skills/gemini-cli/harness-planning/SKILL.md +17 -40
- package/dist/agents/skills/gemini-cli/harness-pulse/SKILL.md +1 -1
- package/dist/agents/skills/gemini-cli/harness-roadmap/SKILL.md +17 -0
- package/dist/agents/skills/gemini-cli/harness-roadmap-pilot/SKILL.md +8 -0
- package/dist/agents/skills/gemini-cli/harness-verification/SKILL.md +9 -12
- package/dist/agents/skills/gemini-cli/initialize-harness-project/SKILL.md +112 -126
- package/dist/agents/skills/gemini-cli/outcome-eval/SKILL.md +1 -0
- package/dist/agents/skills/tests/interaction-channel.test.ts +64 -0
- package/dist/{agents-md-YYCLGMBF.js → agents-md-7OSNWU52.js} +2 -2
- package/dist/{architecture-5KPP4ID4.js → architecture-2XYGJGT3.js} +3 -3
- package/dist/{assess-project-B6ENHUAU.js → assess-project-3KPP2D6M.js} +1 -1
- package/dist/bin/harness-mcp.js +16 -15
- package/dist/bin/harness.js +27 -25
- package/dist/{check-phase-gate-REG3HWYC.js → check-phase-gate-46U5AAXE.js} +4 -3
- package/dist/{chunk-HIPBU2Q4.js → chunk-363V6GPR.js} +5 -5
- package/dist/{chunk-DNX35Q5K.js → chunk-6NCREWWM.js} +2 -2
- package/dist/{chunk-FL5SLUKZ.js → chunk-6YXCZV33.js} +2141 -929
- package/dist/{chunk-UBTMPO5H.js → chunk-AK67S3O4.js} +3 -3
- package/dist/{chunk-ERZCC5VH.js → chunk-BV45354Q.js} +3 -3
- package/dist/{chunk-ZOUJP3EY.js → chunk-ERILXQBP.js} +1153 -484
- package/dist/{chunk-BXKNWSG4.js → chunk-FZ4Q73HA.js} +1 -1
- package/dist/{chunk-PZX2Y3RV.js → chunk-GXCPDAS5.js} +3 -3
- package/dist/{chunk-C2ZGWMTN.js → chunk-HMW3C5DP.js} +2 -2
- package/dist/{chunk-TE5YHHHZ.js → chunk-HXMAIVV2.js} +1 -1
- package/dist/{chunk-TCVQVHEF.js → chunk-IYWT3JJW.js} +1 -1
- package/dist/{chunk-A65QO5KN.js → chunk-KLPX55TI.js} +2953 -1162
- package/dist/{chunk-OQFEFVIN.js → chunk-MSBJ445U.js} +4 -4
- package/dist/{chunk-FUH4J43G.js → chunk-MYVTYFQD.js} +2 -2
- package/dist/{chunk-LMZ2JTNP.js → chunk-PHX45JYN.js} +5 -9
- package/dist/{chunk-ZRKFDE6M.js → chunk-S2RQPPL6.js} +1 -1
- package/dist/chunk-SPE7P4GL.js +61 -0
- package/dist/{chunk-YF3AY6KS.js → chunk-TTWNGOBT.js} +6 -6
- package/dist/{chunk-6B6UN6SG.js → chunk-UTJJJDNN.js} +11 -1
- package/dist/{chunk-WWACIIBA.js → chunk-V3ZQUCRH.js} +1 -1
- package/dist/{chunk-XEOSSKIB.js → chunk-VTU3QOJT.js} +48 -4
- package/dist/{chunk-IF3JK3G5.js → chunk-VTUINXWR.js} +1 -1
- package/dist/{chunk-27AJKSQY.js → chunk-WBVCN35T.js} +1 -1
- package/dist/{chunk-RGQXHCCH.js → chunk-WL3O2PKJ.js} +6 -6
- package/dist/{chunk-YQAFKOOL.js → chunk-XFHZ7GCI.js} +1 -1
- package/dist/chunk-Y6SUQC32.js +9 -0
- package/dist/{chunk-XZ2Q4UTW.js → chunk-ZEYAOYJL.js} +1 -1
- package/dist/{ci-workflow-JEYYCBOC.js → ci-workflow-XJN4UO3Q.js} +2 -2
- package/dist/{dist-3HW4LCHE.js → dist-IPVFYGY2.js} +71 -15
- package/dist/{docs-YRMW7UED.js → docs-RSFZTNPG.js} +3 -3
- package/dist/{engine-E7H5U3AC.js → engine-DJIHY4JT.js} +2 -2
- package/dist/{entropy-3JBKCGJJ.js → entropy-EAEKJKRM.js} +2 -2
- package/dist/{feedback-U6VJ6AUJ.js → feedback-JT5X3DBC.js} +1 -1
- package/dist/{generate-agent-definitions-S2FRZF4Y.js → generate-agent-definitions-CC6KED3E.js} +4 -4
- package/dist/hooks/adoption-tracker.js +6 -5
- package/dist/hooks/protect-config.js +15 -5
- package/dist/{index-builder-3AOQ3ZII.js → index-builder-52IJJHJC.js} +2 -2
- package/dist/index.d.ts +8 -8
- package/dist/index.js +27 -25
- package/dist/{loader-RGQJYERV.js → loader-UWNVTYBY.js} +2 -2
- package/dist/{mcp-NDTFZXWF.js → mcp-TCLNK7HG.js} +18 -17
- package/dist/{performance-FPWKC4CN.js → performance-R6O3N3UB.js} +3 -3
- package/dist/{review-pipeline-WUHG7PHO.js → review-pipeline-MOOWXWSQ.js} +2 -2
- package/dist/{runtime-Q3ORD7LW.js → runtime-RIWQUKBQ.js} +2 -2
- package/dist/{security-JDHTRBGC.js → security-U76C54W6.js} +1 -1
- package/dist/{skill-executor-MOCUIAYS.js → skill-executor-BGUCAVBU.js} +2 -2
- package/dist/state-events-F44CUUZG.js +25 -0
- package/dist/{validate-G3LKBOYA.js → validate-IUAV7MOU.js} +3 -3
- package/dist/{validate-cross-check-4ITA72DF.js → validate-cross-check-M2ZM6RFY.js} +2 -2
- package/package.json +7 -7
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Initialize Harness Project
|
|
2
2
|
|
|
3
|
-
> Scaffold a new harness-compliant project, migrate an existing project to the next adoption level, or bootstrap an existing project that just got the harness marketplace plugin installed (no `harness setup`).
|
|
3
|
+
> Scaffold a new harness-compliant project, migrate an existing project to the next adoption level, or bootstrap an existing project that just got the harness marketplace plugin installed (no `harness setup`). Ground the project in a strategic anchor (`STRATEGY.md`) first, then assess current state, scaffold or migrate, configure, validate, instrument (baselines / telemetry / Tier-0 integrations), and finalize.
|
|
4
4
|
|
|
5
5
|
## When to Use
|
|
6
6
|
|
|
@@ -31,6 +31,52 @@ Detect plugin-only state by checking whether `harness` resolves on PATH (`comman
|
|
|
31
31
|
|
|
32
32
|
**Prompt the human in plain text** — every framework confirmation, migration check, and telemetry-identity question in this skill is plain text only. Do not elevate to `AskUserQuestion`: the framework list (~10 options) exceeds its 4-option cap and natural headers like "Confirm framework" exceed its 12-char cap, rendering the call as ERR.
|
|
33
33
|
|
|
34
|
+
### Phase 0: GROUND — Capture the Strategic Anchor
|
|
35
|
+
|
|
36
|
+
Run this before anything else — it is the first thing init does and the first question it asks the human.
|
|
37
|
+
Offer to capture `STRATEGY.md`, the durable upstream product anchor read by `harness-brainstorming`,
|
|
38
|
+
`harness-ideate`, and `harness-roadmap-pilot`. _Think first (strategy), build second (scaffold)._
|
|
39
|
+
|
|
40
|
+
Ask in plain text in your reply, same as the design-system step (Phase 3 step 5b) — never via `emit_interaction`, `AskUserQuestion`, or any tool (the human won't see a tool-routed prompt; only plain text reaches them across Claude Code, Cursor, Codex, and Gemini CLI). State your recommendation, then STOP and wait for the human's reply:
|
|
41
|
+
|
|
42
|
+
```markdown
|
|
43
|
+
### Capture strategic anchor (STRATEGY.md) now?
|
|
44
|
+
|
|
45
|
+
| | A) Yes — run the strategy interview | B) No — this project does not need a strategy doc | C) Not sure yet |
|
|
46
|
+
| ---------- | -------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | --------------------------------------------------------------------- |
|
|
47
|
+
| **Pros** | Grounds brainstorm/ideate/roadmap-pilot in product-level context; durable across milestones and phases (peer of README.md) | Permanent decline recorded; init does not re-ask on rerun | Decision deferred without commitment; can run /harness:strategy later |
|
|
48
|
+
| **Cons** | Adds an interview (10-20 minutes) to init | Re-running init will not re-offer; user must run /harness:strategy manually | No decline flag set; init may re-offer on rerun |
|
|
49
|
+
| **Risk** | Low | Low | Low |
|
|
50
|
+
| **Effort** | Medium | Low | Low |
|
|
51
|
+
|
|
52
|
+
**Recommendation:** A) Yes — run the strategy interview (confidence: medium) — strategy grounds brainstorm/ideate/roadmap-pilot; value compounds as the project grows.
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Before prompting, check whether `STRATEGY.md` already exists at repo root. Three cases:
|
|
56
|
+
|
|
57
|
+
- **Absent (most common on init).** Present the prompt above. Apply the answer:
|
|
58
|
+
- **Yes:** delegate to `harness-strategy` (which routes via its own Phase 0 to the first-run interview). It writes a valid `STRATEGY.md` at repo root, doc-validated via the `write_strategy` / `validate_strategy` MCP tools. Do **not** run a project-level `harness validate` here — `harness.config.json` does not exist until SCAFFOLD. When `harness-strategy` completes, proceed to Phase 1.
|
|
59
|
+
- **No:** record the decline in working memory. `Phase 3: CONFIGURE` step 0 persists `init.strategy.declined: true` to `.harness/state.json` after SCAFFOLD creates it. Do **not** touch `.harness/` here — it does not exist yet.
|
|
60
|
+
- **Not sure:** record nothing. `/harness:strategy` remains available standalone, and a future re-run of init will re-offer.
|
|
61
|
+
|
|
62
|
+
- **Present and valid.** Skip the prompt silently. Surface a one-line note: `STRATEGY.md detected — downstream skills will pick it up as grounding`. No decline is recorded.
|
|
63
|
+
|
|
64
|
+
- **Present but invalid.** Surface the validation error via the `validate_strategy` MCP tool (the MCP server already has `@harness-engineering/core` loaded, so this resolves even for plugin-only adopters with no `node_modules`). Offer three paths (mirror `harness-strategy` Phase 0):
|
|
65
|
+
- **a) Fix now via `/harness:strategy` update** → delegate to `harness-strategy` with the broken section pre-selected.
|
|
66
|
+
- **b) Move file to `STRATEGY.md.bak.<YYYY-MM-DD-HHmm>` and run a fresh interview** → rename, then delegate to `harness-strategy` Phase 1.
|
|
67
|
+
- **c) Ignore for this init and proceed** → record the decline (persisted in `Phase 3: CONFIGURE` step 0) and continue. Init does NOT block on a present-but-invalid `STRATEGY.md`.
|
|
68
|
+
|
|
69
|
+
**Guards:**
|
|
70
|
+
|
|
71
|
+
- Phase 0 runs for **all** project shapes, including test suites — strategy is offered before the Phase 1
|
|
72
|
+
step 5 test-suite classification and the step-6 dispatch, identical in reach to the legacy step.
|
|
73
|
+
- A present-valid (skip) or present-invalid (offer fix) `STRATEGY.md` is the migration path for an existing
|
|
74
|
+
strategy doc — distinct from the `.harness/`-based adoption-level classification in `Phase 1: ASSESS`,
|
|
75
|
+
which Phase 0 must not pre-empt.
|
|
76
|
+
- **No / Not-sure proceeds immediately into `Phase 1: ASSESS`. Phase 0 never blocks init.**
|
|
77
|
+
|
|
78
|
+
This mirrors the ask-once-record-the-answer pattern also used by the i18n and design-system prompts in Phase 3.
|
|
79
|
+
|
|
34
80
|
### Phase 1: ASSESS — Determine Current State
|
|
35
81
|
|
|
36
82
|
1. **Check for existing harness configuration.** Look for `.harness/` directory, `AGENTS.md`, `harness.config.json`, and any skill definitions. Their presence determines whether this is a new project or a migration.
|
|
@@ -82,6 +128,14 @@ Detect plugin-only state by checking whether `harness` resolves on PATH (`comman
|
|
|
82
128
|
|
|
83
129
|
### Phase 3: CONFIGURE — Customize for the Project
|
|
84
130
|
|
|
131
|
+
0. **Persist the Phase 0 grounding decision.** SCAFFOLD has now created `.harness/`. If the user **declined**
|
|
132
|
+
the strategic anchor in `Phase 0: GROUND` (answered "No", or chose "ignore" on a present-but-invalid
|
|
133
|
+
`STRATEGY.md`), write `init.strategy.declined: true` to `.harness/state.json` (merge into existing JSON;
|
|
134
|
+
do not clobber). If the user answered "Yes" or "Not sure" in Phase 0, write nothing. This is the deferred
|
|
135
|
+
half of the Phase 0 offer: Phase 0 captures the answer, this step records the decline once `.harness/`
|
|
136
|
+
exists — so the relocation never fabricates `.harness/` early and never misclassifies a new project as a
|
|
137
|
+
migration in `Phase 1: ASSESS`.
|
|
138
|
+
|
|
85
139
|
1. **Configure personas.** Run `harness persona generate` to create persona definitions based on the project's stack and team structure. Personas define how agents should behave in this project — coding style, communication preferences, constraint strictness.
|
|
86
140
|
|
|
87
141
|
2. **Customize AGENTS.md.** The generated template needs project-specific content:
|
|
@@ -100,39 +154,21 @@ Detect plugin-only state by checking whether `harness` resolves on PATH (`comman
|
|
|
100
154
|
- **No:** Set `i18n.enabled: false` in `harness.config.json`. The `harness-i18n-process` skill will still fire gentle prompts for unconfigured projects when features touch user-facing strings.
|
|
101
155
|
- **Not sure:** Skip i18n configuration entirely. Do not set `i18n.enabled`. The project can enable i18n later by running `harness-i18n-workflow` directly.
|
|
102
156
|
|
|
103
|
-
5b. **Configure design system (non-test-suite projects).**
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
{
|
|
119
|
-
label: "No — this project has no UI",
|
|
120
|
-
pros: ["No future design nudges", "Permanent decline recorded"],
|
|
121
|
-
cons: ["Re-running init is required if a UI is added later"],
|
|
122
|
-
risk: "low",
|
|
123
|
-
effort: "low"
|
|
124
|
-
},
|
|
125
|
-
{
|
|
126
|
-
label: "Not sure yet",
|
|
127
|
-
pros: ["Decision deferred without commitment", "Can run harness-design-system later"],
|
|
128
|
-
cons: ["No design.enabled flag set; on_new_feature will prompt later"],
|
|
129
|
-
risk: "low",
|
|
130
|
-
effort: "low"
|
|
131
|
-
}
|
|
132
|
-
],
|
|
133
|
-
recommendation: { optionIndex: 0, reason: "Most product/service projects benefit from a centralized design system", confidence: "medium" }
|
|
134
|
-
}
|
|
135
|
-
})
|
|
157
|
+
5b. **Configure design system (non-test-suite projects).** Mirror the i18n step's three-way response shape.
|
|
158
|
+
|
|
159
|
+
**Ask directly in your reply. Do NOT route this question through `emit_interaction`, `AskUserQuestion`, or any tool.** `emit_interaction` records the prompt but does not display it to the human — the client collapses the call to "Called harness" and the rendered text only returns to the model, so the human sees nothing. `AskUserQuestion` is Claude-Code-only and caps headers at 12 chars / 4 options. Plain text in your own message is the only channel that reliably reaches the human across every tool (Claude Code, Cursor, Codex, Gemini CLI). State your recommendation, then STOP and wait for the human's reply:
|
|
160
|
+
|
|
161
|
+
```markdown
|
|
162
|
+
### Will this project have a UI requiring a design system?
|
|
163
|
+
|
|
164
|
+
| | A) Yes — capture design intent now | B) No — this project has no UI | C) Not sure yet |
|
|
165
|
+
|---|---|---|---|
|
|
166
|
+
| **Pros** | Records platforms in harness.config.json; harness-design-system fires automatically on first design-touching feature | No future design nudges; permanent decline recorded | Decision deferred without commitment; can run harness-design-system later |
|
|
167
|
+
| **Cons** | One extra follow-up question (which platforms) | Re-running init is required if a UI is added later | No design.enabled flag set; on_new_feature will prompt later |
|
|
168
|
+
| **Risk** | Low | Low | Low |
|
|
169
|
+
| **Effort** | Low | Low | Low |
|
|
170
|
+
|
|
171
|
+
**Recommendation:** A) Yes — capture design intent now (confidence: medium) — most product/service projects benefit from a centralized design system.
|
|
136
172
|
```
|
|
137
173
|
|
|
138
174
|
Based on the answer:
|
|
@@ -142,60 +178,6 @@ Detect plugin-only state by checking whether `harness` resolves on PATH (`comman
|
|
|
142
178
|
|
|
143
179
|
**Skip this step entirely if Phase 1 step 5 classified the project as a test suite.** Test-suite projects will be dispatched at step 6 below to `initialize-test-suite-project` and have no UI to govern.
|
|
144
180
|
|
|
145
|
-
5c. **Capture strategic anchor (all levels).** Offer a three-way prompt for `STRATEGY.md` — the durable upstream product anchor read by `harness-brainstorming`, `harness-ideate`, and `harness-roadmap-pilot`. Use `emit_interaction`:
|
|
146
|
-
|
|
147
|
-
```json
|
|
148
|
-
emit_interaction({
|
|
149
|
-
type: "question",
|
|
150
|
-
question: {
|
|
151
|
-
text: "Capture strategic anchor (STRATEGY.md) now?",
|
|
152
|
-
options: [
|
|
153
|
-
{
|
|
154
|
-
label: "Yes — run the strategy interview",
|
|
155
|
-
pros: [
|
|
156
|
-
"Grounds brainstorm/ideate/roadmap-pilot in product-level context",
|
|
157
|
-
"Durable across milestones and phases (peer of README.md)"
|
|
158
|
-
],
|
|
159
|
-
cons: ["Adds an interview (10-20 minutes) to init"],
|
|
160
|
-
risk: "low",
|
|
161
|
-
effort: "medium"
|
|
162
|
-
},
|
|
163
|
-
{
|
|
164
|
-
label: "No — this project does not need a strategy doc",
|
|
165
|
-
pros: ["Permanent decline recorded; init does not re-ask on rerun"],
|
|
166
|
-
cons: ["Re-running init will not re-offer; user must run /harness:strategy manually"],
|
|
167
|
-
risk: "low",
|
|
168
|
-
effort: "low"
|
|
169
|
-
},
|
|
170
|
-
{
|
|
171
|
-
label: "Not sure yet",
|
|
172
|
-
pros: ["Decision deferred without commitment", "Can run /harness:strategy later"],
|
|
173
|
-
cons: ["No decline flag set; init may re-offer on rerun"],
|
|
174
|
-
risk: "low",
|
|
175
|
-
effort: "low"
|
|
176
|
-
}
|
|
177
|
-
],
|
|
178
|
-
recommendation: { optionIndex: 0, reason: "Strategy grounds brainstorm/ideate/roadmap-pilot — value compounds as the project grows", confidence: "medium" }
|
|
179
|
-
}
|
|
180
|
-
})
|
|
181
|
-
```
|
|
182
|
-
|
|
183
|
-
Before prompting, check whether `STRATEGY.md` already exists at repo root. Three cases:
|
|
184
|
-
|
|
185
|
-
- **Absent (most common on init).** Present the prompt above. Apply the answer:
|
|
186
|
-
- **Yes:** delegate to `harness-strategy` (which routes via its own Phase 0 to the first-run interview). When `harness-strategy` completes, continue with step 6.
|
|
187
|
-
- **No:** write `init.strategy.declined: true` to `.harness/state.json` (merge into existing JSON; do not clobber). This is the explicit decline location; re-running init detects the flag and skips the prompt.
|
|
188
|
-
- **Not sure:** no state write. `/harness:strategy` remains available standalone, and a future re-run of init will re-offer.
|
|
189
|
-
|
|
190
|
-
- **Present and valid.** Skip the prompt silently. Surface a one-line note: `STRATEGY.md detected — downstream skills will pick it up as grounding`. No `init.strategy.declined` write.
|
|
191
|
-
|
|
192
|
-
- **Present but invalid.** Surface the validation error verbatim (run a Node one-liner that imports `validateStrategy` from `@harness-engineering/core` and pipes the error). Offer three paths (mirror `harness-strategy` Phase 0):
|
|
193
|
-
- **a) Fix now via `/harness:strategy` update** → delegate to `harness-strategy` with the broken section pre-selected.
|
|
194
|
-
- **b) Move file to `STRATEGY.md.bak.<YYYY-MM-DD-HHmm>` and run a fresh interview** → rename, then delegate to `harness-strategy` Phase 1.
|
|
195
|
-
- **c) Ignore for this init and proceed** → record `init.strategy.declined: true` and continue. Init does NOT block on present-but-invalid STRATEGY.md.
|
|
196
|
-
|
|
197
|
-
Mirror the i18n / design-system pattern: ask once, record the answer, never silently skip.
|
|
198
|
-
|
|
199
181
|
6. **Test-suite projects only — dispatch to `initialize-test-suite-project`.** If Phase 1 step 5 classified this as a test suite, invoke `initialize-test-suite-project` now and let it own archetype selection, shared-library decision, layer variants (A self-contained vs B consumer), ESLint flat-config fix, tag taxonomy, reporter stack, custom report, and the "prove the guards fire" verification. Return here for Phase 4 step 4+ (knowledge graph, roadmap, commit). Product and service projects skip this step entirely.
|
|
200
182
|
|
|
201
183
|
### Phase 4: VALIDATE — Confirm Everything Works
|
|
@@ -275,32 +257,21 @@ This phase closes the parity gap that the marketplace plugin install does not co
|
|
|
275
257
|
|
|
276
258
|
### Phase 6: FINALIZE — Roadmap and Commit
|
|
277
259
|
|
|
278
|
-
1. **Set up project roadmap.**
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
{
|
|
294
|
-
label: "No — skip for now",
|
|
295
|
-
pros: ["Smaller initial footprint"],
|
|
296
|
-
cons: ["Run `/harness:roadmap --create` later when ready"],
|
|
297
|
-
risk: "low",
|
|
298
|
-
effort: "low"
|
|
299
|
-
}
|
|
300
|
-
],
|
|
301
|
-
recommendation: { optionIndex: 0, reason: "Validation has just passed — a tangible 'project works' signal is the right moment to introduce planning artifacts", confidence: "medium" }
|
|
302
|
-
}
|
|
303
|
-
})
|
|
260
|
+
1. **Set up project roadmap.** `docs/roadmap.md` tracks features, milestones, and status across your specs and plans.
|
|
261
|
+
|
|
262
|
+
**Ask in plain text in your reply, same as Phase 0: GROUND and Phase 3 step 5b — never via `emit_interaction`, `AskUserQuestion`, or any tool (the human won't see a tool-routed prompt; only plain text reaches them across Claude Code, Cursor, Codex, and Gemini CLI).** State your recommendation, then STOP and wait for the human's reply:
|
|
263
|
+
|
|
264
|
+
```markdown
|
|
265
|
+
### Set up a project roadmap now?
|
|
266
|
+
|
|
267
|
+
| | A) Yes — create docs/roadmap.md now | B) No — skip for now |
|
|
268
|
+
| ---------- | ----------------------------------------------------------------------- | ------------------------------------------------ |
|
|
269
|
+
| **Pros** | Roadmap visible from day one; future specs auto-discovered on next sync | Smaller initial footprint |
|
|
270
|
+
| **Cons** | Adds one file to the initial commit | Run `/harness:roadmap --create` later when ready |
|
|
271
|
+
| **Risk** | Low | Low |
|
|
272
|
+
| **Effort** | Low | Low |
|
|
273
|
+
|
|
274
|
+
**Recommendation:** A) Yes — create docs/roadmap.md now (confidence: medium) — validation has just passed; a tangible "project works" signal is the right moment to introduce planning artifacts.
|
|
304
275
|
```
|
|
305
276
|
|
|
306
277
|
Based on the answer:
|
|
@@ -325,8 +296,8 @@ This phase closes the parity gap that the marketplace plugin install does not co
|
|
|
325
296
|
- **`harness integrations list` / `harness integrations add <name>`** — Phase 5 step 6. Lists Tier-0 (zero-config) and Tier-1 (API-key) MCP integrations and adds them to project `.mcp.json`.
|
|
326
297
|
- **`harness-i18n-workflow configure` + `harness-i18n-workflow scaffold`** — Invoked during Phase 3 if the project will support multiple languages. Sets up i18n configuration and translation file structure.
|
|
327
298
|
- **`harness-design-system` (deferred via `on_new_feature`)** — Phase 3 step 5b records `design.enabled` + `design.platforms` in `harness.config.json` but does NOT run the full design-system skill. Token generation defers to the first design-touching feature, where `harness-design-system` fires via `on_new_feature` and reads `design.enabled` to decide whether to proceed.
|
|
328
|
-
- **`harness-strategy`** — Phase
|
|
329
|
-
- **`
|
|
299
|
+
- **`harness-strategy`** — Phase 0: GROUND delegates to this skill on "Yes". The skill conducts a first-run interview (pushback rules with a 2-round cap per section) and writes a valid `STRATEGY.md` at repo root via `writeStrategyDoc`. On "No" the decline is recorded in working memory and persisted as `init.strategy.declined: true` to `.harness/state.json` by Phase 3: CONFIGURE step 0 (after SCAFFOLD). On "not sure" no state is written; the user can run `/harness:strategy` standalone. When `STRATEGY.md` already exists and is valid the prompt is skipped; when present-but-invalid the user gets the three-path repair offer.
|
|
300
|
+
- **`validate_strategy` / `write_strategy` MCP tools** — used by Phase 0: GROUND to doc-validate `STRATEGY.md` and to detect a present-but-invalid one. Routed through the MCP server (which already has `@harness-engineering/core` loaded) so init does not require local `node_modules` or a `node -e` one-liner.
|
|
330
301
|
- **`initialize-test-suite-project`** — Sub-skill. Invoked during Phase 3 step 6 when Phase 1 step 5 classified the project as a test suite. Owns archetype selection, shared-library vs in-repo decision, layer variants, tag taxonomy, reporter stack, custom report, and "prove the guards fire" verification.
|
|
331
302
|
- **`harness-roadmap` skill** — Phase 6 step 1 invokes this skill (or `/harness:roadmap --create`) when the user opts in to creating `docs/roadmap.md`. The `manage_roadmap` MCP tool does not create roadmaps; it manages entries in an existing one.
|
|
332
303
|
- **`manage_roadmap` MCP tool** — Phase 6 step 1, when `design.enabled === true`, calls `manage_roadmap` with `action: add` to insert a `planned` "Set up design system" item under milestone `Intake` with a summary describing the deferred work.
|
|
@@ -343,7 +314,7 @@ This phase closes the parity gap that the marketplace plugin install does not co
|
|
|
343
314
|
- All generated files are committed in a single atomic commit
|
|
344
315
|
- i18n configuration is set if the human chose to enable it during init
|
|
345
316
|
- For non-test-suite projects, the design-system question was asked and `harness.config.json` reflects the answer: `design.enabled: true` (with `design.platforms` populated) for yes, `design.enabled: false` for no, or absent for not sure.
|
|
346
|
-
- The strategy question (Phase
|
|
317
|
+
- The strategy question (Phase 0: GROUND) was asked unless `STRATEGY.md` already existed and was valid (in which case the prompt was skipped silently with a one-line detection note). The answer was recorded according to the documented semantics: Yes → `STRATEGY.md` exists, doc-validated via `validate_strategy` against `StrategyDocSchema` (no project-level `harness validate` is run in Phase 0); No → `.harness/state.json` contains `init.strategy.declined: true`; Not sure → no `STRATEGY.md` and no `init.strategy.declined` flag.
|
|
347
318
|
- **Phase 5 (INSTRUMENT) outputs:** `.harness/graph/` is populated; `.harness/arch/baselines.json` exists; `harness check-perf` ran without errors (intermediate and above); `.harness/telemetry.json` exists if the human opted into identity tagging; legacy layout warnings were surfaced via `harness migrate --dry-run` and either resolved or explicitly deferred; Tier-0 MCP integrations (context7, sequential-thinking, playwright) are present in the project's `.mcp.json` (or the human declined and that decision is recorded).
|
|
348
319
|
- The roadmap question was asked. If the user answered yes, `docs/roadmap.md` exists and was created via `harness-roadmap` (or the documented `/harness:roadmap --create` fallback).
|
|
349
320
|
- When `design.enabled === true` AND the user answered yes to the roadmap question, `docs/roadmap.md` contains a `planned` entry titled "Set up design system" under milestone `Intake` with a summary describing the deferred work. The entry is absent in all other answer combinations.
|
|
@@ -358,9 +329,9 @@ This phase closes the parity gap that the marketplace plugin install does not co
|
|
|
358
329
|
| "We should start at the advanced level since we want full coverage" | The skill recommends basic for new projects. Each level builds on the previous. Jumping to advanced creates misconfigured rules. |
|
|
359
330
|
| "I will skip the i18n question to keep setup fast" | Phase 3 requires asking about i18n and recording the decision. Skipping creates ambiguity about whether the omission was intentional. |
|
|
360
331
|
| "I will skip the design-system question to keep setup fast" | Phase 3 step 5b requires asking about design and recording the answer in `design.enabled`. Skipping creates ambiguity about whether the omission was intentional and bypasses the linkage between init and the deferred `harness-design-system` invocation on `on_new_feature`. |
|
|
361
|
-
| "I will skip the strategy question — it's just paperwork" | Phase
|
|
362
|
-
| "STRATEGY.md exists already, so I should re-run the interview to refresh it" | When `STRATEGY.md` is present and valid, Phase
|
|
363
|
-
| "STRATEGY.md is present but invalid, so I should block init" | Phase
|
|
332
|
+
| "I will skip the strategy question — it's just paperwork" | Phase 0: GROUND is the only point in the workflow where init asks the user to capture the strategic anchor. Skipping bypasses the grounding signal that brainstorming, ideate, and roadmap-pilot read; downstream skill output degrades silently. Even a "no" or "not sure" answer is better than no answer because it locks in the decision (or absence of one). |
|
|
333
|
+
| "STRATEGY.md exists already, so I should re-run the interview to refresh it" | When `STRATEGY.md` is present and valid, Phase 0: GROUND skips the prompt silently. Refreshing strategy mid-init is out of scope — that is what the standalone `/harness:strategy` update flow is for. Surface a one-line detection note and continue. |
|
|
334
|
+
| "STRATEGY.md is present but invalid, so I should block init" | Phase 0: GROUND explicitly does NOT block. It surfaces the validation error, offers three repair paths (fix now / move-to-bak / ignore), and continues based on the user's choice. Init is the wrong place to gate on strategy correctness. |
|
|
364
335
|
| "Validation passed, so the project is ready" | Phase 5 captures baselines, configures telemetry identity, surfaces legacy warnings, and wires Tier-0 integrations. Validation alone is not sufficient. |
|
|
365
336
|
| "Plugin install means setup is done" | The marketplace plugin ships skills, slash commands, subagents, hooks, and MCP — but it cannot mutate the user's project state. `harness.config.json`, baselines, telemetry identity, and Tier-0 integrations require running this skill once per project. |
|
|
366
337
|
| "Skip Phase 5 if we already ran `harness setup`" | Phase 5 is idempotent. It safely no-ops where setup already wired things and fills gaps where it didn't (common case: setup ran once, then a new Tier-0 integration was added or a layout was migrated). Re-running is the right behavior. |
|
|
@@ -410,6 +381,16 @@ git commit -m "feat: initialize harness project at basic level"
|
|
|
410
381
|
|
|
411
382
|
### Example: New TypeScript Web App with Design and Roadmap
|
|
412
383
|
|
|
384
|
+
**GROUND (Phase 0 — asked first, before ASSESS/SCAFFOLD):**
|
|
385
|
+
|
|
386
|
+
```
|
|
387
|
+
Phase 0 (strategy): "Capture strategic anchor (STRATEGY.md) now?"
|
|
388
|
+
Human: "Yes."
|
|
389
|
+
Delegate to harness-strategy → first-run interview → STRATEGY.md written.
|
|
390
|
+
Result: STRATEGY.md exists at repo root, doc-validated via validate_strategy (StrategyDocSchema);
|
|
391
|
+
no project-level harness validate runs in Phase 0.
|
|
392
|
+
```
|
|
393
|
+
|
|
413
394
|
**ASSESS:**
|
|
414
395
|
|
|
415
396
|
```
|
|
@@ -427,6 +408,8 @@ harness init --level basic --framework nextjs
|
|
|
427
408
|
**CONFIGURE (Phase 3):**
|
|
428
409
|
|
|
429
410
|
```
|
|
411
|
+
Step 0 (persist grounding): user answered "Yes" in Phase 0 → STRATEGY.md already written; nothing to persist.
|
|
412
|
+
|
|
430
413
|
Step 5 (i18n): "Will this project support multiple languages?"
|
|
431
414
|
Human: "No, English only."
|
|
432
415
|
Result: i18n.enabled = false in harness.config.json.
|
|
@@ -439,11 +422,6 @@ Step 5b (design): "Will this project have a UI requiring a design system?"
|
|
|
439
422
|
Inform: "Design tokens will be generated when you start your first design-touching
|
|
440
423
|
feature — harness-design-system fires automatically via on_new_feature."
|
|
441
424
|
|
|
442
|
-
Step 5c (strategy): "Capture strategic anchor (STRATEGY.md) now?"
|
|
443
|
-
Human: "Yes."
|
|
444
|
-
Delegate to harness-strategy → first-run interview → STRATEGY.md written.
|
|
445
|
-
Result: STRATEGY.md exists at repo root; harness validate passes against StrategyDocSchema.
|
|
446
|
-
|
|
447
425
|
Step 6 (test-suite dispatch): skipped (not a test suite).
|
|
448
426
|
```
|
|
449
427
|
|
|
@@ -524,6 +502,14 @@ git commit -m "feat: migrate harness project to intermediate level with layers"
|
|
|
524
502
|
|
|
525
503
|
The user installed `harness-claude` from the marketplace. They have an existing TypeScript repo with no `.harness/` directory. Goal: get them to a working harness install without asking them to `npm install -g`.
|
|
526
504
|
|
|
505
|
+
**GROUND (Phase 0 — asked first, before ASSESS/SCAFFOLD):**
|
|
506
|
+
|
|
507
|
+
```
|
|
508
|
+
Phase 0 (strategy): "Capture strategic anchor (STRATEGY.md) now?"
|
|
509
|
+
Human: "Not sure."
|
|
510
|
+
Result: no STRATEGY.md written, no init.strategy.declined flag; /harness:strategy stays available later.
|
|
511
|
+
```
|
|
512
|
+
|
|
527
513
|
**ASSESS:**
|
|
528
514
|
|
|
529
515
|
```
|
|
@@ -546,10 +532,10 @@ npx @harness-engineering/cli init # auto-detects framework
|
|
|
546
532
|
**CONFIGURE:**
|
|
547
533
|
|
|
548
534
|
```
|
|
535
|
+
Phase 3 step 0 (persist grounding): user was "Not sure" in Phase 0 → nothing to persist.
|
|
549
536
|
Customize AGENTS.md with project-specific content.
|
|
550
537
|
Phase 3 step 5 (i18n): "No, English only." → i18n.enabled = false.
|
|
551
538
|
Phase 3 step 5b (design): "No, this is a backend service." → design.enabled = false.
|
|
552
|
-
Phase 3 step 5c (strategy): "not sure." → no STRATEGY.md, no init.strategy.declined flag; user can run /harness:strategy later.
|
|
553
539
|
Phase 3 step 6 (test-suite dispatch): not a test suite, skipped.
|
|
554
540
|
```
|
|
555
541
|
|
|
@@ -44,6 +44,7 @@ The evaluator resolves the section internally via the fallback chain `## Success
|
|
|
44
44
|
- **Provider path (v1 supported):** the anthropic analysis provider (`ANTHROPIC_API_KEY`). When no provider is configured the call degrades to INCONCLUSIVE/advisory. The openai-compatible _strict_ structured-output path is a known follow-up (see Known Limitations).
|
|
45
45
|
- **Orchestrator:** runs as step 6.5 between Code Review and Ship in `harness.orchestrator.md`.
|
|
46
46
|
- **Persistence:** each `evaluate()` writes one `execution_outcome` node via `ExecutionOutcomeConnector`, consumable by `effectiveness/scorer.ts`.
|
|
47
|
+
- **Relationship to `acceptance-eval`:** `acceptance-eval` is the upstream twin — it gates spec measurability before execution; `outcome-eval` judges implementation satisfaction after. The "authority is never read from the LLM" discipline spans both.
|
|
47
48
|
|
|
48
49
|
## Known Limitations
|
|
49
50
|
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# Acceptance Eval
|
|
2
|
+
|
|
3
|
+
> Pre-execution LLM-judgment: does this spec carry measurable, testable, complete acceptance criteria before work begins? Resolves the spec's acceptance section, critiques observability / testability / completeness (advisory `criteriaFindings`), flags user-visible behaviors with no covering test (advisory `coverageFindings`), and emits a confidence-rated `AcceptanceVerdict` (`MEASURABLE | NOT_MEASURABLE | INCONCLUSIVE`) with a rationale. Merge authority is derived in TypeScript, never trusted from the LLM: a high-confidence `NOT_MEASURABLE` blocks merge; every other verdict is advisory. The upstream twin of the `outcome-eval` ship gate — it keeps the downstream gate fed with judgable specs.
|
|
4
|
+
|
|
5
|
+
## When to Use
|
|
6
|
+
|
|
7
|
+
- On every spec entering the repo under `docs/changes/**` — triggered `on_pr` (and `manual`) — before execution begins.
|
|
8
|
+
- When you need a durable, structured answer to "can this spec's success be objectively judged later?"
|
|
9
|
+
- NOT for judging whether an implementation satisfied its spec post-execution (use `outcome-eval`, the downstream twin).
|
|
10
|
+
- NOT for authoring acceptance criteria — `acceptance-eval` reviews; humans own the thinking layer. It never writes the criteria it judges.
|
|
11
|
+
- NOT for rule-based floors (lint/architecture/entropy) or other craft ceilings — those run elsewhere.
|
|
12
|
+
- NOT when no judgable spec section exists — the verdict degrades to INCONCLUSIVE/advisory and never blocks.
|
|
13
|
+
|
|
14
|
+
## Process
|
|
15
|
+
|
|
16
|
+
### Phase 1: RESOLVE — Find the judgment section
|
|
17
|
+
|
|
18
|
+
The evaluator resolves the section internally via the fallback chain `## Success Criteria` -> `## User-Visible Behavior` -> `## Overview` (reusing `outcome-eval`'s resolver, not a fork), recording the match in `judgedAgainst`. No manual action — pass `specPath` and let `AcceptanceEvaluator` resolve. If no section is judgable, the verdict is INCONCLUSIVE/advisory.
|
|
19
|
+
|
|
20
|
+
### Phase 2: GATHER — Locate test evidence for coverage (optional)
|
|
21
|
+
|
|
22
|
+
For responsibility (b), supply test evidence so the judge can flag user-visible behaviors with no covering test: pass `testGlobs` (globs the tool reads) or `testContent` (snippets you already read). This evidence is optional: omitting it degrades `coverageFindings` to advisory-empty but NEVER affects the (c) measurability gate. Gather it when the spec describes user-visible behavior.
|
|
23
|
+
|
|
24
|
+
### Phase 3: JUDGE — Invoke the evaluator
|
|
25
|
+
|
|
26
|
+
1. Invoke the MCP tool `mcp__harness__acceptance_eval` with `{ specPath }` plus optional `{ testGlobs | testContent, model }`. The tool constructs `AcceptanceEvaluator` cli-side and returns the verdict; the supported v1 provider is the anthropic analysis provider (`ANTHROPIC_API_KEY`).
|
|
27
|
+
2. The LLM returns ONLY `measurability / confidence / criteriaFindings / coverageFindings / rationale`. `authority` is computed in TypeScript from `(measurability, confidence)` via `deriveAcceptanceAuthority` and is never read from the LLM — do not attempt to override it. The tool returns the verdict exactly as the evaluator derives it.
|
|
28
|
+
3. The call is degrade-safe: provider failure (incl. no `ANTHROPIC_API_KEY`), empty test evidence, or a missing judgable section yields INCONCLUSIVE/low/advisory. It never throws and never blocks.
|
|
29
|
+
|
|
30
|
+
### Phase 4: GATE — Render and (conditionally) halt
|
|
31
|
+
|
|
32
|
+
1. Render the verdict: `measurability`, `confidence`, `judgedAgainst`, `rationale`, `criteriaFindings` (a, advisory), and `coverageFindings` (b, advisory).
|
|
33
|
+
2. Authority rule (must match `deriveAcceptanceAuthority`): authority is `blocking` **iff** `measurability === 'NOT_MEASURABLE' && confidence === 'high'`; every other combination — including all `INCONCLUSIVE` and `MEASURABLE` cases, and all `medium`/`low` `NOT_MEASURABLE` — is `advisory`.
|
|
34
|
+
3. **On a blocking verdict: HALT before merge.** Report the missing measurable criteria and stop; the spec must not merge. Resolution requires a human adding measurable success criteria and re-running `acceptance-eval`.
|
|
35
|
+
4. On an advisory verdict: report the `criteriaFindings` and `coverageFindings` for human attention and proceed. Advisory findings do not stop the workflow.
|
|
36
|
+
|
|
37
|
+
## Harness Integration
|
|
38
|
+
|
|
39
|
+
- **`mcp__harness__acceptance_eval`** — MCP tool (the invocation surface). Inputs: `specPath` (required), `testGlobs` / `testContent` (optional (b) evidence), `model` (optional). The handler builds the cli `AnalysisProvider`, constructs `AcceptanceEvaluator`, and returns the `AcceptanceVerdict` with `authority` exactly as derived in TypeScript. (No graph persistence in v1 — the seam is documented in the tool header.)
|
|
40
|
+
- **Evaluator surface:** `AcceptanceEvaluator`, `deriveAcceptanceAuthority`, `acceptanceVerdictSchema`, `AcceptanceVerdict` are exported from `@harness-engineering/intelligence`. The section resolver is imported from `outcome-eval`, not duplicated.
|
|
41
|
+
- **Provider path (v1 supported):** the anthropic analysis provider (`ANTHROPIC_API_KEY`). When no provider is configured the call degrades to INCONCLUSIVE/advisory.
|
|
42
|
+
- **Relationship to `outcome-eval`:** `acceptance-eval` is the upstream twin. It guards spec measurability before execution; `outcome-eval` judges implementation satisfaction after. The same "authority is never read from the LLM" discipline spans both ends of the lifecycle.
|
|
43
|
+
|
|
44
|
+
## Known Limitations
|
|
45
|
+
|
|
46
|
+
- **Coverage findings (b) are heuristic and advisory.** Without `testGlobs`/`testContent` they are advisory-empty; even with evidence they are LLM-judgment over the behavior section plus located tests, not a graph-backed coverage map (deferred — see spec Decision D4). Heuristic misses are low-harm because (b) never blocks.
|
|
47
|
+
- **The (c) gate blocks only on high-confidence NOT_MEASURABLE.** A spec with weak-but-present criteria yields an advisory verdict, not a block — intentional, to avoid taste-blocks-merge false positives.
|
|
48
|
+
- **openai-compatible strict mode** is not the v1 path; the supported provider is anthropic/claude-cli.
|
|
49
|
+
|
|
50
|
+
## Success Criteria
|
|
51
|
+
|
|
52
|
+
See `docs/changes/harness-pm-persona/proposal.md` for the full criteria. This skill satisfies SC1 (skill exists, `tier: 2`, `type: rigid`, four-client artifacts), SC6 (blocking halt on high-confidence NOT_MEASURABLE; advisory-and-proceed otherwise), and SC7 (emits advisory `criteriaFindings` / `coverageFindings`).
|
|
53
|
+
|
|
54
|
+
## Examples
|
|
55
|
+
|
|
56
|
+
### Example: NOT_MEASURABLE with high confidence (blocks)
|
|
57
|
+
|
|
58
|
+
**Input:** a spec whose only "success criteria" are "the feature works well" and "users are happy" — no observable, testable assertions.
|
|
59
|
+
|
|
60
|
+
**Verdict:**
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
measurability: NOT_MEASURABLE
|
|
64
|
+
confidence: high
|
|
65
|
+
judgedAgainst: success-criteria
|
|
66
|
+
authority: blocking
|
|
67
|
+
criteriaFindings:
|
|
68
|
+
- { target: "'works well' / 'users are happy'", message: "No observable assertion — cannot be tested." }
|
|
69
|
+
rationale: "The success section contains only subjective statements; nothing can be judged at outcome time."
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
**Action:** HALT before merge. Report the missing measurable criteria; the spec must not merge until a human adds them.
|
|
73
|
+
|
|
74
|
+
### Example: measurable criteria (advisory, proceeds)
|
|
75
|
+
|
|
76
|
+
**Input:** a spec whose Success Criteria list concrete, testable assertions; one user-visible behavior lacks an obvious covering test.
|
|
77
|
+
|
|
78
|
+
**Verdict:** `measurability: MEASURABLE confidence: high authority: advisory`, with one `coverageFindings` `{target, message}` object (the one user-visible behavior lacking a covering test) surfaced for review. The workflow proceeds.
|
|
79
|
+
|
|
80
|
+
## Gates
|
|
81
|
+
|
|
82
|
+
- **Authority is never read from the LLM.** The verdict's `authority` is always `deriveAcceptanceAuthority(measurability, confidence)` computed in TypeScript. If you find yourself letting the model assert blocking/advisory, STOP — that defeats the entire purpose of this gate.
|
|
83
|
+
- **Block only on high-confidence NOT_MEASURABLE.** `authority === 'blocking'` iff `measurability === 'NOT_MEASURABLE' && confidence === 'high'`. Every other combination is advisory. Do not halt on an advisory verdict.
|
|
84
|
+
- **Never author criteria.** `acceptance-eval` reviews specs; it does not write the acceptance criteria it judges. Humans own the thinking layer.
|
|
85
|
+
- **Never block on infrastructure noise.** A provider failure, an unparseable response, or a missing spec section must resolve to INCONCLUSIVE/advisory, never a thrown error or a block. The evaluator enforces this; do not reintroduce a hard failure in the wrapper.
|
|
86
|
+
|
|
87
|
+
## Escalation
|
|
88
|
+
|
|
89
|
+
- **Blocking verdict the author disputes:** the resolution is for a human to add measurable success criteria (or fix the section the resolver judged) and re-run `acceptance-eval` — not to override the gate.
|
|
90
|
+
- **Repeated INCONCLUSIVE on a real spec:** usually means no judgable section exists. Confirm the spec has a Success Criteria / User-Visible Behavior / Overview section.
|
|
91
|
+
- **No `ANTHROPIC_API_KEY` configured:** every verdict degrades to INCONCLUSIVE/advisory and nothing blocks. Surface this to the human — the gate is effectively disabled until a provider is configured.
|
|
92
|
+
- **Verdict seems wrong (false positive/negative):** capture the spec section and verdict and route to the maintainers; do not loosen the conservative-confidence prompt ad hoc.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
name: acceptance-eval
|
|
2
|
+
version: '0.1.0'
|
|
3
|
+
description: >-
|
|
4
|
+
Pre-execution LLM-judgment skill: does a spec carry measurable, testable,
|
|
5
|
+
complete acceptance criteria before work begins? Resolves the spec's
|
|
6
|
+
acceptance section, critiques observability / testability / completeness
|
|
7
|
+
(advisory), flags user-visible behaviors with no covering test (advisory),
|
|
8
|
+
and emits a confidence-rated AcceptanceVerdict
|
|
9
|
+
(MEASURABLE | NOT_MEASURABLE | INCONCLUSIVE). Authority is derived in
|
|
10
|
+
TypeScript, never from the LLM: a high-confidence NOT_MEASURABLE blocks
|
|
11
|
+
merge; every other verdict is advisory. The upstream twin of the
|
|
12
|
+
outcome-eval ship gate.
|
|
13
|
+
stability: draft
|
|
14
|
+
cognitive_mode: constructive-architect
|
|
15
|
+
triggers:
|
|
16
|
+
- manual
|
|
17
|
+
- on_pr
|
|
18
|
+
platforms:
|
|
19
|
+
- claude-code
|
|
20
|
+
- cursor
|
|
21
|
+
- codex
|
|
22
|
+
- gemini-cli
|
|
23
|
+
tools:
|
|
24
|
+
- Bash
|
|
25
|
+
- Read
|
|
26
|
+
- Glob
|
|
27
|
+
- Grep
|
|
28
|
+
mcp:
|
|
29
|
+
tool: acceptance_eval
|
|
30
|
+
# specPath is the only required input. testGlobs/testContent are the (b)
|
|
31
|
+
# coverage evidence: omitting them degrades coverage findings to
|
|
32
|
+
# advisory-empty but NEVER affects the (c) measurability gate.
|
|
33
|
+
input:
|
|
34
|
+
specPath: 'string (required) — absolute or repo-relative path to the spec markdown to evaluate'
|
|
35
|
+
testGlobs: 'string[] (optional) — globs locating test files that evidence (b) coverage; omission degrades coverage findings to advisory-empty, never the (c) gate'
|
|
36
|
+
testContent: 'string (optional) — pre-read test snippets as (b) coverage evidence; an alternative to testGlobs'
|
|
37
|
+
model: 'string (optional) — model override for the acceptance-eval LLM call'
|
|
38
|
+
type: rigid
|
|
39
|
+
tier: 2
|
|
40
|
+
phases:
|
|
41
|
+
- name: resolve
|
|
42
|
+
description: Resolve the judgment section (Success Criteria -> User-Visible Behavior -> Overview), recorded in judgedAgainst
|
|
43
|
+
required: true
|
|
44
|
+
- name: gather
|
|
45
|
+
description: Locate test evidence for (b) coverage via testGlobs/testContent; optional — omission degrades coverage findings, never the (c) gate
|
|
46
|
+
required: false
|
|
47
|
+
- name: judge
|
|
48
|
+
description: Invoke AcceptanceEvaluator; the LLM returns measurability/confidence/criteriaFindings/coverageFindings/rationale; authority is derived in TS via deriveAcceptanceAuthority
|
|
49
|
+
required: true
|
|
50
|
+
- name: gate
|
|
51
|
+
description: Render the verdict; on a blocking verdict (NOT_MEASURABLE + high confidence) HALT before merge
|
|
52
|
+
required: true
|
|
53
|
+
state:
|
|
54
|
+
persistent: false
|