@ericrisco/rsc 1.0.6 → 1.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/manifest.json CHANGED
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "1.0.6",
2
+ "version": "1.0.8",
3
3
  "counts": {
4
4
  "skills": 258
5
5
  },
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ericrisco/rsc",
3
- "version": "1.0.6",
3
+ "version": "1.0.8",
4
4
  "description": "Eric Risco's agent-skills catalog as a granular, self-recommending CLI installer.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -63,6 +63,60 @@ export function behavioralGate(agg) {
63
63
  return { pass, reasons, mustFix };
64
64
  }
65
65
 
66
+ // The hold-out gate — does a fix TRANSFER, or was it memorized?
67
+ //
68
+ // The fix loop scores the edited skill against a FRESH scenario the fixer never saw. Because the
69
+ // eval engine always runs that scenario twice (with and without the skill), the hold-out already
70
+ // carries its own lift: no pre-fix baseline is needed.
71
+ //
72
+ // The bar is deliberately NOT the main gate's LIFT_MIN. One fresh scenario graded once is noisy, so
73
+ // only the unambiguous signal acts: the edited skill failing to beat the bare agent on unseen work.
74
+ // Anything positive transfers. Raising this to LIFT_MIN would reject good fixes on variance alone
75
+ // (see 02-DOCS/wiki/sdd/specs/generalization-gate.md).
76
+ export function holdoutGate(scored) {
77
+ const agg = scored && scored.aggregate;
78
+ if (!agg || agg.n === 0 || typeof agg.lift !== 'number') {
79
+ return {
80
+ verdict: 'block',
81
+ kind: 'indeterminate',
82
+ lift: null,
83
+ reason: 'Hold-out could not be scored (no gradeable fresh scenario). Failing closed: an unverified edit is not a verified one.',
84
+ };
85
+ }
86
+ if (agg.lift <= 0) {
87
+ return {
88
+ verdict: 'block',
89
+ kind: 'regression',
90
+ lift: agg.lift,
91
+ reason: `Hold-out lift ${agg.lift} <= 0: on a scenario the fixer never saw, the edited skill does not beat no-skill. The edit did not transfer.`,
92
+ };
93
+ }
94
+ return {
95
+ verdict: 'pass',
96
+ kind: 'transfer',
97
+ lift: agg.lift,
98
+ reason: `Hold-out lift ${agg.lift} > 0: the edit still helps on a scenario the fixer never saw.`,
99
+ };
100
+ }
101
+
102
+ export function formatHoldoutVerdict(scored, gate) {
103
+ const agg = (scored && scored.aggregate) || {};
104
+ const lines = [];
105
+ lines.push(`# Hold-out verdict — ${(scored && scored.skillId) || '(unknown skill)'}`);
106
+ lines.push('');
107
+ lines.push(`**${gate.verdict === 'pass' ? 'PASS ✅' : 'BLOCK ❌'}** · ${gate.kind} · lift ${gate.lift == null ? 'n/a' : gate.lift} · absolute ${agg.absoluteScore == null ? 'n/a' : agg.absoluteScore} · n=${agg.n || 0}${agg.dropped ? ` (${agg.dropped} dropped)` : ''}`);
108
+ lines.push('');
109
+ lines.push(gate.reason);
110
+ // The fresh scenario is generated by the same model that fixes. Echoing it here is the only way to
111
+ // audit by hand whether the generator went easy on itself — a risk the spec accepts, not solves.
112
+ if (scored && scored.holdoutScenario) {
113
+ lines.push('');
114
+ lines.push('**Fresh scenario used:**');
115
+ lines.push(`> ${String(scored.holdoutScenario).replace(/\n/g, '\n> ')}`);
116
+ }
117
+ return lines.join('\n');
118
+ }
119
+
66
120
  // raw: { skillId, scenarios: [{index, xIsTreatment, gradeX, gradeY, error?}] }
67
121
  export function scoreFromRaw(raw) {
68
122
  const scenarios = (raw && Array.isArray(raw.scenarios) ? raw.scenarios : []).map((s) => {
@@ -78,7 +132,10 @@ export function scoreFromRaw(raw) {
78
132
  });
79
133
  const agg = aggregate(scenarios);
80
134
  const gate = behavioralGate(agg);
81
- return { skillId: (raw && raw.skillId) || null, scenarios, aggregate: agg, gate };
135
+ const out = { skillId: (raw && raw.skillId) || null, scenarios, aggregate: agg, gate };
136
+ // Carried through untouched so the hold-out verdict can echo the fresh scenario for hand-audit.
137
+ if (raw && raw.holdoutScenario) out.holdoutScenario = raw.holdoutScenario;
138
+ return out;
82
139
  }
83
140
 
84
141
  export function formatScorecard(scored) {
@@ -1,11 +1,15 @@
1
1
  #!/usr/bin/env node
2
2
  // skill-behavior-eval.js — turn the behavior workflow's raw JSON into a scorecard.
3
3
  // Usage:
4
- // node scripts/skill-behavior-eval.js --score <raw.json> (or pipe raw JSON on stdin with -)
5
- // Exits 0 if the behavioral gate passes, 1 if it fails, 2 on usage/parse error.
4
+ // node scripts/skill-behavior-eval.js --score <raw.json> (or pipe raw JSON on stdin with -)
5
+ // node scripts/skill-behavior-eval.js --holdout <raw.json> (fresh-scenario transfer gate)
6
+ // Exits 0 if the gate passes, 1 if it fails/blocks, 2 on usage/parse error.
7
+ //
8
+ // --holdout exists so the fix loop's blocking decision is a comparison of numbers an agent reports,
9
+ // never a judgement an agent makes (constitution P1).
6
10
 
7
11
  import { readFileSync } from 'node:fs';
8
- import { scoreFromRaw, formatScorecard } from './lib/behavior-score.js';
12
+ import { scoreFromRaw, formatScorecard, holdoutGate, formatHoldoutVerdict } from './lib/behavior-score.js';
9
13
 
10
14
  function readInput(argPath) {
11
15
  if (argPath && argPath !== '-') return readFileSync(argPath, 'utf8');
@@ -14,8 +18,9 @@ function readInput(argPath) {
14
18
 
15
19
  function main() {
16
20
  const args = process.argv.slice(2);
17
- if (args[0] !== '--score') {
18
- process.stderr.write('usage: skill-behavior-eval.js --score <raw.json|->\n');
21
+ const mode = args[0];
22
+ if (mode !== '--score' && mode !== '--holdout') {
23
+ process.stderr.write('usage: skill-behavior-eval.js --score|--holdout <raw.json|->\n');
19
24
  process.exit(2);
20
25
  }
21
26
  let raw;
@@ -26,6 +31,11 @@ function main() {
26
31
  process.exit(2);
27
32
  }
28
33
  const scored = scoreFromRaw(raw);
34
+ if (mode === '--holdout') {
35
+ const gate = holdoutGate(scored);
36
+ process.stdout.write(formatHoldoutVerdict(scored, gate) + '\n');
37
+ process.exit(gate.verdict === 'pass' ? 0 : 1);
38
+ }
29
39
  process.stdout.write(formatScorecard(scored) + '\n');
30
40
  process.exit(scored.gate.pass ? 0 : 1);
31
41
  }
@@ -3,7 +3,13 @@
3
3
  The `skill-harden` workflow turns a FAILING behavioral score into edits, without gaming the eval.
4
4
  These are the rules its agents follow.
5
5
 
6
- ## Diagnosis (skill-fault vs eval-fault)
6
+ The three guards below — **diff judge**, **hold-out**, **Generalization gate** — are mandatory *and*
7
+ mechanised: `tests/harden-generalization-gate.test.js` asserts, for each of those three, that the
8
+ loop implements what this file promises. **Adding a fourth guard here means adding its pair to that
9
+ test**; a guard documented but unchecked gives the safety without the protection, which is the
10
+ failure mode this repo has now hit nine times (constitution P2).
11
+
12
+ ## Diagnosis (skill-fault vs eval-fault vs capability)
7
13
 
8
14
  Given the failing `mustFix`, both A/B outputs, and the grader's per-item evidence, decide the FAULT:
9
15
 
@@ -15,8 +21,44 @@ Given the failing `mustFix`, both A/B outputs, and the grader's per-item evidenc
15
21
  2. **Phantom-context must_include** — an item demands workspace artifacts the isolated eval
16
22
  agent cannot have (e.g. `user-profile.md`, a real sibling skill to delegate to), capping the
17
23
  absolute unfairly.
24
+ - **capability** — the gap cannot be closed by words in a skill body. It needs an **executable
25
+ capability**: something that fires on a real execution event and returns to the agent information
26
+ it cannot obtain on its own — a hook, a script, a tool, a sub-agent. Advice that merely tells the
27
+ agent to remember something is **not** a capability: it is not anchored to any observable event, so
28
+ it can fire when it should not and stay silent when it should not.
29
+
30
+ This loop can only ever write **guidance** (SKILL.md, references/). So `capability` is not a fix
31
+ route, it is the honest exit: the loop edits nothing, names the missing capability and the surface it
32
+ would live on, and stops. Writing a paragraph that pretends to cover a missing capability raises the
33
+ score without the capability existing — the worst outcome available.
34
+
35
+ A `capability` verdict **must name** the capability and its surface. One that cannot is an excuse,
36
+ not a diagnosis → choose `skill`.
37
+
38
+ Default to **skill** when unsure: blaming the eval or the surface area is the easier, less honest path.
39
+
40
+ ## The Generalization gate (ex-ante — it binds before the edit is written)
41
+
42
+ This gate governs the **content** of every line the fixer writes, and it travels **in the fixer's own
43
+ instruction**, not in a review afterwards. Writing freely and reverting later burns the round, and the
44
+ loop only has two.
45
+
46
+ Everything added to a skill is global context that will apply to cases the skill has never seen. So:
47
+
48
+ - **Allowed:** a reusable criterion that carries its own **applicability condition** — something the
49
+ agent can evaluate on a case it has never encountered.
50
+ - **Banned outright:** identifiers or titles from the eval scenario; names of its files, symbols or
51
+ fixtures; rules that branch on its specific data; and reciting the finding as if it were a rule
52
+ ("the last round showed X needs Y" — that is an answer, not a criterion).
53
+
54
+ **Litmus test, before every line:** *would this still help on a case in this domain I have never
55
+ seen?* If no → rewrite it as a criterion, or drop it. Not knowledge → not in the skill.
18
56
 
19
- Default to **skill** when unsure: blaming the eval is the easier, less honest path.
57
+ Why ex-ante and not only as review: moving this constraint from after-the-edit to
58
+ before-the-edit is what buys generalization cheaply. The measured version of this result — plus the
59
+ warning that richer feedback **without** this gate makes held-out performance *worse* — is in
60
+ [HarnessCompass](https://arxiv.org/abs/2608.01918); the reasoning it drove is recorded in
61
+ `02-DOCS/wiki/sdd/specs/generalization-gate.md`.
20
62
 
21
63
  ## Eval-fix guard (independent judge)
22
64
 
@@ -27,14 +69,34 @@ skill-fault this round.
27
69
 
28
70
  ## Skill-fix guards (both required)
29
71
 
30
- 1. **Diff judge.** Read the SKILL.md/references diff. Does it add genuine capability (method,
31
- decision rules, concrete guidance) or merely echo the `must_include` wording into the body to
32
- satisfy the grader? Keyword-stuffing → revert the edit; it is not a fix.
33
- 2. **Hold-out.** Re-score on a FRESH scenario from the skill's domain that the fixer never saw.
34
- A real improvement generalizes; a memorized one does not. The hold-out score must also improve.
72
+ 1. **Diff judge.** Read the SKILL.md/references diff and check BOTH: does it add genuine capability
73
+ (method, decision rules, concrete guidance) rather than echoing the `must_include` wording into
74
+ the body? *and* does every added line survive the Generalization gate above? Fail either → revert
75
+ the edit; it is not a fix.
76
+ 2. **Hold-out.** Re-score the EDITED skill on a FRESH scenario from its domain that the fixer never
77
+ saw. A real improvement generalizes; a memorized one does not.
78
+ - The eval engine runs every scenario with **and** without the skill, so the fresh scenario
79
+ already carries its own lift. No pre-fix baseline is needed.
80
+ - The verdict is computed by `node scripts/skill-behavior-eval.js --holdout <raw.json>` — a
81
+ comparison of numbers, deterministic and reproducible, never an agent's opinion (P1). Exit 0
82
+ passes, exit 1 blocks.
83
+ - **Blocks on `lift <= 0`** (`regression`): on unseen work the edited skill does not beat the bare
84
+ agent, so the edit did not transfer. Also blocks when the fresh scenario could not be run or
85
+ graded (`indeterminate`) — failing closed, because an unverified edit is not a verified one.
86
+ - The bar is deliberately **not** the main gate's `lift >= 1.0`: one fresh scenario graded once is
87
+ noisy, so only the unambiguous signal acts. Upgrade path if it proves too noisy in practice:
88
+ N=3 fresh scenarios, majority.
89
+ - **A block reverts the edit.** The round produced nothing and is recorded with its reason.
35
90
 
36
91
  ## Stopping & honesty
37
92
 
38
93
  - Max 2 rounds. On give-up, report the honest final score and a recommendation — never a faked pass.
94
+ - **A main-gate pass whose last surviving edit failed the hold-out does not commit.** It is a fix
95
+ tuned to the cases it was measured against; the loop reverts it and reports why. Conservative on
96
+ purpose: a lost commit costs one re-run, a certified overfit costs the catalog.
97
+ - A `capability` verdict never commits either: it returns the missing capability and its surface, and
98
+ routes the work to `specify`.
99
+ - Every non-commit says which condition failed, in the returned result. A denial without its exit is
100
+ abandonment (P6).
39
101
  - Persistent lift-fail at a high absolute → recommend deprecate/merge: the skill does not justify
40
102
  its own existence.
@@ -39,12 +39,28 @@ const SCORE_SCHEMA = {
39
39
  const FAULT_SCHEMA = {
40
40
  type: 'object',
41
41
  properties: {
42
- fault: { type: 'string', enum: ['skill', 'eval'] },
42
+ // 'capability' is the honest third exit: this loop can only ever write guidance (SKILL.md /
43
+ // references), so a failure that needs an executable capability is out of its reach. Saying so
44
+ // beats writing a paragraph that pretends to cover it.
45
+ fault: { type: 'string', enum: ['skill', 'eval', 'capability'] },
43
46
  rationale: { type: 'string' },
47
+ missingCapability: { type: 'string' },
48
+ surface: { type: 'string' },
44
49
  },
45
50
  required: ['fault', 'rationale'],
46
51
  }
47
52
 
53
+ const HOLDOUT_VERDICT_SCHEMA = {
54
+ type: 'object',
55
+ properties: {
56
+ verdict: { type: 'string', enum: ['pass', 'block'] },
57
+ kind: { type: 'string', enum: ['transfer', 'regression', 'indeterminate'] },
58
+ lift: { type: 'number' },
59
+ reason: { type: 'string' },
60
+ },
61
+ required: ['verdict', 'kind', 'reason'],
62
+ }
63
+
48
64
  const GUARD_SCHEMA = {
49
65
  type: 'object',
50
66
  properties: {
@@ -75,7 +91,26 @@ async function scoreRaw(raw, round) {
75
91
  )
76
92
  }
77
93
 
94
+ // The hold-out gate: score the EDITED skill against a fresh scenario the fixer never saw, and let a
95
+ // deterministic CLI decide. The verdict is a comparison of numbers the agent reports, never a
96
+ // judgement the agent makes (constitution P1).
97
+ async function holdoutVerdict(raw, scenario, round) {
98
+ const payload = JSON.stringify({ ...raw, holdoutScenario: scenario })
99
+ return agent(
100
+ `Write this JSON to /tmp/harden-${skillId}-holdout-r${round}.json exactly, then run ` +
101
+ `\`node scripts/skill-behavior-eval.js --holdout /tmp/harden-${skillId}-holdout-r${round}.json\`. ` +
102
+ `Read its markdown output and return {verdict, kind, lift, reason} verbatim from that output — ` +
103
+ `verdict "pass" if the CLI exit code was 0, "block" if it was 1. Do NOT form your own opinion ` +
104
+ `about whether the fix was good: report what the CLI decided.\n\nJSON:\n${payload}`,
105
+ { label: `holdout:r${round}`, phase: 'Diagnose & Fix', schema: HOLDOUT_VERDICT_SCHEMA },
106
+ )
107
+ }
108
+
78
109
  const history = []
110
+ const holdout = [] // one record per fix round: did the hold-out run, and what did it say
111
+ let lastFixHoldout = null // verdict of the last round that left an edit on disk
112
+ let lastFault = null // last diagnosis, so the report can name a capability we cannot write
113
+ let outcome = null
79
114
  let round = 0
80
115
  let committed = null
81
116
 
@@ -96,9 +131,16 @@ while (true) {
96
131
  const fault = await agent(
97
132
  `Follow scripts/skill-harden-rubric.md (Diagnosis). The skill "${skillId}" FAILED its behavioral gate. ` +
98
133
  `mustFix:\n- ${score.mustFix.join('\n- ')}\n\nGrader signals (both A/B outputs, per-item evidence):\n${evidence}\n\n` +
99
- `Decide fault = 'skill' or 'eval' and give a rationale. Default to 'skill' when unsure.`,
134
+ `Decide fault = 'skill', 'eval' or 'capability' and give a rationale. Default to 'skill' when unsure.\n\n` +
135
+ `Choose 'capability' ONLY when the gap cannot be closed by words in a skill body: it needs an ` +
136
+ `executable capability — something that fires on a real execution event and returns to the agent ` +
137
+ `information it cannot obtain on its own (a hook, a script, a tool, a sub-agent). Advice that only ` +
138
+ `tells the agent to remember something is NOT a capability. If you choose 'capability' you MUST name ` +
139
+ `it in missingCapability and name where it would live in surface; a verdict that cannot name the ` +
140
+ `capability is an excuse, not a diagnosis — pick 'skill' instead.`,
100
141
  { label: `diagnose:r${round}`, phase: 'Diagnose & Fix', schema: FAULT_SCHEMA },
101
142
  )
143
+ lastFault = fault
102
144
 
103
145
  if (fault.fault === 'eval') {
104
146
  const judged = await agent(
@@ -115,31 +157,69 @@ while (true) {
115
157
  }
116
158
  }
117
159
 
160
+ if (fault.fault === 'capability') {
161
+ // Out of this loop's reach: it can only write words. Stop honestly instead of producing guidance
162
+ // that raises the score without the capability existing. Nothing is edited here on purpose.
163
+ outcome = 'capability-out-of-reach'
164
+ log(`${skillId}: needs a capability, not guidance — ${fault.missingCapability || '(unnamed)'} in ${fault.surface || '(unnamed surface)'}`)
165
+ break
166
+ }
167
+
118
168
  if (fault.fault === 'skill') {
119
169
  await agent(
120
170
  `Follow the author-skill discipline. The skill "${skillId}" must genuinely cover this mustFix without ` +
121
171
  `keyword-stuffing:\n- ${score.mustFix.join('\n- ')}\n\nEdit skills/${skillId}/SKILL.md (body) and/or files under ` +
122
172
  `skills/${skillId}/references/ to add the REAL missing capability (method, decision rules, concrete guidance). ` +
123
- `Do not touch evals/. Apply the edits now.`,
173
+ `Do not touch evals/. Apply the edits now.\n\n` +
174
+ `HARD CONSTRAINT — the Generalization gate (scripts/skill-harden-rubric.md defines it; it binds every ` +
175
+ `line you write, and it binds BEFORE you write, not after):\n` +
176
+ `Everything you add is global context that will apply to cases this skill has never seen. Write a ` +
177
+ `reusable criterion that carries its own applicability condition — never the answer to the one ` +
178
+ `scenario that failed. Banned outright: identifiers or titles from the eval scenario, names of its ` +
179
+ `files/symbols/fixtures, rules that branch on its specific data, and reciting the finding as if it ` +
180
+ `were a rule ("the last round showed X needs Y").\n` +
181
+ `Litmus test before every line: "would this still help on a case in this domain I have never seen?" ` +
182
+ `If no, rewrite it as a criterion or drop it. A fix that only lifts this scenario's score will be ` +
183
+ `caught by the hold-out and reverted, and the round will be wasted.`,
124
184
  { label: `fix:r${round}`, phase: 'Diagnose & Fix' },
125
185
  )
126
186
  const diffJudge = await agent(
127
187
  `Follow scripts/skill-harden-rubric.md (Skill-fix guard 1). Run \`git diff -- skills/${skillId}/SKILL.md skills/${skillId}/references\` ` +
128
- `and judge: does the diff add genuine capability, or just echo the mustFix wording to satisfy the grader? ` +
129
- `If it is keyword-stuffing, run \`git checkout -- skills/${skillId}/SKILL.md skills/${skillId}/references\` to revert it. ` +
188
+ `and judge BOTH: (a) does the diff add genuine capability, or just echo the mustFix wording to satisfy the ` +
189
+ `grader? and (b) does every added line survive the Generalization gate — a reusable criterion with its ` +
190
+ `applicability condition, not the answer to the observed scenario (no eval-specific names, no branching on ` +
191
+ `its data, no recited findings)? Fail either check and it is not a fix. ` +
192
+ `If it fails, run \`git checkout -- skills/${skillId}/SKILL.md skills/${skillId}/references\` to revert it. ` +
130
193
  `Return {genuine, rationale}.`,
131
194
  { label: `diff-judge:r${round}`, phase: 'Diagnose & Fix', schema: GUARD_SCHEMA },
132
195
  )
133
196
 
134
- if (diffJudge.genuine) {
135
- // Guard 2: hold-out. Generate a fresh scenario and re-score the A/B on it only.
136
- const holdout = await agent(
197
+ if (!diffJudge.genuine) {
198
+ holdout.push({ round, ran: false, why: 'edit reverted by the diff judge — nothing to validate' })
199
+ } else {
200
+ // Guard 2: hold-out. Score the EDITED skill on a fresh scenario the fixer never saw. The eval
201
+ // engine runs every scenario with AND without the skill, so this already carries its own lift.
202
+ const fresh = await agent(
137
203
  `Invent ONE fresh capability scenario for the "${skillId}" skill's domain that is NOT in its cases.yaml and ` +
138
204
  `does NOT enumerate its own requirements. Return {scenario, mustInclude:[3-6 outcome-level checks]}.`,
139
205
  { label: `holdout-gen:r${round}`, phase: 'Diagnose & Fix', schema: HOLDOUT_SCHEMA },
140
206
  )
141
- const holdoutRaw = await workflow(EVAL, { skillId, scenarios: [holdout] })
142
- await scoreRaw(holdoutRaw, `${round}-holdout`) // recorded in transcript; informs the next loop's eval
207
+ const holdoutRaw = await workflow(EVAL, { skillId, scenarios: [fresh] })
208
+ const verdict = await holdoutVerdict(holdoutRaw, fresh.scenario, round)
209
+ lastFixHoldout = verdict
210
+ holdout.push({ round, ran: true, scenario: fresh.scenario, ...verdict })
211
+
212
+ if (verdict.verdict === 'block') {
213
+ // The edit lifted this skill's own cases.yaml but does not transfer to work the fixer never
214
+ // saw. A memorized fix is not a fix: revert it. This is the guard the rubric always promised
215
+ // and the code never enforced — see 02-DOCS/wiki/sdd/specs/generalization-gate.md.
216
+ log(`${skillId}: hold-out ${verdict.kind} (lift ${verdict.lift}) — reverting round ${round}'s edit`)
217
+ await agent(
218
+ `Run exactly \`git checkout -- skills/${skillId}/SKILL.md skills/${skillId}/references\` and nothing else. ` +
219
+ `The edit failed the hold-out gate (${verdict.kind}) so it must not survive. Return the command's output.`,
220
+ { label: `holdout-revert:r${round}`, phase: 'Diagnose & Fix', effort: 'low' },
221
+ )
222
+ }
143
223
  }
144
224
  }
145
225
 
@@ -147,7 +227,13 @@ while (true) {
147
227
  }
148
228
 
149
229
  const passed = history.length > 0 && history[history.length - 1].pass === true
150
- if (passed && !noCommit) {
230
+ // A main-gate pass whose last surviving edit failed the hold-out is NOT a pass: it is a fix that
231
+ // only works on the cases it was tuned against. Conservative on purpose — a lost commit costs one
232
+ // re-run, a certified overfit costs the catalog.
233
+ const holdoutClean = !lastFixHoldout || lastFixHoldout.verdict === 'pass'
234
+ let notCommittedBecause = null
235
+
236
+ if (passed && holdoutClean && !noCommit) {
151
237
  phase('Commit')
152
238
  const commit = await agent(
153
239
  `The skill "${skillId}" now passes its behavioral gate. Commit ONLY its files: ` +
@@ -156,6 +242,24 @@ if (passed && !noCommit) {
156
242
  { label: `commit:${skillId}`, phase: 'Commit' },
157
243
  )
158
244
  committed = (commit || '').trim()
245
+ } else if (outcome === 'capability-out-of-reach') {
246
+ notCommittedBecause = `needs a capability this loop cannot write: ${lastFault.missingCapability || '(unnamed)'} in ${lastFault.surface || '(unnamed surface)'}. Route it through specify — guidance cannot close this gap.`
247
+ } else if (!passed) {
248
+ notCommittedBecause = 'the behavioral gate still fails. Read the last scorecard and fix by hand, or deprecate the skill.'
249
+ } else if (!holdoutClean) {
250
+ notCommittedBecause = `the last edit passed cases.yaml but failed the hold-out (${lastFixHoldout.kind}, lift ${lastFixHoldout.lift}) and was reverted. Re-run to try a different fix, or write the criterion by hand.`
251
+ } else if (noCommit) {
252
+ notCommittedBecause = 'noCommit was requested.'
159
253
  }
160
254
 
161
- return { skillId, rounds: round + 1, history, committed, passed }
255
+ return {
256
+ skillId,
257
+ rounds: round + 1,
258
+ history,
259
+ holdout,
260
+ outcome: outcome || (passed && holdoutClean ? 'passed' : 'gave-up'),
261
+ missingCapability: lastFault && lastFault.fault === 'capability' ? { capability: lastFault.missingCapability, surface: lastFault.surface, rationale: lastFault.rationale } : null,
262
+ committed,
263
+ passed: passed && holdoutClean,
264
+ notCommittedBecause,
265
+ }
@@ -101,6 +101,28 @@ Render it at the dial's verbosity. Do not log a decision to `decisions.md` — a
101
101
  | Siding with the plan when it contradicts the constitution | The constitution wins by definition — flag CRITICAL. If the principle itself is wrong, that is a `constitution` change the user makes, not a quiet override here. |
102
102
  | Guessing what a vague requirement meant | Guessing defeats the gate. Mark AMBIGUOUS and route to `clarify`; do not encode your guess. |
103
103
 
104
+ ## Result envelope
105
+
106
+ End with the parseable block every SDD phase shares, so the dispatcher can chain without
107
+ interpreting prose (contract: `../sdd/SKILL.md`):
108
+
109
+ ```json result-envelope
110
+ {
111
+ "status": "complete|blocked|failed",
112
+ "executive_summary": "Cross-read of spec/plan/tasks against the constitution; findings ranked.",
113
+ "artifact": "02-DOCS/wiki/sdd/analysis/<slug>.md",
114
+ "next_recommended": "implement",
115
+ "risk": "low|medium|high",
116
+ "skill_resolution": {
117
+ "used": ["analyze"],
118
+ "missing": [],
119
+ "fallback": [],
120
+ "compact_rules": ["Read adversarially across artifacts, not inside one.", "A finding without a location is an opinion."]
121
+ },
122
+ "evidence": ["report path exists", "blockers listed with artifact + location", "constitution conflicts named"]
123
+ }
124
+ ```
125
+
104
126
  ## Next in the chain
105
127
 
106
128
  On `GATE: PASS` (or once the user consciously accepts the remaining MEDIUM/LOW findings), proceed to **`implement`** — execute the tasks with TDD discipline, delegating concrete test tooling to the relevant stack skill (`fastapi`, `nextjs`, `go`, `flutter`, `postgresdb`). On `GATE: BLOCKED`, route each CRITICAL/HIGH finding to its owning phase (`clarify`, `plan`, `tasks`, or `constitution`), let the user resolve, then re-run `analyze`. The gate only opens once.
@@ -113,6 +113,28 @@ After baking back, the spec line becomes a bounded, testable behavior with accep
113
113
 
114
114
  The gate is passed when the spec, constitution and profile were all read (settled questions cited, not re-asked); all ten taxonomy categories were considered; only the build-changing gaps were put to the user, as dial-sized decisions with recommendations; every answer is baked into the spec body with observable acceptance criteria, logged under `## Clarifications` and bounded under `## Out of scope`; and the final re-read opened no new gap.
115
115
 
116
+ ## Result envelope
117
+
118
+ End with the parseable block every SDD phase shares, so the dispatcher can chain without
119
+ interpreting prose (contract: `../sdd/SKILL.md`):
120
+
121
+ ```json result-envelope
122
+ {
123
+ "status": "complete|blocked|failed",
124
+ "executive_summary": "Open points resolved; the spec is de-risked and ready to plan against.",
125
+ "artifact": "02-DOCS/wiki/sdd/specs/<slug>.md",
126
+ "next_recommended": "plan",
127
+ "risk": "low|medium|high",
128
+ "skill_resolution": {
129
+ "used": ["clarify"],
130
+ "missing": [],
131
+ "fallback": [],
132
+ "compact_rules": ["Ask only what changes the spec.", "An unanswered question is recorded, never invented."]
133
+ },
134
+ "evidence": ["answers folded back into the spec", "remaining open points listed with their owner"]
135
+ }
136
+ ```
137
+
116
138
  ## Next in the chain
117
139
 
118
140
  Hand off to **`plan`** — turn the now-sharp spec into a technical implementation plan (architecture, interfaces, data flow, testing strategy, risks), deferring stack specifics to the relevant stack skill. The chain continues: clarify → **plan** → tasks → analyze → implement → verify → review → ship. `debug` is callable any time if what you are "clarifying" turns out to be a runtime fault, not a spec gap.
@@ -132,6 +132,28 @@ This skill's `02-DOCS` record is the constitution at `02-DOCS/wiki/sdd/constitut
132
132
 
133
133
  Every later rsc-sdd phase reads this file before it works. The harness maintains and improves the article over the life of the project; this skill is the place that ratifies and amends it.
134
134
 
135
+ ## Result envelope
136
+
137
+ End with the parseable block every SDD phase shares, so the dispatcher can chain without
138
+ interpreting prose (contract: `../sdd/SKILL.md`):
139
+
140
+ ```json result-envelope
141
+ {
142
+ "status": "complete|blocked|failed",
143
+ "executive_summary": "Constitution written with N numbered, testable rules the later phases inherit.",
144
+ "artifact": "02-DOCS/wiki/sdd/constitution.md",
145
+ "next_recommended": "specify",
146
+ "risk": "low|medium|high",
147
+ "skill_resolution": {
148
+ "used": ["constitution"],
149
+ "missing": [],
150
+ "fallback": [],
151
+ "compact_rules": ["Principles are inherited constraints, not choices to re-make.", "Every rule is testable or it is a preference."]
152
+ },
153
+ "evidence": ["constitution path exists", "rules numbered and testable", "decisions log appended"]
154
+ }
155
+ ```
156
+
135
157
  ## Next in the chain
136
158
 
137
159
  The constitution is the guardrail; now describe what to build. Hand off to **`../specify/SKILL.md`** — turn a fuzzy intent into a spec (what & why, no implementation), grounded in these principles. The full chain: **constitution → specify → clarify → plan → tasks → analyze → implement → verify → review → ship** (with `debug`, `worktrees`, `parallel` callable on demand). The dispatcher is `../sdd/SKILL.md`.
@@ -141,6 +141,28 @@ regardless. (`implement` re-checks this as a hard gate before its first commit.)
141
141
  - Concrete framework/ORM/test-runner mechanics → the stack skill (`../fastapi/SKILL.md`,
142
142
  `../nextjs/SKILL.md`, `../go/SKILL.md`, `../flutter/SKILL.md`, `../postgresdb/SKILL.md`).
143
143
 
144
+ ## Result envelope
145
+
146
+ End with the parseable block every SDD phase shares, so the dispatcher can chain without
147
+ interpreting prose (contract: `../sdd/SKILL.md`):
148
+
149
+ ```json result-envelope
150
+ {
151
+ "status": "complete|blocked|failed",
152
+ "executive_summary": "Technical plan derived from the clarified spec, with the isolation decision made.",
153
+ "artifact": "02-DOCS/wiki/sdd/plans/<slug>.md",
154
+ "next_recommended": "tasks",
155
+ "risk": "low|medium|high",
156
+ "skill_resolution": {
157
+ "used": ["plan"],
158
+ "missing": [],
159
+ "fallback": [],
160
+ "compact_rules": ["The plan answers HOW; the spec owns WHAT.", "Name the isolation choice before the build starts."]
161
+ },
162
+ "evidence": ["plan path exists", "each spec acceptance criterion has an approach", "risks and rollback stated"]
163
+ }
164
+ ```
165
+
144
166
  ## Next in the chain
145
167
 
146
168
  Plan written, indexed, decisions logged → propose isolation (above), then hand off to
@@ -237,6 +237,28 @@ Review is mostly a conversation, but two artifacts persist into the harness wiki
237
237
 
238
238
  Index both in `02-DOCS/wiki/index.md` (the Knowledge map; root `CLAUDE.md` keeps only a short pointer) under the `sdd/` topic — the harness owns that map; this skill just keeps its rows honest.
239
239
 
240
+ ## Result envelope
241
+
242
+ End with the parseable block every SDD phase shares, so the dispatcher can chain without
243
+ interpreting prose (contract: `../sdd/SKILL.md`):
244
+
245
+ ```json result-envelope
246
+ {
247
+ "status": "complete|blocked|failed",
248
+ "executive_summary": "Adversarial review against spec/plan/constitution; verdict and blocking findings.",
249
+ "artifact": "02-DOCS/wiki/sdd/reviews/<slug>.md",
250
+ "next_recommended": "ship",
251
+ "risk": "low|medium|high",
252
+ "skill_resolution": {
253
+ "used": ["review"],
254
+ "missing": [],
255
+ "fallback": [],
256
+ "compact_rules": ["Rank or it is noise: blocker / should-fix / nit.", "A finding without a repro is a question, not a defect."]
257
+ },
258
+ "evidence": ["verdict stated (APPROVE | APPROVE WITH NITS | CHANGES REQUESTED)", "each blocker carries location + repro + fix"]
259
+ }
260
+ ```
261
+
240
262
  ## Next in the chain
241
263
 
242
264
  When the diff carries an **APPROVE** / **APPROVE WITH NITS** verdict and every blocker is resolved, hand off to **ship** — close the branch via PR / merge / cleanup, with **git authorship as Eric, never Claude**. If the review came back **CHANGES REQUESTED**, the loop goes back to **implement** (fix), then **verify** (re-prove green), then back here for re-review. Don't ship a diff that hasn't earned its verdict.