@ericrisco/rsc 1.0.7 → 1.0.8

package/manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.7",
+  "version": "1.0.8",
   "counts": {
     "skills": 258
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ericrisco/rsc",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "Eric Risco's agent-skills catalog as a granular, self-recommending CLI installer.",
   "type": "module",
   "bin": {

package/scripts/lib/behavior-score.js

@@ -63,6 +63,60 @@ export function behavioralGate(agg) {
   return { pass, reasons, mustFix };
 }
 
+// The hold-out gate — does a fix TRANSFER, or was it memorized?
+//
+// The fix loop scores the edited skill against a FRESH scenario the fixer never saw. Because the
+// eval engine always runs that scenario twice (with and without the skill), the hold-out already
+// carries its own lift: no pre-fix baseline is needed.
+//
+// The bar is deliberately NOT the main gate's LIFT_MIN. One fresh scenario graded once is noisy, so
+// only the unambiguous signal acts: the edited skill failing to beat the bare agent on unseen work.
+// Anything positive transfers. Raising this to LIFT_MIN would reject good fixes on variance alone
+// (see 02-DOCS/wiki/sdd/specs/generalization-gate.md).
+export function holdoutGate(scored) {
+  const agg = scored && scored.aggregate;
+  if (!agg || agg.n === 0 || typeof agg.lift !== 'number') {
+    return {
+      verdict: 'block',
+      kind: 'indeterminate',
+      lift: null,
+      reason: 'Hold-out could not be scored (no gradeable fresh scenario). Failing closed: an unverified edit is not a verified one.',
+    };
+  }
+  if (agg.lift <= 0) {
+    return {
+      verdict: 'block',
+      kind: 'regression',
+      lift: agg.lift,
+      reason: `Hold-out lift ${agg.lift} <= 0: on a scenario the fixer never saw, the edited skill does not beat no-skill. The edit did not transfer.`,
+    };
+  }
+  return {
+    verdict: 'pass',
+    kind: 'transfer',
+    lift: agg.lift,
+    reason: `Hold-out lift ${agg.lift} > 0: the edit still helps on a scenario the fixer never saw.`,
+  };
+}
+
+export function formatHoldoutVerdict(scored, gate) {
+  const agg = (scored && scored.aggregate) || {};
+  const lines = [];
+  lines.push(`# Hold-out verdict — ${(scored && scored.skillId) || '(unknown skill)'}`);
+  lines.push('');
+  lines.push(`**${gate.verdict === 'pass' ? 'PASS ✅' : 'BLOCK ❌'}** · ${gate.kind} · lift ${gate.lift == null ? 'n/a' : gate.lift} · absolute ${agg.absoluteScore == null ? 'n/a' : agg.absoluteScore} · n=${agg.n || 0}${agg.dropped ? ` (${agg.dropped} dropped)` : ''}`);
+  lines.push('');
+  lines.push(gate.reason);
+  // The fresh scenario is generated by the same model that fixes. Echoing it here is the only way to
+  // audit by hand whether the generator went easy on itself — a risk the spec accepts, not solves.
+  if (scored && scored.holdoutScenario) {
+    lines.push('');
+    lines.push('**Fresh scenario used:**');
+    lines.push(`> ${String(scored.holdoutScenario).replace(/\n/g, '\n> ')}`);
+  }
+  return lines.join('\n');
+}
+
 // raw: { skillId, scenarios: [{index, xIsTreatment, gradeX, gradeY, error?}] }
 export function scoreFromRaw(raw) {
   const scenarios = (raw && Array.isArray(raw.scenarios) ? raw.scenarios : []).map((s) => {
@@ -78,7 +132,10 @@ export function scoreFromRaw(raw) {
   });
   const agg = aggregate(scenarios);
   const gate = behavioralGate(agg);
-  return { skillId: (raw && raw.skillId) || null, scenarios, aggregate: agg, gate };
+  const out = { skillId: (raw && raw.skillId) || null, scenarios, aggregate: agg, gate };
+  // Carried through untouched so the hold-out verdict can echo the fresh scenario for hand-audit.
+  if (raw && raw.holdoutScenario) out.holdoutScenario = raw.holdoutScenario;
+  return out;
 }
 
 export function formatScorecard(scored) {

package/scripts/skill-behavior-eval.js

@@ -1,11 +1,15 @@
 #!/usr/bin/env node
 // skill-behavior-eval.js — turn the behavior workflow's raw JSON into a scorecard.
 // Usage:
-//   node scripts/skill-behavior-eval.js --score <raw.json>   (or pipe raw JSON on stdin with -)
-// Exits 0 if the behavioral gate passes, 1 if it fails, 2 on usage/parse error.
+//   node scripts/skill-behavior-eval.js --score   <raw.json>   (or pipe raw JSON on stdin with -)
+//   node scripts/skill-behavior-eval.js --holdout <raw.json>   (fresh-scenario transfer gate)
+// Exits 0 if the gate passes, 1 if it fails/blocks, 2 on usage/parse error.
+//
+// --holdout exists so the fix loop's blocking decision is a comparison of numbers an agent reports,
+// never a judgement an agent makes (constitution P1).
 
 import { readFileSync } from 'node:fs';
-import { scoreFromRaw, formatScorecard } from './lib/behavior-score.js';
+import { scoreFromRaw, formatScorecard, holdoutGate, formatHoldoutVerdict } from './lib/behavior-score.js';
 
 function readInput(argPath) {
   if (argPath && argPath !== '-') return readFileSync(argPath, 'utf8');
@@ -14,8 +18,9 @@ function readInput(argPath) {
 
 function main() {
   const args = process.argv.slice(2);
-  if (args[0] !== '--score') {
-    process.stderr.write('usage: skill-behavior-eval.js --score <raw.json|->\n');
+  const mode = args[0];
+  if (mode !== '--score' && mode !== '--holdout') {
+    process.stderr.write('usage: skill-behavior-eval.js --score|--holdout <raw.json|->\n');
     process.exit(2);
   }
   let raw;
@@ -26,6 +31,11 @@ function main() {
     process.exit(2);
   }
   const scored = scoreFromRaw(raw);
+  if (mode === '--holdout') {
+    const gate = holdoutGate(scored);
+    process.stdout.write(formatHoldoutVerdict(scored, gate) + '\n');
+    process.exit(gate.verdict === 'pass' ? 0 : 1);
+  }
   process.stdout.write(formatScorecard(scored) + '\n');
   process.exit(scored.gate.pass ? 0 : 1);
 }

package/scripts/skill-harden-rubric.md

@@ -3,7 +3,13 @@
 The `skill-harden` workflow turns a FAILING behavioral score into edits, without gaming the eval.
 These are the rules its agents follow.
 
-## Diagnosis (skill-fault vs eval-fault)
+The three guards below — **diff judge**, **hold-out**, **Generalization gate** — are mandatory *and*
+mechanised: `tests/harden-generalization-gate.test.js` asserts, for each of those three, that the
+loop implements what this file promises. **Adding a fourth guard here means adding its pair to that
+test**; a guard documented but unchecked gives the safety without the protection, which is the
+failure mode this repo has now hit nine times (constitution P2).
+
+## Diagnosis (skill-fault vs eval-fault vs capability)
 
 Given the failing `mustFix`, both A/B outputs, and the grader's per-item evidence, decide the FAULT:
 
@@ -15,8 +21,44 @@ Given the failing `mustFix`, both A/B outputs, and the grader's per-item evidenc
   2. **Phantom-context must_include** — an item demands workspace artifacts the isolated eval
      agent cannot have (e.g. `user-profile.md`, a real sibling skill to delegate to), capping the
      absolute unfairly.
+- **capability** — the gap cannot be closed by words in a skill body. It needs an **executable
+  capability**: something that fires on a real execution event and returns to the agent information
+  it cannot obtain on its own — a hook, a script, a tool, a sub-agent. Advice that merely tells the
+  agent to remember something is **not** a capability: it is not anchored to any observable event, so
+  it can fire when it should not and stay silent when it should not.
+
+This loop can only ever write **guidance** (SKILL.md, references/). So `capability` is not a fix
+route, it is the honest exit: the loop edits nothing, names the missing capability and the surface it
+would live on, and stops. Writing a paragraph that pretends to cover a missing capability raises the
+score without the capability existing — the worst outcome available.
+
+A `capability` verdict **must name** the capability and its surface. One that cannot is an excuse,
+not a diagnosis → choose `skill`.
+
+Default to **skill** when unsure: blaming the eval or the surface area is the easier, less honest path.
+
+## The Generalization gate (ex-ante — it binds before the edit is written)
+
+This gate governs the **content** of every line the fixer writes, and it travels **in the fixer's own
+instruction**, not in a review afterwards. Writing freely and reverting later burns the round, and the
+loop only has two.
+
+Everything added to a skill is global context that will apply to cases the skill has never seen. So:
+
+- **Allowed:** a reusable criterion that carries its own **applicability condition** — something the
+  agent can evaluate on a case it has never encountered.
+- **Banned outright:** identifiers or titles from the eval scenario; names of its files, symbols or
+  fixtures; rules that branch on its specific data; and reciting the finding as if it were a rule
+  ("the last round showed X needs Y" — that is an answer, not a criterion).
+
+**Litmus test, before every line:** *would this still help on a case in this domain I have never
+seen?* If no → rewrite it as a criterion, or drop it. Not knowledge → not in the skill.
 
-Default to **skill** when unsure: blaming the eval is the easier, less honest path.
+Why ex-ante and not only as review: moving this constraint from after-the-edit to
+before-the-edit is what buys generalization cheaply. The measured version of this result — plus the
+warning that richer feedback **without** this gate makes held-out performance *worse* — is in
+[HarnessCompass](https://arxiv.org/abs/2608.01918); the reasoning it drove is recorded in
+`02-DOCS/wiki/sdd/specs/generalization-gate.md`.
 
 ## Eval-fix guard (independent judge)
 
@@ -27,14 +69,34 @@ skill-fault this round.
 
 ## Skill-fix guards (both required)
 
-1. **Diff judge.** Read the SKILL.md/references diff. Does it add genuine capability (method,
-   decision rules, concrete guidance) or merely echo the `must_include` wording into the body to
-   satisfy the grader? Keyword-stuffing → revert the edit; it is not a fix.
-2. **Hold-out.** Re-score on a FRESH scenario from the skill's domain that the fixer never saw.
-   A real improvement generalizes; a memorized one does not. The hold-out score must also improve.
+1. **Diff judge.** Read the SKILL.md/references diff and check BOTH: does it add genuine capability
+   (method, decision rules, concrete guidance) rather than echoing the `must_include` wording into
+   the body? *and* does every added line survive the Generalization gate above? Fail either → revert
+   the edit; it is not a fix.
+2. **Hold-out.** Re-score the EDITED skill on a FRESH scenario from its domain that the fixer never
+   saw. A real improvement generalizes; a memorized one does not.
+   - The eval engine runs every scenario with **and** without the skill, so the fresh scenario
+     already carries its own lift. No pre-fix baseline is needed.
+   - The verdict is computed by `node scripts/skill-behavior-eval.js --holdout <raw.json>` — a
+     comparison of numbers, deterministic and reproducible, never an agent's opinion (P1). Exit 0
+     passes, exit 1 blocks.
+   - **Blocks on `lift <= 0`** (`regression`): on unseen work the edited skill does not beat the bare
+     agent, so the edit did not transfer. Also blocks when the fresh scenario could not be run or
+     graded (`indeterminate`) — failing closed, because an unverified edit is not a verified one.
+   - The bar is deliberately **not** the main gate's `lift >= 1.0`: one fresh scenario graded once is
+     noisy, so only the unambiguous signal acts. Upgrade path if it proves too noisy in practice:
+     N=3 fresh scenarios, majority.
+   - **A block reverts the edit.** The round produced nothing and is recorded with its reason.
 
 ## Stopping & honesty
 
 - Max 2 rounds. On give-up, report the honest final score and a recommendation — never a faked pass.
+- **A main-gate pass whose last surviving edit failed the hold-out does not commit.** It is a fix
+  tuned to the cases it was measured against; the loop reverts it and reports why. Conservative on
+  purpose: a lost commit costs one re-run, a certified overfit costs the catalog.
+- A `capability` verdict never commits either: it returns the missing capability and its surface, and
+  routes the work to `specify`.
+- Every non-commit says which condition failed, in the returned result. A denial without its exit is
+  abandonment (P6).
 - Persistent lift-fail at a high absolute → recommend deprecate/merge: the skill does not justify
   its own existence.

package/scripts/skill-harden.workflow.js

@@ -39,12 +39,28 @@ const SCORE_SCHEMA = {
 const FAULT_SCHEMA = {
   type: 'object',
   properties: {
-    fault: { type: 'string', enum: ['skill', 'eval'] },
+    // 'capability' is the honest third exit: this loop can only ever write guidance (SKILL.md /
+    // references), so a failure that needs an executable capability is out of its reach. Saying so
+    // beats writing a paragraph that pretends to cover it.
+    fault: { type: 'string', enum: ['skill', 'eval', 'capability'] },
     rationale: { type: 'string' },
+    missingCapability: { type: 'string' },
+    surface: { type: 'string' },
   },
   required: ['fault', 'rationale'],
 }
 
+const HOLDOUT_VERDICT_SCHEMA = {
+  type: 'object',
+  properties: {
+    verdict: { type: 'string', enum: ['pass', 'block'] },
+    kind: { type: 'string', enum: ['transfer', 'regression', 'indeterminate'] },
+    lift: { type: 'number' },
+    reason: { type: 'string' },
+  },
+  required: ['verdict', 'kind', 'reason'],
+}
+
 const GUARD_SCHEMA = {
   type: 'object',
   properties: {
@@ -75,7 +91,26 @@ async function scoreRaw(raw, round) {
   )
 }
 
+// The hold-out gate: score the EDITED skill against a fresh scenario the fixer never saw, and let a
+// deterministic CLI decide. The verdict is a comparison of numbers the agent reports, never a
+// judgement the agent makes (constitution P1).
+async function holdoutVerdict(raw, scenario, round) {
+  const payload = JSON.stringify({ ...raw, holdoutScenario: scenario })
+  return agent(
+    `Write this JSON to /tmp/harden-${skillId}-holdout-r${round}.json exactly, then run ` +
+    `\`node scripts/skill-behavior-eval.js --holdout /tmp/harden-${skillId}-holdout-r${round}.json\`. ` +
+    `Read its markdown output and return {verdict, kind, lift, reason} verbatim from that output — ` +
+    `verdict "pass" if the CLI exit code was 0, "block" if it was 1. Do NOT form your own opinion ` +
+    `about whether the fix was good: report what the CLI decided.\n\nJSON:\n${payload}`,
+    { label: `holdout:r${round}`, phase: 'Diagnose & Fix', schema: HOLDOUT_VERDICT_SCHEMA },
+  )
+}
+
 const history = []
+const holdout = []          // one record per fix round: did the hold-out run, and what did it say
+let lastFixHoldout = null   // verdict of the last round that left an edit on disk
+let lastFault = null        // last diagnosis, so the report can name a capability we cannot write
+let outcome = null
 let round = 0
 let committed = null
 
@@ -96,9 +131,16 @@ while (true) {
   const fault = await agent(
     `Follow scripts/skill-harden-rubric.md (Diagnosis). The skill "${skillId}" FAILED its behavioral gate. ` +
     `mustFix:\n- ${score.mustFix.join('\n- ')}\n\nGrader signals (both A/B outputs, per-item evidence):\n${evidence}\n\n` +
-    `Decide fault = 'skill' or 'eval' and give a rationale. Default to 'skill' when unsure.`,
+    `Decide fault = 'skill', 'eval' or 'capability' and give a rationale. Default to 'skill' when unsure.\n\n` +
+    `Choose 'capability' ONLY when the gap cannot be closed by words in a skill body: it needs an ` +
+    `executable capability — something that fires on a real execution event and returns to the agent ` +
+    `information it cannot obtain on its own (a hook, a script, a tool, a sub-agent). Advice that only ` +
+    `tells the agent to remember something is NOT a capability. If you choose 'capability' you MUST name ` +
+    `it in missingCapability and name where it would live in surface; a verdict that cannot name the ` +
+    `capability is an excuse, not a diagnosis — pick 'skill' instead.`,
     { label: `diagnose:r${round}`, phase: 'Diagnose & Fix', schema: FAULT_SCHEMA },
   )
+  lastFault = fault
 
   if (fault.fault === 'eval') {
     const judged = await agent(
@@ -115,31 +157,69 @@ while (true) {
     }
   }
 
+  if (fault.fault === 'capability') {
+    // Out of this loop's reach: it can only write words. Stop honestly instead of producing guidance
+    // that raises the score without the capability existing. Nothing is edited here on purpose.
+    outcome = 'capability-out-of-reach'
+    log(`${skillId}: needs a capability, not guidance — ${fault.missingCapability || '(unnamed)'} in ${fault.surface || '(unnamed surface)'}`)
+    break
+  }
+
   if (fault.fault === 'skill') {
     await agent(
       `Follow the author-skill discipline. The skill "${skillId}" must genuinely cover this mustFix without ` +
       `keyword-stuffing:\n- ${score.mustFix.join('\n- ')}\n\nEdit skills/${skillId}/SKILL.md (body) and/or files under ` +
       `skills/${skillId}/references/ to add the REAL missing capability (method, decision rules, concrete guidance). ` +
-      `Do not touch evals/. Apply the edits now.`,
+      `Do not touch evals/. Apply the edits now.\n\n` +
+      `HARD CONSTRAINT — the Generalization gate (scripts/skill-harden-rubric.md defines it; it binds every ` +
+      `line you write, and it binds BEFORE you write, not after):\n` +
+      `Everything you add is global context that will apply to cases this skill has never seen. Write a ` +
+      `reusable criterion that carries its own applicability condition — never the answer to the one ` +
+      `scenario that failed. Banned outright: identifiers or titles from the eval scenario, names of its ` +
+      `files/symbols/fixtures, rules that branch on its specific data, and reciting the finding as if it ` +
+      `were a rule ("the last round showed X needs Y").\n` +
+      `Litmus test before every line: "would this still help on a case in this domain I have never seen?" ` +
+      `If no, rewrite it as a criterion or drop it. A fix that only lifts this scenario's score will be ` +
+      `caught by the hold-out and reverted, and the round will be wasted.`,
       { label: `fix:r${round}`, phase: 'Diagnose & Fix' },
     )
     const diffJudge = await agent(
       `Follow scripts/skill-harden-rubric.md (Skill-fix guard 1). Run \`git diff -- skills/${skillId}/SKILL.md skills/${skillId}/references\` ` +
-      `and judge: does the diff add genuine capability, or just echo the mustFix wording to satisfy the grader? ` +
-      `If it is keyword-stuffing, run \`git checkout -- skills/${skillId}/SKILL.md skills/${skillId}/references\` to revert it. ` +
+      `and judge BOTH: (a) does the diff add genuine capability, or just echo the mustFix wording to satisfy the ` +
+      `grader? and (b) does every added line survive the Generalization gate — a reusable criterion with its ` +
+      `applicability condition, not the answer to the observed scenario (no eval-specific names, no branching on ` +
+      `its data, no recited findings)? Fail either check and it is not a fix. ` +
+      `If it fails, run \`git checkout -- skills/${skillId}/SKILL.md skills/${skillId}/references\` to revert it. ` +
       `Return {genuine, rationale}.`,
       { label: `diff-judge:r${round}`, phase: 'Diagnose & Fix', schema: GUARD_SCHEMA },
     )
 
-    if (diffJudge.genuine) {
-      // Guard 2: hold-out. Generate a fresh scenario and re-score the A/B on it only.
-      const holdout = await agent(
+    if (!diffJudge.genuine) {
+      holdout.push({ round, ran: false, why: 'edit reverted by the diff judge — nothing to validate' })
+    } else {
+      // Guard 2: hold-out. Score the EDITED skill on a fresh scenario the fixer never saw. The eval
+      // engine runs every scenario with AND without the skill, so this already carries its own lift.
+      const fresh = await agent(
         `Invent ONE fresh capability scenario for the "${skillId}" skill's domain that is NOT in its cases.yaml and ` +
         `does NOT enumerate its own requirements. Return {scenario, mustInclude:[3-6 outcome-level checks]}.`,
         { label: `holdout-gen:r${round}`, phase: 'Diagnose & Fix', schema: HOLDOUT_SCHEMA },
       )
-      const holdoutRaw = await workflow(EVAL, { skillId, scenarios: [holdout] })
-      await scoreRaw(holdoutRaw, `${round}-holdout`) // recorded in transcript; informs the next loop's eval
+      const holdoutRaw = await workflow(EVAL, { skillId, scenarios: [fresh] })
+      const verdict = await holdoutVerdict(holdoutRaw, fresh.scenario, round)
+      lastFixHoldout = verdict
+      holdout.push({ round, ran: true, scenario: fresh.scenario, ...verdict })
+
+      if (verdict.verdict === 'block') {
+        // The edit lifted this skill's own cases.yaml but does not transfer to work the fixer never
+        // saw. A memorized fix is not a fix: revert it. This is the guard the rubric always promised
+        // and the code never enforced — see 02-DOCS/wiki/sdd/specs/generalization-gate.md.
+        log(`${skillId}: hold-out ${verdict.kind} (lift ${verdict.lift}) — reverting round ${round}'s edit`)
+        await agent(
+          `Run exactly \`git checkout -- skills/${skillId}/SKILL.md skills/${skillId}/references\` and nothing else. ` +
+          `The edit failed the hold-out gate (${verdict.kind}) so it must not survive. Return the command's output.`,
+          { label: `holdout-revert:r${round}`, phase: 'Diagnose & Fix', effort: 'low' },
+        )
+      }
     }
   }
 
@@ -147,7 +227,13 @@ while (true) {
 }
 
 const passed = history.length > 0 && history[history.length - 1].pass === true
-if (passed && !noCommit) {
+// A main-gate pass whose last surviving edit failed the hold-out is NOT a pass: it is a fix that
+// only works on the cases it was tuned against. Conservative on purpose — a lost commit costs one
+// re-run, a certified overfit costs the catalog.
+const holdoutClean = !lastFixHoldout || lastFixHoldout.verdict === 'pass'
+let notCommittedBecause = null
+
+if (passed && holdoutClean && !noCommit) {
   phase('Commit')
   const commit = await agent(
     `The skill "${skillId}" now passes its behavioral gate. Commit ONLY its files: ` +
@@ -156,6 +242,24 @@ if (passed && !noCommit) {
     { label: `commit:${skillId}`, phase: 'Commit' },
   )
   committed = (commit || '').trim()
+} else if (outcome === 'capability-out-of-reach') {
+  notCommittedBecause = `needs a capability this loop cannot write: ${lastFault.missingCapability || '(unnamed)'} in ${lastFault.surface || '(unnamed surface)'}. Route it through specify — guidance cannot close this gap.`
+} else if (!passed) {
+  notCommittedBecause = 'the behavioral gate still fails. Read the last scorecard and fix by hand, or deprecate the skill.'
+} else if (!holdoutClean) {
+  notCommittedBecause = `the last edit passed cases.yaml but failed the hold-out (${lastFixHoldout.kind}, lift ${lastFixHoldout.lift}) and was reverted. Re-run to try a different fix, or write the criterion by hand.`
+} else if (noCommit) {
+  notCommittedBecause = 'noCommit was requested.'
 }
 
-return { skillId, rounds: round + 1, history, committed, passed }
+return {
+  skillId,
+  rounds: round + 1,
+  history,
+  holdout,
+  outcome: outcome || (passed && holdoutClean ? 'passed' : 'gave-up'),
+  missingCapability: lastFault && lastFault.fault === 'capability' ? { capability: lastFault.missingCapability, surface: lastFault.surface, rationale: lastFault.rationale } : null,
+  committed,
+  passed: passed && holdoutClean,
+  notCommittedBecause,
+}