@damiangilgonzalez/harness 1.3.7

package/scripts/skill-behavior-eval.js

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+// skill-behavior-eval.js — turn the behavior workflow's raw JSON into a scorecard.
+// Usage:
+//   node scripts/skill-behavior-eval.js --score   <raw.json> [--transcripts <dir>]
+//   node scripts/skill-behavior-eval.js --holdout <raw.json>   (fresh-scenario transfer gate)
+// Exits 0 if the gate passes, 1 if it fails/blocks, 2 on usage/parse error.
+//
+// --transcripts runs the integrity check (scripts/lib/eval-integrity.js) over the workflow's agent
+// transcripts: did the "without the skill" arm read the skill, or its grading rubric, or write into
+// docs? Any of those and the verdict is BLOCKED and the numbers are withheld, because a lift
+// measured against a contaminated control is not a measurement.
+//
+// Without --transcripts the scorecard says "integrity NOT CHECKED" on the verdict line. That is
+// deliberate: the flag is optional so existing callers keep working, so silence must not read as
+// clean.
+//
+// --holdout exists so the fix loop's blocking decision is a comparison of numbers an agent reports,
+// never a judgement an agent makes (constitution P1).
+
+import { readFileSync } from 'node:fs';
+import { scoreFromRaw, formatScorecard, holdoutGate, formatHoldoutVerdict } from './lib/behavior-score.js';
+import { checkIntegrity } from './lib/eval-integrity.js';
+
+function readInput(argPath) {
+  if (argPath && argPath !== '-') return readFileSync(argPath, 'utf8');
+  return readFileSync(0, 'utf8'); // stdin
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const mode = args[0];
+  if (mode !== '--score' && mode !== '--holdout') {
+    process.stderr.write('usage: skill-behavior-eval.js --score|--holdout <raw.json|->\n');
+    process.exit(2);
+  }
+  let raw;
+  try {
+    raw = JSON.parse(readInput(args[1] && args[1] !== '--transcripts' ? args[1] : '-'));
+  } catch (e) {
+    process.stderr.write(`parse error: ${e.message}\n`);
+    process.exit(2);
+  }
+  const ti = args.indexOf('--transcripts');
+  const transcriptsDir = ti !== -1 ? args[ti + 1] : null;
+  if (ti !== -1 && !transcriptsDir) {
+    process.stderr.write('--transcripts needs a directory\n');
+    process.exit(2);
+  }
+  const integrity = transcriptsDir
+    ? checkIntegrity({ skillId: (raw && raw.skillId) || '', transcriptsDir })
+    : undefined;
+  const scored = scoreFromRaw(raw, integrity);
+  if (mode === '--holdout') {
+    const gate = holdoutGate(scored);
+    process.stdout.write(formatHoldoutVerdict(scored, gate) + '\n');
+    process.exit(gate.verdict === 'pass' ? 0 : 1);
+  }
+  process.stdout.write(formatScorecard(scored) + '\n');
+  process.exit(scored.gate.pass ? 0 : 1);
+}
+
+main();

package/scripts/skill-behavior-eval.workflow.js

@@ -0,0 +1,158 @@
+export const meta = {
+  name: 'skill-behavior-eval',
+  description: 'Behaviorally score one rsc skill: run each capability scenario with and without the skill, blind-grade both, emit raw signals',
+  phases: [
+    { title: 'Load' },
+    { title: 'Execute & Grade' },
+  ],
+}
+
+const skillId = (typeof args === 'string' ? args : (args && args.skillId)) || ''
+if (!skillId) throw new Error('skill-behavior-eval: pass the skill id as args, e.g. "grants"')
+
+const GRADE_OUTPUT = {
+  type: 'object',
+  properties: {
+    mustInclude: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          item: { type: 'string' },
+          satisfied: { type: 'boolean' },
+          evidence: { type: 'string' },
+        },
+        required: ['item', 'satisfied', 'evidence'],
+      },
+    },
+    quality: {
+      type: 'object',
+      properties: {
+        completeness: { type: 'number' },
+        actionability: { type: 'number' },
+        correctness: { type: 'number' },
+        grounding: { type: 'number' },
+      },
+      required: ['completeness', 'actionability', 'correctness', 'grounding'],
+    },
+  },
+  required: ['mustInclude', 'quality'],
+}
+
+const LOAD_SCHEMA = {
+  type: 'object',
+  properties: {
+    skillId: { type: 'string' },
+    skillBody: { type: 'string' },
+    scenarios: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          scenario: { type: 'string' },
+          mustInclude: { type: 'array', items: { type: 'string' } },
+        },
+        required: ['scenario', 'mustInclude'],
+      },
+    },
+  },
+  required: ['skillId', 'skillBody', 'scenarios'],
+}
+
+const GRADE_SCHEMA = {
+  type: 'object',
+  properties: { gradeX: GRADE_OUTPUT, gradeY: GRADE_OUTPUT },
+  required: ['gradeX', 'gradeY'],
+}
+
+phase('Load')
+const injected = (typeof args === 'object' && args && Array.isArray(args.scenarios)) ? args.scenarios : null
+
+const loaded = injected
+  ? await (async () => {
+      const body = await agent(
+        `Read skills/${skillId}/SKILL.md and return its full text as "skillBody". ` +
+        `Return {skillId: "${skillId}", skillBody, scenarios: []}.`,
+        { label: `load-body:${skillId}`, phase: 'Load', schema: LOAD_SCHEMA, agentType: 'Explore' },
+      )
+      return { skillId, skillBody: (body && body.skillBody) || '', scenarios: injected }
+    })()
+  : await agent(
+      `Read two files and return structured data — do not summarize, return content verbatim where asked.\n` +
+      `1. skills/${skillId}/SKILL.md — return its full text as "skillBody".\n` +
+      `2. skills/${skillId}/evals/cases.yaml — under the "capability:" key is a list. For each entry return ` +
+      `{scenario: <the "scenario" string verbatim>, mustInclude: [<each string under that entry's "must_include">]}.\n` +
+      `Return {skillId: "${skillId}", skillBody, scenarios}. If either file is missing, return skillBody "" and scenarios [].`,
+      { label: `load:${skillId}`, phase: 'Load', schema: LOAD_SCHEMA, agentType: 'Explore' },
+    )
+
+if (!loaded || !Array.isArray(loaded.scenarios) || loaded.scenarios.length === 0) {
+  return { skillId, scenarios: [], error: 'no-capability-scenarios' }
+}
+
+// A distinct write zone per (scenario, arm). Under .rsc/, which is already gitignored, so an eval
+// write can neither dirty the tracked tree nor show up later as an abandoned diff.
+const sandbox = (index, arm) => `.rsc/eval-sandbox/${skillId}/${arm}-${index}/`
+
+phase('Execute & Grade')
+const results = await pipeline(
+  loaded.scenarios,
+  // Stage 1: baseline (no skill) + treatment (skill injected), concurrently.
+  (sc, _orig, index) => parallel([
+    () => agent(
+      `Complete this task fully and concretely. Produce the ACTUAL deliverable the task asks for, ` +
+      `not a description of how you would do it.\n\n` +
+      // The baseline IS the control. Reading the skill destroys the only thing this arm is for, so
+      // the prohibition carries its reason — a rule without one gets rationalised away first. This
+      // is guidance, not a barrier: the barrier is the post-hoc check in scripts/lib/eval-integrity.js,
+      // which BLOCKS the run if this arm touched the skill or its rubric.
+      `CONSTRAINTS — you are the control arm of a measurement:\n` +
+      `- Do NOT read anything under skills/ . You are the "without the skill" arm; if you read the ` +
+      `skill under test there is no control and the measurement is void.\n` +
+      `- Do NOT read any evals/cases.yaml — that is the rubric you are being graded against.\n` +
+      `- The repository is READ-ONLY. Never write to docs/ or to any tracked file.\n` +
+      `- Write every file you produce under ${sandbox(index, 'baseline')} .\n\n` +
+      `TASK:\n${sc.scenario}`,
+      { label: `baseline:${index}`, phase: 'Execute & Grade' },
+    ),
+    () => agent(
+      `You have a skill loaded — follow it. Its reference files live under ` +
+      `skills/${loaded.skillId}/references/; read them if the skill points you there.\n\n` +
+      `=== SKILL: ${loaded.skillId} ===\n${loaded.skillBody}\n=== END SKILL ===\n\n` +
+      `Complete this task fully and concretely. Produce the ACTUAL deliverable, not a description.\n\n` +
+      // Both arms once wrote the same path (docs/wiki/sdd/specs/magic-link-login.md) and one
+      // clobbered the other mid-run, so the grader scored a file that no longer existed. A distinct
+      // sandbox per arm makes that collision impossible by construction rather than by luck.
+      `CONSTRAINTS:\n` +
+      `- The repository is READ-ONLY. Never write to docs/ or to any tracked file: this is a ` +
+      `measurement, and it must not mutate the project it runs inside.\n` +
+      `- Write every file you produce under ${sandbox(index, 'treatment')} .\n\n` +
+      `TASK:\n${sc.scenario}`,
+      { label: `treatment:${index}`, phase: 'Execute & Grade' },
+    ),
+  ]).then(([baselineOut, treatmentOut]) => ({ sc, index, baselineOut, treatmentOut })),
+  // Stage 2: blind grade. X/Y slot varied by index parity; mapping recorded for the scorer.
+  (prev) => {
+    if (!prev) return null
+    const { sc, index, baselineOut, treatmentOut } = prev
+    const xIsTreatment = index % 2 === 0
+    const outX = (xIsTreatment ? treatmentOut : baselineOut) || '(empty)'
+    const outY = (xIsTreatment ? baselineOut : treatmentOut) || '(empty)'
+    const checklist = sc.mustInclude.map((m, i) => `  ${i + 1}. ${m}`).join('\n')
+    return agent(
+      `You are an adversarial, independent grader. Two AI outputs answer the same task. You do NOT ` +
+      `know which (if any) used a helper skill — judge only what is on the page.\n\n` +
+      `TASK:\n${sc.scenario}\n\n` +
+      `REQUIRED ELEMENTS (must_include):\n${checklist}\n\n` +
+      `For EACH output and EACH required element, set satisfied true/false and quote the exact line ` +
+      `that satisfies it (empty string if unmet). Be skeptical: if unsure, mark unmet. Then rate four ` +
+      `quality axes 0-10: completeness, actionability, correctness, grounding (no invented facts).\n\n` +
+      `=== OUTPUT X ===\n${outX}\n=== OUTPUT Y ===\n${outY}\n`,
+      { label: `grade:${index}`, phase: 'Execute & Grade', schema: GRADE_SCHEMA },
+    )
+      .then((g) => ({ index, xIsTreatment, gradeX: g.gradeX, gradeY: g.gradeY }))
+      .catch(() => ({ index, xIsTreatment, error: 'grade-failed' }))
+  },
+)
+
+return { skillId: loaded.skillId, scenarios: results.filter(Boolean) }

package/scripts/skill-behavior-rubric.md

@@ -0,0 +1,63 @@
+# skill-behavior-rubric — how a skill's OUTPUT is graded (the behavioral gate)
+
+Sibling of `skill-rubric.md`. That one scores the skill as a **document**; this one scores what
+the skill **produces when run**. A skill ships only if BOTH gates are green.
+
+## How the behavioral eval works
+
+For each `capability` scenario in the skill's `evals/cases.yaml`, the engine
+(`scripts/skill-behavior-eval.workflow.js`) runs the scenario twice — once with the skill's
+`SKILL.md` injected (**treatment**), once without it (**baseline**) — and an independent, blind
+grader scores both outputs. The grader never learns which output used the skill; outputs are
+presented as X and Y, their slot varied by scenario index parity so a "first output" bias can't
+systematically favor the skill.
+
+## What the grader returns (per output)
+
+1. **must_include coverage.** For each `must_include` item: `satisfied` true/false **and a quoted
+   line** from the output that satisfies it. Evidence, not vibes. If unsure, mark unmet.
+2. **Quality, four axes 0-10:** `completeness`, `actionability`, `correctness`, `grounding`
+   (no invented facts / APIs / numbers).
+
+The grader returns raw signals only. It does **not** compute the score — that keeps the number
+deterministic and auditable.
+
+## Orientation signal (the brújula)
+
+User-facing, conversational skills should keep the user oriented. When a skill's capability scenario ends a turn the user acts on, the grader expects the output to:
+
+- **Situate** the user (where they are / what is built vs missing).
+- **Teach the why**, scaled to the dial (`technical_level` / `accompaniment_level`).
+- **Propose a next step phrased as a question** — never end in seco.
+
+Encode this as `must_include` items in those skills' capability scenarios (see `skills/orient/evals/cases.yaml` for the canonical pattern). It scores through the existing coverage axis — no separate formula. Skills that own a purely mechanical task (lint, a single rename) are exempt and should not be penalized.
+
+## How signals become the /10 (computed by `scripts/lib/behavior-score.js`)
+
+- Per output: `score = 0.6 × (satisfied / total × 10) + 0.4 × mean(quality axes)`.
+  Empty output → 0. No checklist → quality only.
+- Per scenario: `absolute` = treatment's score; `delta` = treatment − baseline.
+- Per skill: `absolute_score` = mean of absolutes; `lift` = mean of deltas.
+
+## The gate (both required)
+
+- `absolute_score ≥ 8.5` — the skill genuinely produces a good result.
+- `lift ≥ +1.0` — the skill measurably beats no-skill. `lift ≤ 0` is an automatic FAIL even at a
+  high absolute: the skill adds nothing the bare agent didn't already do.
+
+## Anti-gaming
+
+- Independent, blind grader; default-skeptical on coverage.
+- Errors fail closed: a scenario whose run or grade errored is dropped; if all drop, FAIL.
+- A skill with no `capability` scenario cannot be behaviorally evaluated → FAIL until one exists.
+- v1 uses a single grader per scenario. Upgrade path: N=3 graders, majority vote.
+
+## Running it
+
+```
+# 1) execute + grade — invoke the Workflow tool with the engine script and the skill id:
+#      scriptPath: scripts/skill-behavior-eval.workflow.js   args: "<skill-id>"
+#    capture the workflow's returned object to a JSON file (e.g. /tmp/<skill>-raw.json)
+# 2) score + gate (exit 0 pass / 1 fail):
+node scripts/skill-behavior-eval.js --score /tmp/<skill>-raw.json
+```

package/scripts/skill-harden-rubric.md

@@ -0,0 +1,102 @@
+# skill-harden-rubric — how the fix-loop diagnoses and guards a fix
+
+The `skill-harden` workflow turns a FAILING behavioral score into edits, without gaming the eval.
+These are the rules its agents follow.
+
+The three guards below — **diff judge**, **hold-out**, **Generalization gate** — are mandatory *and*
+mechanised: `tests/harden-generalization-gate.test.js` asserts, for each of those three, that the
+loop implements what this file promises. **Adding a fourth guard here means adding its pair to that
+test**; a guard documented but unchecked gives the safety without the protection, which is the
+failure mode this repo has now hit ten times (constitution P2).
+
+## Diagnosis (skill-fault vs eval-fault vs capability)
+
+Given the failing `mustFix`, both A/B outputs, and the grader's per-item evidence, decide the FAULT:
+
+- **skill** — the treatment output genuinely misses real capability (weak method, missing depth,
+  no concrete output). Fix the SKILL.md body and/or references/.
+- **eval** — the failure is the eval's, not the skill's. Two known biases:
+  1. **Self-describing scenario** — the `scenario` text enumerates its own `must_include`, so a
+     bare agent gets the same guidance the skill would give and the lift collapses artificially.
+  2. **Phantom-context must_include** — an item demands workspace artifacts the isolated eval
+     agent cannot have (e.g. `user-profile.md`, a real sibling skill to delegate to), capping the
+     absolute unfairly.
+- **capability** — the gap cannot be closed by words in a skill body. It needs an **executable
+  capability**: something that fires on a real execution event and returns to the agent information
+  it cannot obtain on its own — a hook, a script, a tool, a sub-agent. Advice that merely tells the
+  agent to remember something is **not** a capability: it is not anchored to any observable event, so
+  it can fire when it should not and stay silent when it should not.
+
+This loop can only ever write **guidance** (SKILL.md, references/). So `capability` is not a fix
+route, it is the honest exit: the loop edits nothing, names the missing capability and the surface it
+would live on, and stops. Writing a paragraph that pretends to cover a missing capability raises the
+score without the capability existing — the worst outcome available.
+
+A `capability` verdict **must name** the capability and its surface. One that cannot is an excuse,
+not a diagnosis → choose `skill`.
+
+Default to **skill** when unsure: blaming the eval or the surface area is the easier, less honest path.
+
+## The Generalization gate (ex-ante — it binds before the edit is written)
+
+This gate governs the **content** of every line the fixer writes, and it travels **in the fixer's own
+instruction**, not in a review afterwards. Writing freely and reverting later burns the round, and the
+loop only has two.
+
+Everything added to a skill is global context that will apply to cases the skill has never seen. So:
+
+- **Allowed:** a reusable criterion that carries its own **applicability condition** — something the
+  agent can evaluate on a case it has never encountered.
+- **Banned outright:** identifiers or titles from the eval scenario; names of its files, symbols or
+  fixtures; rules that branch on its specific data; and reciting the finding as if it were a rule
+  ("the last round showed X needs Y" — that is an answer, not a criterion).
+
+**Litmus test, before every line:** *would this still help on a case in this domain I have never
+seen?* If no → rewrite it as a criterion, or drop it. Not knowledge → not in the skill.
+
+Why ex-ante and not only as review: moving this constraint from after-the-edit to
+before-the-edit is what buys generalization cheaply. The measured version of this result — plus the
+warning that richer feedback **without** this gate makes held-out performance *worse* — is in
+[HarnessCompass](https://arxiv.org/abs/2608.01918); the reasoning it drove is recorded in
+`docs/wiki/sdd/specs/generalization-gate.md`.
+
+## Eval-fix guard (independent judge)
+
+An eval edit ships ONLY if a judge certifies it corrects one of the two biases above and does NOT
+lower the bar — i.e. it makes the scenario less self-describing or removes a phantom-context item,
+but never deletes a legitimate quality criterion to make a weak skill pass. Rejected → treat as a
+skill-fault this round.
+
+## Skill-fix guards (both required)
+
+1. **Diff judge.** Read the SKILL.md/references diff and check BOTH: does it add genuine capability
+   (method, decision rules, concrete guidance) rather than echoing the `must_include` wording into
+   the body? *and* does every added line survive the Generalization gate above? Fail either → revert
+   the edit; it is not a fix.
+2. **Hold-out.** Re-score the EDITED skill on a FRESH scenario from its domain that the fixer never
+   saw. A real improvement generalizes; a memorized one does not.
+   - The eval engine runs every scenario with **and** without the skill, so the fresh scenario
+     already carries its own lift. No pre-fix baseline is needed.
+   - The verdict is computed by `node scripts/skill-behavior-eval.js --holdout <raw.json>` — a
+     comparison of numbers, deterministic and reproducible, never an agent's opinion (P1). Exit 0
+     passes, exit 1 blocks.
+   - **Blocks on `lift <= 0`** (`regression`): on unseen work the edited skill does not beat the bare
+     agent, so the edit did not transfer. Also blocks when the fresh scenario could not be run or
+     graded (`indeterminate`) — failing closed, because an unverified edit is not a verified one.
+   - The bar is deliberately **not** the main gate's `lift >= 1.0`: one fresh scenario graded once is
+     noisy, so only the unambiguous signal acts. Upgrade path if it proves too noisy in practice:
+     N=3 fresh scenarios, majority.
+   - **A block reverts the edit.** The round produced nothing and is recorded with its reason.
+
+## Stopping & honesty
+
+- Max 2 rounds. On give-up, report the honest final score and a recommendation — never a faked pass.
+- **A main-gate pass whose last surviving edit failed the hold-out does not commit.** It is a fix
+  tuned to the cases it was measured against; the loop reverts it and reports why. Conservative on
+  purpose: a lost commit costs one re-run, a certified overfit costs the catalog.
+- A `capability` verdict never commits either: it returns the missing capability and its surface, and
+  routes the work to `specify`.
+- Every non-commit says which condition failed, in the returned result. A denial without its exit is
+  abandonment (P6).
+- Persistent lift-fail at a high absolute → recommend deprecate/merge: the skill does not justify
+  its own existence.

package/scripts/skill-harden.workflow.js

@@ -0,0 +1,265 @@
+export const meta = {
+  name: 'skill-harden',
+  description: 'Detect AND fix one rsc skill: behaviorally score it, diagnose the failure, edit the right artifact with anti-gaming guards, re-verify, and commit on pass (max 2 rounds)',
+  phases: [
+    { title: 'Evaluate' },
+    { title: 'Diagnose & Fix' },
+    { title: 'Commit' },
+  ],
+}
+
+// args may arrive as an object {skillId, noCommit} or as a string "debug" / "debug noCommit".
+function parseArgs(a) {
+  if (typeof a === 'object' && a) return { skillId: a.skillId || '', noCommit: a.noCommit === true }
+  if (typeof a === 'string') {
+    const toks = a.trim().split(/[\s,]+/).filter(Boolean)
+    const id = toks.find((t) => !/^(no[-_]?commit|--no-commit)$/i.test(t)) || ''
+    const nc = toks.some((t) => /^(no[-_]?commit|--no-commit)$/i.test(t))
+    return { skillId: id, noCommit: nc }
+  }
+  return { skillId: '', noCommit: false }
+}
+const { skillId, noCommit } = parseArgs(args)
+if (!skillId) throw new Error('skill-harden: pass the skill id as args, e.g. "debug" or "debug noCommit"')
+const EVAL = { scriptPath: 'scripts/skill-behavior-eval.workflow.js' }
+
+const MAX_ROUNDS = 2
+
+const SCORE_SCHEMA = {
+  type: 'object',
+  properties: {
+    pass: { type: 'boolean' },
+    absolute: { type: 'number' },
+    lift: { type: 'number' },
+    mustFix: { type: 'array', items: { type: 'string' } },
+  },
+  required: ['pass', 'absolute', 'lift', 'mustFix'],
+}
+
+const FAULT_SCHEMA = {
+  type: 'object',
+  properties: {
+    // 'capability' is the honest third exit: this loop can only ever write guidance (SKILL.md /
+    // references), so a failure that needs an executable capability is out of its reach. Saying so
+    // beats writing a paragraph that pretends to cover it.
+    fault: { type: 'string', enum: ['skill', 'eval', 'capability'] },
+    rationale: { type: 'string' },
+    missingCapability: { type: 'string' },
+    surface: { type: 'string' },
+  },
+  required: ['fault', 'rationale'],
+}
+
+const HOLDOUT_VERDICT_SCHEMA = {
+  type: 'object',
+  properties: {
+    verdict: { type: 'string', enum: ['pass', 'block'] },
+    kind: { type: 'string', enum: ['transfer', 'regression', 'indeterminate'] },
+    lift: { type: 'number' },
+    reason: { type: 'string' },
+  },
+  required: ['verdict', 'kind', 'reason'],
+}
+
+const GUARD_SCHEMA = {
+  type: 'object',
+  properties: {
+    genuine: { type: 'boolean' },
+    rationale: { type: 'string' },
+  },
+  required: ['genuine', 'rationale'],
+}
+
+const HOLDOUT_SCHEMA = {
+  type: 'object',
+  properties: {
+    scenario: { type: 'string' },
+    mustInclude: { type: 'array', items: { type: 'string' } },
+  },
+  required: ['scenario', 'mustInclude'],
+}
+
+// Score a raw eval result via the authoritative CLI (keeps math in behavior-score.js).
+async function scoreRaw(raw, round) {
+  return agent(
+    `Write this JSON to /tmp/harden-${skillId}-r${round}.json exactly, then run ` +
+    `\`node scripts/skill-behavior-eval.js --score /tmp/harden-${skillId}-r${round}.json\`. ` +
+    `Read its markdown output and return {pass, absolute, lift, mustFix}: pass = the CLI exit code was 0 ` +
+    `(scorecard says PASS), absolute and lift from the Verdict line, mustFix = the bullet list under "Must fix" ` +
+    `(empty array if none).\n\nJSON:\n${JSON.stringify(raw)}`,
+    { label: `score:r${round}`, phase: 'Evaluate', schema: SCORE_SCHEMA },
+  )
+}
+
+// The hold-out gate: score the EDITED skill against a fresh scenario the fixer never saw, and let a
+// deterministic CLI decide. The verdict is a comparison of numbers the agent reports, never a
+// judgement the agent makes (constitution P1).
+async function holdoutVerdict(raw, scenario, round) {
+  const payload = JSON.stringify({ ...raw, holdoutScenario: scenario })
+  return agent(
+    `Write this JSON to /tmp/harden-${skillId}-holdout-r${round}.json exactly, then run ` +
+    `\`node scripts/skill-behavior-eval.js --holdout /tmp/harden-${skillId}-holdout-r${round}.json\`. ` +
+    `Read its markdown output and return {verdict, kind, lift, reason} verbatim from that output — ` +
+    `verdict "pass" if the CLI exit code was 0, "block" if it was 1. Do NOT form your own opinion ` +
+    `about whether the fix was good: report what the CLI decided.\n\nJSON:\n${payload}`,
+    { label: `holdout:r${round}`, phase: 'Diagnose & Fix', schema: HOLDOUT_VERDICT_SCHEMA },
+  )
+}
+
+const history = []
+const holdout = []          // one record per fix round: did the hold-out run, and what did it say
+let lastFixHoldout = null   // verdict of the last round that left an edit on disk
+let lastFault = null        // last diagnosis, so the report can name a capability we cannot write
+let outcome = null
+let round = 0
+let committed = null
+
+while (true) {
+  phase('Evaluate')
+  const raw = await workflow(EVAL, skillId)
+  if (raw && raw.error === 'no-capability-scenarios') {
+    return { skillId, error: 'no-capability-scenarios', history, committed: null }
+  }
+  const score = await scoreRaw(raw, round)
+  history.push({ absolute: score.absolute, lift: score.lift, pass: score.pass })
+
+  if (score.pass) break
+  if (round >= MAX_ROUNDS) break
+
+  phase('Diagnose & Fix')
+  const evidence = JSON.stringify(raw.scenarios || [])
+  const fault = await agent(
+    `Follow scripts/skill-harden-rubric.md (Diagnosis). The skill "${skillId}" FAILED its behavioral gate. ` +
+    `mustFix:\n- ${score.mustFix.join('\n- ')}\n\nGrader signals (both A/B outputs, per-item evidence):\n${evidence}\n\n` +
+    `Decide fault = 'skill', 'eval' or 'capability' and give a rationale. Default to 'skill' when unsure.\n\n` +
+    `Choose 'capability' ONLY when the gap cannot be closed by words in a skill body: it needs an ` +
+    `executable capability — something that fires on a real execution event and returns to the agent ` +
+    `information it cannot obtain on its own (a hook, a script, a tool, a sub-agent). Advice that only ` +
+    `tells the agent to remember something is NOT a capability. If you choose 'capability' you MUST name ` +
+    `it in missingCapability and name where it would live in surface; a verdict that cannot name the ` +
+    `capability is an excuse, not a diagnosis — pick 'skill' instead.`,
+    { label: `diagnose:r${round}`, phase: 'Diagnose & Fix', schema: FAULT_SCHEMA },
+  )
+  lastFault = fault
+
+  if (fault.fault === 'eval') {
+    const judged = await agent(
+      `Follow scripts/skill-harden-rubric.md (Eval-fix guard). Proposed: edit skills/${skillId}/evals/cases.yaml to ` +
+      `correct an eval bias (self-describing scenario, or a phantom-context must_include item). ` +
+      `Rationale from diagnosis: ${fault.rationale}\n\n` +
+      `Decide genuine=true ONLY if the change corrects a real bias WITHOUT lowering the bar. If genuine, APPLY the ` +
+      `edit to cases.yaml now (Edit tool); if not, change nothing. Return {genuine, rationale}.`,
+      { label: `eval-judge:r${round}`, phase: 'Diagnose & Fix', schema: GUARD_SCHEMA },
+    )
+    if (!judged.genuine) {
+      // Eval blamed but not justified -> fall through to a skill fix this round.
+      fault.fault = 'skill'
+    }
+  }
+
+  if (fault.fault === 'capability') {
+    // Out of this loop's reach: it can only write words. Stop honestly instead of producing guidance
+    // that raises the score without the capability existing. Nothing is edited here on purpose.
+    outcome = 'capability-out-of-reach'
+    log(`${skillId}: needs a capability, not guidance — ${fault.missingCapability || '(unnamed)'} in ${fault.surface || '(unnamed surface)'}`)
+    break
+  }
+
+  if (fault.fault === 'skill') {
+    await agent(
+      `Follow the author-skill discipline. The skill "${skillId}" must genuinely cover this mustFix without ` +
+      `keyword-stuffing:\n- ${score.mustFix.join('\n- ')}\n\nEdit skills/${skillId}/SKILL.md (body) and/or files under ` +
+      `skills/${skillId}/references/ to add the REAL missing capability (method, decision rules, concrete guidance). ` +
+      `Do not touch evals/. Apply the edits now.\n\n` +
+      `HARD CONSTRAINT — the Generalization gate (scripts/skill-harden-rubric.md defines it; it binds every ` +
+      `line you write, and it binds BEFORE you write, not after):\n` +
+      `Everything you add is global context that will apply to cases this skill has never seen. Write a ` +
+      `reusable criterion that carries its own applicability condition — never the answer to the one ` +
+      `scenario that failed. Banned outright: identifiers or titles from the eval scenario, names of its ` +
+      `files/symbols/fixtures, rules that branch on its specific data, and reciting the finding as if it ` +
+      `were a rule ("the last round showed X needs Y").\n` +
+      `Litmus test before every line: "would this still help on a case in this domain I have never seen?" ` +
+      `If no, rewrite it as a criterion or drop it. A fix that only lifts this scenario's score will be ` +
+      `caught by the hold-out and reverted, and the round will be wasted.`,
+      { label: `fix:r${round}`, phase: 'Diagnose & Fix' },
+    )
+    const diffJudge = await agent(
+      `Follow scripts/skill-harden-rubric.md (Skill-fix guard 1). Run \`git diff -- skills/${skillId}/SKILL.md skills/${skillId}/references\` ` +
+      `and judge BOTH: (a) does the diff add genuine capability, or just echo the mustFix wording to satisfy the ` +
+      `grader? and (b) does every added line survive the Generalization gate — a reusable criterion with its ` +
+      `applicability condition, not the answer to the observed scenario (no eval-specific names, no branching on ` +
+      `its data, no recited findings)? Fail either check and it is not a fix. ` +
+      `If it fails, run \`git checkout -- skills/${skillId}/SKILL.md skills/${skillId}/references\` to revert it. ` +
+      `Return {genuine, rationale}.`,
+      { label: `diff-judge:r${round}`, phase: 'Diagnose & Fix', schema: GUARD_SCHEMA },
+    )
+
+    if (!diffJudge.genuine) {
+      holdout.push({ round, ran: false, why: 'edit reverted by the diff judge — nothing to validate' })
+    } else {
+      // Guard 2: hold-out. Score the EDITED skill on a fresh scenario the fixer never saw. The eval
+      // engine runs every scenario with AND without the skill, so this already carries its own lift.
+      const fresh = await agent(
+        `Invent ONE fresh capability scenario for the "${skillId}" skill's domain that is NOT in its cases.yaml and ` +
+        `does NOT enumerate its own requirements. Return {scenario, mustInclude:[3-6 outcome-level checks]}.`,
+        { label: `holdout-gen:r${round}`, phase: 'Diagnose & Fix', schema: HOLDOUT_SCHEMA },
+      )
+      const holdoutRaw = await workflow(EVAL, { skillId, scenarios: [fresh] })
+      const verdict = await holdoutVerdict(holdoutRaw, fresh.scenario, round)
+      lastFixHoldout = verdict
+      holdout.push({ round, ran: true, scenario: fresh.scenario, ...verdict })
+
+      if (verdict.verdict === 'block') {
+        // The edit lifted this skill's own cases.yaml but does not transfer to work the fixer never
+        // saw. A memorized fix is not a fix: revert it. This is the guard the rubric always promised
+        // and the code never enforced — see docs/wiki/sdd/specs/generalization-gate.md.
+        log(`${skillId}: hold-out ${verdict.kind} (lift ${verdict.lift}) — reverting round ${round}'s edit`)
+        await agent(
+          `Run exactly \`git checkout -- skills/${skillId}/SKILL.md skills/${skillId}/references\` and nothing else. ` +
+          `The edit failed the hold-out gate (${verdict.kind}) so it must not survive. Return the command's output.`,
+          { label: `holdout-revert:r${round}`, phase: 'Diagnose & Fix', effort: 'low' },
+        )
+      }
+    }
+  }
+
+  round++
+}
+
+const passed = history.length > 0 && history[history.length - 1].pass === true
+// A main-gate pass whose last surviving edit failed the hold-out is NOT a pass: it is a fix that
+// only works on the cases it was tuned against. Conservative on purpose — a lost commit costs one
+// re-run, a certified overfit costs the catalog.
+const holdoutClean = !lastFixHoldout || lastFixHoldout.verdict === 'pass'
+let notCommittedBecause = null
+
+if (passed && holdoutClean && !noCommit) {
+  phase('Commit')
+  const commit = await agent(
+    `The skill "${skillId}" now passes its behavioral gate. Commit ONLY its files: ` +
+    `run \`git add skills/${skillId}\` then commit with a message describing the hardening. ` +
+    `Author is Eric — do NOT add any Claude co-author or generated footer. Return the commit hash as plain text.`,
+    { label: `commit:${skillId}`, phase: 'Commit' },
+  )
+  committed = (commit || '').trim()
+} else if (outcome === 'capability-out-of-reach') {
+  notCommittedBecause = `needs a capability this loop cannot write: ${lastFault.missingCapability || '(unnamed)'} in ${lastFault.surface || '(unnamed surface)'}. Route it through specify — guidance cannot close this gap.`
+} else if (!passed) {
+  notCommittedBecause = 'the behavioral gate still fails. Read the last scorecard and fix by hand, or deprecate the skill.'
+} else if (!holdoutClean) {
+  notCommittedBecause = `the last edit passed cases.yaml but failed the hold-out (${lastFixHoldout.kind}, lift ${lastFixHoldout.lift}) and was reverted. Re-run to try a different fix, or write the criterion by hand.`
+} else if (noCommit) {
+  notCommittedBecause = 'noCommit was requested.'
+}
+
+return {
+  skillId,
+  rounds: round + 1,
+  history,
+  holdout,
+  outcome: outcome || (passed && holdoutClean ? 'passed' : 'gave-up'),
+  missingCapability: lastFault && lastFault.fault === 'capability' ? { capability: lastFault.missingCapability, surface: lastFault.surface, rationale: lastFault.rationale } : null,
+  committed,
+  passed: passed && holdoutClean,
+  notCommittedBecause,
+}