@ericrisco/rsc 1.0.12 → 1.0.14

package/manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.12",
+  "version": "1.0.14",
   "counts": {
     "skills": 264
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ericrisco/rsc",
-  "version": "1.0.12",
+  "version": "1.0.14",
   "description": "Eric Risco's agent-skills catalog as a granular, self-recommending CLI installer.",
   "type": "module",
   "bin": {

package/scripts/lib/behavior-score.js

@@ -41,7 +41,25 @@ export function aggregate(scenarios) {
   return { absoluteScore, lift, n: valid.length, dropped };
 }
 
-export function behavioralGate(agg) {
+// integrity is OPTIONAL and absent means "not checked", never "clean" — callers that predate the
+// integrity layer (holdoutGate, raw stdin scoring) keep their exact behaviour.
+//
+// A violated integrity is BLOCKED, not FAIL, and the distinction is the whole point: FAIL says "the
+// skill does not clear the bar" and sends someone to edit the skill. When the control read the
+// answer key, the only true statement is "this run measures nothing" — a different action entirely.
+export function behavioralGate(agg, integrity) {
+  if (integrity && integrity.ok === false) {
+    const vs = Array.isArray(integrity.violations) ? integrity.violations : [];
+    return {
+      pass: false,
+      blocked: true,
+      reasons: [integrity.reason || 'integrity check failed'],
+      mustFix: vs.length
+        ? vs.map((v) => `${v.kind}: ${v.agent} touched ${v.pattern} (x${v.count}) — re-run with the arm isolated.`)
+        : ['Re-run the eval with integrity verifiable; the scores from this run are not a measurement.'],
+      violations: vs,
+    };
+  }
   const reasons = [];
   const mustFix = [];
   if (!agg || agg.n === 0) {
@@ -118,7 +136,7 @@ export function formatHoldoutVerdict(scored, gate) {
 }
 
 // raw: { skillId, scenarios: [{index, xIsTreatment, gradeX, gradeY, error?}] }
-export function scoreFromRaw(raw) {
+export function scoreFromRaw(raw, integrity) {
   const scenarios = (raw && Array.isArray(raw.scenarios) ? raw.scenarios : []).map((s) => {
     if (!s || s.error || !s.gradeX || !s.gradeY) {
       return { index: s ? s.index : null, error: (s && s.error) || 'missing-grade', absolute: null, delta: null };
@@ -131,8 +149,9 @@ export function scoreFromRaw(raw) {
     return { index: s.index, treatment: t, baseline: b, absolute: d.absolute, delta: d.delta };
   });
   const agg = aggregate(scenarios);
-  const gate = behavioralGate(agg);
+  const gate = behavioralGate(agg, integrity);
   const out = { skillId: (raw && raw.skillId) || null, scenarios, aggregate: agg, gate };
+  if (integrity) out.integrity = integrity;
   // Carried through untouched so the hold-out verdict can echo the fresh scenario for hand-audit.
   if (raw && raw.holdoutScenario) out.holdoutScenario = raw.holdoutScenario;
   return out;
@@ -143,7 +162,18 @@ export function formatScorecard(scored) {
   const lines = [];
   lines.push(`# Behavioral scorecard — ${skillId || '(unknown skill)'}`);
   lines.push('');
-  lines.push(`**Verdict:** ${gate.pass ? 'PASS ✅' : 'FAIL ❌'}  ·  absolute ${agg.absoluteScore == null ? 'n/a' : agg.absoluteScore}/10 (gate >= ${ABS_MIN})  ·  lift ${agg.lift == null ? 'n/a' : agg.lift} (gate >= ${LIFT_MIN})  ·  n=${agg.n}${agg.dropped ? ` (${agg.dropped} dropped)` : ''}`);
+  if (gate.blocked) {
+    // Deliberately NO absolute and NO lift here. Printing them beside a block invites reading them
+    // anyway, and they are not measurements — that is exactly what the block is saying.
+    lines.push(`**Verdict:** BLOCKED ⛔  ·  ${gate.reasons[0]}`);
+    lines.push('');
+    lines.push('The scores from this run are withheld: they are not a measurement of this skill.');
+    lines.push('');
+    lines.push('**Integrity violations:**');
+    for (const f of gate.mustFix) lines.push(`- ${f}`);
+    return lines.join('\n');
+  }
+  lines.push(`**Verdict:** ${gate.pass ? 'PASS ✅' : 'FAIL ❌'}  ·  absolute ${agg.absoluteScore == null ? 'n/a' : agg.absoluteScore}/10 (gate >= ${ABS_MIN})  ·  lift ${agg.lift == null ? 'n/a' : agg.lift} (gate >= ${LIFT_MIN})  ·  n=${agg.n}${agg.dropped ? ` (${agg.dropped} dropped)` : ''}${scored.integrity ? '  ·  integrity verified' : '  ·  integrity NOT CHECKED (pass --transcripts)'}`);
   lines.push('');
   lines.push('| Scenario | Treatment | Baseline | Delta |');
   lines.push('|---|---|---|---|');

package/scripts/lib/eval-integrity.js

@@ -0,0 +1,154 @@
+// eval-integrity.js — is a behavioral eval run measuring anything at all?
+//
+// skill-behavior-eval promises to run each capability scenario "with and without the skill". The
+// "without" arm runs with full filesystem access inside this repo, where the skill under test IS a
+// file, and nothing stopped it reading that file. Measured in the 2026-08-18 run: the `verify`
+// baseline read skills/verify/SKILL.md 7 times AND skills/verify/evals/cases.yaml 4 times — the
+// very must_include list it was being graded against. It then produced "SUSTITUIDA", a word that
+// exists nowhere but the skill written the day before.
+//
+// A lift measured against a control that read the answer key is a lower bound of unknown size.
+// Publishing it as a number is the claim-without-evidence this repo keeps paying for (P2), and it
+// feeds skill-harden's --holdout, which decides what enters a 264-skill catalog.
+//
+// This module answers the question deterministically, from the transcripts themselves — never from
+// what an agent says about its own behaviour, which is the way this fails open.
+//
+// Design note: the pure functions take text, the reader touches disk. That split is what makes the
+// interesting logic testable without fixtures on disk.
+
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+
+// Prompt markers the workflow actually emits. tests/eval-integrity.test.js asserts each of these
+// still appears verbatim in scripts/skill-behavior-eval.workflow.js — without that tie, rewording a
+// prompt silently retires the classifier and every run starts reporting "clean".
+export const ROLE_MARKERS = {
+  grader: 'adversarial, independent grader',
+  treatment: '=== END SKILL ===',
+  loader: 'return its full text as',
+};
+
+// Write verbs that indicate a transcript actually WROTE a path rather than merely naming it. A
+// mention is not a mutation: the grader quotes repo paths constantly and is not violating anything.
+const WRITE_SIGNALS = [
+  'Write', 'Edit', 'MultiEdit', 'NotebookEdit',
+  'cat >', 'tee ', '>>', 'mv ', 'cp ', 'rm ',
+];
+
+// Paths an eval arm may never write. 02-DOCS is the project's brain and is untracked (P9), so a
+// stray write there has no undo.
+export const PROTECTED_PATHS = ['02-DOCS/wiki/'];
+
+/**
+ * Which arm produced this transcript. Order matters: a grader transcript quotes the skill body, so
+ * it would otherwise look like a treatment.
+ */
+export function classifyAgent(transcriptText) {
+  const t = String(transcriptText || '');
+  if (t.includes(ROLE_MARKERS.grader)) return 'grader';
+  if (t.includes(ROLE_MARKERS.treatment)) return 'treatment';
+  if (t.includes(ROLE_MARKERS.loader)) return 'loader';
+  return 'baseline';
+}
+
+const countOf = (text, needle) => {
+  let n = 0;
+  let i = text.indexOf(needle);
+  while (i !== -1) { n++; i = text.indexOf(needle, i + needle.length); }
+  return n;
+};
+
+/**
+ * findViolations({skillId, agents}) — agents: [{name, role, text}]
+ * Returns {ok, violations:[{kind, agent, pattern, count}]}.
+ *
+ * Only the baseline is judged on reads: the treatment is SUPPOSED to have the skill, and the loader
+ * reads both files by design. Every arm is judged on writes.
+ */
+export function findViolations({ skillId, agents }) {
+  if (!skillId) throw new Error('findViolations: skillId is required');
+  const violations = [];
+  const skillPath = `skills/${skillId}/SKILL.md`;
+  const rubricPath = `skills/${skillId}/evals/cases.yaml`;
+
+  for (const a of agents || []) {
+    const text = String(a.text || '');
+    if (a.role === 'baseline') {
+      // Graded separately: reading the skill removes the control; reading the rubric is reading the
+      // answer key, which also inflates the baseline's own score.
+      const skillHits = countOf(text, skillPath);
+      if (skillHits > 0) {
+        violations.push({ kind: 'baseline-read-skill', agent: a.name, pattern: skillPath, count: skillHits });
+      }
+      const rubricHits = countOf(text, rubricPath);
+      if (rubricHits > 0) {
+        violations.push({ kind: 'baseline-read-rubric', agent: a.name, pattern: rubricPath, count: rubricHits });
+      }
+    }
+    if (a.role === 'baseline' || a.role === 'treatment') {
+      for (const p of PROTECTED_PATHS) {
+        if (!text.includes(p)) continue;
+        // Require a write verb near the path, or a mention becomes a violation and the grader's
+        // quotations would block every run.
+        const wrote = WRITE_SIGNALS.some((verb) => {
+          let i = text.indexOf(p);
+          while (i !== -1) {
+            const window = text.slice(Math.max(0, i - 400), i + 400);
+            if (window.includes(verb)) return true;
+            i = text.indexOf(p, i + p.length);
+          }
+          return false;
+        });
+        if (wrote) {
+          violations.push({ kind: 'wrote-protected-path', agent: a.name, pattern: p, count: countOf(text, p) });
+        }
+      }
+    }
+  }
+  return { ok: violations.length === 0, violations };
+}
+
+/** Read a workflow transcript directory into [{name, role, text}]. Throws if unreadable. */
+export function readAgents(dir) {
+  if (!dir) throw new Error('readAgents: transcripts dir is required');
+  const st = statSync(dir); // throws ENOENT — deliberately not caught here
+  if (!st.isDirectory()) throw new Error(`readAgents: not a directory: ${dir}`);
+  return readdirSync(dir)
+    .filter((f) => /^agent-.*\.jsonl$/.test(f))
+    .map((f) => {
+      const text = readFileSync(join(dir, f), 'utf8');
+      return { name: f, role: classifyAgent(text), text };
+    });
+}
+
+/**
+ * checkIntegrity({skillId, transcriptsDir}) → {ok, blocked, reason, violations, counts}
+ *
+ * FAILS CLOSED. A missing dir, an unreadable dir, or zero baseline transcripts all return
+ * blocked — never a silent pass. "We could not look" must never render as "we looked and it was
+ * fine", which is the single failure mode this whole file exists for.
+ */
+export function checkIntegrity({ skillId, transcriptsDir }) {
+  let agents;
+  try {
+    agents = readAgents(transcriptsDir);
+  } catch (e) {
+    return { ok: false, blocked: true, reason: `integrity not verifiable: ${e.message}`, violations: [], counts: {} };
+  }
+  const counts = agents.reduce((acc, a) => { acc[a.role] = (acc[a.role] || 0) + 1; return acc; }, {});
+  if (!counts.baseline) {
+    return {
+      ok: false, blocked: true,
+      reason: `integrity not verifiable: no baseline transcript found in ${transcriptsDir}`,
+      violations: [], counts,
+    };
+  }
+  const { ok, violations } = findViolations({ skillId, agents });
+  if (ok) return { ok: true, blocked: false, reason: 'integrity verified', violations: [], counts };
+  return {
+    ok: false, blocked: true,
+    reason: `${violations.length} integrity violation(s) — this run measures nothing`,
+    violations, counts,
+  };
+}

package/scripts/skill-behavior-eval.js

@@ -1,15 +1,25 @@
 #!/usr/bin/env node
 // skill-behavior-eval.js — turn the behavior workflow's raw JSON into a scorecard.
 // Usage:
-//   node scripts/skill-behavior-eval.js --score   <raw.json>   (or pipe raw JSON on stdin with -)
+//   node scripts/skill-behavior-eval.js --score   <raw.json> [--transcripts <dir>]
 //   node scripts/skill-behavior-eval.js --holdout <raw.json>   (fresh-scenario transfer gate)
 // Exits 0 if the gate passes, 1 if it fails/blocks, 2 on usage/parse error.
 //
+// --transcripts runs the integrity check (scripts/lib/eval-integrity.js) over the workflow's agent
+// transcripts: did the "without the skill" arm read the skill, or its grading rubric, or write into
+// 02-DOCS? Any of those and the verdict is BLOCKED and the numbers are withheld, because a lift
+// measured against a contaminated control is not a measurement.
+//
+// Without --transcripts the scorecard says "integrity NOT CHECKED" on the verdict line. That is
+// deliberate: the flag is optional so existing callers keep working, so silence must not read as
+// clean.
+//
 // --holdout exists so the fix loop's blocking decision is a comparison of numbers an agent reports,
 // never a judgement an agent makes (constitution P1).
 
 import { readFileSync } from 'node:fs';
 import { scoreFromRaw, formatScorecard, holdoutGate, formatHoldoutVerdict } from './lib/behavior-score.js';
+import { checkIntegrity } from './lib/eval-integrity.js';
 
 function readInput(argPath) {
   if (argPath && argPath !== '-') return readFileSync(argPath, 'utf8');
@@ -25,12 +35,21 @@ function main() {
   }
   let raw;
   try {
-    raw = JSON.parse(readInput(args[1]));
+    raw = JSON.parse(readInput(args[1] && args[1] !== '--transcripts' ? args[1] : '-'));
   } catch (e) {
     process.stderr.write(`parse error: ${e.message}\n`);
     process.exit(2);
   }
-  const scored = scoreFromRaw(raw);
+  const ti = args.indexOf('--transcripts');
+  const transcriptsDir = ti !== -1 ? args[ti + 1] : null;
+  if (ti !== -1 && !transcriptsDir) {
+    process.stderr.write('--transcripts needs a directory\n');
+    process.exit(2);
+  }
+  const integrity = transcriptsDir
+    ? checkIntegrity({ skillId: (raw && raw.skillId) || '', transcriptsDir })
+    : undefined;
+  const scored = scoreFromRaw(raw, integrity);
   if (mode === '--holdout') {
     const gate = holdoutGate(scored);
     process.stdout.write(formatHoldoutVerdict(scored, gate) + '\n');

package/scripts/skill-behavior-eval.workflow.js

@@ -90,6 +90,10 @@ if (!loaded || !Array.isArray(loaded.scenarios) || loaded.scenarios.length === 0
   return { skillId, scenarios: [], error: 'no-capability-scenarios' }
 }
 
+// A distinct write zone per (scenario, arm). Under .rsc/, which is already gitignored, so an eval
+// write can neither dirty the tracked tree nor show up later as an abandoned diff.
+const sandbox = (index, arm) => `.rsc/eval-sandbox/${skillId}/${arm}-${index}/`
+
 phase('Execute & Grade')
 const results = await pipeline(
   loaded.scenarios,
@@ -97,7 +101,18 @@ const results = await pipeline(
   (sc, _orig, index) => parallel([
     () => agent(
       `Complete this task fully and concretely. Produce the ACTUAL deliverable the task asks for, ` +
-      `not a description of how you would do it.\n\nTASK:\n${sc.scenario}`,
+      `not a description of how you would do it.\n\n` +
+      // The baseline IS the control. Reading the skill destroys the only thing this arm is for, so
+      // the prohibition carries its reason — a rule without one gets rationalised away first. This
+      // is guidance, not a barrier: the barrier is the post-hoc check in scripts/lib/eval-integrity.js,
+      // which BLOCKS the run if this arm touched the skill or its rubric.
+      `CONSTRAINTS — you are the control arm of a measurement:\n` +
+      `- Do NOT read anything under skills/ . You are the "without the skill" arm; if you read the ` +
+      `skill under test there is no control and the measurement is void.\n` +
+      `- Do NOT read any evals/cases.yaml — that is the rubric you are being graded against.\n` +
+      `- The repository is READ-ONLY. Never write to 02-DOCS/ or to any tracked file.\n` +
+      `- Write every file you produce under ${sandbox(index, 'baseline')} .\n\n` +
+      `TASK:\n${sc.scenario}`,
       { label: `baseline:${index}`, phase: 'Execute & Grade' },
     ),
     () => agent(
@@ -105,6 +120,13 @@ const results = await pipeline(
       `skills/${loaded.skillId}/references/; read them if the skill points you there.\n\n` +
       `=== SKILL: ${loaded.skillId} ===\n${loaded.skillBody}\n=== END SKILL ===\n\n` +
       `Complete this task fully and concretely. Produce the ACTUAL deliverable, not a description.\n\n` +
+      // Both arms once wrote the same path (02-DOCS/wiki/sdd/specs/magic-link-login.md) and one
+      // clobbered the other mid-run, so the grader scored a file that no longer existed. A distinct
+      // sandbox per arm makes that collision impossible by construction rather than by luck.
+      `CONSTRAINTS:\n` +
+      `- The repository is READ-ONLY. Never write to 02-DOCS/ or to any tracked file: this is a ` +
+      `measurement, and it must not mutate the project it runs inside.\n` +
+      `- Write every file you produce under ${sandbox(index, 'treatment')} .\n\n` +
       `TASK:\n${sc.scenario}`,
       { label: `treatment:${index}`, phase: 'Execute & Grade' },
     ),

package/skills/author-skill/SKILL.md

@@ -148,6 +148,7 @@ A skill ships only when every box is checked or a miss is consciously justified.
 - [ ] **Concrete tooling delegated** to the stack skills rather than reinvented.
 - [ ] **evals present** — `cases.yaml` (≥5 `should_trigger` incl. non-obvious, ≥4 `should_not_trigger` each with a real-sibling `route_to`, ≥1 `capability` with a `must_include` rubric) + an honest `README.md`. `scripts/eval-lint.sh` passes — but it only checks presence and the counts (≥5/≥4/≥1) and that those keys are lists; the `route_to`-points-at-a-real-sibling, non-obvious phrasings, and `must_include` quality are yours to verify here, not the linter's.
 - [ ] **verify.sh** present iff the skill has a checkable artifact; process skills rely on evals.
+- [ ] **Every `must_include` item discriminates** — answerable by the scenario's task, plausibly caused by the skill and plausibly missed without it. An item *both* arms fail measures nothing and lowers the absolute; it has turned a real PASS into a FAIL here. → `references/eval-authoring.md`.
 - [ ] **Sibling links resolve** — every `../x/SKILL.md` points to a skill that exists.
 - [ ] **Wired** — `tags` + `recommends` set, `npm run manifest` re-run, and `npm run validate` / `npm run manifest:check` pass (manifest current, no dangling recommends).
 

package/skills/author-skill/references/eval-authoring.md

@@ -63,6 +63,40 @@ The `must_include` points are the differentiators — the specific things a *goo
 
 For a process skill (no `verify.sh`), the capability scenario is the *primary* rigor — it is how you prove the safety rails actually change behavior. Make it count.
 
+## A `must_include` item must discriminate, or it subtracts
+
+Measured on 2026-08-18, running the behavioral eval on five skills for the first time. Two items
+added to `testing-py` and `testing-web` came back **unsatisfied in both arms** — treatment and
+baseline alike. They demanded the answer *name* a tool (`mutmut`, Stryker); both arms did the
+behaviour (planted a bug, checked the suite caught it) without naming one. Contributing nothing to
+lift and still counting against the coverage term, they dragged `testing-py` from **PASS (9.0,
+lift +2.2) to FAIL (7.5, lift +1.7)**. The rule they encoded was fine; the item was not.
+
+Two tests before an item earns its place:
+
+1. **Answerable by the scenario's task.** If the scenario says "write this test file", then "names
+   the mutation tool" is not part of that deliverable and no competent answer will contain it.
+2. **Discriminating.** The skill must plausibly cause it and a bare agent must plausibly miss it.
+   An item both arms satisfy measures the model; one both arms fail measures nothing and lowers the
+   absolute.
+
+**The symptom, so you can catch it from a scorecard:** an item unsatisfied in *both* arms is
+suspect. Nine times out of ten it is written as a spelling ("mentions X") rather than a behaviour
+("does not treat coverage as proof the suite detects bugs"). Rewrite it as the behaviour, or delete
+it — do not leave it in as an aspiration, because the aggregate cannot tell an aspiration from a
+failure.
+
+The same run showed the other side: the equivalent item in `testing-go`, where the skill gives a
+concrete *procedure* rather than a tool name, discriminated hard and nearly doubled the lift
+(1.5 → 2.7). Prescriptive guidance produces gradeable behaviour; a tool name produces a keyword
+check.
+
+**And the scenario itself must be executable where it runs.** `verify`'s scenario asked to verify a
+FastAPI orders feature that exists in no checkout, so both arms correctly answered "there is nothing
+to verify" and several items were unsatisfiable by construction — the absolute measured the scenario,
+not the skill. Make a `capability` scenario self-contained: carry the spec, the diff and the reported
+facts inline rather than assuming repo state.
+
 ## README.md — run it honestly
 
 `evals/README.md` documents the two-axis run procedure and is candid about limits:

package/skills/fastapi/scripts/verify.sh

@@ -9,9 +9,20 @@
 # Auto-detects each tool. If a tool is missing it prints a yellow SKIP and continues
 # (it never FAILs on a missing tool). Prefers `uv run <tool>` when `uv` is present,
 # else the bare tool on PATH. Exits non-zero only if a tool actually ran and reported
-# a failure. The coverage threshold is read from the project's pyproject.toml
-# (--cov-fail-under); this script does not hardcode a second threshold. Idempotent:
-# re-running yields the same result (read-only beyond whatever the project's pytest does).
+# a failure. Idempotent: re-running yields the same result (read-only beyond whatever
+# the project's pytest does).
+#
+# COVERAGE IS ONLY A GATE IF YOUR PROJECT MAKES IT ONE. The threshold is deliberately
+# delegated to the project (--cov-fail-under in pyproject.toml / setup.cfg / .coveragerc)
+# rather than hardcoded here. But when the project sets none, `pytest --cov` prints a
+# percentage and exits 0 however far coverage falls — and a layer that prints a number and
+# exits 0 is a report, not a gate. That case is reported as [gap] instead of PASS, because
+# the two are indistinguishable on screen and the broken one can only ever produce green:
+# no failing run will ever surface it.
+#
+# A [gap] does NOT change the exit code — breaking every repo that passes today is not this
+# script's call. This script reports; the `verify` skill judges, and there a gap counts as
+# an unverified criterion, which fails the verdict.
 #
 # Compatible with stock macOS bash 3.2: no `mapfile`, no associative arrays, and every
 # array access is guarded so `set -u` never trips on an "unbound" empty array.
@@ -30,7 +41,28 @@ warn() { printf '%s%s%s\n' "$YELLOW" "$*" "$RESET"; }
 ok()   { printf '%s%s%s\n' "$GREEN" "$*" "$RESET"; }
 fail() { printf '%s%s%s\n' "$RED" "$*" "$RESET"; }
 
-PASSED=0; SKIPPED=0; FAILED=0
+PASSED=0; SKIPPED=0; FAILED=0; GAPS=0
+
+# gap <reason> — the layer ran and had nothing to fail with. Counted and printed, but it
+# deliberately does NOT touch FAILED: see the header note on why the exit code is unchanged.
+gap() { printf '%sGAP: %s%s\n' "$YELLOW" "$*" "$RESET"; GAPS=$((GAPS + 1)); }
+
+# coverage_threshold_configured — does the project actually gate on coverage?
+# Looks for pytest's --cov-fail-under (addopts or a bare flag) and coverage.py's fail_under.
+# Grep rc: 0 = found, 1 = not found, >=2 = the scan itself broke. Treat "broke" as NOT
+# configured so the answer errs toward reporting a gap; a broken scan must never buy a pass.
+coverage_threshold_configured() {
+  local f rc
+  for f in "$TARGET/pyproject.toml" "$TARGET/setup.cfg" "$TARGET/tox.ini" "$TARGET/.coveragerc" "$TARGET/pytest.ini"; do
+    [ -f "$f" ] || continue
+    grep -qE 'cov-fail-under|fail_under' "$f" && return 0
+    rc=$?
+    if [ "$rc" -ge 2 ]; then
+      warn "NOTE: could not read '${f}' while checking for a coverage threshold; treating it as absent"
+    fi
+  done
+  return 1
+}
 
 have() { command -v "$1" >/dev/null 2>&1; }
 
@@ -105,12 +137,21 @@ run_step "ruff check"        ruff check "$TARGET"
 run_step "ruff format check" ruff format --check "$TARGET"
 run_step "mypy"              mypy "$TARGET"
 run_step "pytest + coverage" pytest --cov --cov-report=term-missing
+# Only meaningful if pytest actually ran; a SKIP already says nothing was measured.
+if tool_available pytest && ! coverage_threshold_configured; then
+  gap "coverage ran without a threshold (no --cov-fail-under / fail_under found) — this layer cannot fail. Add --cov-fail-under=<n> to pyproject.toml addopts to make it a gate."
+fi
 run_step "pip-audit"         pip-audit
 
-printf '\n%d passed, %d skipped, %d failed\n' "$PASSED" "$SKIPPED" "$FAILED"
+printf '\n%d passed, %d skipped, %d failed, %d gap(s)\n' "$PASSED" "$SKIPPED" "$FAILED" "$GAPS"
+if [ "$GAPS" -gt 0 ]; then
+  warn "A GAP is neither a pass nor a failure: the layer ran and could not have failed. Report it as unverified."
+fi
 if [ "$FAILED" -gt 0 ]; then
   fail "verify.sh: failures detected"
   exit 1
 fi
-ok "verify.sh: ok"
+# "every check that could fail, passed" rather than a bare ok: with a GAP present, the
+# stronger claim would be false in exactly the way this script now exists to prevent.
+ok "verify.sh: ok (every check that could fail, passed)"
 exit 0

package/skills/go/scripts/verify.sh

@@ -11,6 +11,17 @@
 # Real problems (unformatted code, vet/test/vuln failures) exit non-zero.
 # Read-only: never mutates your source.
 #
+# ONE LAYER HERE IS A REPORT, NOT A GATE. `go test -cover` takes no threshold flag, so
+# coverage can print any number — 4%, 0% — and still exit 0. It is therefore reported as
+# [gap]: the layer ran and had nothing to fail with. Said out loud because a layer that
+# prints a number and exits 0 looks exactly like a layer that passed, and that shape can
+# only ever produce green — no failing run will ever tell you about it. Want a real
+# coverage gate in Go? Parse `go tool cover -func` and compare, or use a CI action; this
+# script does not pretend to be one.
+#
+# A [gap] does NOT change the exit code: this script reports, the `verify` skill judges,
+# and there it counts as an unverified criterion, which fails the verdict.
+#
 # Portability: runs on stock macOS bash 3.2 (no mapfile, no associative arrays,
 # no unguarded array expansions under `set -u`).
 
@@ -24,12 +35,16 @@ else
 fi
 
 failed=0
+gaps=0
 
 have()  { command -v "$1" >/dev/null 2>&1; }
 warn()  { printf '%s[skip]%s %s\n' "$YELLOW" "$RESET" "$*"; }
 fail()  { printf '%s[fail]%s %s\n' "$RED" "$RESET" "$*"; failed=1; }
 ok()    { printf '%s[ ok ]%s %s\n' "$GREEN" "$RESET" "$*"; }
 info()  { printf -- '----- %s\n' "$*"; }
+# A layer that ran but had nothing to fail with. Deliberately does NOT set `failed`:
+# breaking every repo that passes today is not this script's call to make.
+gap()   { printf '%s[gap ]%s %s\n' "$YELLOW" "$RESET" "$*"; gaps=$((gaps + 1)); }
 
 # Must run from a module root.
 if [ ! -f go.mod ]; then
@@ -89,6 +104,10 @@ if have go; then
     warn "cgo disabled or C compiler '$cgo_cc' not found; running plain go test -cover"
     if go test -cover ./...; then ok "tests pass (no race)"; else fail "tests failed"; fi
   fi
+  # The tests above are a gate; the coverage number printed alongside them is not.
+  # `go test -cover` has no threshold flag, so coverage cannot fail this script no matter
+  # how far it falls. Reported so nobody reads the percentage as a checked constraint.
+  gap "coverage: reported, not gated — 'go test -cover' takes no threshold, so this layer cannot fail"
 else
   warn "go not found - skipping tests"
 fi
@@ -102,8 +121,14 @@ else
 fi
 
 echo
+if [ "$gaps" -ne 0 ]; then
+  printf '%s%d layer(s) ran without being able to fail (see [gap] above).%s\n' "$YELLOW" "$gaps" "$RESET"
+  printf 'Not a pass and not a failure: report them as unverified rather than green.\n'
+fi
 if [ "$failed" -ne 0 ]; then
   printf '%sFAIL:%s one or more checks failed.\n' "$RED" "$RESET"
   exit 1
 fi
-printf '%sPASS:%s all checks passed.\n' "$GREEN" "$RESET"
+# Deliberately "checks that could fail, passed" and not "all checks passed": with a [gap]
+# present the second sentence would be false in the way this whole change exists to stop.
+printf '%sPASS:%s every check that could fail, passed.\n' "$GREEN" "$RESET"

package/skills/implement/SKILL.md

@@ -76,6 +76,24 @@ code tells you nothing about whether it can fail. **Refactor only on green** kee
 reversible against a passing bar. **Checkpoint after each task** is what makes this resumable and
 reviewable instead of a 2000-line surprise diff.
 
+### Four things you may never do to reach green
+
+The loop only means anything if green cannot be manufactured. These are absolute, and each one is a
+way people reach green without fixing anything:
+
+1. **Never weaken a test to make it pass.** Not broadening an assertion, not adding a skip, not
+   raising a tolerance, not deleting it. A test that looks wrong is a **spec conversation** — surface
+   it, don't bury it.
+2. **Never edit a test and the implementation in the same step to reach green.** Change one, run,
+   then the other. Simultaneous edits let you redefine correctness to match your bug without
+   noticing you did it.
+3. **Never mock the unit under test**, or mock so much that the test only exercises the mocks. Mock
+   boundaries you don't own — network, clock, filesystem, third-party SDK — never the logic you are
+   being paid to verify.
+4. **Never chase the coverage number.** Coverage detects untested code; it is not a target. A test
+   added to touch lines, with no meaningful assertion, is gaming — and mutation testing exists
+   precisely to catch it (`../testing-py/SKILL.md`, `../testing-web/SKILL.md`, `../testing-go/SKILL.md`).
+
 ### The done-check is the contract
 
 Each task carries a done-check from the `tasks` phase ("returns 409 on duplicate email", "list

package/skills/review/SKILL.md

@@ -128,9 +128,41 @@ gap deliberately with `--accept-partial-lenses` (it is recorded). Every approval
 
 **Lenses by risk tier** — tier 0 never reaches you (the gate passes docs/copy silently);
 tier 1 → run the single most relevant pass from the table above yourself; tier 2 → dispatch
-**three parallel fresh-context subagents** (correctness · security · tests-as-evidence), each
-given only the diff and told to *refute* readiness, not confirm it. Fresh context is the point:
-a reviewer who inherits the implementer's context inherits its blind spots.
+**three parallel fresh-context subagents** (correctness · security · tests-as-evidence), each told
+to *refute* readiness, not confirm it. Fresh context is the point: a reviewer who inherits the
+implementer's context inherits its blind spots.
+
+**Give each refuter exactly four inputs — no more, no less.** The diff alone is not enough, and
+that is not a detail: with only the diff, the one failure class no test suite can structurally
+catch — *the change is correct and is not what the approved spec asked for* — is invisible to the
+whole panel.
+
+1. **The task contract** — the original request **plus every scope change a human explicitly
+   approved since.** Not just the first message: without the approved changes a legitimate scope
+   revision reads as a spec gap and you get a confident false positive.
+2. **The approved spec** (`02-DOCS/wiki/sdd/specs/<slug>.md`).
+3. **The exact source state** — commit SHA, or a tree hash when git is absent. A verdict belongs to
+   the state that was reviewed, not to the project.
+4. **The entry point** — the one command that reruns the checks.
+
+**And say what they do not get:** your conversation, your reasoning, your defences, and your draft
+verdict. If a claim needs your justification to stand, it is not proven.
+
+**Blind first, compare second.** Each refuter records what it attacked and what it found *before*
+being shown your conclusions. Only then may it compare and add findings — the blind record is
+append-only after that, never rewritten. Skip this and the fresh context is spent confirming your
+framing, which is the one thing it was bought to avoid.
+
+**The attack list is the deliverable, not just the findings.** "Nothing found" without saying where
+you looked is indistinguishable from not having looked — the same rule that governs a dismissal.
+
+Two extra finding classes the four inputs unlock, neither reachable from a diff:
+
+- **Contract vs spec** — what would a caller reasonably expect, given the stated deployment, that
+  no clause covers? An exclusion the spec deliberately approved is **not** a finding; an exclusion
+  the spec describes *inaccurately* is.
+- **Mapping, both directions** — an acceptance criterion with no test that can be made to fail, and
+  a test pinning behaviour no criterion asked for.
 
 **A finding blocks only if it survives all three filters** — no exceptions, and eagerness to
 find something is not evidence:

package/skills/specify/evals/cases.yaml

@@ -78,3 +78,5 @@ capability:
       - "Captures observable behaviour (what the user sees/can do) and explicit non-goals / out-of-scope rather than implementation (no file format library, storage, or job-queue detail)."
       - "Turns answers into binary acceptance criteria and defers unresolved items (e.g. data scope, retention, format) to Points to clarify instead of fabricating decisions."
       - "Notes that the spec is complete-enough-to-plan with open points remaining, and hands off to clarify — not to plan or implementation."
+      - "After the user answers a question, folds the answer in, states what changed, shows the revised spec, and asks for approval AGAIN — it does not treat the answer as approval of the spec, including when the user picked the recommended option."
+      - "If told to run the whole chain up front, records the approval as autopilot rather than item-by-item, and says the spec is the artifact to review after the fact."

package/skills/testing-go/evals/cases.yaml

@@ -42,3 +42,5 @@ capability:
       - A hand-written interface fake or net/http/httptest server for the HTTP dependency, not a mock framework by default
       - Runs with -race for the concurrent path and shows a go test -coverprofile / cross-package coverage command
       - A benchmark using for b.Loop() with b.ReportAllocs() rather than a b.N loop
+      - "Does not present the coverage percentage as proof the suite detects bugs: states Go has no mature mutation tool, gives the manual procedure, and distinguishes mutation (tests that would not notice a bug) from fuzzing (inputs the code mishandles)."
+      - "Requires a hand-rolled mutant runner to prove it executed each mutant, and says why — a kill reported without running can only inflate the score, so no failing run would ever reveal it."

package/skills/verify/SKILL.md

@@ -80,6 +80,7 @@ If `config.yaml` provides `testing.commands.verify`, run those commands first or
 Rules for RUN:
 
 - The stack `verify.sh` **skips missing tools** (yellow SKIP) rather than failing on them — so a SKIP is not a pass. Note every SKIP; a skipped test suite means that criterion is **unverified**, which is not the same as verified-passing.
+- A `GAP` is the third thing, and it is the one that looks most like a pass: **the layer ran and had nothing to fail with.** The fastapi gate reports it when coverage runs with no `--cov-fail-under` configured; the Go gate reports it always, because `go test -cover` takes no threshold. The script deliberately does not fail on a GAP — that judgement is yours. **Treat a GAP as an unverified criterion, which fails the verdict**, and record it (see the mapping below). A gate that reports and a gate that checks are indistinguishable on screen, and only the broken one is guaranteed to stay green.
 - A non-zero exit from a tool that actually ran is a hard FAIL. Record the failing tool and its output.
 - Run from the directory the stack skill documents (Go runs from the module root; others from the subproject root). Don't guess paths.
 - If no `scripts/verify.sh` exists for a touched stack, that's a gap — say the gate is incomplete and recommend the user add the stack skill, rather than hand-rolling a one-off check that drifts from the real gate.
@@ -157,6 +158,15 @@ source_state: 3f9a1c2 (clean)
 
 `SUSTITUIDA` is the dangerous one. "Ran the suite twice" is not *suite health: stable* — it is a substitute that cannot see whole-suite order dependence, and a reader who can't tell the two apart reads "found nothing" where the truth is "did not look with that instrument".
 
+**Where a `GAP` goes — and do not invent a fourth label.** A layer that ran without being able to fail is not `NO-APLICA` (the surface exists) and not `HERRAMIENTA-AUSENTE` (the tool ran). It is `SUSTITUIDA`: what ran was **a report in place of a gate**, and what it cannot detect is **coverage falling**. Write it that way:
+
+```text
+- **SUSTITUIDA** — coverage: ran without a threshold (GAP). A report, not a gate;
+  cannot detect coverage regressing. Fix: --cov-fail-under=<n> in pyproject addopts.
+```
+
+**And while you are there: the global percentage is the wrong number.** A project-wide floor barely moves when your change lands untested — 200 new uncovered lines in a 20k-line codebase is a rounding error against an 85% floor, so the gate passes and the change is untested. What matters is whether **the lines you touched** are exercised: `diff-cover coverage.xml --fail-under=100` gates exactly those. This is guidance, not something the generated `verify.sh` scripts do — recommend it, don't assume it ran.
+
 **A dismissed finding carries evidence, one line each.** A fix is self-evidencing — the test now passes. A dismissal is not: "not a real problem" is indistinguishable from "did not check". Name the command, the `file:line`, or the test that disproves it, or write "ninguno".
 
 Append-only spirit: don't overwrite a prior run's record; a new run is a new dated file. The record is the receipt the `review` and `ship` phases trust.

package/skills/verify/evals/cases.yaml

@@ -61,14 +61,44 @@ should_not_trigger:
 # and refuses to call it done without evidence; an ungrounded answer tends to
 # eyeball the code, trust stale results, or declare success prematurely.
 capability:
-  - scenario: "Implementation of the 'duplicate-submit is a no-op' order feature just finished in a FastAPI service. The user says 'it's done, let's merge'. Verify it."
+  - scenario: >
+      A change is finished and the user says "it's done, let's merge". Verify it, and produce the
+      verification record.
+
+      Everything you need is here — do NOT assume any of it exists on disk; this scenario is
+      self-contained on purpose.
+
+      APPROVED SPEC (acceptance criteria):
+        AC1. Submitting an order returns 201 with the created order id.
+        AC2. Submitting the SAME order twice creates exactly one order; the second call returns
+             200 with the first order's id.
+        AC3. An order with a negative quantity is rejected with 422 and nothing is stored.
+
+      TASK DONE-CHECKS from the plan:
+        T1. POST /orders creates an order.            done-check: a test asserts 201 + id.
+        T2. Duplicate POST is a no-op.                done-check: a test asserts one row after two POSTs.
+        T3. Negative quantity is rejected.            done-check: a test asserts 422 + no write.
+
+      WHAT THE IMPLEMENTER REPORTS:
+        - Files changed: app/orders.py, tests/test_orders.py
+        - "I ran the suite: 2 passed. Tests written: test_create_order_returns_201,
+           test_negative_quantity_rejected."
+        - "mypy wasn't installed so the type step printed SKIP, and coverage ran without a
+           threshold configured. Everything else was green."
+        - "The retry loop can spin forever" was raised in review; the implementer replied
+          "not a real problem".
     must_include:
       - "Reads the accompaniment level from 02-DOCS/wiki/harness/user-profile.md and adapts verbosity (verdict legibility unchanged by the dial)."
-      - "LOCATES the spec acceptance criteria and the task done-checks, and identifies the FastAPI subproject as the touched stack from git diff/manifests."
-      - "RUNS the fastapi stack scripts/verify.sh (ruff/black, mypy, pytest+coverage, pip-audit) from the documented root and captures the output verbatim rather than reusing a previous run."
-      - "Treats any tool SKIP (missing tool) as UNVERIFIED, not as a pass, and notes the constitution's coverage floor as part of the gate."
-      - "WALKS every task done-check and every acceptance criterion, marking each with concrete evidence (a passing test / observable result) or as an open FAIL item — specifically flags the duplicate-submit criterion as UNVERIFIED if no test exercises the duplicate-POST path."
+      - "LOCATES the three acceptance criteria and the three task done-checks from the material given, and maps each to what the implementer actually reports."
+      - "Refuses to accept the implementer's summary as the gate: says the checks must be re-run and the output read, rather than trusting a reported '2 passed'."
+      - "Treats the mypy SKIP as UNVERIFIED rather than a pass, and does not let 'everything else was green' stand in for the skipped layer."
+      - "WALKS all three done-checks and all three acceptance criteria, and catches that AC2/T2 (duplicate POST is a no-op) has NO test at all — the two named tests cover AC1 and AC3 only — marking it UNVERIFIED rather than passing it."
       - "Does NOT fix a failing test inline; if a check fails it records it and hands off to debug, and refuses to fix-then-claim."
       - "RECORDS a dated verification file under 02-DOCS/wiki/sdd/verifications/ and indexes it in the root CLAUDE.md Knowledge map."
       - "Returns a VERDICT that is PASS only if EVERY item has passing evidence; a single unverified acceptance criterion makes the whole verdict FAIL — and refuses to call it 'done' otherwise."
       - "On PASS, points to the review phase; on FAIL, routes failing checks to debug and missing coverage back to implement."
+      - "Treats a coverage GAP (the layer ran with no threshold configured, so it could not have failed) as an unverified criterion that fails the verdict — not as a pass — and records it as SUSTITUIDA: a report in place of a gate, unable to detect coverage regressing."
+      - "Before trusting any home-grown gate that reports green, asks for it to be watched failing on a known-bad input, and states the limit: one negative control proves one bad case reaches the failure path, not that the gate covers the rule it claims to enforce."
+      - "Splits layers that did not run into NO-APLICA / HERRAMIENTA-AUSENTE / SUSTITUIDA rather than one undifferentiated SKIP list, and never writes SUSTITUIDA as a pass."
+      - "Binds the record to a source state (short commit SHA plus clean/dirty) and takes every number from one fresh run made after the last edit, refusing to reuse mid-task figures."
+      - "Gives any dismissed finding its own evidence line (command, file:line, or the test that disproves it) rather than asserting it was not a real problem."