@damiangilgonzalez/harness 1.3.7

package/scripts/knowledge-doctor.js

@@ -0,0 +1,131 @@
+#!/usr/bin/env node
+// knowledge-doctor.js — report what the wiki claims that no longer holds.
+//
+// Usage: node scripts/knowledge-doctor.js [--json]
+//
+// Report-only, on demand. NOT in `rsc doctor`, NOT in any hook, NOT in the release gates — decided in
+// clarify (2026-08-18): the precision a check needs scales with how intrusive it is, and an asked-for
+// report tolerates noise an automatic one does not. Accepted cost, stated here so nobody has to
+// rediscover it: drift is not detected on its own. Somebody has to run this.
+//
+// It NEVER writes. Not a file, not a mkdir. `docs/` is untracked (P9): there is no undo here.
+
+import { readFileSync, readdirSync, statSync, existsSync } from 'node:fs';
+import { join, dirname, relative } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { execFileSync } from 'node:child_process';
+import { diagnose, CLASSES } from './lib/knowledge-doctor.js';
+
+const REPO = join(dirname(fileURLToPath(import.meta.url)), '..');
+const WIKI = join(REPO, 'docs', 'wiki');
+
+// withFileTypes, NOT statSync — this matches scripts/drift-check.js:167, the precedent the plan cited
+// as exact and then diverged from. statSync FOLLOWS symlinks, so a symlink under the wiki let the walk
+// read outside the declared zone (reporting the logical path, hiding the escape), a dangling symlink
+// crashed the whole report with ENOENT, and a symlink cycle crashed it with ELOOP. dirent.isDirectory()
+// does none of that: a symlink is neither a file nor a directory here, so it is skipped.
+const walk = (dir) => readdirSync(dir, { withFileTypes: true }).flatMap((e) => {
+  const p = join(dir, e.name);
+  if (e.isDirectory()) return walk(p);
+  if (e.isFile() && p.endsWith('.md')) return [p];
+  return []; // symlinks, sockets, anything else: not our business, and not a crash
+});
+
+// Injected so the pure library never touches git. Cached: the same ref is asked about repeatedly.
+const refCache = new Map();
+const refExists = (ref) => {
+  if (refCache.has(ref)) return refCache.get(ref);
+  let ok = false;
+  for (const spec of [`${ref}^{commit}`, `refs/tags/${ref}`]) {
+    try { execFileSync('git', ['cat-file', '-e', spec], { cwd: REPO, stdio: 'ignore' }); ok = true; break; }
+    catch { /* not this kind of ref */ }
+  }
+  refCache.set(ref, ok);
+  return ok;
+};
+
+function main() {
+  const asJson = process.argv.includes('--json');
+  if (!existsSync(WIKI)) {
+    // Not a failure: a repo without the harness has nothing to audit.
+    console.log('knowledge-doctor: no hay docs/wiki/ — nada que auditar.');
+    process.exit(0);
+  }
+  const files = walk(WIKI);
+  const indexPath = join(WIKI, 'index.md');
+  const indexText = existsSync(indexPath) ? readFileSync(indexPath, 'utf8') : null;
+
+  // AC7 says an unreadable document is a candidate with its reason, never a silent skip. The first
+  // version read "unreadable" as "frontmatter unparseable" only, so a chmod-000 file or a dangling
+  // symlink threw out of readFileSync and took the WHOLE report with it — worse than a silent skip.
+  const unreadable = [];
+  const docs = files.map((p) => {
+    const rel = relative(WIKI, p);
+    let text;
+    try {
+      text = readFileSync(p, 'utf8');
+    } catch (e) {
+      unreadable.push({
+        path: relative(REPO, p), line: 1, cls: 'fuera-de-sitio', uncertain: true,
+        signal: `no se pudo leer el fichero (${e.code || e.message})`,
+        expected: 'un documento legible', found: e.code || 'error de lectura',
+      });
+      return null;
+    }
+    return {
+      path: relative(REPO, p),
+      rel,
+      dir: dirname(rel) === '.' ? '' : dirname(rel),
+      text,
+      // Only artifacts the chain consumes are expected in the Knowledge map; an article is not.
+      indexable: rel.startsWith('sdd/specs/') || rel.startsWith('sdd/verifications/'),
+    };
+  }).filter(Boolean);
+
+  // Project artifacts outside the wiki: skill ids and script basenames. The library cannot read the
+  // repo (it is pure by design), so the caller that can, does.
+  const extraSlugs = [];
+  for (const [dir, strip] of [['skills', null], ['scripts', /\.(js|mjs|sh)$/], ['scripts/lib', /\.(js|mjs)$/]]) {
+    try {
+      for (const e of readdirSync(join(REPO, dir), { withFileTypes: true })) {
+        if (strip && e.isFile()) extraSlugs.push(e.name.replace(strip, ''));
+        else if (!strip && e.isDirectory()) extraSlugs.push(e.name);
+      }
+    } catch { /* a missing directory is not an error here */ }
+  }
+
+  const report = diagnose({ docs, indexText, refExists, extraSlugs });
+  // Unreadable files are candidates too, and they sort first: a document nobody can open is the one
+  // whose staleness nobody can check.
+  if (unreadable.length) report.candidates = [...unreadable, ...report.candidates];
+  if (asJson) { console.log(JSON.stringify(report, null, 2)); process.exit(0); }
+
+  const byClass = (c) => report.candidates.filter((x) => x.cls === c);
+  console.log(`# knowledge-doctor — ${report.scanned} documentos\n`);
+  if (!report.candidates.length) {
+    console.log('Sin candidatos. Nada que triar.\n');
+  } else {
+    console.log(`${report.candidates.length} candidato(s), ordenados por coste de equivocarse:\n`);
+    // Filenames are interpolated into markdown, and a filename may contain newlines and backticks. A
+    // crafted name rendered a structurally valid FORGED candidate while the tally below still read 0 —
+    // and the consumer of this report (knowledge-ops) archives and overwrites files. Neutralise the
+    // three characters that can break out of the line.
+    const safe = (s) => String(s).replace(/[\r\n]+/g, '⏎').replace(/`/g, "'");
+    for (const c of report.candidates) {
+      console.log(`- **${c.cls}** · \`${safe(c.path)}:${c.line}\`${c.uncertain ? ' · _incierto_' : ''}`);
+      console.log(`  - señal: ${safe(c.signal)}`);
+      console.log(`  - esperado: ${safe(c.expected)} · encontrado: ${safe(c.found)}`);
+    }
+    console.log('');
+    for (const [label, cls] of [['se contradice', CLASSES.CONTRADICTS], ['ya no aplica', CLASSES.STALE], ['fuera de sitio', CLASSES.MISPLACED]]) {
+      console.log(`- ${label}: ${byClass(cls).length}`);
+    }
+    console.log('');
+  }
+  console.log('## Lo que NO se miró\n');
+  for (const n of report.notLookedAt) console.log(`- ${n}`);
+  console.log('\nEl juicio es de `knowledge-ops` o tuyo: esto estrecha, no cura. No se ha escrito nada.');
+  process.exit(0);
+}
+
+main();

package/scripts/lib/backups.js

@@ -0,0 +1,165 @@
+import {
+  cpSync,
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  readlinkSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { dirname, isAbsolute, join, relative, sep } from 'node:path';
+
+const SCHEMA_VERSION = 1;
+
+export function backupsDir(cwd = process.cwd()) {
+  return join(cwd, '.rsc', 'backups');
+}
+
+export function createBackup({ cwd = process.cwd(), operation, target, paths, cliVersion, now = new Date() }) {
+  const uniquePaths = [...new Set((paths || []).filter(Boolean))];
+  const id = uniqueSnapshotId({ cwd, now, operation, target });
+  const root = join(backupsDir(cwd), id);
+  const filesRoot = join(root, 'files');
+  mkdirSync(filesRoot, { recursive: true });
+
+  const entries = uniquePaths.map((absPath) => snapshotEntry({ cwd, root, absPath }));
+  const manifest = {
+    schemaVersion: SCHEMA_VERSION,
+    id,
+    createdAt: now.toISOString(),
+    operation,
+    target,
+    cwd,
+    cliVersion,
+    entries,
+  };
+  writeFileSync(join(root, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
+  return manifest;
+}
+
+export function listBackups({ cwd = process.cwd() } = {}) {
+  const dir = backupsDir(cwd);
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .map((id) => readManifest({ cwd, id }))
+    .filter(Boolean)
+    .sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id));
+}
+
+export function restoreBackup({ cwd = process.cwd(), id, dryRun = false }) {
+  const snapshot = resolveSnapshot({ cwd, id });
+  const changed = [];
+  for (const entry of snapshot.entries) {
+    const absPath = safeJoin(cwd, entry.path);
+    changed.push(absPath);
+    if (dryRun) continue;
+    restoreEntry({ cwd, snapshot, entry, absPath });
+  }
+  return { snapshot, changed };
+}
+
+function snapshotEntry({ cwd, root, absPath }) {
+  const rel = safeRelative(cwd, absPath);
+  if (!existsSync(absPath)) return { path: rel, existed: false, kind: 'missing' };
+
+  const stat = lstatSync(absPath);
+  if (stat.isSymbolicLink()) {
+    return { path: rel, existed: true, kind: 'symlink', linkTarget: readlinkSync(absPath) };
+  }
+
+  const contentPath = join('files', rel);
+  const contentAbs = join(root, contentPath);
+  mkdirSync(dirname(contentAbs), { recursive: true });
+  if (stat.isDirectory()) {
+    cpSync(absPath, contentAbs, { recursive: true });
+    return { path: rel, existed: true, kind: 'dir', contentPath };
+  }
+
+  cpSync(absPath, contentAbs);
+  return { path: rel, existed: true, kind: 'file', contentPath };
+}
+
+function restoreEntry({ cwd, snapshot, entry, absPath }) {
+  if (!entry.existed) {
+    rmSync(absPath, { recursive: true, force: true });
+    return;
+  }
+
+  rmSync(absPath, { recursive: true, force: true });
+  mkdirSync(dirname(absPath), { recursive: true });
+  if (entry.kind === 'symlink') {
+    symlinkSync(entry.linkTarget, absPath, process.platform === 'win32' ? 'junction' : undefined);
+    return;
+  }
+
+  const contentAbs = join(backupsDir(cwd), snapshot.id, entry.contentPath);
+  cpSync(contentAbs, absPath, { recursive: entry.kind === 'dir' });
+}
+
+function resolveSnapshot({ cwd, id }) {
+  if (!id) throw new Error('restore requires a snapshot id or latest');
+  if (id === 'latest') {
+    const latest = listBackups({ cwd })[0];
+    if (!latest) throw new Error('no backups found');
+    return latest;
+  }
+
+  const manifest = readManifest({ cwd, id });
+  if (!manifest) throw new Error(`backup not found: ${id}`);
+  return manifest;
+}
+
+function readManifest({ cwd, id }) {
+  const path = join(backupsDir(cwd), id, 'manifest.json');
+  if (!existsSync(path)) return undefined;
+  return JSON.parse(readFileSync(path, 'utf8'));
+}
+
+function uniqueSnapshotId({ cwd, now, operation, target }) {
+  const base = snapshotId({ now, operation, target });
+  let id = base;
+  let counter = 2;
+  while (existsSync(join(backupsDir(cwd), id))) {
+    id = `${base}-${counter}`;
+    counter += 1;
+  }
+  return id;
+}
+
+function snapshotId({ now, operation, target }) {
+  const stamp = now.toISOString().replace(/[-:]/g, '').replace(/\..*/, '').replace('T', '-');
+  return `${stamp}-${safeId(operation)}-${safeId(target || 'all')}`;
+}
+
+function safeId(value) {
+  return String(value).replace(/[^a-z0-9._-]+/gi, '-').replace(/^-+|-+$/g, '').toLowerCase();
+}
+
+function safeRelative(cwd, absPath) {
+  const rel = relative(cwd, absPath);
+  if (!rel || rel.startsWith('..') || rel.split(sep).includes('..')) {
+    throw new Error(`path is outside project root: ${absPath}`);
+  }
+  return rel.split(sep).join('/');
+}
+
+// A manifest path is split on BOTH separators, because the input is file content — readManifest()
+// reads it off disk — and not every manifest was written by safeRelative() on this platform. A path
+// stored with Windows separators (`..\..\somewhere`) has no '/' to split on, so the old check saw a
+// single segment, found no '..', and handed the raw string to join() — which honours backslashes on
+// Windows. The guard only guarded on POSIX.
+//
+// Splitting into real segments does both jobs at once: `..` becomes visible to the check, and the
+// path is rebuilt by join() rather than passed through, so the separator the manifest happened to
+// use stops mattering. Found while fixing the Windows hook-duplication bug: same family.
+const pathSegments = (value) => value.split(/[/\\]+/).filter(Boolean);
+
+function safeJoin(cwd, relPath) {
+  if (!relPath || isAbsolute(relPath) || pathSegments(relPath).includes('..')) {
+    throw new Error(`path is outside project root: ${relPath}`);
+  }
+  return join(cwd, ...pathSegments(relPath));
+}

package/scripts/lib/behavior-score.js

@@ -0,0 +1,190 @@
+// behavior-score.js — pure scoring math for the behavioral skill gate.
+// No I/O, no LLM, no randomness: every function is deterministic and unit-tested.
+// The Workflow engine returns raw grader signals; this lib turns them into the /10.
+
+export const ABS_MIN = 8.5;
+export const LIFT_MIN = 1.0;
+export const COVERAGE_WEIGHT = 0.6;
+export const QUALITY_WEIGHT = 0.4;
+
+export function round1(x) {
+  return Math.round(x * 10) / 10;
+}
+
+// Compose one output's 0-10 from grader signals.
+// signals: { mustInclude: [{satisfied:boolean}], quality: {completeness,actionability,correctness,grounding} }
+export function composeOutputScore(signals) {
+  const mi = Array.isArray(signals && signals.mustInclude) ? signals.mustInclude : [];
+  const total = mi.length;
+  const satisfied = mi.filter((m) => m && m.satisfied).length;
+  const q = (signals && signals.quality) || {};
+  const qVals = [q.completeness, q.actionability, q.correctness, q.grounding]
+    .map((v) => (typeof v === 'number' ? v : 0));
+  const quality = qVals.reduce((a, b) => a + b, 0) / qVals.length; // 0-10
+  if (total === 0) return round1(quality); // no checklist -> quality only
+  const coverage = (satisfied / total) * 10; // 0-10
+  return round1(COVERAGE_WEIGHT * coverage + QUALITY_WEIGHT * quality);
+}
+
+// treatmentScore/baselineScore are composed 0-10 numbers.
+export function deriveScenario(treatmentScore, baselineScore) {
+  return { absolute: round1(treatmentScore), delta: round1(treatmentScore - baselineScore) };
+}
+
+// scenarios: array of {absolute, delta} | null
+export function aggregate(scenarios) {
+  const valid = scenarios.filter((s) => s && typeof s.absolute === 'number');
+  const dropped = scenarios.length - valid.length;
+  if (valid.length === 0) return { absoluteScore: null, lift: null, n: 0, dropped };
+  const absoluteScore = round1(valid.reduce((a, s) => a + s.absolute, 0) / valid.length);
+  const lift = round1(valid.reduce((a, s) => a + s.delta, 0) / valid.length);
+  return { absoluteScore, lift, n: valid.length, dropped };
+}
+
+// integrity is OPTIONAL and absent means "not checked", never "clean" — callers that predate the
+// integrity layer (holdoutGate, raw stdin scoring) keep their exact behaviour.
+//
+// A violated integrity is BLOCKED, not FAIL, and the distinction is the whole point: FAIL says "the
+// skill does not clear the bar" and sends someone to edit the skill. When the control read the
+// answer key, the only true statement is "this run measures nothing" — a different action entirely.
+export function behavioralGate(agg, integrity) {
+  if (integrity && integrity.ok === false) {
+    const vs = Array.isArray(integrity.violations) ? integrity.violations : [];
+    return {
+      pass: false,
+      blocked: true,
+      reasons: [integrity.reason || 'integrity check failed'],
+      mustFix: vs.length
+        ? vs.map((v) => `${v.kind}: ${v.agent} touched ${v.pattern} (x${v.count}) — re-run with the arm isolated.`)
+        : ['Re-run the eval with integrity verifiable; the scores from this run are not a measurement.'],
+      violations: vs,
+    };
+  }
+  const reasons = [];
+  const mustFix = [];
+  if (!agg || agg.n === 0) {
+    reasons.push('No gradeable capability scenarios (none present or all dropped).');
+    mustFix.push('Add at least one capability scenario with a must_include rubric, then re-run.');
+    return { pass: false, reasons, mustFix };
+  }
+  let pass = true;
+  if (agg.absoluteScore < ABS_MIN) {
+    pass = false;
+    reasons.push(`absolute_score ${agg.absoluteScore} < ${ABS_MIN}`);
+    mustFix.push(`Raise produced-output quality: absolute ${agg.absoluteScore}, needs >= ${ABS_MIN}.`);
+  }
+  if (agg.lift < LIFT_MIN) {
+    pass = false;
+    reasons.push(`lift ${agg.lift} < ${LIFT_MIN}`);
+    mustFix.push(`Skill barely beats no-skill (lift ${agg.lift}). Make the body add value a bare agent lacks.`);
+  }
+  return { pass, reasons, mustFix };
+}
+
+// The hold-out gate — does a fix TRANSFER, or was it memorized?
+//
+// The fix loop scores the edited skill against a FRESH scenario the fixer never saw. Because the
+// eval engine always runs that scenario twice (with and without the skill), the hold-out already
+// carries its own lift: no pre-fix baseline is needed.
+//
+// The bar is deliberately NOT the main gate's LIFT_MIN. One fresh scenario graded once is noisy, so
+// only the unambiguous signal acts: the edited skill failing to beat the bare agent on unseen work.
+// Anything positive transfers. Raising this to LIFT_MIN would reject good fixes on variance alone
+// (see docs/wiki/sdd/specs/generalization-gate.md).
+export function holdoutGate(scored) {
+  const agg = scored && scored.aggregate;
+  if (!agg || agg.n === 0 || typeof agg.lift !== 'number') {
+    return {
+      verdict: 'block',
+      kind: 'indeterminate',
+      lift: null,
+      reason: 'Hold-out could not be scored (no gradeable fresh scenario). Failing closed: an unverified edit is not a verified one.',
+    };
+  }
+  if (agg.lift <= 0) {
+    return {
+      verdict: 'block',
+      kind: 'regression',
+      lift: agg.lift,
+      reason: `Hold-out lift ${agg.lift} <= 0: on a scenario the fixer never saw, the edited skill does not beat no-skill. The edit did not transfer.`,
+    };
+  }
+  return {
+    verdict: 'pass',
+    kind: 'transfer',
+    lift: agg.lift,
+    reason: `Hold-out lift ${agg.lift} > 0: the edit still helps on a scenario the fixer never saw.`,
+  };
+}
+
+export function formatHoldoutVerdict(scored, gate) {
+  const agg = (scored && scored.aggregate) || {};
+  const lines = [];
+  lines.push(`# Hold-out verdict — ${(scored && scored.skillId) || '(unknown skill)'}`);
+  lines.push('');
+  lines.push(`**${gate.verdict === 'pass' ? 'PASS ✅' : 'BLOCK ❌'}** · ${gate.kind} · lift ${gate.lift == null ? 'n/a' : gate.lift} · absolute ${agg.absoluteScore == null ? 'n/a' : agg.absoluteScore} · n=${agg.n || 0}${agg.dropped ? ` (${agg.dropped} dropped)` : ''}`);
+  lines.push('');
+  lines.push(gate.reason);
+  // The fresh scenario is generated by the same model that fixes. Echoing it here is the only way to
+  // audit by hand whether the generator went easy on itself — a risk the spec accepts, not solves.
+  if (scored && scored.holdoutScenario) {
+    lines.push('');
+    lines.push('**Fresh scenario used:**');
+    lines.push(`> ${String(scored.holdoutScenario).replace(/\n/g, '\n> ')}`);
+  }
+  return lines.join('\n');
+}
+
+// raw: { skillId, scenarios: [{index, xIsTreatment, gradeX, gradeY, error?}] }
+export function scoreFromRaw(raw, integrity) {
+  const scenarios = (raw && Array.isArray(raw.scenarios) ? raw.scenarios : []).map((s) => {
+    if (!s || s.error || !s.gradeX || !s.gradeY) {
+      return { index: s ? s.index : null, error: (s && s.error) || 'missing-grade', absolute: null, delta: null };
+    }
+    const treatment = s.xIsTreatment ? s.gradeX : s.gradeY;
+    const baseline = s.xIsTreatment ? s.gradeY : s.gradeX;
+    const t = composeOutputScore(treatment);
+    const b = composeOutputScore(baseline);
+    const d = deriveScenario(t, b);
+    return { index: s.index, treatment: t, baseline: b, absolute: d.absolute, delta: d.delta };
+  });
+  const agg = aggregate(scenarios);
+  const gate = behavioralGate(agg, integrity);
+  const out = { skillId: (raw && raw.skillId) || null, scenarios, aggregate: agg, gate };
+  if (integrity) out.integrity = integrity;
+  // Carried through untouched so the hold-out verdict can echo the fresh scenario for hand-audit.
+  if (raw && raw.holdoutScenario) out.holdoutScenario = raw.holdoutScenario;
+  return out;
+}
+
+export function formatScorecard(scored) {
+  const { skillId, scenarios, aggregate: agg, gate } = scored;
+  const lines = [];
+  lines.push(`# Behavioral scorecard — ${skillId || '(unknown skill)'}`);
+  lines.push('');
+  if (gate.blocked) {
+    // Deliberately NO absolute and NO lift here. Printing them beside a block invites reading them
+    // anyway, and they are not measurements — that is exactly what the block is saying.
+    lines.push(`**Verdict:** BLOCKED ⛔  ·  ${gate.reasons[0]}`);
+    lines.push('');
+    lines.push('The scores from this run are withheld: they are not a measurement of this skill.');
+    lines.push('');
+    lines.push('**Integrity violations:**');
+    for (const f of gate.mustFix) lines.push(`- ${f}`);
+    return lines.join('\n');
+  }
+  lines.push(`**Verdict:** ${gate.pass ? 'PASS ✅' : 'FAIL ❌'}  ·  absolute ${agg.absoluteScore == null ? 'n/a' : agg.absoluteScore}/10 (gate >= ${ABS_MIN})  ·  lift ${agg.lift == null ? 'n/a' : agg.lift} (gate >= ${LIFT_MIN})  ·  n=${agg.n}${agg.dropped ? ` (${agg.dropped} dropped)` : ''}${scored.integrity ? '  ·  integrity verified' : '  ·  integrity NOT CHECKED (pass --transcripts)'}`);
+  lines.push('');
+  lines.push('| Scenario | Treatment | Baseline | Delta |');
+  lines.push('|---|---|---|---|');
+  for (const s of scenarios) {
+    if (s.error) { lines.push(`| ${s.index} | — | — | dropped (${s.error}) |`); continue; }
+    lines.push(`| ${s.index} | ${s.treatment} | ${s.baseline} | ${s.delta >= 0 ? '+' : ''}${s.delta} |`);
+  }
+  if (!gate.pass) {
+    lines.push('');
+    lines.push('**Must fix:**');
+    for (const f of gate.mustFix) lines.push(`- ${f}`);
+  }
+  return lines.join('\n');
+}