crewforth 3.1.1 → 3.2.0-rc.2

package/kit/eval/lib/findings/findings.mjs

@@ -0,0 +1,410 @@
+#!/usr/bin/env node
+// findings.mjs — the evidence-backed findings core: one document shape, and a validator that holds every
+// finding to the lines it cites.
+//
+//   node findings.mjs validate [--root DIR] [--verified] [--json] FILE|-    rc 0 = every entry verified · 1 = at least one problem
+//   node findings.mjs stamp    [--root DIR] [--verified] [--redact] FILE|-  the document on stdout with a sha256 on each verified entry
+//   node findings.mjs write    [--root DIR] [--verified] NAME FILE|-        validate, then write DIR/.crewforth/NAME.json
+//   node findings.mjs schema                                   the schema, as shipped
+//       rc 2 = usage, or a document that could not be read as a JSON object. Nothing is written on a non-zero rc.
+//
+// Host-independent on purpose: plain JSON in, plain JSON and an exit code out, Node's standard library only. It
+// reads no hook payload, no settings file and no environment variable, so the same file serves a scanner, a CI
+// job and an assistant; whatever is specific to one host belongs in the caller.
+//
+// The shape lives in findings.schema.json beside this file and is READ from there, so there is one definition.
+// What a schema cannot say is checked here, against the project at --root (default: the working directory):
+//
+//   path      the cited file is a relative, forward-slash path with no '.' or '..' segment, spelled exactly as
+//             the directory lists it (a case-insensitive disk would otherwise verify a path that another
+//             machine cannot open), and it resolves inside the project after symlinks
+//   missing   the file is there and is a regular file
+//   range     line..endLine exist in it
+//   stale     the anchor still matches: sha256 of the cited lines, and/or the quoted text is in them
+//   machine-path   no string in the document carries this machine's project or home directory
+//   verification   a finding's verdict is the one its verifiers' verdicts give, its verifiers are distinct agents on
+//                  distinct lenses, and only a TRUE_POSITIVE keeps a severity
+//   --verified     every finding of a severity in the schema's x-verifiers table has that many verifiers, and every
+//                  verifier is a line of the refuter ledger, .claude/state/crew-refuter.tsv under --root, which only
+//                  Crewforth's SubagentStop hook writes. The path is fixed on purpose: a ledger the caller could name
+//                  is one the caller could write.
+//
+// A document is either standalone ("schema": "crewforth.findings/1", held to the whole schema) or a host
+// document with its own "schema": "crewforth.<name>/<n>" and a "findings" array in the same shape. In both,
+// EVERY "evidence" array is verified wherever it sits, so a host document can put evidence on its own objects
+// (an edge, a module) and have it held to the same standard.
+//
+// Not checked: that the cited lines actually support the finding's claim. That is a judgement; this proves only
+// that the lines exist and are the ones the producer read.
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+
+const SELF = fileURLToPath(import.meta.url);
+export const SCHEMA_ID = 'crewforth.findings/1';
+export const OUT_DIR = '.crewforth';
+const HOST_SCHEMA = /^crewforth\.[a-z][a-z0-9-]*\/[1-9][0-9]*$/;
+const OUT_NAME = /^[a-z0-9][a-z0-9-]*$/;
+const MAX_FILE_BYTES = 32 * 1024 * 1024;
+
+export function loadSchema() {
+  return JSON.parse(fs.readFileSync(path.join(path.dirname(SELF), 'findings.schema.json'), 'utf8'));
+}
+
+// ---- shape: the subset of JSON Schema that findings.schema.json uses --------------------------------------
+// $ref into $defs, type, const, enum, pattern, min/maxLength, minimum, minItems, items, required, properties,
+// additionalProperties:false, anyOf. A keyword outside this list in the schema is a bug the selfcheck pins.
+export const SHAPE_KEYWORDS = ['$ref', 'type', 'const', 'enum', 'pattern', 'minLength', 'maxLength', 'minimum',
+  'minItems', 'items', 'required', 'properties', 'additionalProperties', 'anyOf'];
+
+const kind = (v) => (v === null ? 'null' : Array.isArray(v) ? 'array'
+  : Number.isInteger(v) ? 'integer' : typeof v);
+const show = (v) => { const s = JSON.stringify(v); return s === undefined ? String(v) : s.length > 80 ? s.slice(0, 77) + '...' : s; };
+
+function shape(v, s, at, schema, out) {
+  if (s.$ref) s = s.$ref.replace(/^#\//, '').split('/').reduce((o, k) => o[k], schema);
+  const bad = (message) => out.push({ at, code: 'shape', message });
+  if ('const' in s && v !== s.const) return bad(`must be ${show(s.const)}, is ${show(v)}`);
+  if (s.enum && !s.enum.includes(v)) return bad(`must be one of ${s.enum.join(' | ')}, is ${show(v)}`);
+  if (s.type && kind(v) !== s.type) return bad(`must be of type ${s.type}, is ${show(v)}`);
+  if (typeof v === 'string') {
+    if (s.minLength !== undefined && v.length < s.minLength) bad(`is shorter than ${s.minLength}`);
+    if (s.maxLength !== undefined && v.length > s.maxLength) bad(`is longer than ${s.maxLength} (${v.length})`);
+    if (s.pattern && !new RegExp(s.pattern).test(v)) bad(`must match ${s.pattern}, is ${show(v)}`);
+  }
+  if (typeof v === 'number' && s.minimum !== undefined && v < s.minimum) bad(`must be at least ${s.minimum}, is ${v}`);
+  if (Array.isArray(v)) {
+    if (s.minItems !== undefined && v.length < s.minItems) bad(`needs at least ${s.minItems} entr${s.minItems === 1 ? 'y' : 'ies'}, has ${v.length}`);
+    if (s.items) v.forEach((x, i) => shape(x, s.items, `${at}[${i}]`, schema, out));
+  }
+  if (kind(v) === 'object') {
+    for (const k of s.required || []) if (!(k in v)) bad(`"${k}" is required`);
+    for (const k of Object.keys(v)) {
+      if (s.properties && k in s.properties) shape(v[k], s.properties[k], at ? `${at}.${k}` : k, schema, out);
+      else if (s.additionalProperties === false) bad(`"${k}" is not a field of this object`);
+    }
+    if (s.anyOf && !s.anyOf.some((alt) => { const o = []; shape(v, alt, at, schema, o); return o.length === 0; })) {
+      bad(`needs one of: ${s.anyOf.map((alt) => (alt.required || []).join('+')).join(' | ')}`);
+    }
+  }
+}
+
+// ---- evidence: a cited range, held against the project ----------------------------------------------------
+const squash = (t) => t.replace(/\s+/g, ' ').trim();
+
+function unsafePath(file) {
+  if (file.includes('\0')) return 'contains a NUL';
+  if (file.includes('\\')) return 'uses a backslash; write forward slashes';
+  if (file.startsWith('/') || /^[A-Za-z]:/.test(file)) return 'is absolute; write it relative to the project root';
+  for (const seg of file.split('/')) {
+    if (seg === '' || seg === '.' || seg === '..') return `has ${seg === '' ? 'an empty' : `a "${seg}"`} segment`;
+  }
+  return '';
+}
+
+// One project, opened once: directory listings and file lines are cached for the run.
+export function openProject(root) {
+  const real = fs.realpathSync(root);
+  const dirs = new Map();
+  const files = new Map();
+  const list = (dir) => {
+    if (!dirs.has(dir)) { let names = null; try { names = new Set(fs.readdirSync(dir)); } catch { /* not a directory */ } dirs.set(dir, names); }
+    return dirs.get(dir);
+  };
+  // { lines: Buffer[] } or { code, message }
+  const read = (file) => {
+    if (files.has(file)) return files.get(file);
+    const done = (r) => { files.set(file, r); return r; };
+    const why = unsafePath(file);
+    if (why) return done({ code: 'path', message: `${show(file)} ${why}` });
+    let abs = real;
+    for (const seg of file.split('/')) {
+      const names = list(abs);
+      if (!names || !names.has(seg)) return done({ code: 'missing', message: `${file}: no "${seg}" with that exact spelling in the project` });
+      abs = path.join(abs, seg);
+    }
+    let target;
+    try { target = fs.realpathSync(abs); } catch { return done({ code: 'missing', message: `${file}: a link that leads nowhere` }); }
+    if (target !== real && !target.startsWith(real + path.sep)) return done({ code: 'path', message: `${file} resolves outside the project` });
+    const st = fs.statSync(target);
+    if (!st.isFile()) return done({ code: 'missing', message: `${file} is not a regular file` });
+    if (st.size > MAX_FILE_BYTES) return done({ code: 'unreadable', message: `${file} is ${st.size} bytes; the limit is ${MAX_FILE_BYTES}` });
+    const buf = fs.readFileSync(target);
+    const lines = [];
+    for (let from = 0; from < buf.length;) {
+      let nl = buf.indexOf(0x0a, from);
+      if (nl === -1) nl = buf.length;
+      let to = nl;
+      if (to > from && buf[to - 1] === 0x0d) to--;
+      lines.push(buf.subarray(from, to));
+      from = nl + 1;
+    }
+    return done({ lines });
+  };
+  return { root: real, read };
+}
+
+// The anchor's bytes: the cited lines joined by LF, CR at line ends already removed, no trailing newline.
+function rangeOf(project, ev) {
+  const f = project.read(ev.file);
+  if (!f.lines) return f;
+  const end = ev.endLine === undefined ? ev.line : ev.endLine;
+  if (end < ev.line) return { code: 'range', message: `${ev.file}: endLine ${end} is before line ${ev.line}` };
+  if (end > f.lines.length) return { code: 'range', message: `${ev.file}:${ev.line}${end > ev.line ? '-' + end : ''} is past the end; the file has ${f.lines.length} line(s)` };
+  const parts = [];
+  for (let i = ev.line; i <= end; i++) { if (i > ev.line) parts.push(Buffer.from('\n')); parts.push(f.lines[i - 1]); }
+  const bytes = Buffer.concat(parts);
+  return { bytes, sha256: crypto.createHash('sha256').update(bytes).digest('hex'), where: `${ev.file}:${ev.line}${end > ev.line ? '-' + end : ''}` };
+}
+
+// null when the entry holds; otherwise { code, message }. `ev` has already passed the shape check.
+export function checkEvidence(project, ev) {
+  const r = rangeOf(project, ev);
+  if (!r.bytes) return r;
+  if (ev.sha256 !== undefined && ev.sha256 !== r.sha256) {
+    return { code: 'stale', message: `${r.where} sha256 is ${r.sha256}, the document says ${ev.sha256}` };
+  }
+  if (ev.text !== undefined) {
+    const want = squash(ev.text);
+    if (want === '') return { code: 'shape', message: `${r.where} "text" is only whitespace` };
+    if (!squash(r.bytes.toString('utf8')).includes(want)) return { code: 'stale', message: `${r.where} does not contain the quoted text ${show(want)}` };
+  }
+  return null;
+}
+
+// Every "evidence" member in the document, with its JSON path. The key is the contract: a host document that
+// puts evidence anywhere gets it verified, and one that names the member differently gets nothing — which the
+// counts in the report make visible.
+function evidenceArrays(v, at, out) {
+  if (Array.isArray(v)) v.forEach((x, i) => evidenceArrays(x, `${at}[${i}]`, out));
+  else if (v && typeof v === 'object') {
+    for (const k of Object.keys(v)) {
+      const here = at ? `${at}.${k}` : k;
+      if (k === 'evidence') out.push({ at: here, value: v[k] });
+      else evidenceArrays(v[k], here, out);
+    }
+  }
+  return out;
+}
+
+function strings(v, at, out) {
+  if (typeof v === 'string') out.push({ at, value: v });
+  else if (Array.isArray(v)) v.forEach((x, i) => strings(x, `${at}[${i}]`, out));
+  else if (v && typeof v === 'object') for (const k of Object.keys(v)) strings(v[k], at ? `${at}.${k}` : k, out);
+  return out;
+}
+
+// The spellings of this machine's project and home directory that must not travel in a document. A root or a
+// home of three characters or fewer ("/", "C:\") is skipped: it would match every path there is.
+function machinePaths(rootGiven, rootReal) {
+  const set = new Set();
+  for (const p of [path.resolve(rootGiven), rootReal, os.homedir()]) {
+    if (!p || p.length <= 3) continue;
+    set.add(p); set.add(p.split(path.sep).join('/'));
+  }
+  return [...set];
+}
+
+// { ok, schema, counts: { findings, evidence, verified }, problems: [{ at, code, message }], held: [entry…] }
+export function validate(doc, root = '.', opts = {}) {
+  const problems = [];
+  const counts = { findings: 0, evidence: 0, verified: 0 };
+  const held = [];
+  const report = () => ({ ok: problems.length === 0, schema: doc && doc.schema, counts, problems, held });
+  if (kind(doc) !== 'object') { problems.push({ at: '', code: 'shape', message: `the document must be a JSON object, is ${show(doc)}` }); return report(); }
+  const schema = loadSchema();
+  const FINDING = { $ref: '#/$defs/finding' }; const EVIDENCE = { $ref: '#/$defs/evidence' };
+  if (doc.schema === SCHEMA_ID) shape(doc, schema, '', schema, problems);
+  else {
+    if (typeof doc.schema !== 'string' || !HOST_SCHEMA.test(doc.schema)) problems.push({ at: 'schema', code: 'shape', message: `must be "${SCHEMA_ID}" or a host document's "crewforth.<name>/<n>", is ${show(doc.schema)}` });
+    if (!Array.isArray(doc.findings)) problems.push({ at: 'findings', code: 'shape', message: `must be an array, is ${show(doc.findings)}` });
+    else doc.findings.forEach((f, i) => shape(f, FINDING, `findings[${i}]`, schema, problems));
+  }
+  if (Array.isArray(doc.findings)) {
+    counts.findings = doc.findings.length;
+    const seen = new Map();
+    doc.findings.forEach((f, i) => {
+      if (!f || typeof f.id !== 'string') return;
+      if (seen.has(f.id)) problems.push({ at: `findings[${i}].id`, code: 'duplicate-id', message: `${show(f.id)} is also findings[${seen.get(f.id)}]` });
+      else seen.set(f.id, i);
+    });
+  }
+  const project = openProject(root);
+  for (const m of machinePaths(root, project.root)) {
+    for (const s of strings(doc, '', [])) {
+      if (s.value.includes(m)) problems.push({ at: s.at, code: 'machine-path', message: `carries this machine's directory ${show(m)}` });
+    }
+  }
+  const inFinding = /^findings\[\d+\]\.evidence$/;
+  for (const arr of evidenceArrays(doc, '', [])) {
+    if (!Array.isArray(arr.value)) { problems.push({ at: arr.at, code: 'shape', message: `must be an array, is ${show(arr.value)}` }); continue; }
+    arr.value.forEach((ev, i) => {
+      const at = `${arr.at}[${i}]`;
+      counts.evidence++;
+      // A finding's evidence was already shaped, and reported, with the finding; evidence anywhere else is
+      // reported here. Either way an entry that is not well-formed is not taken to the project.
+      const o = []; shape(ev, EVIDENCE, at, schema, o);
+      if (o.length) { if (!inFinding.test(arr.at)) problems.push(...o); return; }
+      const p = checkEvidence(project, ev);
+      if (p) problems.push({ at, ...p }); else { counts.verified++; held.push(ev); }
+    });
+  }
+  if (Array.isArray(doc.findings)) verification(doc.findings, schema, project, opts, problems);
+  return report();
+}
+
+// ---- verification: what refuters concluded, and the ledger it must come from -------------------------------
+export const LEDGER = path.join('.claude', 'state', 'crew-refuter.tsv');
+// The verdict a set of verifiers gives: one that shows the claim wrong outweighs the rest, an open question
+// outweighs agreement. The producer writes it down; it does not get to choose it.
+export function verdictOf(verdicts) {
+  if (verdicts.includes('FALSE_POSITIVE')) return 'FALSE_POSITIVE';
+  if (verdicts.includes('CANNOT_VERIFY')) return 'CANNOT_VERIFY';
+  return 'TRUE_POSITIVE';
+}
+// { rows: [{ agent, finding, lens, verdict }] } or { error }
+export function readLedger(file) {
+  let text;
+  try { text = fs.readFileSync(file, 'utf8'); } catch { return { error: `no refuter ledger at ${LEDGER.split(path.sep).join('/')}` }; }
+  const rows = [];
+  for (const line of text.split('\n')) {
+    const c = line.replace(/\r$/, '').split('\t');
+    if (c.length < 6 || c[0] === 'ts') continue;
+    rows.push({ agent: c[2], finding: c[3], lens: c[4], verdict: c[5] });
+  }
+  return { rows };
+}
+function verification(findings, schema, project, opts, problems) {
+  const table = opts.verified ? (schema['x-verifiers'] || {}) : {};
+  let ledger = null;
+  findings.forEach((f, i) => {
+    if (!f || typeof f !== 'object') return;
+    const at = `findings[${i}]`;
+    const v = f.verification;
+    const ok = v && typeof v === 'object' && Array.isArray(v.verifiers) && v.verifiers.every((x) => x && typeof x === 'object');
+    if (ok) {
+      const want = verdictOf(v.verifiers.map((x) => x.verdict));
+      if (v.verdict !== want) problems.push({ at: `${at}.verification.verdict`, code: 'verification', message: `is ${show(v.verdict)}; its verifiers give ${want}` });
+      const agents = v.verifiers.map((x) => x.agent); const lenses = v.verifiers.map((x) => x.lens);
+      if (new Set(agents).size !== agents.length) problems.push({ at: `${at}.verification.verifiers`, code: 'verification', message: 'names one agent twice: one context counted as two verifiers' });
+      if (new Set(lenses).size !== lenses.length) problems.push({ at: `${at}.verification.verifiers`, code: 'verification', message: 'repeats a lens: the same method twice is repetition, not confirmation' });
+      if (f.severity !== undefined && v.verdict !== 'TRUE_POSITIVE') problems.push({ at: `${at}.severity`, code: 'verification', message: `a finding judged ${v.verdict} carries no severity` });
+    }
+    if (!opts.verified) return;
+    const need = typeof f.severity === 'string' ? table[f.severity] : undefined;
+    if (typeof need === 'number' && need > 0) {
+      if (!ok) { problems.push({ at, code: 'unverified', message: `a ${f.severity} finding needs ${need} verifier(s) from the refuter ledger, and has none` }); return; }
+      if (v.verifiers.length < need) problems.push({ at: `${at}.verification.verifiers`, code: 'unverified', message: `a ${f.severity} finding needs ${need} verifier(s), has ${v.verifiers.length}` });
+    }
+    if (!ok) return;
+    if (ledger === null) ledger = readLedger(opts.ledger || path.join(project.root, LEDGER));
+    if (ledger.error) { problems.push({ at: `${at}.verification`, code: 'ledger', message: ledger.error }); return; }
+    v.verifiers.forEach((x, k) => {
+      const hit = ledger.rows.some((r) => r.agent === x.agent && r.finding === f.id && r.lens === x.lens && r.verdict === x.verdict);
+      if (!hit) problems.push({ at: `${at}.verification.verifiers[${k}]`, code: 'ledger', message: `no ledger line says agent ${show(x.agent)} judged ${show(f.id)} on ${show(x.lens)} as ${show(x.verdict)}` });
+    });
+  });
+}
+
+// The document with the cited lines' sha256 written onto every entry that verifies. An entry that does not
+// verify is never given a hash: stamping would turn an unchecked claim into one that looks checked.
+// `redact` then drops "text" from those entries, for lines that should not be quoted (a secret, personal data).
+export function stamp(doc, root = '.', { redact = false, ...opts } = {}) {
+  const r = validate(doc, root, opts);
+  const project = openProject(root);
+  for (const ev of r.held) {
+    ev.sha256 = rangeOf(project, ev).sha256;
+    if (redact) delete ev.text;
+  }
+  return r;
+}
+
+// The bytes a document is stored as: findings in id order, two-space indent, LF, one trailing newline. Members
+// keep the order the producer gave them; a producer that wants a byte-stable file emits them in a fixed order.
+export function serialize(doc) {
+  const out = { ...doc };
+  if (Array.isArray(out.findings)) out.findings = [...out.findings].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+  return JSON.stringify(out, null, 2) + '\n';
+}
+
+// Validate, then write <root>/.crewforth/<name>.json. An unverified document is never written. Returns the
+// validation report with `file` (relative to the root) when it was.
+export function write(doc, root, name, opts = {}) {
+  if (!OUT_NAME.test(name)) throw new Error(`the name must match ${OUT_NAME}, is ${show(name)}`);
+  const r = validate(doc, root, opts);
+  if (!r.ok) return r;
+  const real = fs.realpathSync(root);
+  const dir = path.join(real, OUT_DIR);
+  fs.mkdirSync(dir, { recursive: true });
+  if (!fs.lstatSync(dir).isDirectory()) throw new Error(`${OUT_DIR} is not a directory of this project`);
+  const tmp = path.join(dir, `.${name}.${process.pid}.tmp`);
+  fs.writeFileSync(tmp, serialize(doc));
+  fs.renameSync(tmp, path.join(dir, `${name}.json`));
+  return { ...r, file: `${OUT_DIR}/${name}.json` };
+}
+
+// ---- command line -----------------------------------------------------------------------------------------
+function usage(msg) {
+  if (msg) console.error(`findings: ${msg}`);
+  console.error('usage: findings.mjs validate [--root DIR] [--verified] [--json] FILE|-\n'
+    + '       findings.mjs stamp    [--root DIR] [--verified] [--redact] FILE|-\n'
+    + '       findings.mjs write    [--root DIR] [--verified] NAME FILE|-\n'
+    + '       findings.mjs schema');
+  return 2;
+}
+
+function summary(r, log) {
+  for (const p of r.problems) log(`  FAIL ${p.at || '(document)'}: ${p.code} — ${p.message}`);
+  log(`findings: ${r.counts.findings} finding(s) · ${r.counts.evidence} evidence entr${r.counts.evidence === 1 ? 'y' : 'ies'} checked · ${r.counts.verified} verified · ${r.problems.length} problem(s)`);
+}
+
+export function main(argv) {
+  const [op, ...rest] = argv;
+  if (op === 'schema' && rest.length === 0) { process.stdout.write(JSON.stringify(loadSchema(), null, 2) + '\n'); return 0; }
+  if (!['validate', 'stamp', 'write'].includes(op)) return usage(op ? `no such operation: ${op}` : '');
+  let root = '.'; let json = false; let redact = false; let verified = false; const pos = [];
+  for (let i = 0; i < rest.length; i++) {
+    const a = rest[i];
+    if (a === '--root') { root = rest[++i]; if (root === undefined) return usage('--root needs a directory'); }
+    else if (a === '--json' && op === 'validate') json = true;
+    else if (a === '--redact' && op === 'stamp') redact = true;
+    else if (a === '--verified') verified = true;
+    else if (a !== '-' && a.startsWith('-')) return usage(`${op} has no option ${a}`);
+    else pos.push(a);
+  }
+  if (pos.length !== (op === 'write' ? 2 : 1)) return usage(`${op} takes ${op === 'write' ? 'a name and a document' : 'one document'}`);
+  const src = pos[pos.length - 1];
+  let doc;
+  try {
+    if (!fs.statSync(root).isDirectory()) throw new Error('not a directory');
+  } catch (e) { console.error(`findings: --root ${root}: ${e.message}`); return 2; }
+  try { doc = JSON.parse(fs.readFileSync(src === '-' ? 0 : src, 'utf8')); }
+  catch (e) { console.error(`findings: ${src === '-' ? 'stdin' : src} could not be read as JSON: ${e.message}`); return 2; }
+
+  if (op === 'validate') {
+    const r = validate(doc, root, { verified });
+    if (json) { const { held, ...pub } = r; process.stdout.write(JSON.stringify(pub, null, 2) + '\n'); }
+    else summary(r, (l) => console.log(l));
+    return r.ok ? 0 : 1;
+  }
+  if (op === 'stamp') {
+    const r = stamp(doc, root, { redact, verified });
+    summary(r, (l) => console.error(l));
+    if (!r.ok) return 1;
+    process.stdout.write(serialize(doc));
+    return 0;
+  }
+  let r;
+  try { r = write(doc, root, pos[0], { verified }); } catch (e) { return usage(e.message); }
+  summary(r, (l) => console.error(l));
+  if (!r.ok) { console.error('findings: nothing was written'); return 1; }
+  console.log(r.file);
+  return 0;
+}
+
+let direct = false;
+try { direct = !!process.argv[1] && fs.realpathSync(process.argv[1]) === fs.realpathSync(SELF); } catch { /* imported */ }
+if (direct) process.exitCode = main(process.argv.slice(2));

package/kit/eval/lib/findings/findings.schema.json

@@ -0,0 +1,122 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "crewforth.findings/1",
+  "title": "Crewforth findings document",
+  "description": "A list of findings, each one tied to the lines of the project it was read from. This file is the one definition of the shape: findings.mjs reads it at run time, so a field cannot be accepted by the validator and absent here, or the reverse. Whether an evidence entry is TRUE (the file is there, the lines are there, the anchor still matches) is not expressible in JSON Schema; findings.mjs checks that against the project and its header says how.",
+  "type": "object",
+  "required": ["schema", "findings"],
+  "additionalProperties": false,
+  "properties": {
+    "schema": { "const": "crewforth.findings/1" },
+    "producer": {
+      "type": "object",
+      "required": ["name"],
+      "additionalProperties": false,
+      "properties": {
+        "name": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$" },
+        "version": { "type": "string", "minLength": 1, "maxLength": 64 }
+      }
+    },
+    "source": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "commit": { "type": "string", "pattern": "^[0-9a-f]{7,64}$" }
+      }
+    },
+    "findings": { "type": "array", "items": { "$ref": "#/$defs/finding" } },
+    "notMeasured": { "type": "array", "items": { "$ref": "#/$defs/notMeasured" } }
+  },
+  "x-verifiers": {
+    "description": "Under --verified: a finding of one of these severities needs this many verifiers, each its own agent on its own lens, and keeps the severity only when the verdict is TRUE_POSITIVE. A severity not listed needs none.",
+    "medium": 1, "high": 3, "critical": 3
+  },
+  "$defs": {
+    "finding": {
+      "type": "object",
+      "required": ["id", "rule", "title", "confidence", "evidence"],
+      "additionalProperties": false,
+      "properties": {
+        "id": {
+          "description": "Stable across runs and unique in the document.",
+          "type": "string",
+          "pattern": "^[a-z0-9][a-z0-9._:-]*$",
+          "maxLength": 200
+        },
+        "rule": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$", "maxLength": 80 },
+        "title": { "type": "string", "minLength": 1, "maxLength": 200 },
+        "confidence": {
+          "description": "proven: the cited lines show it on their own. inferred: the cited lines are what the conclusion was drawn from.",
+          "enum": ["proven", "inferred"]
+        },
+        "severity": { "enum": ["info", "low", "medium", "high", "critical"] },
+        "detail": { "type": "string", "maxLength": 4000 },
+        "evidence": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/evidence" } },
+        "data": {
+          "description": "The producer's own fields. Not interpreted here.",
+          "type": "object"
+        },
+        "verification": { "$ref": "#/$defs/verification" }
+      }
+    },
+    "verification": {
+      "description": "What independent refuters concluded. verdict is not chosen by the producer: it is FALSE_POSITIVE when any verifier said so, else CANNOT_VERIFY when any did, else TRUE_POSITIVE, and the validator checks it. Under --verified every verifier must be a line of the refuter ledger.",
+      "type": "object",
+      "required": ["verdict", "verifiers"],
+      "additionalProperties": false,
+      "properties": {
+        "verdict": { "$ref": "#/$defs/verdict" },
+        "verifiers": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/verifier" } }
+      }
+    },
+    "verifier": {
+      "type": "object",
+      "required": ["agent", "lens", "verdict"],
+      "additionalProperties": false,
+      "properties": {
+        "agent": { "description": "The refuter agent's id, as the ledger records it.", "type": "string", "pattern": "^[A-Za-z0-9._-]{1,128}$" },
+        "lens": { "enum": ["reachability", "protection", "reproduction", "exclusion", "blast-radius", "claim"] },
+        "verdict": { "$ref": "#/$defs/verdict" }
+      }
+    },
+    "verdict": { "enum": ["TRUE_POSITIVE", "FALSE_POSITIVE", "CANNOT_VERIFY"] },
+    "evidence": {
+      "description": "One cited range of one file. An entry needs an anchor, text or sha256, because a line number alone only says the file is that long.",
+      "type": "object",
+      "required": ["file", "line"],
+      "additionalProperties": false,
+      "anyOf": [{ "required": ["text"] }, { "required": ["sha256"] }],
+      "properties": {
+        "file": {
+          "description": "Relative to the project root, forward slashes, no '.' or '..' segment.",
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 1024
+        },
+        "line": { "type": "integer", "minimum": 1 },
+        "endLine": { "type": "integer", "minimum": 1 },
+        "text": {
+          "description": "A quote from the cited lines. Compared with runs of whitespace collapsed.",
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        },
+        "sha256": {
+          "description": "SHA-256 of the cited lines joined by LF, CR at line ends removed, no trailing newline.",
+          "type": "string",
+          "pattern": "^[0-9a-f]{64}$"
+        },
+        "note": { "type": "string", "maxLength": 400 }
+      }
+    },
+    "notMeasured": {
+      "type": "object",
+      "required": ["what", "reason"],
+      "additionalProperties": false,
+      "properties": {
+        "what": { "type": "string", "minLength": 1, "maxLength": 400 },
+        "reason": { "type": "string", "minLength": 1, "maxLength": 400 }
+      }
+    }
+  }
+}

package/kit/eval/lib/findings/findings.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# findings.sh — the shell entry to findings.mjs. It does one thing: find a Node that RUNS and hand over.
+#
+#   bash findings.sh validate|stamp|write|schema …     the arguments and exit codes of findings.mjs (0, 1, 2)
+#       rc 3 = no usable Node here, so nothing was checked. Not a pass: a caller must not read 3 as verified.
+#
+# A name that resolves is not a runtime that works: the Windows Store's stub satisfies `command -v` and then
+# exits 49. So node is asked for its version and the answer is read. When PATH has none, the panel's own
+# resolver is asked (it knows the places PATH does not reach, and the runtime Crewforth may have fetched). This
+# file never installs anything.
+set -u
+# A caller on Windows may name this file with backslashes (Node's path.join does); cut at either separator,
+# or the directory falls back to "." and findings.mjs is looked for in the caller's cwd.
+d="${BASH_SOURCE%[/\\]*}"; [ "$d" = "${BASH_SOURCE}" ] && d=.
+FLOOR=18
+
+usable(){ local v; v="$("$1" -p 'process.versions.node.split(".")[0]' 2>/dev/null)" || return 1
+  case "$v" in ''|*[!0-9]*) return 1 ;; esac; [ "$v" -ge "$FLOOR" ]; }
+
+NODE=""
+if usable node; then NODE=node
+elif [ -f "$d/../../../studio/ensure-node.sh" ]; then
+  n="$(bash "$d/../../../studio/ensure-node.sh" 2>/dev/null)" && [ -n "$n" ] && usable "$n" && NODE="$n"
+fi
+if [ -z "$NODE" ]; then
+  echo "findings: no Node $FLOOR+ that runs was found, so nothing was checked (this is not a pass)" >&2
+  exit 3
+fi
+exec "$NODE" "$d/findings.mjs" "$@"

package/kit/eval/lib/settings-default.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# settings-default.sh — put one of Crewforth's defaults into the project's .claude/settings.json, only where nothing
+# sets it yet. The project's settings win over the user's, so writing a default over a user's own choice would
+# silently replace it: a value set in the project's settings, its local settings or the user's own is kept.
+#
+#   bash settings-default.sh --install <path> <defaults.json> [ENV_VAR]     from the project root
+#     <path>           the setting, as a dot path: statusLine, env.CLAUDE_CODE_SUBAGENT_MODEL
+#     <defaults.json>  a JSON object holding just that setting (data beside the caller: one definition)
+#     ENV_VAR          also kept when this variable is set in the environment (a value exported in the shell)
+#
+# Prints one word: installed · kept (it is set) · none (no settings.json) · failed (the file is left as it was).
+# A managed settings layer is not read. The merge is Crewforth's own JSON reader.
+set -u
+_here="${BASH_SOURCE%[/\\]*}"; [ "$_here" = "${BASH_SOURCE}" ] && _here=.
+
+[ "${1:-}" = --install ] && [ -n "${2:-}" ] && [ -n "${3:-}" ] || { echo "usage: settings-default.sh --install <path> <defaults.json> [ENV_VAR]" >&2; exit 2; }
+KEY="$2"; K="$3"; VAR="${4:-}"
+P=".claude/settings.json"; SJ="$_here/settings-json.awk"
+[ -f "$P" ] && [ -f "$SJ" ] || { echo none; exit 0; }
+case "$VAR" in '') ;; [A-Za-z_]*) case "$VAR" in *[!A-Za-z0-9_]*) echo failed; exit 0 ;; esac; [ -n "${!VAR:-}" ] && { echo kept; exit 0; } ;; *) echo failed; exit 0 ;; esac
+for f in "$P" ".claude/settings.local.json" "${CLAUDE_CONFIG_DIR:-$HOME/.claude}/settings.json"; do
+  [ -f "$f" ] && awk -v op=get -v path="$KEY" -f "$SJ" "$f" >/dev/null 2>&1 && { echo kept; exit 0; }
+done
+[ -f "$K" ] || { echo failed; exit 0; }
+t="$P.default.$$"
+r=failed
+# The project's file goes FIRST: the merge refreshes the first file's .claude/hooks/ entries from the first file,
+# so put second it would lose every Crewforth hook (measured: the installed project's own smoke failed 30 checks).
+# The second file holds only the default, which the first lacks: that is the one change (an object such as env is
+# merged member by member, the project's own members kept). Written through the file, so a symlink and its mode
+# survive.
+if awk -v op=merge -v retired= -f "$SJ" "$P" "$K" > "$t" 2>/dev/null && [ -s "$t" ] \
+   && awk -v op=validate -f "$SJ" "$t" 2>/dev/null && awk -v op=get -v path="$KEY" -f "$SJ" "$t" >/dev/null 2>&1; then
+  cat "$t" > "$P" && r=installed
+fi
+rm -f "$t"; echo "$r"

package/kit/eval/lib/settings-json.awk

@@ -4,10 +4,14 @@
 #   awk -v op=validate -f settings-json.awk FILE              rc 0 = FILE is a JSON object, 1 = it is not
 #   awk -v op=get -v path=hooks.Stop.0 -f settings-json.awk FILE   the value there, as the merge would print it
 #   awk -v op=len -v path=hooks.Stop -f settings-json.awk FILE     how many entries an array/object holds
+#   awk -v op=gets -v path='a.b|c.d' -f settings-json.awk FILE   several values in one read, tab-separated, "" if absent
 #   awk -v op=strings -v path=permissions.deny -f settings-json.awk FILE   each string element of that array, one per
 #       line, quotes off, escapes as written (rc 1 = not an array)
 #       path = dot-separated keys, a number indexes an array (0-based), empty = the root. rc 1 = no such path
 #       (or, for len, a scalar there), 10 = FILE is not a JSON object.
+#   awk -v op=get -v path=P -v setk=M -v setv=RAWJSON -f settings-json.awk FILE   the object at P with member M set
+#       to RAWJSON (added when absent): a payload rewritten in one field, every other field and escape kept
+#       (rc 1 = nothing at P, or not an object)
 #   awk -v op=overlay -v key=K [-v setk=M -v setv=RAWJSON] -f settings-json.awk FILE1 FILE2
 #       FILE1 with FILE1.K shallow-merged from FILE2.K (FILE2 wins, FILE1's other members and order kept), then
 #       member M set to RAWJSON. rc 10 = FILE1 (or its K) is not an object, 11 = FILE2 has no object at K.
@@ -69,6 +73,10 @@ function at(id,path,  n,sg,i){ n=split(path,sg,".")
   return id }
 function mk(t){ NN++; T[NN]=t; N[NN]=0; return NN }
 function add(o,k,c){ N[o]++; K[o,N[o]]=k; C[o,N[o]]=c }
+# Member k of object o set to the raw JSON v, added at the end when o has none (overlay and get share it).
+function setm(o,k,v,  sn,i,x){ sn=mk("s"); V[sn]=v; x=0
+  for(i=1;i<=N[o];i++) if(K[o,i]=="\"" k "\""){ C[o,i]=sn; x=1 }
+  if(!x) add(o,"\"" k "\"",sn) }
 function canon(id,  i,j,n,s,t,ord){
   if(T[id]=="s") return V[id]
   if(T[id]=="a"){ s="["; for(i=1;i<=N[id];i++) s=s (i>1 ? "," : "") canon(C[id,i]); return s "]" }
@@ -124,8 +132,13 @@ END{
       print (it && !isnull(it) ? V[it] : 0) + V[cr] + (cc && !isnull(cc) ? V[cc] : 0); exit 0 }
     exit 0 }
   if(op=="validate") exit !doc(1)
+  # gets: several paths in one read (path = "a.b|c.d"), their values on one line, tab-separated; an absent one is
+  # empty. One process for what a hot caller (the status line) would otherwise read with one awk per field.
+  if(op=="gets"){ if(!(r=doc(1))) exit 10; n=split(path,pp,"|"); o=""
+    for(i=1;i<=n;i++){ x=at(r,pp[i]); o=o (i>1?"\t":"") (x ? put(x,"") : "") } print o; exit 0 }
   if(op=="get" || op=="len"){ if(!(r=doc(1))) exit 10; if(!(r=at(r,path))) exit 1
-    if(op=="get"){ print put(r,""); exit 0 } if(T[r]=="s") exit 1; print N[r]; exit 0 }
+    if(op=="get"){ if(setk!=""){ if(T[r]!="o") exit 1; setm(r,setk,setv) } print put(r,""); exit 0 }
+    if(T[r]=="s") exit 1; print N[r]; exit 0 }
   if(op=="strings"){ if(!(r=doc(1))) exit 10; if(!(r=at(r,path))) exit 1; if(T[r]!="a") exit 1
     for(i=1;i<=N[r];i++){ c=C[r,i]; if(T[c]=="s" && substr(V[c],1,1)=="\"") print substr(V[c],2,length(V[c])-2) }
     exit 0 }
@@ -133,9 +146,7 @@ END{
     qv=get(q,kk); if(T[qv]!="o") exit 11; tv=get(t,kk); if(tv && T[tv]!="o") exit 10
     nv=mk("o"); for(i=1;i<=N[tv];i++){ x=get(qv,K[tv,i]); add(nv,K[tv,i], x ? x : C[tv,i]) }
     for(i=1;i<=N[qv];i++) if(!get(nv,K[qv,i])) add(nv,K[qv,i],C[qv,i])
-    if(setk!=""){ sn=mk("s"); V[sn]=setv; x=0
-      for(i=1;i<=N[nv];i++) if(K[nv,i]=="\"" setk "\""){ C[nv,i]=sn; x=1 }
-      if(!x) add(nv,"\"" setk "\"",sn) }
+    if(setk!="") setm(nv,setk,setv)
     x=0; for(i=1;i<=N[t];i++) if(K[t,i]==kk){ C[t,i]=nv; x=1 }
     if(!x) add(t,kk,nv)
     print put(t,""); exit 0 }