hullproof 0.0.0-stage → 0.5.0

package/src/prove/spec.ts

@@ -0,0 +1,256 @@
+// The test spec language of the proof sandbox: validation, templates and assertions. A spec is data. The engine runs it; no model
+// written code ever runs. This file has no imports beyond types, because the same file runs inside the sandbox (see exec-main.ts).
+import { createHash } from 'node:crypto';
+import type { Json } from '../types/hunt.ts';
+import type { Assertion, Spec, SpecStep, StepRecord } from '../types/prove.ts';
+
+export const SPEC_LIMITS = { steps: 24, actors: 6, assertions: 8, captures: 4, body: 4096, header: 512, text: 300, path: 300, idLen: 40 } as const;
+const ID = /^[a-z][a-z0-9_]{0,39}$/;
+const VAR = /\{\{([a-z][a-z0-9_]{0,39})\}\}/g;
+// A path on the app itself: a slash first, no scheme, no host, no dot dot, no backslash. Templates are allowed.
+const PATH = /^\/(?!\/)[A-Za-z0-9._~!$&'()*+,;=:@%/?#\-{}]*$/;
+const METHODS = ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'];
+const HEADER_DENY = new Set(['host', 'connection', 'content-length', 'transfer-encoding', 'expect', 'upgrade', 'te', 'trailer', 'keep-alive']);
+const HEADER_DENY_PREFIX = ['proxy-', 'sec-', 'x-forwarded-', 'x-real-ip'];
+// Strings that look built to damage data, read files or run code. A guard against a bad spec, not a boundary: the sandbox is the boundary.
+const HARMFUL = /\b(?:drop|truncate|alter)\s+(?:table|database|schema)\b|\bdelete\s+from\b|\bunion\s+select\b|\brm\s+-|\bshutdown\b|;\s*--|\$\(|`|\.\.[\\/]|<script|\bexec\s*\(|\bcurl\s|\bwget\s/i;
+export const SECRET_NAME = /(?:pass(?:word|wd)?|secret|token|api[_-]?key|authorization|cookie|session|bearer|jwt|credential|private)/i;
+
+const isObj = (v: unknown): v is Record<string, unknown> => !!v && typeof v === 'object' && !Array.isArray(v);
+
+// Sorted keys, no spaces: the form every hash in this phase is taken over.
+export function canon(v: unknown): string {
+  const sort = (x: unknown): unknown => {
+    if (Array.isArray(x)) return x.map(sort);
+    if (isObj(x)) { const o: Record<string, unknown> = {}; for (const k of Object.keys(x).sort()) if (x[k] !== undefined) o[k] = sort(x[k]); return o; }
+    return x;
+  };
+  return JSON.stringify(sort(v));
+}
+export const sha256 = (s: string): string => createHash('sha256').update(s).digest('hex');
+export const specSha256 = (s: Spec): string => sha256(canon(s));
+
+export function varsIn(text: string): string[] { return [...text.matchAll(VAR)].map((m) => m[1]); }
+
+function strings(v: unknown, out: string[] = []): string[] {
+  if (typeof v === 'string') out.push(v);
+  else if (Array.isArray(v)) v.forEach((x) => strings(x, out));
+  else if (isObj(v)) for (const [k, x] of Object.entries(v)) { out.push(k); strings(x, out); }
+  return out;
+}
+function keysOf(v: unknown, out: string[] = []): string[] {
+  if (Array.isArray(v)) v.forEach((x) => keysOf(x, out));
+  else if (isObj(v)) for (const [k, x] of Object.entries(v)) { out.push(k); keysOf(x, out); }
+  return out;
+}
+
+export type SpecCheck = { ok: true; spec: Spec } | { ok: false; errors: string[] };
+
+function checkHeaders(h: unknown, where: string, err: (m: string) => void): Record<string, string> | undefined {
+  if (h === undefined) return undefined;
+  if (!isObj(h)) { err(`${where}: headers must be an object`); return undefined; }
+  const out: Record<string, string> = {};
+  for (const [k, v] of Object.entries(h)) {
+    const name = k.toLowerCase();
+    if (!/^[a-z][a-z0-9-]{0,39}$/.test(name)) { err(`${where}: header name ${JSON.stringify(k).slice(0, 40)} is not allowed`); continue; }
+    if (HEADER_DENY.has(name) || HEADER_DENY_PREFIX.some((p) => name.startsWith(p))) { err(`${where}: header ${name} may not be set`); continue; }
+    if (typeof v !== 'string' || v.length > SPEC_LIMITS.header || /[\u0000-\u001f\u007f]/.test(v)) { err(`${where}: header ${name} must be a short single line string`); continue; }
+    out[name] = v;
+  }
+  return out;
+}
+
+function checkAssertion(a: unknown, where: string, err: (m: string) => void): Assertion | null {
+  if (!isObj(a) || typeof a.kind !== 'string') { err(`${where}: an assertion needs a kind`); return null; }
+  const text = (x: unknown): x is string => typeof x === 'string' && x.length > 0 && x.length <= SPEC_LIMITS.text;
+  if (a.kind === 'status') {
+    const ok = (n: unknown): boolean => Number.isInteger(n) && (n as number) >= 100 && (n as number) <= 599;
+    if (a.op === 'in' ? !(Array.isArray(a.value) && a.value.length > 0 && a.value.length <= 8 && a.value.every(ok)) : !((a.op === 'eq' || a.op === 'ne') && ok(a.value))) { err(`${where}: a status assertion needs op eq, ne or in and status codes`); return null; }
+    return { kind: 'status', op: a.op as 'eq' | 'ne' | 'in', value: a.value as number | number[] };
+  }
+  if (a.kind === 'json') {
+    if (typeof a.path !== 'string' || !/^[A-Za-z0-9_$-]+(?:\.[A-Za-z0-9_$-]+)*$/.test(a.path) || a.path.length > 120) { err(`${where}: a json assertion needs a dotted path such as items.0.owner`); return null; }
+    if (SECRET_NAME.test(a.path)) { err(`${where}: an assertion may not read a field named like a secret (${a.path}); it is redacted in the transcript`); return null; }
+    if (!['eq', 'ne', 'contains', 'exists', 'absent'].includes(String(a.op))) { err(`${where}: a json assertion needs op eq, ne, contains, exists or absent`); return null; }
+    const needs = a.op === 'eq' || a.op === 'ne' || a.op === 'contains';
+    const v = a.value;
+    if (needs && !(typeof v === 'string' ? v.length <= SPEC_LIMITS.text : typeof v === 'number' || typeof v === 'boolean' || v === null)) { err(`${where}: a json ${a.op} assertion needs a value`); return null; }
+    return { kind: 'json', path: a.path, op: a.op as 'eq', ...(needs ? { value: v as string } : {}) };
+  }
+  if (a.kind === 'text') {
+    if ((a.op !== 'contains' && a.op !== 'absent') || !text(a.value)) { err(`${where}: a text assertion needs op contains or absent and a value`); return null; }
+    return { kind: 'text', op: a.op, value: a.value };
+  }
+  if (a.kind === 'header') {
+    if (typeof a.name !== 'string' || !/^[A-Za-z][A-Za-z0-9-]{0,39}$/.test(a.name) || SECRET_NAME.test(a.name)) { err(`${where}: a header assertion needs a plain header name that is not a secret`); return null; }
+    if (!['eq', 'contains', 'exists', 'absent'].includes(String(a.op))) { err(`${where}: a header assertion needs op eq, contains, exists or absent`); return null; }
+    const needs = a.op === 'eq' || a.op === 'contains';
+    if (needs && !text(a.value)) { err(`${where}: a header ${a.op} assertion needs a value`); return null; }
+    return { kind: 'header', name: a.name.toLowerCase(), op: a.op as 'eq', ...(needs ? { value: a.value as string } : {}) };
+  }
+  err(`${where}: unknown assertion kind`);
+  return null;
+}
+
+// True when an attack assertion shows something got through: a 2xx status or a value that is there.
+const exposes = (a: Assertion): boolean =>
+  (a.kind === 'status' && (a.op === 'eq' || a.op === 'in') && [a.value].flat().every((n) => n >= 200 && n < 300)) ||
+  (a.kind === 'json' && (a.op === 'eq' || a.op === 'contains' || a.op === 'exists')) || (a.kind === 'text' && a.op === 'contains');
+const succeeds = (a: Assertion): boolean => a.kind === 'status' && (a.op === 'eq' || a.op === 'in') && [a.value].flat().every((n) => n >= 200 && n < 300);
+
+// Validates a spec read from JSON. Returns a clean copy (unknown keys dropped) or every problem found. The positive control rule is
+// here: a spec with no control step that must succeed, placed before the attack, is invalid.
+export function validateSpec(raw: unknown): SpecCheck {
+  const errors: string[] = [];
+  const err = (m: string): void => { errors.push(m); };
+  if (!isObj(raw)) return { ok: false, errors: ['the spec must be a JSON object'] };
+  if (raw.schema_version !== '1.0.0') err('schema_version must be "1.0.0"');
+  const str = (k: string, max: number): string => { const v = raw[k]; if (typeof v !== 'string' || !v.trim() || v.length > max) { err(`${k} is missing or too long`); return ''; } return v.replace(/[\u0000-\u001f\u007f]+/g, ' ').trim(); };
+  const spec_id = str('spec_id', SPEC_LIMITS.idLen);
+  if (spec_id && !ID.test(spec_id)) err('spec_id must be lowercase letters, digits and underscores');
+  const title = str('title', 160);
+  const claim = str('claim', 500);
+  const lead_key = raw.lead_key === null || raw.lead_key === undefined ? null : typeof raw.lead_key === 'string' && raw.lead_key.length <= 400 ? raw.lead_key : (err('lead_key must be a string or null'), null);
+  const t = raw.target;
+  let target = { entry_point: '', file: '', line: 1 };
+  if (isObj(t) && typeof t.entry_point === 'string' && t.entry_point && t.entry_point.length <= 300 && typeof t.file === 'string' && t.file && t.file.length <= 300 && Number.isInteger(t.line) && (t.line as number) >= 1) target = { entry_point: t.entry_point, file: t.file, line: t.line as number };
+  else err('target needs entry_point, file and line');
+  const req = raw.requirement_ids;
+  const requirement_ids = Array.isArray(req) && req.length <= 6 && req.every((r) => typeof r === 'string' && /^SEC-[A-Z]+-\d{3}$/.test(r)) ? [...new Set(req as string[])].sort() : (err('requirement_ids must be a list of at most 6 ids like SEC-API-001 (it may be empty)'), []);
+
+  const actors: Spec['actors'] = {};
+  if (!isObj(raw.actors) || !Object.keys(raw.actors).length || Object.keys(raw.actors).length > SPEC_LIMITS.actors) err(`actors needs 1 to ${SPEC_LIMITS.actors} entries`);
+  else {
+    for (const [name, a] of Object.entries(raw.actors)) {
+      if (!ID.test(name)) { err(`actor name ${JSON.stringify(name).slice(0, 30)} is not allowed`); continue; }
+      if (!isObj(a)) { err(`actor ${name} must be an object`); continue; }
+      const h = checkHeaders(a.headers, `actor ${name}`, err);
+      actors[name] = h ? { headers: h } : {};
+    }
+  }
+
+  const steps: SpecStep[] = [];
+  if (!Array.isArray(raw.steps) || raw.steps.length < 2 || raw.steps.length > SPEC_LIMITS.steps) err(`steps needs 2 to ${SPEC_LIMITS.steps} entries`);
+  else {
+    const ids = new Set<string>();
+    raw.steps.forEach((s, i) => {
+      const w = `step ${i + 1}`;
+      if (!isObj(s)) return err(`${w}: must be an object`);
+      if (typeof s.id !== 'string' || !ID.test(s.id) || ids.has(s.id)) return err(`${w}: id must be unique lowercase letters, digits and underscores`);
+      ids.add(s.id);
+      if (s.phase !== 'setup' && s.phase !== 'control' && s.phase !== 'attack') return err(`${w}: phase must be setup, control or attack`);
+      if (typeof s.actor !== 'string' || !(s.actor in actors)) return err(`${w}: actor is not declared in actors`);
+      if (typeof s.method !== 'string' || !METHODS.includes(s.method)) return err(`${w}: method must be one of ${METHODS.join(', ')}`);
+      if (typeof s.path !== 'string' || s.path.length > SPEC_LIMITS.path || !PATH.test(s.path) || s.path.includes('..')) return err(`${w}: path must start with one slash and name a path on the app, never a host or address`);
+      const headers = checkHeaders(s.headers, w, err);
+      let body: SpecStep['body'] = null;
+      if (s.body !== undefined && s.body !== null) {
+        if (!isObj(s.body) || canon(s.body).length > SPEC_LIMITS.body) return err(`${w}: body must be a small JSON object`);
+        body = s.body as { [k: string]: Json };
+        if (s.method === 'GET' || s.method === 'HEAD') return err(`${w}: a ${s.method} request has no body`);
+      }
+      const captures: SpecStep['captures'] = [];
+      if (s.captures !== undefined) {
+        if (!Array.isArray(s.captures) || s.captures.length > SPEC_LIMITS.captures) return err(`${w}: captures needs at most ${SPEC_LIMITS.captures} entries`);
+        for (const c of s.captures) {
+          if (!isObj(c) || typeof c.name !== 'string' || !ID.test(c.name) || !['json', 'header', 'status'].includes(String(c.from))) return err(`${w}: a capture needs a name and from json, header or status`);
+          if (c.from !== 'status' && (typeof c.path !== 'string' || !c.path || c.path.length > 120)) return err(`${w}: a ${String(c.from)} capture needs a path`);
+          captures.push({ name: c.name, from: c.from as 'json', ...(c.from === 'status' ? {} : { path: c.path as string }) });
+        }
+      }
+      if (!Array.isArray(s.assert) || s.assert.length > SPEC_LIMITS.assertions) return err(`${w}: assert needs a list of at most ${SPEC_LIMITS.assertions} assertions`);
+      const assert = s.assert.map((a) => checkAssertion(a, w, err)).filter((a): a is Assertion => a !== null);
+      steps.push({ id: s.id, phase: s.phase, actor: s.actor, method: s.method as SpecStep['method'], path: s.path, ...(headers ? { headers } : {}), ...(body ? { body } : {}), ...(captures.length ? { captures } : {}), assert });
+    });
+  }
+
+  // Only words the spec can supply from an earlier capture: a variable must be captured before the step that uses it.
+  const known = new Set<string>();
+  const secretVars = new Set<string>();
+  for (const s of steps) {
+    const used = [...varsIn(s.path), ...Object.values(s.headers ?? {}).flatMap(varsIn), ...strings(s.body).flatMap(varsIn), ...s.assert.flatMap((a) => ('value' in a && typeof a.value === 'string' ? varsIn(a.value) : []))];
+    for (const v of new Set(used)) if (!known.has(v)) errors.push(`step ${s.id}: {{${v}}} is used before any step captures it`);
+    for (const a of s.assert) if ('value' in a && typeof a.value === 'string') for (const v of varsIn(a.value)) if (secretVars.has(v)) errors.push(`step ${s.id}: an assertion may not compare with {{${v}}}, a secret that is redacted in the transcript`);
+    for (const c of s.captures ?? []) { known.add(c.name); if (SECRET_NAME.test(c.name) || SECRET_NAME.test(c.path ?? '')) secretVars.add(c.name); }
+  }
+  for (const [n, a] of Object.entries(actors)) for (const v of Object.values(a.headers ?? {}).flatMap(varsIn)) if (!steps.some((s) => s.captures?.some((c) => c.name === v))) errors.push(`actor ${n}: {{${v}}} is never captured`);
+
+  // Harmless payloads only.
+  for (const s of steps) {
+    const all = [s.path, ...Object.values(s.headers ?? {}), ...strings(s.body), ...s.assert.flatMap((a) => ('value' in a && typeof a.value === 'string' ? [a.value] : []))];
+    if (all.some((x) => HARMFUL.test(x))) errors.push(`step ${s.id}: payload_not_harmless: the request holds text built to damage data, read files or run code. Use canary strings and read style requests.`);
+    for (const k of keysOf(s.body)) if (HARMFUL.test(k)) errors.push(`step ${s.id}: payload_not_harmless in a body key`);
+  }
+
+  // The positive control rule.
+  const firstAttack = steps.findIndex((s) => s.phase === 'attack');
+  const firstControl = steps.findIndex((s) => s.phase === 'control');
+  if (steps.length && firstAttack < 0) errors.push('a spec needs at least one attack step');
+  if (steps.length && (firstControl < 0 || (firstAttack >= 0 && firstControl > firstAttack))) errors.push('positive control missing: a control step that must succeed has to come before the first attack step');
+  else if (!steps.some((s) => s.phase === 'control' && s.assert.some(succeeds))) errors.push('positive control missing: a control step needs a status assertion with a 2xx code, so a working harness is shown first');
+  for (const s of steps) if ((s.phase === 'control' || s.phase === 'attack') && !s.assert.length) errors.push(`step ${s.id}: a ${s.phase} step needs at least one assertion`);
+  if (steps.some((s) => s.phase === 'attack') && !steps.filter((s) => s.phase === 'attack').some((s) => s.assert.some(exposes))) errors.push('an attack step has to assert what got through (a 2xx status or a value that is there); asserting a denial proves nothing');
+  if (steps.length && steps.some((s, i) => s.phase === 'setup' && steps.slice(0, i).some((p) => p.phase !== 'setup'))) errors.push('setup steps come first');
+  if (!isObj(raw.expect) || raw.expect.outcome !== 'flaw_present') err('expect.outcome must be "flaw_present"');
+
+  if (errors.length) return { ok: false, errors };
+  return { ok: true, spec: { schema_version: '1.0.0', spec_id, title, claim, lead_key, target, requirement_ids, actors, steps, expect: { outcome: 'flaw_present' } } };
+}
+
+// ---- templates ---------------------------------------------------------------------------------------------------------------------
+export type Vars = Record<string, Json>;
+// Replaces {{name}} with the captured value. A name that is not captured is an error. With `redact`, a secret variable shows as [redacted].
+export function fill(text: string, vars: Vars, secrets: ReadonlySet<string>, redact: boolean): string {
+  return text.replace(VAR, (_m, n: string) => {
+    if (!(n in vars)) throw new Error(`unresolved variable ${n}`);
+    if (redact && secrets.has(n)) return '[redacted]';
+    const v = vars[n];
+    return typeof v === 'string' ? v : JSON.stringify(v);
+  });
+}
+export function fillJson(v: Json, vars: Vars, secrets: ReadonlySet<string>, redact: boolean): Json {
+  if (typeof v === 'string') return fill(v, vars, secrets, redact);
+  if (Array.isArray(v)) return v.map((x) => fillJson(x, vars, secrets, redact));
+  if (v && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, fillJson(x, vars, secrets, redact)]));
+  return v;
+}
+
+// ---- assertions, evaluated on the stored form of a response (so a replay gives the same answer) -------------------------------------
+export function pathGet(v: Json | undefined, path: string): Json | undefined {
+  let cur: Json | undefined = v;
+  for (const part of path.split('.')) {
+    if (cur === null || cur === undefined) return undefined;
+    if (Array.isArray(cur)) { if (!/^\d+$/.test(part)) return undefined; cur = cur[Number(part)]; }
+    else if (typeof cur === 'object') cur = Object.prototype.hasOwnProperty.call(cur, part) ? cur[part] : undefined;
+    else return undefined;
+  }
+  return cur;
+}
+const same = (a: Json | undefined, b: Json): boolean => canon(a) === canon(b);
+
+export function evalAssertion(r: NonNullable<StepRecord['response']>, a: Assertion, vars: Vars, secrets: ReadonlySet<string>): { ok: boolean; desc: string } {
+  const want = (s: string): string => fill(s, vars, secrets, true);
+  if (a.kind === 'status') {
+    const ok = a.op === 'eq' ? r.status === a.value : a.op === 'ne' ? r.status !== a.value : (a.value as number[]).includes(r.status);
+    return { ok, desc: `status ${a.op} ${JSON.stringify(a.value)}` };
+  }
+  if (a.kind === 'json') {
+    const got = pathGet(r.json, a.path);
+    const w = a.value === undefined ? undefined : typeof a.value === 'string' ? want(a.value) : a.value;
+    const ok = a.op === 'exists' ? got !== undefined : a.op === 'absent' ? got === undefined
+      : got === undefined ? a.op === 'ne' && r.json !== undefined
+      : a.op === 'eq' ? same(got, w as Json) || (typeof got === 'number' && typeof w === 'string' && String(got) === w)
+      : a.op === 'ne' ? !(same(got, w as Json) || (typeof got === 'number' && typeof w === 'string' && String(got) === w))
+      : typeof got === 'string' ? got.includes(String(w)) : Array.isArray(got) && got.some((x) => same(x, w as Json));
+    return { ok, desc: `json ${a.path} ${a.op}${w === undefined ? '' : ' ' + JSON.stringify(w)}` };
+  }
+  if (a.kind === 'text') {
+    const hay = r.text ?? (r.json === undefined ? '' : canon(r.json));
+    const has = hay.includes(want(a.value));
+    return { ok: a.op === 'contains' ? has : !has, desc: `body text ${a.op} ${JSON.stringify(a.value)}` };
+  }
+  const got = r.headers[a.name];
+  const w = a.value === undefined ? undefined : want(a.value);
+  const ok = a.op === 'exists' ? got !== undefined : a.op === 'absent' ? got === undefined : got !== undefined && (a.op === 'eq' ? got === w : got.includes(w!));
+  return { ok, desc: `header ${a.name} ${a.op}${w === undefined ? '' : ' ' + JSON.stringify(w)}` };
+}

package/src/prove/store.ts

@@ -0,0 +1,48 @@
+// Where proof files live: <out>/<commit>/prove/. Specs go in, run transcripts and the result come out. Written through a temporary
+// file and a rename, with the engine's one JSON writer, so the same value is the same bytes.
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { ProveMeta, ProveResult, Recipe, RunRecord, Spec } from '../types/index.ts';
+import { cmp, stableJson } from '../util.ts';
+import { validateSpec } from './spec.ts';
+
+export const proveDir = (scanDir: string): string => join(scanDir, 'prove');
+const write = (p: string, text: string): void => { mkdirSync(join(p, '..'), { recursive: true }); writeFileSync(`${p}.tmp`, text); renameSync(`${p}.tmp`, p); };
+const readJson = <T>(p: string): T => { try { return JSON.parse(readFileSync(p, 'utf8')) as T; } catch (e) { throw new Error(`${p} cannot be read: ${(e as Error).message}`); } };
+
+export const specFile = (dir: string, id: string): string => join(dir, 'specs', `${id}.json`);
+export const writeSpec = (dir: string, s: Spec): void => write(specFile(dir, s.spec_id), stableJson(s));
+export const runFile = (dir: string, id: string, run: number): string => join(dir, 'runs', id, `run-${run}.json`);
+export const writeRun = (dir: string, r: RunRecord): void => write(runFile(dir, r.spec_id, r.run), stableJson(r));
+export const clearRuns = (dir: string): void => rmSync(join(dir, 'runs'), { recursive: true, force: true });
+export const writeMeta = (dir: string, m: ProveMeta): void => write(join(dir, 'meta.json'), stableJson(m));
+export const readMeta = (dir: string): ProveMeta | null => (existsSync(join(dir, 'meta.json')) ? readJson<ProveMeta>(join(dir, 'meta.json')) : null);
+export const writeRecipe = (dir: string, r: Recipe): void => write(join(dir, 'recipe.json'), stableJson(r));
+export const writeProveResult = (dir: string, r: ProveResult): void => write(join(dir, 'result.json'), stableJson(r));
+export const readProveResult = (dir: string): ProveResult | null => (existsSync(join(dir, 'result.json')) ? readJson<ProveResult>(join(dir, 'result.json')) : null);
+
+// Every spec file, validated. A file that fails validation is dropped and counted, with the reason.
+export function loadSpecs(dir: string): { specs: Spec[]; dropped: { file: string; reason: string }[] } {
+  const specs: Spec[] = [];
+  const dropped: { file: string; reason: string }[] = [];
+  const d = join(dir, 'specs');
+  if (!existsSync(d)) return { specs, dropped };
+  const seen = new Set<string>();
+  for (const f of readdirSync(d).filter((n) => n.endsWith('.json')).sort(cmp)) {
+    let raw: unknown;
+    try { raw = JSON.parse(readFileSync(join(d, f), 'utf8')); } catch { dropped.push({ file: f, reason: 'not valid JSON' }); continue; }
+    const v = validateSpec(raw);
+    if (!v.ok) { dropped.push({ file: f, reason: v.errors.slice(0, 3).join('; ') }); continue; }
+    if (`${v.spec.spec_id}.json` !== f) { dropped.push({ file: f, reason: `spec_id ${v.spec.spec_id} does not match the file name` }); continue; }
+    if (seen.has(v.spec.spec_id)) { dropped.push({ file: f, reason: 'duplicate spec_id' }); continue; }
+    seen.add(v.spec.spec_id);
+    specs.push(v.spec);
+  }
+  return { specs, dropped };
+}
+
+export function readRuns(dir: string, id: string): RunRecord[] {
+  const d = join(dir, 'runs', id);
+  if (!existsSync(d)) return [];
+  return readdirSync(d).filter((n) => /^run-\d+\.json$/.test(n)).sort(cmp).map((n) => readJson<RunRecord>(join(d, n))).sort((a, b) => a.run - b.run);
+}

package/src/report/build.ts

@@ -0,0 +1,304 @@
+// Builds results.json (one row per requirement in scope) and the findings record (schema 1.1.0) from the check results
+// and scanner hits. FAIL comes only from a FAIL check or a mapped rule hit; PASS (static) only from a check on a repo
+// authority requirement; every other requirement in scope is NOT ASSESSED with the sub state that closes it.
+import { createHash } from 'node:crypto';
+import { cmp, stableJson, stableJsonCompact } from '../util.ts';
+import { ENGINE_VERSION } from '../version.ts';
+import type {
+  CheckResult, CheckResultsFile, FailRecord, FileLineNote, FindingsFile, FindingRecord, Hit, ObservationRecord, ResultRow,
+  ResultWord, RouteState, Severity, ToolMeta,
+} from '../types/index.ts';
+import { checkMessage, ruleMessage } from './messages.ts';
+import { highest, inScope, type Control, type Kit, type Scope } from './kit.ts';
+import { TEST_ARTIFACT_IDS, testCodeReason } from './test-code.ts';
+import { isSecretHit, leadReason, leadTier, leadTierForCheck, loadTiers } from './tiers.ts';
+
+export interface GitState { commit: string; tree: string; clean: boolean }
+export interface BuildInput {
+  kit: Kit; scope: Scope; git: GitState; checks: CheckResultsFile; hits: Hit[]; metas: ToolMeta[];
+  checkShas: Record<string, string>;  // check id -> sha256 of its definition file
+  rulesDirs: string[];               // Semgrep rule folders (engine, then the rule pack), for the plain English text
+}
+
+const MAX_NOTE = 600;
+// Checks whose FAIL means a control is missing everywhere searched (class missing_control, with the search recorded).
+const ABSENCE_CHECKS = new Set(['check.web.security-headers-configured', 'check.auth.login-rate-limit', 'check.supply.lockfile-committed']);
+// A fingerprint path has no spaces or colons.
+const fpPath = (f: string): string => f.replace(/[\s:]/g, '_');
+const clip = (s: string, n: number): string => (s.length <= n ? s : s.slice(0, n - 1).trimEnd() + '…');
+const slug = (s: string, max = 12): string => s.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').split('-').filter(Boolean).slice(0, max).join('-') || 'x';
+const where = (e: { file: string; line?: number }): string => (e.line ? `${e.file}:${e.line}` : e.file);
+
+// The sub state for a requirement nothing settled, from its authority and how it is verified.
+export function routeFor(c: Control): RouteState {
+  if (c.authority === 'dashboard') return 'NEEDS DASHBOARD';
+  if (c.authority === 'runtime') return 'NEEDS DYNAMIC TEST';
+  return 'UNKNOWN';
+}
+
+export const OWNER_ACTION: Record<RouteState, string> = {
+  'NEEDS DASHBOARD': 'Export the provider setting (screenshot or export, dated, naming the project) and record it.',
+  'NEEDS BUILD': 'Run a production build from this commit and scan the output.',
+  'NEEDS DYNAMIC TEST': 'Test the running app or a staging copy and record the requests and responses.',
+  'ASK OWNER': 'Answer the question in writing.',
+  'ATTESTATION': 'Sign and date a statement.',
+  'UNKNOWN': 'No engine check or rule settles this from the files. A code review (a person, or the Hullproof audit) must read the code and record the result.',
+};
+
+interface Fail { source: string; sec_ids: string[]; evidence: { file: string; line?: number; note: string }[] }
+
+// A finding kept for the report and never in results.json or the gate: test code (src/report/test-code.ts), or a lead
+// from a rule whose confidence tier is lead (src/report/tiers.ts).
+interface TestItem { hit?: Hit; check?: CheckResult; ids: string[]; reason: string; kind: 'test' | 'lead'; total?: number }
+
+// Splits hits and failed checks into what can fail a requirement and what is only reported. In order:
+// 1. A secret (a Gitleaks hit or a rule mapped to a SEC-SECRETS ID) stays in the gate wherever it is, unless the scanner marked the hit as an
+//    example value (h.noise: src/scanners/secret-noise.ts), which moves to an example value item.
+// 2. In a test path, a requirement about shipped test material (TEST_ARTIFACT_IDS) keeps failing; every other ID moves
+//    to a test code item.
+// 3. A hit of a rule whose tier is lead moves to a lead item.
+export function separateTestCode(input: BuildInput): { input: BuildInput; test: TestItem[] } {
+  const test: TestItem[] = [];
+  const hits: Hit[] = [];
+  const tiers = loadTiers(input.rulesDirs ?? []);
+  for (const h of input.hits) {
+    // A secret stays in the gate, unless the scanner found the value to be an example (a placeholder, or a shapeless rule in docs or fixtures).
+    if (isSecretHit(h) && h.noise) { test.push({ hit: h, ids: h.sec_ids, reason: h.noise, kind: 'test' }); continue; }
+    const reason = isSecretHit(h) ? null : testCodeReason(h.file);
+    if (reason) {
+      const keep = h.sec_ids.filter((id) => TEST_ARTIFACT_IDS.has(id));
+      if (keep.length) hits.push({ ...h, sec_ids: keep });
+      if (keep.length < h.sec_ids.length || !h.sec_ids.length) test.push({ hit: h, ids: h.sec_ids.filter((id) => !TEST_ARTIFACT_IDS.has(id)), reason, kind: 'test' });
+      continue;
+    }
+    const lead = leadTier(h, tiers);
+    if (lead) { test.push({ hit: h, ids: h.sec_ids, reason: leadReason(lead, tiers), kind: 'lead' }); continue; }
+    hits.push(h);
+  }
+  const results: CheckResult[] = [];
+  for (const r of input.checks.results) {
+    // A check that reads the code map is a lead until it is measured: its FAIL is reported and never fails a requirement.
+    const leadCheck = r.result === 'FAIL' ? leadTierForCheck(r.check_id, tiers) : null;
+    if (leadCheck) {
+      const byFile = new Map<string, typeof r.evidence>();
+      for (const e of r.evidence) byFile.set(e.file, [...(byFile.get(e.file) ?? []), e]);
+      const reason = leadReason(leadCheck, tiers);
+      // Past COLLAPSE_FILES files the check gets one record for all its places (the list stays in checks.json), so a repository
+      // with a thousand routes does not push the findings block past the kit limit.
+      if (byFile.size > COLLAPSE_FILES) test.push({ check: r, ids: r.sec_ids, reason, kind: 'lead', total: r.evidence.length });
+      else for (const f of [...byFile.keys()].sort(cmp)) test.push({ check: { ...r, evidence: byFile.get(f)! }, ids: r.sec_ids, reason, kind: 'lead' });
+      continue;
+    }
+    const secret = r.sec_ids.some((id) => id.startsWith('SEC-SECRETS-'));
+    const inTest = r.result === 'FAIL' && !secret && !r.sec_ids.some((id) => TEST_ARTIFACT_IDS.has(id)) ? r.evidence.filter((e) => testCodeReason(e.file)) : [];
+    const shipped = r.evidence.filter((e) => !inTest.includes(e));
+    if (inTest.length) test.push({ check: { ...r, evidence: inTest }, ids: r.sec_ids, reason: testCodeReason(inTest[0].file)!, kind: 'test' });
+    // A FAIL whose evidence is all in test code no longer fails; one with no evidence (an absence check) still does.
+    if (!inTest.length || shipped.length) results.push(inTest.length ? { ...r, evidence: shipped } : r);
+  }
+  return { input: { ...input, hits, checks: { ...input.checks, results } }, test };
+}
+
+function failsFor(input: BuildInput): Fail[] {
+  const out: Fail[] = [];
+  for (const r of input.checks.results) if (r.result === 'FAIL') out.push({ source: r.check_id, sec_ids: r.sec_ids, evidence: r.evidence });
+  for (const h of input.hits) if (h.sec_ids.length) out.push({ source: `${h.tool} rule ${h.rule_id}`, sec_ids: h.sec_ids, evidence: [{ file: h.file, line: h.line || undefined, note: h.rule_id }] });
+  return out;
+}
+
+export function buildResults(all: BuildInput): ResultRow[] {
+  const input = separateTestCode(all).input;
+  const { kit, scope } = input;
+  const fails = failsFor(input);
+  const checkWords = new Map<string, CheckResult[]>();
+  for (const r of input.checks.results) for (const id of r.sec_ids) checkWords.set(id, [...(checkWords.get(id) ?? []), r]);
+  const skippedBy = new Map<string, string[]>();
+  for (const m of input.metas) if (m.status !== 'ran') for (const id of m.not_assessed_sec_ids) skippedBy.set(id, [...(skippedBy.get(id) ?? []), `${m.tool} (${m.status})`]);
+
+  const rows: ResultRow[] = [];
+  for (const c of kit.controls.filter((x) => inScope(x, scope)).sort((a, b) => cmp(a.id, b.id))) {
+    const mine = fails.filter((f) => f.sec_ids.includes(c.id));
+    if (mine.length) {
+      const where1 = mine.flatMap((f) => f.evidence.slice(0, 3).map((e) => `${f.source} at ${where(e)}`));
+      rows.push({ id: c.id, result: 'FAIL', notes: clip(`Failed by ${[...new Set(where1)].join('; ')}.`, MAX_NOTE) });
+      continue;
+    }
+    const words = checkWords.get(c.id) ?? [];
+    const pass = words.find((w) => w.result === 'PASS (static)');
+    if (pass && c.authority === 'repo') {
+      rows.push({ id: c.id, result: 'PASS (static)', notes: clip(`Settled by ${pass.check_id} on the files of this commit.`, MAX_NOTE) });
+      continue;
+    }
+    // A check that ran and found nothing wrong names the sub state that would settle the rest.
+    const routed = words.map((w) => w.result).find((w) => w.startsWith('NOT ASSESSED: ')) as ResultWord | undefined;
+    const state = (routed ? routed.slice('NOT ASSESSED: '.length) : routeFor(c)) as RouteState;
+    const parts = [routed ? `${words.map((w) => w.check_id).join(', ')} found nothing wrong in the files; that cannot settle it.` : '', OWNER_ACTION[state]];
+    const skipped = skippedBy.get(c.id);
+    if (skipped) parts.push(`Not run: ${skipped.join(', ')}.`);
+    rows.push({ id: c.id, result: `NOT ASSESSED: ${state}`, notes: clip(parts.filter(Boolean).join(' '), MAX_NOTE) });
+  }
+  return rows;
+}
+
+// Requirement IDs of a fail split into those in scope (they make a fail record) and the rest.
+function split(ids: string[], input: BuildInput): { scoped: string[]; known: string[]; unknown: string[] } {
+  const known = ids.filter((id) => input.kit.byId.has(id));
+  return {
+    scoped: known.filter((id) => inScope(input.kit.byId.get(id)!, input.scope)).sort(cmp),
+    known: known.sort(cmp),
+    unknown: ids.filter((id) => !input.kit.byId.has(id)).sort(cmp),
+  };
+}
+
+const refs = (ev: { file: string; line?: number; note: string }[]): FileLineNote[] =>
+  ev.slice(0, 8).map((e) => ({ file: e.file, line: e.line && e.line > 0 ? e.line : 1, note: clip(e.note, 300) }));
+
+export interface Dropped { source: string; ids: string[]; where: string }
+
+// Past the kit's size warning even on one line, a rule that matched more than COLLAPSE_FILES files gets one record for
+// all its files instead of one per file; every hit stays in scan/<tool>.json, and the record says how many files.
+// ponytail: fixed file count per rule; lower it or group per folder if a repository still passes the kit limit.
+const COLLAPSE_FILES = 20;
+export function buildFindings(all: BuildInput): { file: FindingsFile; dropped: Dropped[] } {
+  const first = buildFindingsWith(all, null);
+  return stableJsonCompact(first.file).length <= KIT_WARN_BLOCK_CHARS ? first : buildFindingsWith(all, COLLAPSE_FILES);
+}
+
+function buildFindingsWith(all: BuildInput, collapse: number | null): { file: FindingsFile; dropped: Dropped[] } {
+  const { input, test } = separateTestCode(all);
+  // tool and rule -> files it matched, in shipped code and in test code apart.
+  const filesOf = new Map<string, Set<string>>();
+  const note = (k: string, f: string): void => { filesOf.set(k, (filesOf.get(k) ?? new Set()).add(f)); };
+  for (const h of input.hits) note(`${h.tool}\u0000${h.rule_id}`, h.file);
+  for (const t of test) if (t.hit) note(`${t.kind}\u0000${t.hit.tool}\u0000${t.hit.rule_id}`, t.hit.file);
+  const many = (k: string): number => (collapse !== null && (filesOf.get(k)?.size ?? 0) > collapse ? filesOf.get(k)!.size : 0);
+  const records: FindingRecord[] = [];
+  const dropped: Dropped[] = [];
+  const add = (o: {
+    source: string; ruleId: string; tool: string; toolVersion: string; configSha: string | null; ids: string[];
+    evidence: { file: string; line?: number; note: string }[]; msg: ReturnType<typeof checkMessage>; absence: CheckResult['absence_search']; line?: number;
+  }): void => {
+    const s = split(o.ids, input);
+    const file0 = fpPath(o.evidence[0]?.file ?? '-');
+    if (s.unknown.length) dropped.push({ source: o.source, ids: s.unknown, where: where(o.evidence[0] ?? { file: '-' }) });
+    if (s.scoped.length === 0) {
+      // Outside this run's edition or stage, or unmapped: shown as an observation, never in the gate.
+      const obs: ObservationRecord = {
+        fingerprint: `OBS:${file0}:${slug(`${o.ruleId} ${o.line ?? ''}`)}`, kind: 'observation', sec_ids: s.known, title: clip(o.msg.title, 160),
+        where: refs(o.evidence).slice(0, 4),
+        note: clip(s.known.length ? `Outside this run's scope (${input.scope.edition} edition, ${input.scope.stage} stage): ${s.known.join(', ')}. ${o.msg.fix}` : `No requirement is mapped to ${o.source}${s.unknown.length ? ` in this kit (named: ${s.unknown.join(', ')})` : ''}. ${o.msg.fix}`, 400),
+        raised_by: ['hullproof-engine'],
+      };
+      records.push(obs);
+      return;
+    }
+    const sev: Severity = highest(s.scoped.map((id) => input.kit.byId.get(id)!.severity));
+    const rec: FailRecord = {
+      fingerprint: `${s.scoped[0]}:${file0}:${slug(`${o.ruleId} ${o.line ?? ''}`)}`,
+      kind: 'fail', sec_ids: s.scoped, title: clip(o.msg.title, 160),
+      class: o.tool === 'hullproof-check' && ABSENCE_CHECKS.has(o.ruleId) && o.absence ? 'missing_control' : 'finding',
+      gap_in_standard: false, severity: sev, confidence: 'SUSPECTED', demonstrated: false,
+      root_cause: clip(o.msg.root, 120), raised_by: ['hullproof-engine'], impact_statement: clip(o.msg.impact, 600),
+      remediation: { strategy: clip(o.msg.fix, 800), confirm: clip(o.msg.confirm, 400) },
+      evidence: refs(o.evidence),
+      machine_evidence: { tool: o.tool, rule_id: o.ruleId, tool_version: o.toolVersion.replace(/[^A-Za-z0-9._+-]/g, '-') || 'unknown', config_sha256: o.configSha },
+    };
+    if (rec.class === 'missing_control') {
+      rec.absence_search = { command: clip(`${o.ruleId}: ${o.absence!.pattern} over ${o.absence!.file_globs.join(' ') || 'the selected files'}`, 300), files_searched: o.absence!.files_searched, count: o.absence!.matches, practice: 'present_unsafe' };
+    }
+    records.push(rec);
+  };
+
+  for (const r of input.checks.results) {
+    if (r.result !== 'FAIL') continue;
+    add({ source: r.check_id, ruleId: r.check_id, tool: 'hullproof-check', toolVersion: ENGINE_VERSION, configSha: input.checkShas[r.check_id] ?? null,
+      ids: r.sec_ids, evidence: r.evidence, msg: checkMessage(r.check_id), absence: r.absence_search });
+  }
+  // Scanner hits: one record per tool, rule and file, with up to 8 lines.
+  const groups = new Map<string, Hit[]>();
+  for (const h of input.hits) {
+    const k = `${h.tool}\u0000${h.rule_id}\u0000${many(`${h.tool}\u0000${h.rule_id}`) ? '*' : h.file}`;
+    groups.set(k, [...(groups.get(k) ?? []), h]);
+  }
+  for (const k of [...groups.keys()].sort(cmp)) {
+    const hs = groups.get(k)!;
+    const h = hs[0];
+    const n = many(`${h.tool}\u0000${h.rule_id}`);
+    const meta = input.metas.find((m) => m.tool === h.tool);
+    const label = n ? `${h.tool} rule ${h.rule_id}, which matched ${n} files (all in scan/${h.tool}.json)` : `${h.tool} rule ${h.rule_id}`;
+    add({ source: `${h.tool} rule ${h.rule_id}`, ruleId: h.rule_id, tool: h.tool, toolVersion: meta?.version ?? 'unknown', configSha: meta?.config_sha256 ?? null,
+      ids: [...new Set(hs.flatMap((x) => x.sec_ids))], evidence: hs.map((x) => ({ file: x.file, line: x.line || undefined, note: clip(label, 300) })),
+      msg: ruleMessage(h.tool, h.rule_id, input.rulesDirs), absence: null, line: n ? undefined : h.line });
+  }
+
+  // Test code and leads: one observation per tool, rule and file, marked by a fingerprint slug that starts with
+  // test-code or lead.
+  const testGroups = new Map<string, TestItem[]>();
+  for (const t of test) {
+    const k = t.hit
+      ? `${t.kind}\u0000${t.hit.tool}\u0000${t.hit.rule_id}\u0000${many(`${t.kind}\u0000${t.hit.tool}\u0000${t.hit.rule_id}`) ? '*' : t.hit.file}`
+      : `${t.kind}\u0000hullproof-check\u0000${t.check!.check_id}\u0000${t.check!.evidence[0].file}`;
+    testGroups.set(k, [...(testGroups.get(k) ?? []), t]);
+  }
+  for (const k of [...testGroups.keys()].sort(cmp)) {
+    const ts = testGroups.get(k)!;
+    const [kind, tool, ruleId, key] = k.split('\u0000');
+    const file = key === '*' ? ts[0].hit!.file : key;
+    const msg = ts[0].hit ? ruleMessage(tool, ruleId, input.rulesDirs) : checkMessage(ruleId);
+    const ev = ts.flatMap((t) => (t.hit ? [{ file: t.hit.file, line: t.hit.line || undefined, note: `${tool} rule ${ruleId}` }] : t.check!.evidence));
+    const ids = [...new Set(ts.flatMap((t) => t.ids))].filter((id) => input.kit.byId.has(id)).sort(cmp);
+    const isLead = kind === 'lead';
+    const isExample = !isLead && ts.every((t) => !!t.hit?.noise);
+    records.push({
+      fingerprint: `OBS:${fpPath(file)}:${slug(`${isLead ? 'lead' : 'test code'} ${ruleId} ${ts[0].hit?.line ?? ''}`, 14)}`, kind: 'observation', sec_ids: ids,
+      title: clip(`${isLead ? 'Lead' : isExample ? 'Example value' : 'Test code'}: ${msg.title}`, 160), where: refs(ev).slice(0, 4),
+      note: clip(isLead
+        ? `Lead from ${ts[0].hit ? 'rule' : 'check'} ${ruleId}, whose confidence tier is lead: ${ts[0].reason}. Read the code; it never holds the gate.${ids.length ? ` If real it fails ${ids.join(', ')}.` : ''}${ts[0].total ? ` ${ts[0].total} places in all; the first 4 are listed here and every one is in checks.json.` : ''}`
+        : `${isExample ? 'Example value' : 'Test code'} (${ts[0].reason}): kept as a lead, never in the gate.${ids.length ? ` In shipped code it would fail ${ids.join(', ')}.` : ''} ${msg.fix}`, 400),
+      raised_by: ['hullproof-engine'],
+    });
+  }
+
+  // Fingerprints are unique: a second record with the same tail gets a short hash of its source.
+  const seen = new Set<string>();
+  for (const r of records) {
+    let fp = r.fingerprint;
+    const tail = fp.slice(fp.indexOf(':') + 1);
+    if (seen.has(tail)) fp = `${fp}-${createHash('sha256').update(JSON.stringify(r)).digest('hex').slice(0, 6).replace(/[^a-z0-9]/g, 'a')}`;
+    seen.add(fp.slice(fp.indexOf(':') + 1));
+    r.fingerprint = fp;
+  }
+  records.sort((a, b) => cmp(a.fingerprint, b.fingerprint));
+  const needsVerifier = records.some((r) => r.kind === 'fail' && (r.severity === 'BLOCKER' || r.severity === 'CRITICAL' || r.sec_ids.some((id) => ['BLOCKER', 'CRITICAL'].includes(input.kit.byId.get(id)!.severity))));
+  const file: FindingsFile = {
+    schema_version: '1.1.0',
+    report: {
+      commit: input.git.commit, tree: input.git.tree, tree_clean: input.git.clean, stage: input.scope.stage, profile: 'standard', scope: 'full',
+      // No model and no verifier run in Phase 1. A BLOCKER or CRITICAL record needs a verifier outcome in a complete run,
+      // so such a run is recorded incomplete with the kit's reason for exactly this: deterministic_only (schema 1.1.0).
+      run_status: needsVerifier ? 'incomplete' : 'complete',
+      ...(needsVerifier ? { incomplete_reason: 'deterministic_only' as const } : {}),
+      agent_calls: 0, budget: null,
+    },
+    records,
+  };
+  return { file, dropped: dropped.sort((a, b) => cmp(a.source + a.where, b.source + b.where)) };
+}
+
+// The kit validator warns on a findings block over 150000 characters and rejects one over 200000 (rule F3). Indented JSON
+// is written while it stays under the warning size; past it the same sorted JSON goes on one line, which is about 40
+// percent shorter on the targets measured (juice-shop with a large rule pack: 207103 indented).
+// ponytail: compact JSON only raises the ceiling; group records per rule instead of per file if a repository passes it.
+export const KIT_WARN_BLOCK_CHARS = 150000;
+export function findingsJson(f: FindingsFile): string {
+  const pretty = stableJson(f);
+  return pretty.length <= KIT_WARN_BLOCK_CHARS ? pretty : stableJsonCompact(f);
+}
+
+export function findingsMarkdown(f: FindingsFile, json: string): string {
+  return [
+    '# Findings record', '',
+    `Written by the Hullproof Engine ${ENGINE_VERSION} for commit ${f.report.commit}. ${f.records.some((r) => r.raised_by.includes('hullproof-hunt')) ? 'Every record comes from a check, a scanner rule or a lead the engine confirmed with evidence it can re run; no model decides any record.' : 'Every record comes from a check or a scanner rule, with no model.'} Validate it with tools/hullproof/ci/validate-findings.mjs from the kit.`, '',
+    '```hullproof-findings', json.trimEnd(), '```', '',
+  ].join('\n');
+}