hullproof 0.0.0-stage → 0.5.0

package/dist/prove/run.js

@@ -0,0 +1,136 @@
+import { ENGINE_VERSION } from '../version.js';
+import { cmp } from '../util.js';
+import { judge, runRecord } from './executor.js';
+import { deriveRecipe, recipeSha256 } from './recipe.js';
+import { makeSandboxCopy } from './safety.js';
+import { DEFAULT_LIMITS } from './runner.js';
+import { canon, sha256, specSha256 } from './spec.js';
+import { clearRuns, loadSpecs, readMeta, readRuns, writeMeta, writeProveResult, writeRecipe, writeRun } from './store.js';
+export const RUNS_REQUIRED = 3;
+function verifyRun(spec, specSha, rr) {
+    const bad = (detail) => ({ run: rr.run, outcome: 'error', transcript_sha256: rr.transcript_sha256, detail });
+    if (rr.spec_sha256 !== specSha)
+        return bad('the spec changed after this run, so its transcript no longer belongs to it');
+    const { transcript_sha256: _h, ...body } = rr;
+    if (sha256(canon(body)) !== rr.transcript_sha256)
+        return bad('the stored transcript does not match its hash; it was changed after the run');
+    if (!rr.steps.length)
+        return rr.outcome === 'error' ? { run: rr.run, outcome: 'error', transcript_sha256: rr.transcript_sha256, detail: rr.detail } : bad('a run with no steps can only be an error');
+    const j = judge(spec, rr.steps);
+    if (!j.done || !j.outcome)
+        return bad('the stored transcript stops before the spec reaches a verdict');
+    if (j.outcome !== rr.outcome)
+        return bad(`the stored outcome (${rr.outcome}) does not match what the transcript shows (${j.outcome})`);
+    return { run: rr.run, outcome: j.outcome, transcript_sha256: rr.transcript_sha256, detail: j.detail };
+}
+function combine(runs, required) {
+    if (runs.length < required)
+        return { outcome: 'error', reason: `${runs.length} of ${required} runs finished, so nothing is proven` };
+    const set = new Set(runs.map((r) => r.outcome));
+    if (set.size > 1)
+        return { outcome: 'unstable', reason: `the runs disagreed (${runs.map((r) => r.outcome).join(', ')}), so the lead stays a lead` };
+    const o = runs[0].outcome;
+    return { outcome: o, reason: o === 'proven' ? `${required} of ${required} runs: the positive control passed and the flaw showed` : runs[0].detail };
+}
+export function buildResult(meta, specs, dropped, runsOf) {
+    const out = [];
+    for (const s of specs.slice().sort((a, b) => cmp(a.spec_id, b.spec_id))) {
+        const sha = specSha256(s);
+        const base = { spec_id: s.spec_id, spec_sha256: sha, lead_key: s.lead_key, title: s.title, claim: s.claim, entry_point: s.target.entry_point, requirement_ids: s.requirement_ids, sink: { file: s.target.file, line: s.target.line } };
+        if (meta.not_provable) {
+            out.push({ ...base, outcome: 'not_provable', reason: `not provable here (${meta.not_provable.code}): ${meta.not_provable.reason}`, runs: [] });
+            continue;
+        }
+        const runs = runsOf(s.spec_id).map((r) => verifyRun(s, sha, r));
+        out.push({ ...base, ...combine(runs, meta.runs_required), runs });
+    }
+    const n = (o) => out.filter((x) => x.outcome === o).length;
+    return {
+        schema_version: '1.0.0', engine_version: ENGINE_VERSION, commit: meta.commit, runner: meta.runner, runs_required: meta.runs_required,
+        recipe_sha256: meta.recipe_sha256, not_provable: meta.not_provable, specs: out, dropped, left_behind: meta.left_behind, copy_excluded: meta.copy_excluded,
+        counts: { specs: out.length, proven: n('proven'), not_proven: n('not_proven'), invalid: n('invalid'), unstable: n('unstable'), error: n('error'), not_provable: n('not_provable'), dropped: dropped.length },
+    };
+}
+const finish = (i, meta, specs, dropped) => {
+    writeMeta(i.proveDir, meta);
+    const r = buildResult(meta, specs, dropped, (id) => readRuns(i.proveDir, id));
+    writeProveResult(i.proveDir, r);
+    return r;
+};
+export async function runProve(i) {
+    const limits = i.limits ?? DEFAULT_LIMITS;
+    const required = i.runs ?? RUNS_REQUIRED;
+    const { specs, dropped } = loadSpecs(i.proveDir);
+    clearRuns(i.proveDir);
+    const meta = { schema_version: '1.0.0', commit: i.commit, runner: i.runner.name, runs_required: required, recipe_sha256: null, not_provable: null, left_behind: 0, copy_excluded: [] };
+    const stop = (np) => finish(i, { ...meta, not_provable: np }, specs, dropped);
+    if (!specs.length) {
+        i.log('hullproof prove: no valid specs to run');
+        return finish(i, meta, specs, dropped);
+    }
+    const rec = deriveRecipe(i.repo, i.stack, i.map, { allowInstallScripts: i.allowInstallScripts });
+    if (!rec.ok)
+        return stop({ code: rec.code, reason: rec.reason });
+    const recipe = rec.recipe;
+    meta.recipe_sha256 = recipeSha256(recipe);
+    writeRecipe(i.proveDir, recipe);
+    const av = await i.runner.available();
+    if (!av.ok)
+        return stop({ code: 'no_runtime', reason: av.reason });
+    const copy = makeSandboxCopy(i.repo, { keepNodeModules: recipe.vendored_modules });
+    if ('error' in copy)
+        return stop({ code: 'too_large', reason: copy.error });
+    meta.copy_excluded = copy.excluded;
+    let notProvable = null;
+    let leftBehind = 0;
+    try {
+        i.log(`hullproof prove: ${i.runner.name}, ${copy.files} files copied, ${copy.excluded.length} left out, preparing the image`);
+        const prep = await i.runner.prepare({ recipe, copyDir: copy.dir, sourceDir: i.repo, limits });
+        if (!prep.ok) {
+            notProvable = { code: prep.code, reason: prep.reason };
+        }
+        else {
+            try {
+                outer: for (const spec of specs) {
+                    const sha = specSha256(spec);
+                    for (let run = 1; run <= required; run++) {
+                        const st = await prep.prepared.start();
+                        if (!st.ok) {
+                            notProvable = { code: st.code, reason: st.reason };
+                            break outer;
+                        }
+                        try {
+                            const r = await st.instance.runSpec(spec);
+                            const body = r.ok ? { outcome: r.outcome, detail: r.detail, steps: r.steps } : { outcome: 'error', detail: r.reason, steps: [] };
+                            writeRun(i.proveDir, runRecord(spec, sha, run, body));
+                            i.log(`hullproof prove: ${spec.spec_id} run ${run}/${required}: ${body.outcome}`);
+                        }
+                        finally {
+                            leftBehind += (await st.instance.stop()).left_behind.length;
+                        }
+                    }
+                }
+            }
+            finally {
+                const t = await prep.prepared.dispose();
+                leftBehind += t.left_behind.length;
+                if (t.left_behind.length)
+                    i.log(`hullproof prove: WARNING: left behind after teardown: ${t.left_behind.join(', ')}`);
+            }
+        }
+    }
+    finally {
+        copy.cleanup();
+    }
+    return finish(i, { ...meta, not_provable: notProvable, left_behind: leftBehind }, specs, dropped);
+}
+// Rebuilds result.json from the saved specs and run files. No sandbox, no app, no model.
+export function replayProve(proveDir) {
+    const meta = readMeta(proveDir);
+    if (!meta)
+        return `no saved proof run in ${proveDir}`;
+    const { specs, dropped } = loadSpecs(proveDir);
+    const r = buildResult(meta, specs, dropped, (id) => readRuns(proveDir, id));
+    writeProveResult(proveDir, r);
+    return r;
+}

package/dist/prove/runner.js

@@ -0,0 +1 @@
+export const DEFAULT_LIMITS = { memory: '512m', cpus: '1', pids: 128, instanceMs: 180_000, execMs: 60_000, healthMs: 30_000 };

package/dist/prove/safety.js

@@ -0,0 +1,126 @@
+// The safety rules of the proof sandbox that do not depend on a container runtime: the typed confirmation, the loopback only
+// target rule, the copy of the repository without secrets, and the dummy environment. Design: docs/PHASE-4-DESIGN.md, safety rules 1 to 3.
+import { createHash } from 'node:crypto';
+import { existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, copyFileSync, realpathSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, relative, sep } from 'node:path';
+import { loopbackBase } from './executor.js';
+export const CONFIRM_PHRASE = 'I own this app and approve running it';
+// Rule 1: an explicit flag and the exact typed phrase. Nothing else counts: no environment variable, no default, no CI setting.
+export function checkConfirmation(allowRun, typed) {
+    if (!allowRun)
+        return { ok: false, reason: 'hullproof prove runs the app, so it needs --allow-run' };
+    if (typed === null)
+        return { ok: false, reason: `hullproof prove needs the owner's typed confirmation: type "${CONFIRM_PHRASE}"` };
+    if (typed.trim() !== CONFIRM_PHRASE)
+        return { ok: false, reason: `the confirmation does not match. Type exactly: ${CONFIRM_PHRASE}` };
+    return { ok: true };
+}
+// Rule 2: the only address a request may go to is the sandbox's own loopback on the recipe's port.
+export function assertSandboxTarget(url, port) {
+    const u = loopbackBase(url);
+    if (Number(u.port) !== port)
+        throw new Error(`refused: ${u.origin} is not the sandbox app port ${port}`);
+}
+// Rule 3: names that never go into the copy. Environment files, keys, credentials, tool folders, git history.
+const SECRET_FILE = /^(?:\.env(?:\..*)?|\.npmrc|\.netrc|\.pypirc|\.git-credentials|id_(?:rsa|dsa|ecdsa|ed25519)(?:\.pub)?|credentials(?:\..*)?|.*\.(?:pem|key|p12|pfx|jks|keystore|kdbx)|service-account.*\.json|.*secrets?\.(?:json|ya?ml|toml))$/i;
+const SKIP_DIR = new Set(['.git', '.hullproof', '.DS_Store', '.next', '.cache', 'coverage']);
+export const isSecretFile = (name) => SECRET_FILE.test(name);
+const MAX_FILE = 10 * 1024 * 1024;
+const MAX_TOTAL = 200 * 1024 * 1024;
+// Copies the repository to a fresh temporary folder, leaving out secrets and (unless the repository vendors its modules) node_modules.
+// Symbolic links are never followed or copied. Returns what was left out so the report can say so.
+export function makeSandboxCopy(repo, o) {
+    const root = realpathSync(repo);
+    const dir = mkdtempSync(join(tmpdir(), 'hp-prove-'));
+    const excluded = [];
+    let files = 0;
+    let total = 0;
+    const walk = (from) => {
+        for (const name of readdirSync(from).sort()) {
+            const p = join(from, name);
+            const rel = relative(root, p).split(sep).join('/');
+            const st = lstatSync(p);
+            if (st.isSymbolicLink()) {
+                excluded.push(`${rel} (symbolic link)`);
+                continue;
+            }
+            if (st.isDirectory()) {
+                if (SKIP_DIR.has(name))
+                    continue;
+                if (name === 'node_modules' && !o.keepNodeModules) {
+                    excluded.push(`${rel}/ (dependencies are installed in the build phase)`);
+                    continue;
+                }
+                mkdirSync(join(dir, rel), { recursive: true });
+                const e = walk(p);
+                if (e)
+                    return e;
+                continue;
+            }
+            if (!st.isFile())
+                continue;
+            if (isSecretFile(name)) {
+                excluded.push(`${rel} (secret file)`);
+                continue;
+            }
+            if (st.size > MAX_FILE) {
+                excluded.push(`${rel} (over 10 MB)`);
+                continue;
+            }
+            total += st.size;
+            if (total > MAX_TOTAL)
+                return 'the repository is over 200 MB, which is too large to copy into a sandbox';
+            mkdirSync(join(dir, rel, '..'), { recursive: true });
+            copyFileSync(p, join(dir, rel));
+            files++;
+        }
+        return null;
+    };
+    const err = walk(root);
+    if (err) {
+        rmSync(dir, { recursive: true, force: true });
+        return { error: err };
+    }
+    return { dir, files, excluded: excluded.sort(), cleanup: () => rmSync(dir, { recursive: true, force: true }) };
+}
+// The environment variable names the repository reads: process.env.NAME and process.env['NAME'] in source files, and the names
+// (not the values) in .env.example. A real .env file is never opened.
+const SRC = /\.(?:[cm]?[jt]sx?)$/;
+const PY_SRC = /\.py$/;
+export function envNames(repo) {
+    const names = new Set();
+    const walk = (d, depth) => {
+        if (depth > 8)
+            return;
+        for (const n of readdirSync(d)) {
+            if (n === 'node_modules' || n === '.git' || n === '.hullproof' || n === '.next' || n === '__pycache__' || n === '.venv' || n === 'venv')
+                continue;
+            const p = join(d, n);
+            const st = lstatSync(p);
+            if (st.isSymbolicLink())
+                continue;
+            if (st.isDirectory())
+                walk(p, depth + 1);
+            else if (st.isFile() && st.size < 1_000_000) {
+                if (SRC.test(n))
+                    for (const m of readFileSync(p, 'utf8').matchAll(/process\.env(?:\.([A-Z][A-Z0-9_]*)|\[\s*['"]([A-Z][A-Z0-9_]*)['"]\s*\])/g))
+                        names.add(m[1] ?? m[2]);
+                else if (PY_SRC.test(n))
+                    for (const m of readFileSync(p, 'utf8').matchAll(/os\.(?:environ(?:\.get)?\s*[[(]\s*|getenv\s*\(\s*)['"]([A-Z][A-Z0-9_]*)['"]/g))
+                        names.add(m[1]);
+                else if (n === '.env.example')
+                    for (const l of readFileSync(p, 'utf8').split(/\r?\n/)) {
+                        const m = /^\s*(?:export\s+)?([A-Z][A-Z0-9_]*)\s*=/.exec(l);
+                        if (m)
+                            names.add(m[1]);
+                    }
+            }
+        }
+    };
+    if (existsSync(repo))
+        walk(repo, 0);
+    return [...names].sort();
+}
+// A fixed dummy value per name, so two runs get the same environment. It looks like what it stands for and opens nothing.
+export const dummyValue = (name) => `dummy-${name.toLowerCase().replace(/_/g, '-')}-${createHash('sha256').update(name).digest('hex').slice(0, 8)}`;

package/dist/prove/spec.js

@@ -0,0 +1,357 @@
+// The test spec language of the proof sandbox: validation, templates and assertions. A spec is data. The engine runs it; no model
+// written code ever runs. This file has no imports beyond types, because the same file runs inside the sandbox (see exec-main.ts).
+import { createHash } from 'node:crypto';
+export const SPEC_LIMITS = { steps: 24, actors: 6, assertions: 8, captures: 4, body: 4096, header: 512, text: 300, path: 300, idLen: 40 };
+const ID = /^[a-z][a-z0-9_]{0,39}$/;
+const VAR = /\{\{([a-z][a-z0-9_]{0,39})\}\}/g;
+// A path on the app itself: a slash first, no scheme, no host, no dot dot, no backslash. Templates are allowed.
+const PATH = /^\/(?!\/)[A-Za-z0-9._~!$&'()*+,;=:@%/?#\-{}]*$/;
+const METHODS = ['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'];
+const HEADER_DENY = new Set(['host', 'connection', 'content-length', 'transfer-encoding', 'expect', 'upgrade', 'te', 'trailer', 'keep-alive']);
+const HEADER_DENY_PREFIX = ['proxy-', 'sec-', 'x-forwarded-', 'x-real-ip'];
+// Strings that look built to damage data, read files or run code. A guard against a bad spec, not a boundary: the sandbox is the boundary.
+const HARMFUL = /\b(?:drop|truncate|alter)\s+(?:table|database|schema)\b|\bdelete\s+from\b|\bunion\s+select\b|\brm\s+-|\bshutdown\b|;\s*--|\$\(|`|\.\.[\\/]|<script|\bexec\s*\(|\bcurl\s|\bwget\s/i;
+export const SECRET_NAME = /(?:pass(?:word|wd)?|secret|token|api[_-]?key|authorization|cookie|session|bearer|jwt|credential|private)/i;
+const isObj = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
+// Sorted keys, no spaces: the form every hash in this phase is taken over.
+export function canon(v) {
+    const sort = (x) => {
+        if (Array.isArray(x))
+            return x.map(sort);
+        if (isObj(x)) {
+            const o = {};
+            for (const k of Object.keys(x).sort())
+                if (x[k] !== undefined)
+                    o[k] = sort(x[k]);
+            return o;
+        }
+        return x;
+    };
+    return JSON.stringify(sort(v));
+}
+export const sha256 = (s) => createHash('sha256').update(s).digest('hex');
+export const specSha256 = (s) => sha256(canon(s));
+export function varsIn(text) { return [...text.matchAll(VAR)].map((m) => m[1]); }
+function strings(v, out = []) {
+    if (typeof v === 'string')
+        out.push(v);
+    else if (Array.isArray(v))
+        v.forEach((x) => strings(x, out));
+    else if (isObj(v))
+        for (const [k, x] of Object.entries(v)) {
+            out.push(k);
+            strings(x, out);
+        }
+    return out;
+}
+function keysOf(v, out = []) {
+    if (Array.isArray(v))
+        v.forEach((x) => keysOf(x, out));
+    else if (isObj(v))
+        for (const [k, x] of Object.entries(v)) {
+            out.push(k);
+            keysOf(x, out);
+        }
+    return out;
+}
+function checkHeaders(h, where, err) {
+    if (h === undefined)
+        return undefined;
+    if (!isObj(h)) {
+        err(`${where}: headers must be an object`);
+        return undefined;
+    }
+    const out = {};
+    for (const [k, v] of Object.entries(h)) {
+        const name = k.toLowerCase();
+        if (!/^[a-z][a-z0-9-]{0,39}$/.test(name)) {
+            err(`${where}: header name ${JSON.stringify(k).slice(0, 40)} is not allowed`);
+            continue;
+        }
+        if (HEADER_DENY.has(name) || HEADER_DENY_PREFIX.some((p) => name.startsWith(p))) {
+            err(`${where}: header ${name} may not be set`);
+            continue;
+        }
+        if (typeof v !== 'string' || v.length > SPEC_LIMITS.header || /[\u0000-\u001f\u007f]/.test(v)) {
+            err(`${where}: header ${name} must be a short single line string`);
+            continue;
+        }
+        out[name] = v;
+    }
+    return out;
+}
+function checkAssertion(a, where, err) {
+    if (!isObj(a) || typeof a.kind !== 'string') {
+        err(`${where}: an assertion needs a kind`);
+        return null;
+    }
+    const text = (x) => typeof x === 'string' && x.length > 0 && x.length <= SPEC_LIMITS.text;
+    if (a.kind === 'status') {
+        const ok = (n) => Number.isInteger(n) && n >= 100 && n <= 599;
+        if (a.op === 'in' ? !(Array.isArray(a.value) && a.value.length > 0 && a.value.length <= 8 && a.value.every(ok)) : !((a.op === 'eq' || a.op === 'ne') && ok(a.value))) {
+            err(`${where}: a status assertion needs op eq, ne or in and status codes`);
+            return null;
+        }
+        return { kind: 'status', op: a.op, value: a.value };
+    }
+    if (a.kind === 'json') {
+        if (typeof a.path !== 'string' || !/^[A-Za-z0-9_$-]+(?:\.[A-Za-z0-9_$-]+)*$/.test(a.path) || a.path.length > 120) {
+            err(`${where}: a json assertion needs a dotted path such as items.0.owner`);
+            return null;
+        }
+        if (SECRET_NAME.test(a.path)) {
+            err(`${where}: an assertion may not read a field named like a secret (${a.path}); it is redacted in the transcript`);
+            return null;
+        }
+        if (!['eq', 'ne', 'contains', 'exists', 'absent'].includes(String(a.op))) {
+            err(`${where}: a json assertion needs op eq, ne, contains, exists or absent`);
+            return null;
+        }
+        const needs = a.op === 'eq' || a.op === 'ne' || a.op === 'contains';
+        const v = a.value;
+        if (needs && !(typeof v === 'string' ? v.length <= SPEC_LIMITS.text : typeof v === 'number' || typeof v === 'boolean' || v === null)) {
+            err(`${where}: a json ${a.op} assertion needs a value`);
+            return null;
+        }
+        return { kind: 'json', path: a.path, op: a.op, ...(needs ? { value: v } : {}) };
+    }
+    if (a.kind === 'text') {
+        if ((a.op !== 'contains' && a.op !== 'absent') || !text(a.value)) {
+            err(`${where}: a text assertion needs op contains or absent and a value`);
+            return null;
+        }
+        return { kind: 'text', op: a.op, value: a.value };
+    }
+    if (a.kind === 'header') {
+        if (typeof a.name !== 'string' || !/^[A-Za-z][A-Za-z0-9-]{0,39}$/.test(a.name) || SECRET_NAME.test(a.name)) {
+            err(`${where}: a header assertion needs a plain header name that is not a secret`);
+            return null;
+        }
+        if (!['eq', 'contains', 'exists', 'absent'].includes(String(a.op))) {
+            err(`${where}: a header assertion needs op eq, contains, exists or absent`);
+            return null;
+        }
+        const needs = a.op === 'eq' || a.op === 'contains';
+        if (needs && !text(a.value)) {
+            err(`${where}: a header ${a.op} assertion needs a value`);
+            return null;
+        }
+        return { kind: 'header', name: a.name.toLowerCase(), op: a.op, ...(needs ? { value: a.value } : {}) };
+    }
+    err(`${where}: unknown assertion kind`);
+    return null;
+}
+// True when an attack assertion shows something got through: a 2xx status or a value that is there.
+const exposes = (a) => (a.kind === 'status' && (a.op === 'eq' || a.op === 'in') && [a.value].flat().every((n) => n >= 200 && n < 300)) ||
+    (a.kind === 'json' && (a.op === 'eq' || a.op === 'contains' || a.op === 'exists')) || (a.kind === 'text' && a.op === 'contains');
+const succeeds = (a) => a.kind === 'status' && (a.op === 'eq' || a.op === 'in') && [a.value].flat().every((n) => n >= 200 && n < 300);
+// Validates a spec read from JSON. Returns a clean copy (unknown keys dropped) or every problem found. The positive control rule is
+// here: a spec with no control step that must succeed, placed before the attack, is invalid.
+export function validateSpec(raw) {
+    const errors = [];
+    const err = (m) => { errors.push(m); };
+    if (!isObj(raw))
+        return { ok: false, errors: ['the spec must be a JSON object'] };
+    if (raw.schema_version !== '1.0.0')
+        err('schema_version must be "1.0.0"');
+    const str = (k, max) => { const v = raw[k]; if (typeof v !== 'string' || !v.trim() || v.length > max) {
+        err(`${k} is missing or too long`);
+        return '';
+    } return v.replace(/[\u0000-\u001f\u007f]+/g, ' ').trim(); };
+    const spec_id = str('spec_id', SPEC_LIMITS.idLen);
+    if (spec_id && !ID.test(spec_id))
+        err('spec_id must be lowercase letters, digits and underscores');
+    const title = str('title', 160);
+    const claim = str('claim', 500);
+    const lead_key = raw.lead_key === null || raw.lead_key === undefined ? null : typeof raw.lead_key === 'string' && raw.lead_key.length <= 400 ? raw.lead_key : (err('lead_key must be a string or null'), null);
+    const t = raw.target;
+    let target = { entry_point: '', file: '', line: 1 };
+    if (isObj(t) && typeof t.entry_point === 'string' && t.entry_point && t.entry_point.length <= 300 && typeof t.file === 'string' && t.file && t.file.length <= 300 && Number.isInteger(t.line) && t.line >= 1)
+        target = { entry_point: t.entry_point, file: t.file, line: t.line };
+    else
+        err('target needs entry_point, file and line');
+    const req = raw.requirement_ids;
+    const requirement_ids = Array.isArray(req) && req.length <= 6 && req.every((r) => typeof r === 'string' && /^SEC-[A-Z]+-\d{3}$/.test(r)) ? [...new Set(req)].sort() : (err('requirement_ids must be a list of at most 6 ids like SEC-API-001 (it may be empty)'), []);
+    const actors = {};
+    if (!isObj(raw.actors) || !Object.keys(raw.actors).length || Object.keys(raw.actors).length > SPEC_LIMITS.actors)
+        err(`actors needs 1 to ${SPEC_LIMITS.actors} entries`);
+    else {
+        for (const [name, a] of Object.entries(raw.actors)) {
+            if (!ID.test(name)) {
+                err(`actor name ${JSON.stringify(name).slice(0, 30)} is not allowed`);
+                continue;
+            }
+            if (!isObj(a)) {
+                err(`actor ${name} must be an object`);
+                continue;
+            }
+            const h = checkHeaders(a.headers, `actor ${name}`, err);
+            actors[name] = h ? { headers: h } : {};
+        }
+    }
+    const steps = [];
+    if (!Array.isArray(raw.steps) || raw.steps.length < 2 || raw.steps.length > SPEC_LIMITS.steps)
+        err(`steps needs 2 to ${SPEC_LIMITS.steps} entries`);
+    else {
+        const ids = new Set();
+        raw.steps.forEach((s, i) => {
+            const w = `step ${i + 1}`;
+            if (!isObj(s))
+                return err(`${w}: must be an object`);
+            if (typeof s.id !== 'string' || !ID.test(s.id) || ids.has(s.id))
+                return err(`${w}: id must be unique lowercase letters, digits and underscores`);
+            ids.add(s.id);
+            if (s.phase !== 'setup' && s.phase !== 'control' && s.phase !== 'attack')
+                return err(`${w}: phase must be setup, control or attack`);
+            if (typeof s.actor !== 'string' || !(s.actor in actors))
+                return err(`${w}: actor is not declared in actors`);
+            if (typeof s.method !== 'string' || !METHODS.includes(s.method))
+                return err(`${w}: method must be one of ${METHODS.join(', ')}`);
+            if (typeof s.path !== 'string' || s.path.length > SPEC_LIMITS.path || !PATH.test(s.path) || s.path.includes('..'))
+                return err(`${w}: path must start with one slash and name a path on the app, never a host or address`);
+            const headers = checkHeaders(s.headers, w, err);
+            let body = null;
+            if (s.body !== undefined && s.body !== null) {
+                if (!isObj(s.body) || canon(s.body).length > SPEC_LIMITS.body)
+                    return err(`${w}: body must be a small JSON object`);
+                body = s.body;
+                if (s.method === 'GET' || s.method === 'HEAD')
+                    return err(`${w}: a ${s.method} request has no body`);
+            }
+            const captures = [];
+            if (s.captures !== undefined) {
+                if (!Array.isArray(s.captures) || s.captures.length > SPEC_LIMITS.captures)
+                    return err(`${w}: captures needs at most ${SPEC_LIMITS.captures} entries`);
+                for (const c of s.captures) {
+                    if (!isObj(c) || typeof c.name !== 'string' || !ID.test(c.name) || !['json', 'header', 'status'].includes(String(c.from)))
+                        return err(`${w}: a capture needs a name and from json, header or status`);
+                    if (c.from !== 'status' && (typeof c.path !== 'string' || !c.path || c.path.length > 120))
+                        return err(`${w}: a ${String(c.from)} capture needs a path`);
+                    captures.push({ name: c.name, from: c.from, ...(c.from === 'status' ? {} : { path: c.path }) });
+                }
+            }
+            if (!Array.isArray(s.assert) || s.assert.length > SPEC_LIMITS.assertions)
+                return err(`${w}: assert needs a list of at most ${SPEC_LIMITS.assertions} assertions`);
+            const assert = s.assert.map((a) => checkAssertion(a, w, err)).filter((a) => a !== null);
+            steps.push({ id: s.id, phase: s.phase, actor: s.actor, method: s.method, path: s.path, ...(headers ? { headers } : {}), ...(body ? { body } : {}), ...(captures.length ? { captures } : {}), assert });
+        });
+    }
+    // Only words the spec can supply from an earlier capture: a variable must be captured before the step that uses it.
+    const known = new Set();
+    const secretVars = new Set();
+    for (const s of steps) {
+        const used = [...varsIn(s.path), ...Object.values(s.headers ?? {}).flatMap(varsIn), ...strings(s.body).flatMap(varsIn), ...s.assert.flatMap((a) => ('value' in a && typeof a.value === 'string' ? varsIn(a.value) : []))];
+        for (const v of new Set(used))
+            if (!known.has(v))
+                errors.push(`step ${s.id}: {{${v}}} is used before any step captures it`);
+        for (const a of s.assert)
+            if ('value' in a && typeof a.value === 'string')
+                for (const v of varsIn(a.value))
+                    if (secretVars.has(v))
+                        errors.push(`step ${s.id}: an assertion may not compare with {{${v}}}, a secret that is redacted in the transcript`);
+        for (const c of s.captures ?? []) {
+            known.add(c.name);
+            if (SECRET_NAME.test(c.name) || SECRET_NAME.test(c.path ?? ''))
+                secretVars.add(c.name);
+        }
+    }
+    for (const [n, a] of Object.entries(actors))
+        for (const v of Object.values(a.headers ?? {}).flatMap(varsIn))
+            if (!steps.some((s) => s.captures?.some((c) => c.name === v)))
+                errors.push(`actor ${n}: {{${v}}} is never captured`);
+    // Harmless payloads only.
+    for (const s of steps) {
+        const all = [s.path, ...Object.values(s.headers ?? {}), ...strings(s.body), ...s.assert.flatMap((a) => ('value' in a && typeof a.value === 'string' ? [a.value] : []))];
+        if (all.some((x) => HARMFUL.test(x)))
+            errors.push(`step ${s.id}: payload_not_harmless: the request holds text built to damage data, read files or run code. Use canary strings and read style requests.`);
+        for (const k of keysOf(s.body))
+            if (HARMFUL.test(k))
+                errors.push(`step ${s.id}: payload_not_harmless in a body key`);
+    }
+    // The positive control rule.
+    const firstAttack = steps.findIndex((s) => s.phase === 'attack');
+    const firstControl = steps.findIndex((s) => s.phase === 'control');
+    if (steps.length && firstAttack < 0)
+        errors.push('a spec needs at least one attack step');
+    if (steps.length && (firstControl < 0 || (firstAttack >= 0 && firstControl > firstAttack)))
+        errors.push('positive control missing: a control step that must succeed has to come before the first attack step');
+    else if (!steps.some((s) => s.phase === 'control' && s.assert.some(succeeds)))
+        errors.push('positive control missing: a control step needs a status assertion with a 2xx code, so a working harness is shown first');
+    for (const s of steps)
+        if ((s.phase === 'control' || s.phase === 'attack') && !s.assert.length)
+            errors.push(`step ${s.id}: a ${s.phase} step needs at least one assertion`);
+    if (steps.some((s) => s.phase === 'attack') && !steps.filter((s) => s.phase === 'attack').some((s) => s.assert.some(exposes)))
+        errors.push('an attack step has to assert what got through (a 2xx status or a value that is there); asserting a denial proves nothing');
+    if (steps.length && steps.some((s, i) => s.phase === 'setup' && steps.slice(0, i).some((p) => p.phase !== 'setup')))
+        errors.push('setup steps come first');
+    if (!isObj(raw.expect) || raw.expect.outcome !== 'flaw_present')
+        err('expect.outcome must be "flaw_present"');
+    if (errors.length)
+        return { ok: false, errors };
+    return { ok: true, spec: { schema_version: '1.0.0', spec_id, title, claim, lead_key, target, requirement_ids, actors, steps, expect: { outcome: 'flaw_present' } } };
+}
+// Replaces {{name}} with the captured value. A name that is not captured is an error. With `redact`, a secret variable shows as [redacted].
+export function fill(text, vars, secrets, redact) {
+    return text.replace(VAR, (_m, n) => {
+        if (!(n in vars))
+            throw new Error(`unresolved variable ${n}`);
+        if (redact && secrets.has(n))
+            return '[redacted]';
+        const v = vars[n];
+        return typeof v === 'string' ? v : JSON.stringify(v);
+    });
+}
+export function fillJson(v, vars, secrets, redact) {
+    if (typeof v === 'string')
+        return fill(v, vars, secrets, redact);
+    if (Array.isArray(v))
+        return v.map((x) => fillJson(x, vars, secrets, redact));
+    if (v && typeof v === 'object')
+        return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, fillJson(x, vars, secrets, redact)]));
+    return v;
+}
+// ---- assertions, evaluated on the stored form of a response (so a replay gives the same answer) -------------------------------------
+export function pathGet(v, path) {
+    let cur = v;
+    for (const part of path.split('.')) {
+        if (cur === null || cur === undefined)
+            return undefined;
+        if (Array.isArray(cur)) {
+            if (!/^\d+$/.test(part))
+                return undefined;
+            cur = cur[Number(part)];
+        }
+        else if (typeof cur === 'object')
+            cur = Object.prototype.hasOwnProperty.call(cur, part) ? cur[part] : undefined;
+        else
+            return undefined;
+    }
+    return cur;
+}
+const same = (a, b) => canon(a) === canon(b);
+export function evalAssertion(r, a, vars, secrets) {
+    const want = (s) => fill(s, vars, secrets, true);
+    if (a.kind === 'status') {
+        const ok = a.op === 'eq' ? r.status === a.value : a.op === 'ne' ? r.status !== a.value : a.value.includes(r.status);
+        return { ok, desc: `status ${a.op} ${JSON.stringify(a.value)}` };
+    }
+    if (a.kind === 'json') {
+        const got = pathGet(r.json, a.path);
+        const w = a.value === undefined ? undefined : typeof a.value === 'string' ? want(a.value) : a.value;
+        const ok = a.op === 'exists' ? got !== undefined : a.op === 'absent' ? got === undefined
+            : got === undefined ? a.op === 'ne' && r.json !== undefined
+                : a.op === 'eq' ? same(got, w) || (typeof got === 'number' && typeof w === 'string' && String(got) === w)
+                    : a.op === 'ne' ? !(same(got, w) || (typeof got === 'number' && typeof w === 'string' && String(got) === w))
+                        : typeof got === 'string' ? got.includes(String(w)) : Array.isArray(got) && got.some((x) => same(x, w));
+        return { ok, desc: `json ${a.path} ${a.op}${w === undefined ? '' : ' ' + JSON.stringify(w)}` };
+    }
+    if (a.kind === 'text') {
+        const hay = r.text ?? (r.json === undefined ? '' : canon(r.json));
+        const has = hay.includes(want(a.value));
+        return { ok: a.op === 'contains' ? has : !has, desc: `body text ${a.op} ${JSON.stringify(a.value)}` };
+    }
+    const got = r.headers[a.name];
+    const w = a.value === undefined ? undefined : want(a.value);
+    const ok = a.op === 'exists' ? got !== undefined : a.op === 'absent' ? got === undefined : got !== undefined && (a.op === 'eq' ? got === w : got.includes(w));
+    return { ok, desc: `header ${a.name} ${a.op}${w === undefined ? '' : ' ' + JSON.stringify(w)}` };
+}