hullproof 0.0.0-stage → 0.5.0

package/src/prove/container.ts

@@ -0,0 +1,237 @@
+// The container backend. It speaks the docker command line, so Colima, Docker Desktop and Podman work through the same commands.
+// Safety rules 4 and 6 of the design are built into runArgs() and checked again by assertSafeRunArgs() before any container starts:
+// non root, read only root filesystem, no host mounts (the copy goes in at build time, nothing is mounted), no capabilities, no
+// privilege escalation, memory, cpu, process and time limits, and no network while tests run. The build phase may reach the package
+// registry (lockfile install, no install scripts). A Postgres store runs on an internal network that has no route out.
+import { spawn } from 'node:child_process';
+import { copyFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { randomBytes } from 'node:crypto';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import type { Recipe } from '../types/index.ts';
+import { assertSandboxTarget } from './safety.ts';
+import type { Exec, Instance, Limits, PrepareRequest, PrepareResult, Prepared, Runner, SpecRun, StartResult, Teardown } from './runner.ts';
+
+export interface ContainerRuntime { readonly bin: string; run(args: string[], o?: { input?: string; timeoutMs?: number }): Promise<Exec> }
+
+const MAX_OUT = 4 * 1024 * 1024;
+export class CliRuntime implements ContainerRuntime {
+  readonly bin: string;
+  constructor(bin: string = 'docker') { this.bin = bin; }
+  run(args: string[], o: { input?: string; timeoutMs?: number } = {}): Promise<Exec> {
+    return new Promise((resolve) => {
+      let out = ''; let err = ''; let timedOut = false; let done = false;
+      const fin = (e: Exec): void => { if (!done) { done = true; resolve(e); } };
+      let child;
+      try { child = spawn(this.bin, args, { stdio: ['pipe', 'pipe', 'pipe'] }); } catch (e) { return fin({ code: null, stdout: '', stderr: (e as Error).message, timedOut: false }); }
+      const timer = o.timeoutMs ? setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, o.timeoutMs) : null;
+      child.stdout.on('data', (d: Buffer) => { if (out.length < MAX_OUT) out += d.toString('utf8'); });
+      child.stderr.on('data', (d: Buffer) => { if (err.length < MAX_OUT) err += d.toString('utf8'); });
+      child.on('error', (e: Error) => { if (timer) clearTimeout(timer); fin({ code: null, stdout: out, stderr: (e as NodeJS.ErrnoException).code === 'ENOENT' ? `${this.bin}: command not found` : e.message, timedOut }); });
+      child.on('close', (code: number | null) => { if (timer) clearTimeout(timer); fin({ code, stdout: out, stderr: err, timedOut }); });
+      child.stdin.on('error', () => undefined);
+      child.stdin.end(o.input ?? '');
+    });
+  }
+}
+
+export const LABEL = 'hullproof.prove';
+export const APP_USER = '10001:10001';
+export const EXEC_DIR = '/app/.hullproof-exec';
+const EXEC_FILES = ['spec.ts', 'executor.ts', 'exec-main.ts'];
+// The step executor runs inside the container as TypeScript, so its sources are always read from src/prove (shipped with the package),
+// also when the engine itself runs from dist/.
+const HERE = fileURLToPath(new URL('../../src/prove/', import.meta.url));
+const PG_IMAGE = 'postgres:16-alpine';
+const PG_USER = '70:70';
+const clip = (s: string, n = 600): string => { const t = s.trim().replace(/\s+/g, ' '); return t.length <= n ? t : '…' + t.slice(t.length - n); };
+const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
+
+// ---- commands, as pure functions so they can be tested without a runtime --------------------------------------------------------
+// A Python app runs in a Python image that also holds the node binary, because the step executor and the health probe are Node scripts
+// that run inside the sandbox. The binary is copied from the same image the Node recipe uses, at build time, so nothing else is fetched.
+export const NODE_IMAGE = 'node:22-slim';
+export function dockerfile(r: Recipe): string {
+  const py = r.stack === 'python-flask';
+  return [
+    ...(py ? [`FROM ${NODE_IMAGE} AS node`] : []),
+    `FROM ${r.base_image}`,
+    ...(py ? ['COPY --from=node /usr/local/bin/node /usr/local/bin/node', 'ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_NO_CACHE_DIR=1 HOME=/tmp'] : []),
+    'WORKDIR /app', 'COPY . /app', ...(r.install ? [`RUN ${r.install.join(' ')}`] : []), `USER ${APP_USER}`, '',
+  ].join('\n');
+}
+export const buildArgs = (tag: string, runId: string, ctx: string): string[] => ['build', '--label', `${LABEL}=${runId}`, '--tag', tag, '--file', join(ctx, 'Dockerfile.hullproof'), ctx];
+
+export interface RunSpecArgs { name: string; runId: string; image: string; network: string; user: string; tmpfs: string[]; env: Record<string, string>; limits: Limits; cmd: string[] }
+export function runArgs(a: RunSpecArgs): string[] {
+  const args = [
+    'run', '--detach', '--init', '--name', a.name, '--label', `${LABEL}=${a.runId}`,
+    '--network', a.network, '--read-only', '--user', a.user, '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges',
+    '--pids-limit', String(a.limits.pids), '--memory', a.limits.memory, '--memory-swap', a.limits.memory, '--cpus', a.limits.cpus,
+    '--ulimit', 'nofile=1024:1024', '--ulimit', 'core=0',
+    ...a.tmpfs.flatMap((t) => ['--tmpfs', t]),
+    ...Object.entries(a.env).flatMap(([k, v]) => ['--env', `${k}=${v}`]),
+    a.image, ...a.cmd,
+  ];
+  assertSafeRunArgs(args.slice(0, args.length - 1 - a.cmd.length), a.network);
+  return args;
+}
+
+const FORBIDDEN = new Set(['--privileged', '-v', '--volume', '--volumes-from', '--mount', '-p', '--publish', '-P', '--publish-all', '--device', '--cap-add', '--pid', '--ipc', '--uts', '--userns', '--add-host', '--env-file', '--cgroupns', '--group-add', '--sysctl']);
+// Refuses anything that would weaken the sandbox. `head` is the argument list up to, and not including, the image name. The builder
+// calls this on its own output; tests call it on bad input.
+export function assertSafeRunArgs(head: string[], allowedNetwork: string): void {
+  const bad: string[] = [];
+  const val = (flag: string): string | undefined => { const i = head.indexOf(flag); return i >= 0 ? head[i + 1] : undefined; };
+  for (const a of head) if (FORBIDDEN.has(a) || /^--(?:privileged|volume|mount|publish|device|cap-add|pid|ipc|uts|userns|add-host|env-file)=/.test(a)) bad.push(a);
+  if (head.some((a) => /unconfined/.test(a))) bad.push('unconfined security option');
+  if (!head.includes('--read-only')) bad.push('missing --read-only');
+  if (val('--cap-drop') !== 'ALL') bad.push('capabilities are not all dropped');
+  if (!head.some((a, i) => a === '--security-opt' && head[i + 1] === 'no-new-privileges')) bad.push('missing no-new-privileges');
+  for (const f of ['--pids-limit', '--memory', '--cpus']) if (!val(f)) bad.push(`missing ${f}`);
+  const user = val('--user');
+  if (!user || /^(?:0|root)(?::|$)/.test(user)) bad.push('the container would run as root');
+  const net = val('--network');
+  if (net !== 'none' && net !== allowedNetwork) bad.push(`network ${net ?? 'default'} is not none or the internal network`);
+  if (net !== 'none' && !/^hp-prove-net-/.test(net ?? '')) bad.push('a network other than none must be a hullproof internal network');
+  if (bad.length) throw new Error(`unsafe container arguments refused: ${bad.join(', ')}`);
+}
+
+const healthScript = (port: number, path: string): string => `fetch('http://127.0.0.1:${port}${path}',{signal:AbortSignal.timeout(2000)}).then(r=>process.exit(r.status<500?0:2),()=>process.exit(1))`;
+
+export class ContainerRunner implements Runner {
+  readonly name = 'container';
+  private readonly rt: ContainerRuntime;
+  private readonly o: { sleepMs?: number };
+  constructor(rt: ContainerRuntime, o: { sleepMs?: number } = {}) { this.rt = rt; this.o = o; }
+
+  async available(): Promise<{ ok: true } | { ok: false; reason: string }> {
+    const r = await this.rt.run(['info'], { timeoutMs: 20_000 });
+    if (r.code === 0) return { ok: true };
+    const missing = /not found|ENOENT/.test(r.stderr);
+    return { ok: false, reason: missing ? `no container runtime: ${this.rt.bin} was not found. Install one, for example: brew install colima docker, then colima start.` : `the container runtime is not running (${clip(r.stderr, 160) || 'no answer'}). For Colima run: colima start.` };
+  }
+
+  async prepare(req: PrepareRequest): Promise<PrepareResult> {
+    const runId = randomBytes(5).toString('hex');
+    const image = `hullproof-prove-${runId}`;
+    mkdirSync(join(req.copyDir, '.hullproof-exec'), { recursive: true });
+    for (const f of EXEC_FILES) copyFileSync(join(HERE, f), join(req.copyDir, '.hullproof-exec', f));
+    writeFileSync(join(req.copyDir, 'Dockerfile.hullproof'), dockerfile(req.recipe));
+    const b = await this.rt.run(buildArgs(image, runId, req.copyDir), { timeoutMs: 600_000 });
+    if (b.code !== 0) { await this.rt.run(['rmi', '-f', image]); return { ok: false, code: 'build_failed', reason: `the image did not build${b.timedOut ? ' in time' : ''}: ${clip(b.stderr || b.stdout)}` }; }
+    return { ok: true, prepared: new ContainerPrepared(this.rt, runId, image, req, this.o.sleepMs ?? 500) };
+  }
+}
+
+class ContainerPrepared implements Prepared {
+  private n = 0;
+  private readonly live = new Set<ContainerInstance>();
+  private readonly rt: ContainerRuntime;
+  private readonly runId: string;
+  private readonly image: string;
+  private readonly req: PrepareRequest;
+  private readonly pause: number;
+  constructor(rt: ContainerRuntime, runId: string, image: string, req: PrepareRequest, pause: number) { this.rt = rt; this.runId = runId; this.image = image; this.req = req; this.pause = pause; }
+
+  async start(): Promise<StartResult> {
+    const { recipe, limits } = this.req;
+    const id = `${this.runId}-${++this.n}`;
+    const name = `hp-prove-${id}`;
+    const inst = new ContainerInstance(this.rt, name, recipe, limits);
+    this.live.add(inst);
+    const fail = async (code: 'start_failed' | 'health_failed' | 'build_failed', reason: string): Promise<StartResult> => { const logs = await this.rt.run(['logs', '--tail', '15', name]); await inst.stop(); this.live.delete(inst); return { ok: false, code, reason: `${reason}${logs.stdout + logs.stderr ? `. Last output: ${clip(logs.stdout + ' ' + logs.stderr, 400)}` : ''}` }; };
+    const data = (s: string): string => s.replace(/\{DATA\}/g, '/data');
+    let network = 'none';
+    const env: Record<string, string> = {};
+    const tmpfs = ['/tmp:rw,noexec,nosuid,nodev,size=64m'];
+    if (recipe.data_store.kind === 'sqlite') tmpfs.push(`/data:rw,noexec,nosuid,nodev,size=64m,uid=10001,gid=10001`);
+    if (recipe.data_store.kind === 'postgres') {
+      network = `hp-prove-net-${id}`;
+      const pg = `hp-prove-pg-${id}`;
+      const net = await this.rt.run(['network', 'create', '--internal', '--label', `${LABEL}=${this.runId}`, network]);
+      if (net.code !== 0) { this.live.delete(inst); return { ok: false, code: 'start_failed', reason: `the internal network could not be created: ${clip(net.stderr)}` }; }
+      inst.network = network;
+      inst.extra.push(pg);
+      const pgRun = await this.rt.run(runArgs({ name: pg, runId: this.runId, image: PG_IMAGE, network, user: PG_USER, tmpfs: ['/var/lib/postgresql/data:rw,nosuid,nodev,size=128m,uid=70,gid=70', '/var/run/postgresql:rw,nosuid,nodev,size=8m,uid=70,gid=70', '/tmp:rw,nosuid,nodev,size=32m'], env: { POSTGRES_PASSWORD: 'dummy', POSTGRES_DB: 'app' }, limits, cmd: [] }));
+      if (pgRun.code !== 0) return fail('start_failed', `Postgres did not start: ${clip(pgRun.stderr)}`);
+      let up = false;
+      for (let t = Date.now(); Date.now() - t < limits.healthMs && !up; await this.sleep()) up = (await this.rt.run(['exec', pg, 'pg_isready', '-h', '127.0.0.1', '-U', 'postgres', '-d', 'app'], { timeoutMs: 5000 })).code === 0;
+      if (!up) return fail('health_failed', 'Postgres did not become ready in time');
+      for (const f of [...recipe.data_store.migrations, ...(recipe.data_store.seed_sql ? [recipe.data_store.seed_sql] : [])]) {
+        const sql = readFileSync(join(this.req.sourceDir, f), 'utf8');
+        const r = await this.rt.run(['exec', '-i', pg, 'psql', '-v', 'ON_ERROR_STOP=1', '-q', '-U', 'postgres', '-d', 'app'], { input: sql, timeoutMs: 60_000 });
+        if (r.code !== 0) return fail('start_failed', `${f} did not apply: ${clip(r.stderr, 300)}`);
+      }
+      env.__PGURL = `postgres://postgres:dummy@${pg}:5432/app`;
+    }
+    for (const [k, v] of Object.entries(recipe.env)) env[k] = data(v).replace(/\{PGURL\}/g, env.__PGURL ?? '');
+    delete env.__PGURL;
+    const script = `${recipe.seed ? recipe.seed + ' && ' : ''}exec ${recipe.start}`;
+    const run = await this.rt.run(runArgs({ name, runId: this.runId, image: this.image, network, user: APP_USER, tmpfs, env, limits, cmd: ['sh', '-c', script] }));
+    if (run.code !== 0) return fail('start_failed', `the container did not start: ${clip(run.stderr)}`);
+    for (let t = Date.now(); Date.now() - t < limits.healthMs; await this.sleep()) {
+      const h = await this.rt.run(['exec', name, 'node', '-e', healthScript(recipe.port, recipe.health_path)], { timeoutMs: 6000 });
+      if (h.code === 0) { inst.arm(); return { ok: true, instance: inst }; }
+      const st = await this.rt.run(['inspect', '--format', '{{.State.Running}}', name]);
+      if (st.stdout.trim() === 'false') return fail('start_failed', 'the app stopped before it answered (the seed step or the start command failed)');
+    }
+    return fail('health_failed', `the app did not answer on ${recipe.health_path} within ${Math.round(limits.healthMs / 1000)} seconds`);
+  }
+  private sleep(): Promise<void> { return sleep(this.pause); }
+
+  async dispose(): Promise<Teardown> {
+    for (const i of this.live) await i.stop();
+    await this.rt.run(['rmi', '-f', this.image]);
+    const left: string[] = [];
+    for (const [what, args] of [['container', ['ps', '-a', '-q']], ['image', ['images', '-q']], ['network', ['network', 'ls', '-q']], ['volume', ['volume', 'ls', '-q']]] as const) {
+      const r = await this.rt.run([...args, '--filter', `label=${LABEL}=${this.runId}`]);
+      for (const l of r.stdout.split('\n').map((x) => x.trim()).filter(Boolean)) left.push(`${what} ${l}`);
+    }
+    return { left_behind: left };
+  }
+}
+
+class ContainerInstance implements Instance {
+  readonly baseUrl: string;
+  network: string | null = null;
+  readonly extra: string[] = [];
+  private stopped = false;
+  private timer: NodeJS.Timeout | null = null;
+  private readonly rt: ContainerRuntime;
+  private readonly name: string;
+  private readonly recipe: Recipe;
+  private readonly limits: Limits;
+  constructor(rt: ContainerRuntime, name: string, recipe: Recipe, limits: Limits) {
+    this.rt = rt; this.name = name; this.recipe = recipe; this.limits = limits;
+    this.baseUrl = `http://127.0.0.1:${recipe.port}`;
+  }
+  arm(): void { this.timer = setTimeout(() => { void this.stop(); }, this.limits.instanceMs); this.timer.unref(); }
+
+  async exec(cmd: string[], o: { input?: string; timeoutMs?: number } = {}): Promise<Exec> {
+    if (this.stopped) return { code: null, stdout: '', stderr: 'the instance time limit was reached or it was stopped', timedOut: false };
+    const ms = o.timeoutMs ?? this.limits.execMs;
+    // `timeout -s KILL` inside the container ends the process there. Killing the docker client alone would leave it running.
+    return this.rt.run(['exec', '-i', this.name, 'timeout', '-s', 'KILL', String(Math.max(1, Math.ceil(ms / 1000))), ...cmd], { input: o.input, timeoutMs: ms + 10_000 });
+  }
+
+  async runSpec(spec: import('../types/index.ts').Spec): Promise<SpecRun> {
+    assertSandboxTarget(this.baseUrl, this.recipe.port);
+    const r = await this.exec(['node', '--experimental-strip-types', `${EXEC_DIR}/exec-main.ts`, this.baseUrl], { input: JSON.stringify(spec) });
+    if (r.timedOut || r.code === 137) return { ok: false, reason: 'the step executor ran past the time limit and was killed' };
+    const line = r.stdout.trim().split('\n').pop() ?? '';
+    try {
+      const j = JSON.parse(line) as { ok: boolean; error?: string; outcome?: SpecRun extends { ok: true; outcome: infer O } ? O : never; detail?: string; steps?: import('../types/index.ts').StepRecord[] };
+      if (!j.ok) return { ok: false, reason: j.error ?? 'the step executor failed' };
+      return { ok: true, outcome: j.outcome!, detail: j.detail ?? '', steps: j.steps ?? [] };
+    } catch { return { ok: false, reason: `the step executor gave no result (${clip(r.stderr, 200) || 'no output'})` }; }
+  }
+
+  async stop(): Promise<Teardown> {
+    if (this.timer) clearTimeout(this.timer);
+    this.stopped = true;
+    for (const c of [this.name, ...this.extra]) await this.rt.run(['rm', '-f', '--volumes', c]);
+    if (this.network) await this.rt.run(['network', 'rm', this.network]);
+    return { left_behind: [] };
+  }
+}

package/src/prove/exec-main.ts

@@ -0,0 +1,16 @@
+// Runs inside the sandbox: `node --experimental-strip-types /hp/exec-main.ts http://127.0.0.1:PORT` with a spec as JSON on standard
+// input. Prints one JSON line: the outcome, the detail and the redacted step records. Refuses any address but the sandbox's own loopback.
+import { executeSpec } from './executor.ts';
+import { validateSpec } from './spec.ts';
+
+const chunks: Buffer[] = [];
+for await (const c of process.stdin) chunks.push(c as Buffer);
+try {
+  const v = validateSpec(JSON.parse(Buffer.concat(chunks).toString('utf8')));
+  if (!v.ok) { process.stdout.write(JSON.stringify({ ok: false, error: `invalid spec: ${v.errors.join('; ')}` }) + '\n'); process.exit(0); }
+  const r = await executeSpec(v.spec, { baseUrl: process.argv[2] ?? '' });
+  process.stdout.write(JSON.stringify({ ok: true, ...r }) + '\n');
+} catch (e) {
+  process.stdout.write(JSON.stringify({ ok: false, error: (e as Error).message.slice(0, 300) }) + '\n');
+}
+process.exit(0);

package/src/prove/executor.ts

@@ -0,0 +1,130 @@
+// The step executor. It turns a validated spec into HTTP requests against one local address and records what came back. It runs
+// inside the sandbox (through exec-main.ts), next to the app, so the request never leaves the sandbox network. Imports only
+// node built ins and spec.ts. Responses are stored in redacted form and every assertion is judged on that stored form, so a replay
+// from the stored transcript reaches the same verdict with no sandbox.
+import type { Json } from '../types/hunt.ts';
+import type { RunOutcome, Spec, SpecStep, StepRecord } from '../types/prove.ts';
+import { canon, evalAssertion, fill, fillJson, pathGet, SECRET_NAME, sha256, type Vars } from './spec.ts';
+
+export const LIMITS = { requestMs: 8000, bodyChars: 16384, textStore: 4096 } as const;
+const HOSTS = new Set(['127.0.0.1', 'localhost', '[::1]']);
+
+// The one place an address is accepted. Anything that is not the sandbox's own loopback is refused before a request is built.
+export function loopbackBase(url: string): URL {
+  let u: URL;
+  try { u = new URL(url); } catch { throw new Error(`refused: ${JSON.stringify(url).slice(0, 80)} is not an address`); }
+  if (u.protocol !== 'http:' || !HOSTS.has(u.hostname) || u.username || u.password || !u.port || u.pathname !== '/' || u.search || u.hash) {
+    throw new Error(`refused: ${u.origin} is not a loopback address of the sandbox (http://127.0.0.1:PORT only)`);
+  }
+  return u;
+}
+
+const JWT = /\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]*/g;
+const LONG = /\b[A-Fa-f0-9]{40,}\b|\b[A-Za-z0-9_]{32,}\b/g;
+export const redactText = (s: string): string => s.replace(JWT, '[redacted]').replace(LONG, '[redacted]');
+export function redactJson(v: Json, key = ''): Json {
+  if (SECRET_NAME.test(key) && v !== null && typeof v !== 'object') return '[redacted]';
+  if (typeof v === 'string') return redactText(v);
+  if (Array.isArray(v)) return v.map((x) => redactJson(x, key));
+  if (v && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, redactJson(x, k)]));
+  return v;
+}
+
+// What is kept of a response header: a short allow list. No cookie, no authorization, nothing that could hold a credential.
+const KEEP = new Set(['content-type', 'location', 'cache-control', 'www-authenticate', 'access-control-allow-origin', 'x-content-type-options']);
+
+function judgeStep(spec: Spec, s: SpecStep, rec: StepRecord, vars: Vars, secrets: ReadonlySet<string>): void {
+  rec.assertions = rec.response ? s.assert.map((a) => evalAssertion(rec.response!, a, vars, secrets)) : [];
+}
+
+export interface Judgement { done: boolean; outcome: RunOutcome | null; detail: string }
+
+// One verdict function for the live run and for a replay. Reads only the stored records.
+export function judge(spec: Spec, recs: StepRecord[]): Judgement & { recs: StepRecord[] } {
+  const vars: Vars = {};
+  const secrets = new Set<string>();
+  const out: StepRecord[] = [];
+  for (let i = 0; i < recs.length; i++) {
+    const s = spec.steps[i];
+    const rec = recs[i];
+    if (!s || rec.id !== s.id) return { done: true, outcome: 'error', detail: `the stored step ${i + 1} does not match the spec`, recs: out };
+    for (const c of s.captures ?? []) if (SECRET_NAME.test(c.name) || SECRET_NAME.test(c.path ?? '')) secrets.add(c.name);
+    const r = { ...rec, assertions: [...rec.assertions] };
+    judgeStep(spec, s, r, vars, secrets);
+    out.push(r);
+    Object.assign(vars, rec.captures);
+    const failed = r.assertions.find((a) => !a.ok);
+    if (rec.error && !(failed && s.phase !== 'attack')) return { done: true, outcome: 'error', detail: `step ${s.id}: ${rec.error}`, recs: out };
+    if (s.phase === 'setup' && failed) return { done: true, outcome: 'invalid', detail: `setup step ${s.id} did not hold (${failed.desc}), so the harness or the seed is not working`, recs: out };
+    if (s.phase === 'control' && failed) return { done: true, outcome: 'invalid', detail: `positive control failed at step ${s.id} (${failed.desc}), so nothing can be said about the flaw`, recs: out };
+    if (s.phase === 'attack' && failed) return { done: true, outcome: 'not_proven', detail: `the app refused as it should at step ${s.id} (${failed.desc} did not hold)`, recs: out };
+  }
+  if (recs.length === spec.steps.length) return { done: true, outcome: 'proven', detail: 'the positive control passed and every attack assertion held', recs: out };
+  return { done: false, outcome: null, detail: '', recs: out };
+}
+
+const parseJson = (t: string): Json | undefined => { try { return JSON.parse(t) as Json; } catch { return undefined; } };
+
+export interface ExecOptions { baseUrl: string; fetchFn?: typeof fetch; requestMs?: number }
+
+// Runs the steps in order and stops as soon as the verdict is known. Returns the redacted records and the verdict.
+export async function executeSpec(spec: Spec, o: ExecOptions): Promise<{ outcome: RunOutcome; detail: string; steps: StepRecord[] }> {
+  const base = loopbackBase(o.baseUrl);
+  const f = o.fetchFn ?? fetch;
+  const vars: Vars = {};
+  const secrets = new Set<string>();
+  const recs: StepRecord[] = [];
+  for (const s of spec.steps) {
+    for (const c of s.captures ?? []) if (SECRET_NAME.test(c.name) || SECRET_NAME.test(c.path ?? '')) secrets.add(c.name);
+    const rec: StepRecord = { id: s.id, phase: s.phase, actor: s.actor, error: null, request: { method: s.method, path: s.path, headers: {}, body: null }, response: null, captures: {}, assertions: [] };
+    try {
+      const path = fill(s.path, vars, secrets, false);
+      if (!path.startsWith('/') || path.startsWith('//')) throw new Error('the path became something other than a path on the app');
+      // The actor's headers apply once every variable in them has been captured (an actor that is not signed in yet sends none).
+      const headers: Record<string, string> = {};
+      for (const [k, v] of Object.entries(spec.actors[s.actor]?.headers ?? {})) { try { headers[k] = fill(v, vars, secrets, false); } catch { /* not signed in yet */ } }
+      for (const [k, v] of Object.entries(s.headers ?? {})) headers[k] = fill(v, vars, secrets, false);
+      const body = s.body ? fillJson(s.body, vars, secrets, false) : null;
+      if (body && !headers['content-type']) headers['content-type'] = 'application/json';
+      rec.request = {
+        method: s.method, path: fill(s.path, vars, secrets, true),
+        headers: Object.fromEntries(Object.entries({ ...(spec.actors[s.actor]?.headers ?? {}), ...(s.headers ?? {}) }).map(([k, v]) => [k, SECRET_NAME.test(k) ? '[redacted]' : v]).sort(([a], [b]) => (a < b ? -1 : 1))),
+        body: s.body ? redactJson(fillJson(s.body, vars, secrets, true)) : null,
+      };
+      const res = await f(new URL(path, base), { method: s.method, headers, body: body ? JSON.stringify(body) : undefined, redirect: 'manual', signal: AbortSignal.timeout(o.requestMs ?? LIMITS.requestMs) });
+      const full = (await res.text()).slice(0, LIMITS.bodyChars + 1);
+      const truncated = full.length > LIMITS.bodyChars;
+      const text = full.slice(0, LIMITS.bodyChars);
+      const rawJson = truncated ? undefined : parseJson(text);
+      const rh: Record<string, string> = {};
+      res.headers.forEach((v, k) => { if (KEEP.has(k.toLowerCase())) rh[k.toLowerCase()] = redactText(v); });
+      rec.response = {
+        status: res.status, headers: rh, truncated,
+        json: rawJson === undefined ? undefined : redactJson(rawJson),
+        text: rawJson === undefined ? redactText(text).slice(0, LIMITS.textStore) : undefined,
+      };
+      // Captures read the raw response. What is stored is redacted. A capture that finds nothing is an error, but a failed assertion
+      // on the same step is reported first (a failed sign in is an invalid setup, not a broken run).
+      for (const c of s.captures ?? []) {
+        const v: Json | undefined = c.from === 'status' ? res.status : c.from === 'header' ? (res.headers.get(c.path!) ?? undefined) : pathGet(rawJson, c.path!);
+        if (v === undefined || (typeof v === 'object' && v !== null)) { rec.error ??= `capture ${c.name} found nothing at ${c.from} ${c.path ?? ''}`.trim(); continue; }
+        vars[c.name] = v;
+        rec.captures[c.name] = secrets.has(c.name) ? '[redacted]' : redactJson(v);
+      }
+    } catch (e) {
+      rec.error = (e as Error).name === 'TimeoutError' ? 'the request timed out' : (e as Error).message.slice(0, 200);
+    }
+    recs.push(rec);
+    const j = judge(spec, recs);
+    recs[recs.length - 1] = j.recs[j.recs.length - 1];
+    if (j.done) return { outcome: j.outcome!, detail: j.detail, steps: recs };
+  }
+  const j = judge(spec, recs);
+  return { outcome: j.outcome ?? 'error', detail: j.detail, steps: recs };
+}
+
+// The record stored for one run. The hash covers every other field.
+export function runRecord(spec: Spec, specSha: string, run: number, r: { outcome: RunOutcome; detail: string; steps: StepRecord[] }): import('../types/prove.ts').RunRecord {
+  const body = { schema_version: '1.0.0' as const, spec_id: spec.spec_id, spec_sha256: specSha, run, outcome: r.outcome, detail: r.detail, steps: r.steps };
+  return { ...body, transcript_sha256: sha256(canon(body)) };
+}

package/src/prove/local.ts

@@ -0,0 +1,100 @@
+// UNSAFE. A plain local process backend, for the engine's own fixture apps in CI and nothing else. There is no isolation here: the
+// app runs as the current user on the host. For that reason this runner refuses any repository that is not inside a fixture folder of
+// this engine, and the command line never offers it. It is labelled unsafe in its name, in the result file and in the report.
+import { spawn, type ChildProcess } from 'node:child_process';
+import { mkdtempSync, realpathSync, rmSync } from 'node:fs';
+import { createServer } from 'node:net';
+import { tmpdir } from 'node:os';
+import { join, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { executeSpec } from './executor.ts';
+import { assertSandboxTarget } from './safety.ts';
+import { validateSpec } from './spec.ts';
+import type { Exec, Instance, PrepareRequest, PrepareResult, Prepared, Runner, SpecRun, StartResult, Teardown } from './runner.ts';
+
+const ENGINE = fileURLToPath(new URL('../../', import.meta.url));
+export const FIXTURE_ROOTS = [join(ENGINE, 'test/fixtures/prove'), join(ENGINE, 'bench/fixtures')];
+const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
+
+const freePort = (): Promise<number> => new Promise((res, rej) => { const s = createServer(); s.listen(0, '127.0.0.1', () => { const p = (s.address() as { port: number }).port; s.close(() => res(p)); }); s.on('error', rej); });
+const inside = (p: string, root: string): boolean => p === root || p.startsWith(root + sep);
+
+export class LocalProcessRunner implements Runner {
+  readonly name = 'local-process-unsafe';
+  private readonly roots: string[];
+  constructor(o: { acknowledgeUnsafe: true; fixtureRoots?: string[] }) {
+    if (o.acknowledgeUnsafe !== true) throw new Error('the local process runner is unsafe and must be created with acknowledgeUnsafe: true');
+    this.roots = (o.fixtureRoots ?? FIXTURE_ROOTS).map((r) => { try { return realpathSync(r); } catch { return r; } });
+  }
+  async available(): Promise<{ ok: true }> { return { ok: true }; }
+
+  async prepare(req: PrepareRequest): Promise<PrepareResult> {
+    let src: string;
+    try { src = realpathSync(req.sourceDir); } catch { return { ok: false, code: 'refused_unsafe_backend', reason: 'the source folder does not exist' }; }
+    if (!this.roots.some((r) => inside(src, r))) return { ok: false, code: 'refused_unsafe_backend', reason: 'the local process backend has no isolation, so it only runs the fixture apps of this engine and refuses any other folder' };
+    if (req.recipe.install) return { ok: false, code: 'refused_unsafe_backend', reason: 'the local process backend installs nothing' };
+    if (req.recipe.data_store.kind === 'postgres') return { ok: false, code: 'refused_unsafe_backend', reason: 'the local process backend has no Postgres' };
+    return { ok: true, prepared: new LocalPrepared(req) };
+  }
+}
+
+class LocalPrepared implements Prepared {
+  private readonly req: PrepareRequest;
+  private readonly live = new Set<LocalInstance>();
+  constructor(req: PrepareRequest) { this.req = req; }
+
+  // A free port is chosen first and the app binds it a moment later, so another process can take it in between. Start again on a new
+  // port, twice at most, when the app exits before it answers.
+  async start(): Promise<StartResult> {
+    let last = await this.startOnce();
+    for (let i = 0; i < 2 && !last.ok && last.code === 'start_failed' && this.req.recipe.port_from_env; i++) last = await this.startOnce();
+    return last;
+  }
+
+  private async startOnce(): Promise<StartResult> {
+    const { recipe, copyDir, limits } = this.req;
+    const data = mkdtempSync(join(tmpdir(), 'hp-local-data-'));
+    const port = recipe.port_from_env ? await freePort() : recipe.port;
+    const env: Record<string, string> = { PATH: process.env.PATH ?? '', HOME: data, TMPDIR: data };
+    for (const [k, v] of Object.entries(recipe.env)) env[k] = v.replace(/\{DATA\}/g, data);
+    if (recipe.port_from_env) env.PORT = String(port);
+    let log = '';
+    const child = spawn('sh', ['-c', `${recipe.seed ? recipe.seed + ' && ' : ''}exec ${recipe.start}`], { cwd: copyDir, env, detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
+    child.stdout.on('data', (d: Buffer) => { log = (log + d.toString()).slice(-2000); });
+    child.stderr.on('data', (d: Buffer) => { log = (log + d.toString()).slice(-2000); });
+    let exited = false;
+    child.on('exit', () => { exited = true; });
+    const inst = new LocalInstance(child, data, port);
+    this.live.add(inst);
+    const deadline = Date.now() + limits.healthMs;
+    while (Date.now() < deadline && !exited) {
+      try { const r = await fetch(`http://127.0.0.1:${port}${recipe.health_path}`, { signal: AbortSignal.timeout(1500) }); if (r.status < 500) return { ok: true, instance: inst }; } catch { /* not up yet */ }
+      await sleep(100);
+    }
+    await inst.stop();
+    return { ok: false, code: exited ? 'start_failed' : 'health_failed', reason: `${exited ? 'the app exited before it answered' : 'the app did not answer in time'}: ${log.trim().replace(/\s+/g, ' ').slice(-300)}` };
+  }
+  async dispose(): Promise<Teardown> { for (const i of this.live) await i.stop(); return { left_behind: [...this.live].flatMap((i) => i.alive() ? [`process ${i.pid}`] : []) }; }
+}
+
+class LocalInstance implements Instance {
+  readonly baseUrl: string;
+  readonly pid: number;
+  private readonly child: ChildProcess;
+  private readonly data: string;
+  private readonly port: number;
+  constructor(child: ChildProcess, data: string, port: number) { this.child = child; this.data = data; this.port = port; this.pid = child.pid ?? 0; this.baseUrl = `http://127.0.0.1:${port}`; }
+  alive(): boolean { try { process.kill(this.pid, 0); return true; } catch { return false; } }
+  async exec(): Promise<Exec> { return { code: null, stdout: '', stderr: 'the local process backend does not run commands', timedOut: false }; }
+  async runSpec(spec: import('../types/index.ts').Spec): Promise<SpecRun> {
+    const v = validateSpec(spec);
+    if (!v.ok) return { ok: false, reason: `invalid spec: ${v.errors.join('; ')}` };
+    try { assertSandboxTarget(this.baseUrl, this.port); return { ok: true, ...(await executeSpec(v.spec, { baseUrl: this.baseUrl })) }; } catch (e) { return { ok: false, reason: (e as Error).message }; }
+  }
+  async stop(): Promise<Teardown> {
+    try { process.kill(-this.pid, 'SIGKILL'); } catch { /* already gone */ }
+    for (let i = 0; i < 20 && this.alive(); i++) await sleep(25);
+    rmSync(this.data, { recursive: true, force: true });
+    return { left_behind: this.alive() ? [`process ${this.pid}`] : [] };
+  }
+}

package/src/prove/probes.ts

@@ -0,0 +1,68 @@
+// Escape attempts. Each probe is a short script run inside a started sandbox instance with the same limits as the app. The sandbox
+// has held when every probe reports ok. They are the exit test of step 4a and `hullproof prove --self-test` runs them on demand.
+import { randomBytes } from 'node:crypto';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { Recipe } from '../types/index.ts';
+import type { Instance } from './runner.ts';
+
+export interface ProbeResult { name: string; ok: boolean; detail: string }
+
+const NET = `
+const net=require('net'),dns=require('dns');
+const conn=(h,p)=>new Promise(r=>{const s=net.connect({host:h,port:p,timeout:3000});s.on('connect',()=>{s.destroy();r(true)});s.on('error',()=>r(false));s.on('timeout',()=>{s.destroy();r(false)})});
+(async()=>{const o={};o.tcp_cloudflare=await conn('1.1.1.1',443);o.tcp_google=await conn('8.8.8.8',53);o.dns=await new Promise(r=>dns.lookup('example.com',e=>r(!e)));
+o.fetch=await fetch('http://example.com',{signal:AbortSignal.timeout(3000)}).then(()=>true,()=>false);console.log(JSON.stringify(o));})()`;
+const FILES = `
+const fs=require('fs');const out={};
+for(const p of [process.argv[1],'/Users','/host','/mnt/host','/var/run/docker.sock','/run/docker.sock','/root/.ssh']){try{fs.readFileSync(p);out[p]='READ';}catch(e){try{const l=fs.readdirSync(p);out[p]='LISTED:'+l.length;}catch(e2){out[p]=e2.code;}}}
+console.log(JSON.stringify(out))`;
+const WRITES = `
+const fs=require('fs');const out={};
+for(const p of ['/app/hp_probe','/etc/hp_probe','/usr/hp_probe','/hp_probe','/tmp/hp_probe']){try{fs.writeFileSync(p,'x');out[p]='WROTE';}catch(e){out[p]=e.code;}}
+out.uid=process.getuid();console.log(JSON.stringify(out))`;
+const FORK = `
+const {spawn}=require('child_process'),fs=require('fs');let refused=0,peak=0;const kids=[];
+const rd=(f)=>{try{return fs.readFileSync(f,'utf8').trim()}catch{return null}};
+const cur=()=>Number(rd('/sys/fs/cgroup/pids.current')||rd('/sys/fs/cgroup/pids/pids.current')||0);
+const max=rd('/sys/fs/cgroup/pids.max')||rd('/sys/fs/cgroup/pids/pids.max');
+const go=async()=>{for(let i=0;i<400;i++){const k=spawn(process.execPath,['-e','setTimeout(()=>{},60000)'],{stdio:'ignore'});k.on('error',()=>refused++);kids.push(k);peak=Math.max(peak,cur());await new Promise(r=>setTimeout(r,8));}
+await new Promise(r=>setTimeout(r,800));console.log(JSON.stringify({attempted:400,refused,peak_processes:peak,limit:max}));kids.forEach(k=>{try{k.kill('SIGKILL')}catch{}});process.exit(0);};go()`;
+const MEM = `const a=[];try{for(let i=0;i<64;i++)a.push(Buffer.alloc(64*1024*1024,1));console.log('ALLOCATED');}catch(e){console.log('REFUSED');}`;
+
+const j = (s: string): Record<string, unknown> => { try { return JSON.parse(s.trim().split('\n').pop() ?? '{}') as Record<string, unknown>; } catch { return {}; } };
+
+export async function runEscapeProbes(inst: Instance, hostMarker: string): Promise<ProbeResult[]> {
+  const out: ProbeResult[] = [];
+  const net = j((await inst.exec(['node', '-e', NET], { timeoutMs: 30_000 })).stdout);
+  out.push({ name: 'network call to the internet', ok: Object.keys(net).length === 4 && Object.values(net).every((v) => v === false), detail: JSON.stringify(net) });
+  const files = j((await inst.exec(['node', '-e', FILES, hostMarker], { timeoutMs: 15_000 })).stdout);
+  out.push({ name: 'host file read', ok: Object.keys(files).length > 0 && Object.values(files).every((v) => v === 'ENOENT' || v === 'EACCES'), detail: JSON.stringify(files) });
+  const w = j((await inst.exec(['node', '-e', WRITES], { timeoutMs: 15_000 })).stdout);
+  const ro = ['/app/hp_probe', '/etc/hp_probe', '/usr/hp_probe', '/hp_probe'].every((p) => w[p] === 'EROFS' || w[p] === 'EACCES');
+  out.push({ name: 'write to the root filesystem, and run as root', ok: ro && typeof w.uid === 'number' && w.uid !== 0, detail: JSON.stringify(w) });
+  const fork = j((await inst.exec(['node', '-e', FORK], { timeoutMs: 40_000 })).stdout);
+  const alive = (await inst.exec(['node', '-e', 'console.log(1)'], { timeoutMs: 15_000 })).code === 0;
+  out.push({ name: 'fork bomb', ok: typeof fork.peak_processes === 'number' && fork.peak_processes > 0 && fork.peak_processes <= Number(fork.limit) && alive, detail: `${JSON.stringify(fork)}; sandbox still answers: ${alive}` });
+  const t0 = Date.now();
+  const loop = await inst.exec(['node', '-e', 'while(true){}'], { timeoutMs: 3000 });
+  const after = (await inst.exec(['node', '-e', 'console.log(1)'], { timeoutMs: 15_000 })).code === 0;
+  out.push({ name: 'long loop', ok: (loop.timedOut || loop.code === 137) && Date.now() - t0 < 30_000 && after, detail: `ended after ${Date.now() - t0} ms with code ${loop.code}; sandbox still answers: ${after}` });
+  const mem = await inst.exec(['node', '-e', MEM], { timeoutMs: 30_000 });
+  out.push({ name: 'memory limit', ok: !mem.stdout.includes('ALLOCATED') && (await inst.exec(['node', '-e', 'console.log(1)'], { timeoutMs: 15_000 })).code === 0, detail: `exit code ${mem.code}` });
+  return out;
+}
+
+// A small app and a recipe for `hullproof prove --self-test`: it needs nothing installed and reads nothing from the user's repository.
+export function selfTestApp(): { dir: string; recipe: Recipe; marker: string; cleanup(): void } {
+  const dir = mkdtempSync(join(tmpdir(), 'hp-selftest-'));
+  const outside = mkdtempSync(join(tmpdir(), 'hp-marker-'));
+  const marker = join(outside, `secret-${randomBytes(4).toString('hex')}.txt`);
+  writeFileSync(marker, 'HOST-SECRET-MARKER');
+  mkdirSync(join(dir, 'app'), { recursive: true });
+  writeFileSync(join(dir, 'server.js'), "require('http').createServer((q,s)=>{s.setHeader('content-type','application/json');s.end(JSON.stringify({ok:true}))}).listen(3000);\n");
+  writeFileSync(join(dir, 'package.json'), '{"name":"hp-selftest","version":"0.0.0","main":"server.js"}\n');
+  const recipe: Recipe = { schema_version: '1.0.0', stack: 'node-express', base_image: 'node:22-slim', install: null, vendored_modules: false, seed: null, start: 'node server.js', port: 3000, port_from_env: false, health_path: '/', data_store: { kind: 'none' }, env: {}, notes: [] };
+  return { dir, recipe, marker, cleanup: () => { rmSync(dir, { recursive: true, force: true }); rmSync(outside, { recursive: true, force: true }); } };
+}