@davidrydberg/proofy 0.0.0-stage → 0.1.0

package/dist/run/engine.d.ts

@@ -0,0 +1,48 @@
+import type { Config } from "../config.js";
+import type { Paths } from "../project.js";
+import type { Proc } from "../sandbox/docker.js";
+import { type Sandbox, type SandboxOptions } from "../sandbox/sandbox.js";
+import type { Approvals } from "./approvals.js";
+import { type PlannedFlow, type RunReport } from "./results.js";
+export interface RunContext {
+    proc: Proc;
+    env: NodeJS.ProcessEnv;
+    now: () => Date;
+    log: (line: string) => void;
+    /** Where results and approvals live: always the working copy. */
+    base: Paths;
+    /** Where the app, tests and fakes come from: the working copy, or an exported pull request. */
+    source: Paths;
+    config: Config;
+    approvals: Approvals;
+    cacheDir: string;
+    sourceInfo: RunReport["source"];
+}
+export declare function planFlows(ctx: Pick<RunContext, "source" | "config" | "approvals">, names: string[]): Promise<PlannedFlow[]>;
+export declare function newRunId(now: Date): string;
+export declare function sandboxOptions(ctx: RunContext, mode: "run" | "dev", project: string, logDir: string): SandboxOptions;
+/** A whole sealed run: fresh sandbox, tests, results, teardown. Results are written even when the sandbox fails. */
+export declare function sealedRun(ctx: RunContext, planned: PlannedFlow[], options: {
+    keep: boolean;
+    project: string;
+}): Promise<RunReport>;
+/** Runs the planned flows in a sandbox that is already up: a sealed run's, or the dev sandbox. */
+export declare function testsInSandbox(ctx: RunContext, sandbox: Sandbox, planned: PlannedFlow[], extraArgs: string[]): Promise<void>;
+export declare function finishRun(ctx: RunContext, run: {
+    id: string;
+    kind: "run" | "dev";
+    startedAt: string;
+    runDir: string;
+    planned: PlannedFlow[];
+    notes: string[];
+    environmentError?: string;
+    sandbox?: Sandbox;
+    offsets?: Offsets;
+}): Promise<RunReport>;
+export interface Offsets {
+    requests: number;
+    dns: number;
+}
+/** How far the dev sandbox's request logs are, so one dev test only reports its own calls. */
+export declare function logOffsets(sandbox: Sandbox): Promise<Offsets>;
+export declare function reserveRunId(ctx: Pick<RunContext, "base" | "now">): Promise<string>;

package/dist/run/engine.js

@@ -0,0 +1,168 @@
+import { appendFile, cp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { CliError } from "../errors.js";
+import { MOUNT } from "../sandbox/contract.js";
+import { baseUrl } from "../sandbox/compose.js";
+import { installImages, prepare, runTests, saveLogs, start, stop } from "../sandbox/sandbox.js";
+import { fingerprint, sameFingerprint } from "./fingerprint.js";
+import { outDir, pruneRuns } from "./history.js";
+import { renderMarkdown } from "./report.js";
+import { collectResults } from "./results.js";
+import { parseRequirements } from "./spec.js";
+export async function planFlows(ctx, names) {
+    const planned = [];
+    for (const name of names) {
+        const print = await fingerprint(ctx.source, ctx.config, name);
+        const specText = await readFile(join(ctx.source.specs, `${name}.md`), "utf8");
+        planned.push({ name, approved: sameFingerprint(ctx.approvals.flows[name], print), fingerprint: print, requirements: parseRequirements(specText) });
+    }
+    return planned;
+}
+export function newRunId(now) {
+    const pad = (n) => String(n).padStart(2, "0");
+    return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
+}
+export function sandboxOptions(ctx, mode, project, logDir) {
+    return {
+        sourceDir: ctx.source.root,
+        e2eDir: ctx.source.e2e,
+        logDir,
+        config: ctx.config,
+        mode,
+        project,
+        cacheDir: ctx.cacheDir,
+        env: ctx.env,
+        log: ctx.log,
+    };
+}
+/** A whole sealed run: fresh sandbox, tests, results, teardown. Results are written even when the sandbox fails. */
+export async function sealedRun(ctx, planned, options) {
+    const id = await reserveRunId(ctx);
+    const runDir = join(outDir(ctx.base), id);
+    const startedAt = ctx.now().toISOString();
+    const sandboxOpts = sandboxOptions(ctx, "run", options.project, runDir);
+    let sandbox;
+    let environmentError;
+    const notes = [];
+    try {
+        sandbox = await prepare(ctx.proc, sandboxOpts);
+        notes.push(...sandbox.notes);
+        await installImages(ctx.proc, sandboxOpts, sandbox);
+        sandbox = await start(ctx.proc, sandboxOpts, sandbox);
+        await testsInSandbox(ctx, sandbox, planned, []);
+    }
+    catch (error) {
+        if (!(error instanceof CliError) || error.code !== "ENVIRONMENT_ERROR")
+            throw error;
+        environmentError = [error.message, ...error.suggestions].join("\n");
+    }
+    finally {
+        if (sandbox) {
+            await saveLogs(ctx.proc, sandbox, runDir);
+            if (options.keep) {
+                notes.push(`Sandbox kept running: docker compose -p ${sandbox.project} -f ${sandbox.composeFile} down -v`);
+            }
+            else {
+                ctx.log("tearing down the sandbox");
+                await stop(ctx.proc, sandbox);
+            }
+        }
+    }
+    return finishRun(ctx, { id, kind: "run", startedAt, runDir, planned, notes, ...(environmentError ? { environmentError } : {}), ...(sandbox ? { sandbox } : {}) });
+}
+/** Runs the planned flows in a sandbox that is already up: a sealed run's, or the dev sandbox. */
+export async function testsInSandbox(ctx, sandbox, planned, extraArgs) {
+    const out = join(sandbox.workDir, "out");
+    for (const leftover of ["flows", "playwright", "playwright.json", "playwright.log"]) {
+        await rm(join(out, leftover), { recursive: true, force: true });
+    }
+    await writeFile(join(sandbox.workDir, "runner", "playwright.config.mjs"), playwrightConfig(ctx.config, planned.map((flow) => flow.name)));
+    ctx.log(`running ${planned.map((flow) => flow.name).join(", ")}`);
+    await runTests(ctx.proc, sandboxOptions(ctx, "run", sandbox.project, out), sandbox, extraArgs);
+}
+export async function finishRun(ctx, run) {
+    await mkdir(run.runDir, { recursive: true });
+    if (run.sandbox)
+        await harvest(run.sandbox, run.runDir, run.offsets);
+    const collected = await collectResults(run.runDir, run.planned);
+    const environmentError = run.environmentError ?? collected.environmentError;
+    const report = {
+        id: run.id,
+        kind: run.kind,
+        startedAt: run.startedAt,
+        finishedAt: ctx.now().toISOString(),
+        source: ctx.sourceInfo,
+        flows: environmentError ? collected.flows.map((flow) => ({ ...flow, status: "failed" })) : collected.flows,
+        outside: collected.outside,
+        lookedUpOnly: collected.lookedUpOnly,
+        notes: run.notes,
+        ...(environmentError ? { environmentError } : {}),
+    };
+    await writeFile(join(run.runDir, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
+    await writeFile(join(run.runDir, "report.md"), renderMarkdown(report, (path) => path));
+    await pruneRuns(ctx.base);
+    return report;
+}
+/** How far the dev sandbox's request logs are, so one dev test only reports its own calls. */
+export async function logOffsets(sandbox) {
+    return { requests: await lineCount(join(sandbox.workDir, "out", "requests.jsonl")), dns: await lineCount(join(sandbox.workDir, "out", "dns.jsonl")) };
+}
+async function harvest(sandbox, runDir, offsets = { requests: 0, dns: 0 }) {
+    const out = join(sandbox.workDir, "out");
+    for (const entry of ["flows", "playwright", "playwright.json", "playwright.log"]) {
+        try {
+            await cp(join(out, entry), join(runDir, entry), { recursive: true });
+        }
+        catch {
+            // Not every run produces every file, for example when the sandbox never started.
+        }
+    }
+    for (const [file, skip] of [["requests.jsonl", offsets.requests], ["dns.jsonl", offsets.dns]]) {
+        const lines = (await readText(join(out, file))).split("\n").filter(Boolean).slice(skip);
+        if (lines.length > 0)
+            await appendFile(join(runDir, file), `${lines.join("\n")}\n`);
+    }
+}
+export async function reserveRunId(ctx) {
+    const base = newRunId(ctx.now());
+    await mkdir(outDir(ctx.base), { recursive: true });
+    for (let n = 0;; n++) {
+        const id = n === 0 ? base : `${base}-${n}`;
+        try {
+            await mkdir(join(outDir(ctx.base), id));
+            return id;
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+        }
+    }
+}
+function playwrightConfig(config, flows) {
+    const settings = {
+        testDir: `${MOUNT.e2e}/tests`,
+        testMatch: flows.map((flow) => `**/${flow}.spec.ts`),
+        outputDir: `${MOUNT.out}/playwright`,
+        timeout: config.limits.testMinutes * 60_000,
+        retries: 0,
+        workers: config.limits.parallel,
+        fullyParallel: true,
+        globalSetup: `${MOUNT.pkg}/dist/test/global-setup.js`,
+        reporter: [["json", { outputFile: `${MOUNT.out}/playwright.json` }], ["line"]],
+        // A step that waits for something that never appears fails after 30 seconds with its locator, not after the whole test timeout.
+        use: { baseURL: baseUrl(config), ignoreHTTPSErrors: true, trace: "retain-on-failure", screenshot: "off", video: "off", actionTimeout: 30_000, navigationTimeout: 60_000 },
+        projects: [{ name: "chromium", use: { browserName: "chromium" } }],
+    };
+    return `// Generated by proofy for one run.\nexport default ${JSON.stringify(settings, null, 2)};\n`;
+}
+async function readText(file) {
+    try {
+        return await readFile(file, "utf8");
+    }
+    catch {
+        return "";
+    }
+}
+async function lineCount(file) {
+    return (await readText(file)).split("\n").filter(Boolean).length;
+}

package/dist/run/fingerprint.d.ts

@@ -0,0 +1,11 @@
+import type { Config } from "../config.js";
+import type { Paths } from "../project.js";
+export interface Fingerprint {
+    spec: string;
+    test: string;
+    /** Every fake and the hosts list together: a change to any fake can change any flow's result. */
+    fakes: string;
+}
+export declare function fingerprint(paths: Paths, config: Config, flow: string): Promise<Fingerprint>;
+export declare function fakesHash(paths: Paths, config: Config): Promise<string>;
+export declare function sameFingerprint(a: Fingerprint | undefined, b: Fingerprint | undefined): boolean;

package/dist/run/fingerprint.js

@@ -0,0 +1,40 @@
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+export async function fingerprint(paths, config, flow) {
+    return {
+        spec: await hashFile(join(paths.specs, `${flow}.md`)),
+        test: await hashFile(join(paths.tests, `${flow}.spec.ts`)),
+        fakes: await fakesHash(paths, config),
+    };
+}
+export async function fakesHash(paths, config) {
+    const hash = createHash("sha256");
+    hash.update(JSON.stringify(Object.entries(config.hosts).sort()));
+    let files = [];
+    try {
+        files = (await readdir(paths.standins)).filter((file) => file.endsWith(".mjs")).sort();
+    }
+    catch {
+        // No fakes yet.
+    }
+    for (const file of files) {
+        hash.update(file);
+        hash.update(await readFile(join(paths.standins, file)));
+    }
+    return short(hash.digest("hex"));
+}
+export function sameFingerprint(a, b) {
+    return a !== undefined && b !== undefined && a.spec === b.spec && a.test === b.test && a.fakes === b.fakes;
+}
+async function hashFile(file) {
+    try {
+        return short(createHash("sha256").update(await readFile(file)).digest("hex"));
+    }
+    catch {
+        return "missing";
+    }
+}
+function short(hex) {
+    return hex.slice(0, 16);
+}

package/dist/run/history.d.ts

@@ -0,0 +1,29 @@
+import type { Config } from "../config.js";
+import { type Paths } from "../project.js";
+import type { Approvals } from "./approvals.js";
+import { type Fingerprint } from "./fingerprint.js";
+import { type FlowResult, type RunReport } from "./results.js";
+export declare const KEEP_RUNS = 20;
+export type FlowState = "draft" | "unproven" | "unstable" | "ready" | "approved" | "out-of-date";
+export interface FlowStatus {
+    name: string;
+    state: FlowState;
+    /** Runs with the current files, newest first, up to three. */
+    runs: string[];
+    /** Why a flow with three runs is still not ready, such as a requirement no test checks. */
+    reason?: string;
+    fingerprint: Fingerprint;
+}
+export declare function outDir(paths: Paths): string;
+export declare function runIds(paths: Paths): Promise<string[]>;
+export declare function readReport(paths: Paths, id: string): Promise<RunReport | undefined>;
+/** Newest first. */
+export declare function recentReports(paths: Paths): Promise<RunReport[]>;
+export declare function pruneRuns(paths: Paths): Promise<void>;
+export declare function flowStatuses(paths: Paths, config: Config, names: string[], approvals: Approvals, reports: RunReport[]): Promise<FlowStatus[]>;
+/** Three runs with the current files and identical results per requirement, every requirement checked. */
+export declare function proven(flows: FlowResult[]): {
+    ok: boolean;
+    unstable: boolean;
+    reason?: string;
+};

package/dist/run/history.js

@@ -0,0 +1,85 @@
+import { readdir, readFile, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { exists } from "../project.js";
+import { fingerprint, sameFingerprint } from "./fingerprint.js";
+import { outcome } from "./results.js";
+export const KEEP_RUNS = 20;
+const RUN_ID = /^\d{8}-\d{6}(-\d+)?$/;
+export function outDir(paths) {
+    return join(paths.e2e, "out");
+}
+export async function runIds(paths) {
+    try {
+        return (await readdir(outDir(paths))).filter((name) => RUN_ID.test(name)).sort();
+    }
+    catch {
+        return [];
+    }
+}
+export async function readReport(paths, id) {
+    try {
+        return JSON.parse(await readFile(join(outDir(paths), id, "report.json"), "utf8"));
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Newest first. */
+export async function recentReports(paths) {
+    const ids = (await runIds(paths)).reverse();
+    const reports = await Promise.all(ids.map((id) => readReport(paths, id)));
+    return reports.filter((report) => report !== undefined);
+}
+export async function pruneRuns(paths) {
+    const ids = await runIds(paths);
+    for (const id of ids.slice(0, Math.max(0, ids.length - KEEP_RUNS))) {
+        await rm(join(outDir(paths), id), { recursive: true, force: true });
+    }
+}
+export async function flowStatuses(paths, config, names, approvals, reports) {
+    const out = [];
+    for (const name of names) {
+        const current = await fingerprint(paths, config, name);
+        if (!(await exists(join(paths.tests, `${name}.spec.ts`)))) {
+            out.push({ name, state: "draft", runs: [], fingerprint: current });
+            continue;
+        }
+        const approval = approvals.flows[name];
+        const matching = [];
+        for (const report of reports) {
+            if (report.environmentError)
+                continue;
+            const flow = report.flows.find((candidate) => candidate.name === name);
+            if (flow && sameFingerprint(flow.fingerprint, current))
+                matching.push({ id: report.id, flow });
+            if (matching.length === 3)
+                break;
+        }
+        const runs = matching.map((match) => match.id);
+        if (approval && sameFingerprint(approval, current)) {
+            out.push({ name, state: "approved", runs, fingerprint: current });
+            continue;
+        }
+        const proof = proven(matching.map((match) => match.flow));
+        const state = approval ? "out-of-date" : proof.ok ? "ready" : proof.unstable ? "unstable" : "unproven";
+        out.push({ name, state, runs, fingerprint: current, ...(proof.reason ? { reason: proof.reason } : {}) });
+    }
+    return out;
+}
+/** Three runs with the current files and identical results per requirement, every requirement checked. */
+export function proven(flows) {
+    if (flows.length < 3)
+        return { ok: false, unstable: false, reason: `${flows.length} of 3 runs with the current files` };
+    const outcomes = new Set(flows.map(outcome));
+    if (outcomes.size > 1)
+        return { ok: false, unstable: true, reason: "the last 3 runs gave different results" };
+    const latest = flows[0];
+    const unchecked = latest?.requirements.filter((requirement) => requirement.result === "not-checked").map((requirement) => requirement.id) ?? [];
+    if (unchecked.length > 0)
+        return { ok: false, unstable: false, reason: `no claim checks ${unchecked.join(", ")}` };
+    if (latest && latest.unknownClaims.length > 0)
+        return { ok: false, unstable: false, reason: `claims cite ${latest.unknownClaims.join(", ")}, which the spec does not have` };
+    if (latest && latest.requirements.length === 0)
+        return { ok: false, unstable: false, reason: "the spec has no numbered requirements" };
+    return { ok: true, unstable: false };
+}

package/dist/run/report.d.ts

@@ -0,0 +1,14 @@
+import type { FlowResult, RunReport } from "./results.js";
+export type Verdict = "pass" | "fail" | "untrusted" | "environment";
+/** The run's one-word result, matching the exit code: environment 3, a failed approved flow 1, untrusted 4, pass 0. */
+export declare function verdict(report: RunReport): Verdict;
+export declare const EXIT_FOR: Record<Verdict, number>;
+/**
+ * The report a person reads: per requirement what was asked, what happened and the evidence.
+ * `image` turns a screenshot path relative to the run folder into a link, local or on GitHub.
+ */
+export declare function renderMarkdown(report: RunReport, image: (path: string) => string, options?: {
+    title?: string;
+    maxShots?: number;
+}): string;
+export declare function countResults(flow: FlowResult): string;

package/dist/run/report.js

@@ -0,0 +1,115 @@
+const LABEL = {
+    pass: "pass",
+    fail: "FAIL",
+    "not-covered": "not covered",
+    "not-reached": "FAIL (not reached)",
+    "not-checked": "not checked",
+};
+/** The run's one-word result, matching the exit code: environment 3, a failed approved flow 1, untrusted 4, pass 0. */
+export function verdict(report) {
+    if (report.environmentError)
+        return "environment";
+    if (report.flows.some((flow) => flow.approved && flow.status === "failed"))
+        return "fail";
+    if (report.flows.some((flow) => !flow.approved))
+        return "untrusted";
+    return "pass";
+}
+export const EXIT_FOR = { pass: 0, fail: 1, environment: 3, untrusted: 4 };
+const HEADLINE = {
+    pass: "PASS: every requirement of every approved flow held",
+    fail: "FAIL: an approved flow has a requirement that did not hold",
+    untrusted: "NOT TRUSTED: some flows are not approved, so their results prove nothing yet",
+    environment: "ENVIRONMENT FAILURE: the sandbox did not work, so this run says nothing about the app",
+};
+/**
+ * The report a person reads: per requirement what was asked, what happened and the evidence.
+ * `image` turns a screenshot path relative to the run folder into a link, local or on GitHub.
+ */
+export function renderMarkdown(report, image, options = {}) {
+    const result = verdict(report);
+    const lines = [`## ${options.title ?? "proofy"}: ${HEADLINE[result]}`, ""];
+    const source = [report.source.pr ? `PR #${report.source.pr}` : undefined, report.source.commit ? `commit ${report.source.commit.slice(0, 12)}` : undefined, report.source.dirty ? "with uncommitted changes" : undefined].filter(Boolean);
+    lines.push(`Run ${report.id} (${report.kind}), ${source.join(", ") || "working tree"}. Sealed: the app had no way out; every outside call went to a fake or was refused.`, "");
+    if (report.environmentError) {
+        lines.push("```", report.environmentError, "```", "");
+    }
+    if (report.flows.length > 0) {
+        lines.push("| Flow | Result | Approved |", "|---|---|---|");
+        for (const flow of report.flows) {
+            const counts = countResults(flow);
+            lines.push(`| ${flow.name} | ${flow.status === "passed" ? "passed" : "FAILED"} (${counts}) | ${flow.approved ? "yes" : "no, result not trusted"} |`);
+        }
+        lines.push("");
+    }
+    for (const flow of report.flows) {
+        lines.push(...flowSection(flow, image, options.maxShots));
+    }
+    lines.push(...outsideSection(report));
+    if (report.notes.length > 0) {
+        lines.push("### Notes", "", ...report.notes.map((note) => `- ${note}`), "");
+    }
+    return lines.join("\n");
+}
+function flowSection(flow, image, maxShots = Infinity) {
+    const lines = [`### ${flow.name}`, ""];
+    lines.push("| Req | Result | Requirement | What the test did | Expected | Actual | Relied on fakes |", "|---|---|---|---|---|---|---|");
+    for (const requirement of flow.requirements) {
+        const did = requirement.result === "not-covered" ? `not covered: ${requirement.why ?? ""}` : (requirement.did ?? "");
+        lines.push(`| ${requirement.id} | ${LABEL[requirement.result]} | ${cell(requirement.text)} | ${cell(did)} | ${cell(requirement.expected ?? "")} | ${cell(requirement.actual ?? "")} | ${cell(requirement.fakes.join(", ") || "none")} |`);
+    }
+    lines.push("");
+    const evidence = flow.requirements.filter((requirement) => requirement.evidence !== undefined);
+    if (evidence.length > 0) {
+        lines.push("<details><summary>Evidence</summary>", "");
+        for (const requirement of evidence) {
+            lines.push(`${requirement.id}:`, "", "```json", JSON.stringify(requirement.evidence, null, 2).slice(0, 4000), "```", "");
+        }
+        lines.push("</details>", "");
+    }
+    const broken = flow.tests.filter((test) => test.status !== "passed");
+    for (const test of broken) {
+        lines.push(`**${test.title}**: ${test.status}${test.trace ? ` (trace: \`${test.trace}\`, open with \`npx playwright show-trace\`)` : ""}`, "", "```", (test.error ?? "").slice(0, 1500), "```", "");
+    }
+    if (flow.unknownClaims.length > 0) {
+        lines.push(`Claims cite ${flow.unknownClaims.join(", ")}, which the spec does not have.`, "");
+    }
+    if (flow.shots.length > 0) {
+        lines.push("<details><summary>Screenshots</summary>", "");
+        for (const shot of flow.shots.slice(0, maxShots)) {
+            lines.push(`${shot.label}`, "", `<img src="${image(`flows/${flow.name}/${shot.file}`)}" width="360">`, "");
+        }
+        if (flow.shots.length > maxShots)
+            lines.push(`and ${flow.shots.length - maxShots} more in the run folder`, "");
+        lines.push("</details>", "");
+    }
+    return lines;
+}
+function outsideSection(report) {
+    if (report.outside.length === 0 && report.lookedUpOnly.length === 0)
+        return [];
+    const lines = ["### Outside calls", ""];
+    if (report.outside.length > 0) {
+        lines.push("| Host | Handled as | Requests |", "|---|---|---|");
+        for (const call of report.outside)
+            lines.push(`| ${call.host} | ${call.handledAs} | ${call.count} |`);
+        lines.push("");
+    }
+    const refused = report.outside.filter((call) => call.handledAs === "refused").map((call) => call.host);
+    if (refused.length > 0) {
+        lines.push(`Refused: ${refused.join(", ")}. If the app needs them, list them in e2e/proofy.json hosts.`, "");
+    }
+    if (report.lookedUpOnly.length > 0) {
+        lines.push(`Looked up but never called over HTTP (another protocol, or the call gave up): ${report.lookedUpOnly.join(", ")}.`, "");
+    }
+    return lines;
+}
+export function countResults(flow) {
+    const counts = new Map();
+    for (const requirement of flow.requirements)
+        counts.set(requirement.result, (counts.get(requirement.result) ?? 0) + 1);
+    return [...counts].map(([result, count]) => `${count} ${result}`).join(", ") || "no requirements";
+}
+function cell(text) {
+    return text.replace(/\|/g, "\\|").replace(/\r?\n/g, " ").slice(0, 300);
+}

package/dist/run/results.d.ts

@@ -0,0 +1,70 @@
+import type { Entry } from "../test/index.js";
+import type { Fingerprint } from "./fingerprint.js";
+import type { Requirement } from "./spec.js";
+export type RequirementOutcome = "pass" | "fail" | "not-covered" | "not-reached" | "not-checked";
+export interface RequirementResult {
+    id: string;
+    text: string;
+    result: RequirementOutcome;
+    did?: string;
+    expected?: string;
+    actual?: string;
+    evidence?: unknown;
+    why?: string;
+    fakes: string[];
+}
+export interface TestResult {
+    title: string;
+    status: string;
+    durationMs: number;
+    error?: string;
+    trace?: string;
+}
+export interface FlowResult {
+    name: string;
+    approved: boolean;
+    fingerprint: Fingerprint;
+    status: "passed" | "failed";
+    requirements: RequirementResult[];
+    tests: TestResult[];
+    shots: Array<{
+        test: string;
+        file: string;
+        label: string;
+    }>;
+    /** Claims citing a requirement number the spec does not have. */
+    unknownClaims: string[];
+}
+export interface OutsideCall {
+    host: string;
+    handledAs: string;
+    count: number;
+}
+export interface RunReport {
+    id: string;
+    kind: "run" | "dev";
+    startedAt: string;
+    finishedAt: string;
+    source: {
+        commit?: string;
+        dirty?: boolean;
+        pr?: number;
+    };
+    flows: FlowResult[];
+    outside: OutsideCall[];
+    /** Names the app looked up but never sent an HTTP request to, such as an SMTP host. */
+    lookedUpOnly: string[];
+    notes: string[];
+    environmentError?: string;
+}
+export interface PlannedFlow {
+    name: string;
+    approved: boolean;
+    fingerprint: Fingerprint;
+    requirements: Requirement[];
+}
+/** Reads what the runner and the fake-service container wrote to the run folder. */
+export declare function collectResults(outDir: string, planned: PlannedFlow[]): Promise<Pick<RunReport, "flows" | "outside" | "lookedUpOnly" | "environmentError">>;
+export declare function flowResult(plan: PlannedFlow, entries: Entry[], tests: TestResult[]): FlowResult;
+/** The per-requirement results that must be identical across three runs before a flow can be approved. */
+export declare function outcome(flow: FlowResult): string;