@davidrydberg/proofy 0.0.0-stage → 0.1.0

package/dist/run/results.js

@@ -0,0 +1,129 @@
+import { readFile } from "node:fs/promises";
+import { basename, join } from "node:path";
+/** Reads what the runner and the fake-service container wrote to the run folder. */
+export async function collectResults(outDir, planned) {
+    const pw = (await readJson(join(outDir, "playwright.json")));
+    const tests = new Map();
+    for (const spec of specsOf(pw?.suites ?? [])) {
+        const flow = basename(spec.file).replace(/\.spec\.[cm]?[jt]s$/, "");
+        for (const test of spec.tests) {
+            const result = test.results.at(-1);
+            if (!result)
+                continue;
+            const messages = [...new Set([result.error?.message, ...(result.errors ?? []).map((error) => error.message)].filter((message) => Boolean(message)))];
+            const error = messages.join("\n\n");
+            const trace = result.attachments?.find((attachment) => attachment.name === "trace")?.path;
+            const list = tests.get(flow) ?? [];
+            list.push({
+                title: spec.title,
+                status: result.status,
+                durationMs: result.duration,
+                ...(error ? { error: stripAnsi(error).slice(0, 2000) } : {}),
+                ...(trace ? { trace: trace.replace(/^\/out\//, "") } : {}),
+            });
+            tests.set(flow, list);
+        }
+    }
+    const flows = [];
+    for (const plan of planned) {
+        const entries = await readLines(join(outDir, "flows", plan.name, "claims.jsonl"));
+        const flowTests = tests.get(plan.name) ?? [];
+        flows.push(flowResult(plan, entries, flowTests));
+    }
+    const requests = await readLines(join(outDir, "requests.jsonl"));
+    const counts = new Map();
+    for (const request of requests) {
+        const key = `${request.host} ${request.handledAs}`;
+        const entry = counts.get(key) ?? { host: request.host, handledAs: request.handledAs, count: 0 };
+        entry.count++;
+        counts.set(key, entry);
+    }
+    const contacted = new Set(requests.map((request) => request.host));
+    const lookups = await readLines(join(outDir, "dns.jsonl"));
+    const environmentError = pw?.errors?.map((error) => stripAnsi(error.message ?? "")).find(Boolean);
+    return {
+        flows,
+        outside: [...counts.values()].sort((a, b) => a.host.localeCompare(b.host) || a.handledAs.localeCompare(b.handledAs)),
+        lookedUpOnly: [...new Set(lookups.map((lookup) => lookup.name))].filter((name) => !contacted.has(name)).sort(),
+        ...(environmentError ? { environmentError: environmentError.slice(0, 2000) } : {}),
+    };
+}
+export function flowResult(plan, entries, tests) {
+    const known = new Set(plan.requirements.map((requirement) => requirement.id));
+    const anyTestBroke = tests.length === 0 || tests.some((test) => test.status !== "passed" && !/Requirements not met/.test(test.error ?? ""));
+    const requirements = plan.requirements.map((requirement) => {
+        const claims = entries.filter((entry) => entry.type === "claim" && entry.spec === requirement.id);
+        const failed = claims.find((entry) => !entry.ok);
+        const shown = failed ?? claims[0];
+        if (shown) {
+            const fakes = [...new Set(claims.flatMap((entry) => entry.fakes))].sort();
+            return {
+                id: requirement.id,
+                text: requirement.text,
+                result: failed ? "fail" : "pass",
+                did: shown.did,
+                expected: shown.expected,
+                actual: shown.actual,
+                ...(shown.evidence === undefined ? {} : { evidence: shown.evidence }),
+                fakes,
+            };
+        }
+        const skipped = entries.find((entry) => entry.type === "not-covered" && entry.spec === requirement.id);
+        if (skipped)
+            return { id: requirement.id, text: requirement.text, result: "not-covered", why: skipped.why, fakes: [] };
+        return { id: requirement.id, text: requirement.text, result: anyTestBroke ? "not-reached" : "not-checked", fakes: [] };
+    });
+    const unknownClaims = [...new Set(entries.flatMap((entry) => (entry.type === "shot" || known.has(entry.spec) ? [] : [entry.spec])))];
+    return {
+        name: plan.name,
+        approved: plan.approved,
+        fingerprint: plan.fingerprint,
+        status: requirements.some((requirement) => requirement.result === "fail" || requirement.result === "not-reached") ? "failed" : "passed",
+        requirements,
+        tests,
+        shots: entries.filter((entry) => entry.type === "shot").map(({ test, file, label }) => ({ test, file, label })),
+        unknownClaims,
+    };
+}
+/** The per-requirement results that must be identical across three runs before a flow can be approved. */
+export function outcome(flow) {
+    return flow.requirements.map((requirement) => `${requirement.id}:${requirement.result}`).join(" ");
+}
+function* specsOf(suites) {
+    for (const suite of suites) {
+        yield* suite.specs ?? [];
+        yield* specsOf(suite.suites ?? []);
+    }
+}
+async function readJson(file) {
+    try {
+        return JSON.parse(await readFile(file, "utf8"));
+    }
+    catch {
+        return undefined;
+    }
+}
+async function readLines(file) {
+    let raw;
+    try {
+        raw = await readFile(file, "utf8");
+    }
+    catch {
+        return [];
+    }
+    return raw
+        .split("\n")
+        .filter(Boolean)
+        .flatMap((line) => {
+        try {
+            return [JSON.parse(line)];
+        }
+        catch {
+            return [];
+        }
+    });
+}
+const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "g");
+function stripAnsi(text) {
+    return text.replace(ANSI, "");
+}

package/dist/run/spec.d.ts

@@ -0,0 +1,6 @@
+export interface Requirement {
+    id: string;
+    text: string;
+}
+/** Numbered requirements from a spec file: lines starting with R1., R2: or R3). */
+export declare function parseRequirements(markdown: string): Requirement[];

package/dist/run/spec.js

@@ -0,0 +1,16 @@
+/** Numbered requirements from a spec file: lines starting with R1., R2: or R3). */
+export function parseRequirements(markdown) {
+    const out = [];
+    const seen = new Set();
+    for (const line of markdown.split(/\r?\n/)) {
+        const match = /^\s*(?:[-*]\s+)?\**(R\d+)\**[.:)]\s+(.+?)\s*$/.exec(line);
+        if (!match)
+            continue;
+        const [, id = "", text = ""] = match;
+        if (seen.has(id))
+            continue;
+        seen.add(id);
+        out.push({ id, text });
+    }
+    return out;
+}

package/dist/sandbox/compose.d.ts

@@ -0,0 +1,38 @@
+import type { Config } from "../config.js";
+export type Mode = "run" | "dev";
+export interface ComposeInputs {
+    /** The app's compose files, normalized by `docker compose config --format json`. */
+    model: Record<string, unknown>;
+    config: Config;
+    mode: Mode;
+    subnet: string;
+    standinIp: string;
+    standinImage: string;
+    runnerImage: string;
+    /** "uid:gid" of the person running proofy, so files written into mounted folders stay theirs on Linux. */
+    user: string;
+    dirs: {
+        pkg: string;
+        ca: string;
+        runSetup: string;
+        out: string;
+        e2e: string;
+        work: string;
+        nodeModules: string;
+    };
+}
+export interface Generated {
+    compose: Record<string, unknown>;
+    notes: string[];
+}
+/**
+ * Rewrites the app's own compose model into a sealed sandbox: one internal network with no way out, every
+ * hostname answered by the fake-service container, no published ports, settings only from the proofy config.
+ */
+export declare function sandboxCompose(inputs: ComposeInputs): Generated;
+export declare function baseUrl(config: Config): string;
+/** A /24 per project, away from the ranges Docker and home networks usually take. */
+export declare function subnetFor(seed: string, attempt: number): {
+    subnet: string;
+    standinIp: string;
+};

package/dist/sandbox/compose.js

@@ -0,0 +1,132 @@
+import { CliError } from "../errors.js";
+import { FORWARD, MAILPIT, MAILPIT_IMAGE, MOUNT, RUNNER, STANDIN } from "./contract.js";
+/**
+ * Rewrites the app's own compose model into a sealed sandbox: one internal network with no way out, every
+ * hostname answered by the fake-service container, no published ports, settings only from the proofy config.
+ */
+export function sandboxCompose(inputs) {
+    const { model, config, mode, dirs } = inputs;
+    const notes = [];
+    const userServices = (model.services ?? {});
+    const services = {};
+    for (const name of Object.keys(config.env)) {
+        if (!(name in userServices)) {
+            throw new CliError(`e2e/proofy.json env names service "${name}", which is not in the compose files`, "CONFIG_INVALID", [
+                `Services: ${Object.keys(userServices).join(", ") || "none"}`,
+            ]);
+        }
+    }
+    const named = [
+        ["app", config.app],
+        ["seed", config.seed],
+        ...Object.entries(config.exec).map(([key, value]) => [`exec.${key}`, value]),
+    ];
+    for (const [label, entry] of named) {
+        if (entry && !(entry.service in userServices)) {
+            throw new CliError(`e2e/proofy.json ${label} names service "${entry.service}", which is not in the compose files`, "CONFIG_INVALID", [
+                `Services: ${Object.keys(userServices).join(", ") || "none"}`,
+            ]);
+        }
+    }
+    for (const [name, original] of Object.entries(userServices)) {
+        if (name.startsWith("proofy-") || name === MAILPIT) {
+            throw new CliError(`The compose service name "${name}" is reserved by proofy`, "CONFIG_INVALID", ["Rename the service in your compose file"]);
+        }
+        const service = { ...original };
+        if (service.network_mode !== undefined) {
+            throw new CliError(`Service ${name} sets network_mode, which would bypass the sealed network`, "CONFIG_INVALID", [
+                "Remove network_mode from the compose file proofy uses",
+            ]);
+        }
+        if (service.env_file !== undefined) {
+            notes.push(`${name}: env_file ignored, settings come only from e2e/proofy.json env`);
+            delete service.env_file;
+        }
+        delete service.ports;
+        delete service.container_name;
+        service.networks = { proofy: { aliases: aliasesOf(original.networks) } };
+        service.dns = [inputs.standinIp];
+        const environment = { ...(original.environment ?? {}), ...(config.env[name] ?? {}) };
+        service.environment = environment;
+        service.volumes = [...(original.volumes ?? []), { type: "bind", source: dirs.ca, target: MOUNT.appCa, read_only: true }];
+        service.depends_on = { ...(original.depends_on ?? {}), [STANDIN]: { condition: "service_healthy", required: true } };
+        services[name] = service;
+    }
+    services[STANDIN] = {
+        image: inputs.standinImage,
+        user: inputs.user,
+        command: ["node", `${MOUNT.pkg}/dist/standin/server.js`],
+        volumes: [
+            bind(dirs.pkg, MOUNT.pkg, true),
+            bind(dirs.ca, MOUNT.ca, false),
+            bind(dirs.runSetup, MOUNT.runSetup, true),
+            bind(dirs.out, MOUNT.out, false),
+            bind(dirs.e2e, MOUNT.e2e, true),
+        ],
+        networks: { proofy: { ipv4_address: inputs.standinIp } },
+        healthcheck: { test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8099/health"], interval: "1s", timeout: "2s", retries: 60 },
+    };
+    services[MAILPIT] = { image: MAILPIT_IMAGE, networks: { proofy: {} }, environment: { MP_SMTP_AUTH_ACCEPT_ANY: "1", MP_SMTP_AUTH_ALLOW_INSECURE: "1" } };
+    services[RUNNER] = {
+        image: inputs.runnerImage,
+        user: inputs.user,
+        profiles: [RUNNER],
+        working_dir: MOUNT.work,
+        shm_size: "1gb",
+        dns: [inputs.standinIp],
+        networks: { proofy: {} },
+        environment: { CI: "1", HOME: "/tmp", PROOFY_BASE_URL: baseUrl(config), PROOFY_START_MS: String(config.limits.startMinutes * 60_000) },
+        volumes: [
+            bind(dirs.work, MOUNT.work, false),
+            bind(dirs.pkg, MOUNT.pkg, true),
+            bind(dirs.nodeModules, "/node_modules", true),
+            bind(dirs.e2e, MOUNT.e2e, true),
+            bind(dirs.out, MOUNT.out, false),
+        ],
+    };
+    const networks = { proofy: { internal: true, ipam: { config: [{ subnet: inputs.subnet }] } } };
+    if (mode === "dev" && config.app) {
+        // The only container with a second network: it forwards host ports in and gives nothing a way out.
+        services[FORWARD] = {
+            image: inputs.standinImage,
+            user: inputs.user,
+            command: ["node", `${MOUNT.pkg}/dist/standin/forward.js`, `8080=${config.app.service}:${config.app.port}`, `8025=${MAILPIT}:8025`],
+            volumes: [bind(dirs.pkg, MOUNT.pkg, true)],
+            networks: { proofy: {}, "proofy-host": {} },
+            ports: ["127.0.0.1::8080", "127.0.0.1::8025"],
+        };
+        networks["proofy-host"] = {};
+    }
+    const compose = { services, networks };
+    if (model.volumes)
+        compose.volumes = model.volumes;
+    return { compose, notes };
+}
+export function baseUrl(config) {
+    return config.app ? `http://${config.app.service}:${config.app.port}${config.app.path}` : "";
+}
+/** A /24 per project, away from the ranges Docker and home networks usually take. */
+export function subnetFor(seed, attempt) {
+    let hash = 0;
+    for (const char of `${seed}:${attempt}`)
+        hash = (hash * 31 + char.charCodeAt(0)) >>> 0;
+    const second = 200 + (hash % 50);
+    const third = (hash >>> 8) % 256;
+    return { subnet: `10.${second}.${third}.0/24`, standinIp: `10.${second}.${third}.53` };
+}
+function aliasesOf(networks) {
+    if (!networks || typeof networks !== "object" || Array.isArray(networks))
+        return [];
+    const aliases = new Set();
+    for (const value of Object.values(networks)) {
+        if (value && typeof value === "object" && Array.isArray(value.aliases)) {
+            for (const alias of value.aliases)
+                if (typeof alias === "string")
+                    aliases.add(alias);
+        }
+    }
+    return [...aliases];
+}
+function bind(source, target, readOnly) {
+    return { type: "bind", source, target, read_only: readOnly };
+}

package/dist/sandbox/contract.d.ts

@@ -0,0 +1,19 @@
+export declare const STANDIN = "proofy-standin";
+export declare const MAILPIT = "mailpit";
+export declare const RUNNER = "proofy-runner";
+export declare const FORWARD = "proofy-forward";
+export declare const ADMIN_PORT = 8099;
+export declare const MAILPIT_API = "http://mailpit:8025";
+export declare const STANDIN_ADMIN = "http://proofy-standin:8099";
+/** Mount points inside containers. */
+export declare const MOUNT: {
+    readonly ca: "/proofy-ca";
+    readonly appCa: "/proofy";
+    readonly runSetup: "/proofy-run";
+    readonly out: "/out";
+    readonly e2e: "/e2e";
+    readonly work: "/work";
+    readonly pkg: "/proofy-pkg";
+    readonly exec: "/work/exec";
+};
+export declare const MAILPIT_IMAGE = "axllent/mailpit:v1.27.11";

package/dist/sandbox/contract.js

@@ -0,0 +1,20 @@
+// Names and paths shared by the host side and the containers. Changing one changes both.
+export const STANDIN = "proofy-standin";
+export const MAILPIT = "mailpit";
+export const RUNNER = "proofy-runner";
+export const FORWARD = "proofy-forward";
+export const ADMIN_PORT = 8099;
+export const MAILPIT_API = `http://${MAILPIT}:8025`;
+export const STANDIN_ADMIN = `http://${STANDIN}:${ADMIN_PORT}`;
+/** Mount points inside containers. */
+export const MOUNT = {
+    ca: "/proofy-ca",
+    appCa: "/proofy",
+    runSetup: "/proofy-run",
+    out: "/out",
+    e2e: "/e2e",
+    work: "/work",
+    pkg: "/proofy-pkg",
+    exec: "/work/exec",
+};
+export const MAILPIT_IMAGE = "axllent/mailpit:v1.27.11";

package/dist/sandbox/docker.d.ts

@@ -0,0 +1,17 @@
+export interface ProcResult {
+    code: number;
+    stdout: string;
+    stderr: string;
+    timedOut: boolean;
+}
+export interface ProcOptions {
+    cwd?: string;
+    timeoutMs?: number;
+    /** Streams output as it comes, for long steps such as image builds and test runs. */
+    onOutput?: (chunk: string) => void;
+}
+/** Runs a program without a shell. Injected so tests never start Docker. */
+export type Proc = (program: string, args: string[], options?: ProcOptions) => Promise<ProcResult>;
+export declare const spawnProc: Proc;
+/** The last lines of a command's output, for error messages that point at the cause. */
+export declare function tail(text: string, lines?: number): string;

package/dist/sandbox/docker.js

@@ -0,0 +1,48 @@
+import { spawn } from "node:child_process";
+import { CliError } from "../errors.js";
+export const spawnProc = (program, args, options = {}) => new Promise((resolve, reject) => {
+    const child = spawn(program, args, { cwd: options.cwd, stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    const timer = options.timeoutMs === undefined
+        ? undefined
+        : setTimeout(() => {
+            timedOut = true;
+            child.kill("SIGTERM");
+        }, options.timeoutMs);
+    child.stdout.on("data", (chunk) => {
+        stdout += chunk.toString();
+        options.onOutput?.(chunk.toString());
+    });
+    child.stderr.on("data", (chunk) => {
+        stderr += chunk.toString();
+        options.onOutput?.(chunk.toString());
+    });
+    child.on("error", (error) => {
+        if (timer)
+            clearTimeout(timer);
+        if (error.code === "ENOENT") {
+            reject(new CliError(`${program} is not installed or not on PATH`, "ENVIRONMENT_ERROR", [installHint(program)]));
+        }
+        else {
+            reject(error);
+        }
+    });
+    child.on("close", (code) => {
+        if (timer)
+            clearTimeout(timer);
+        resolve({ code: code ?? 1, stdout, stderr, timedOut });
+    });
+});
+function installHint(program) {
+    if (program === "docker")
+        return "Install Docker (Docker Desktop, OrbStack or Docker Engine) and start it";
+    if (program === "gh")
+        return "Install the GitHub CLI: https://cli.github.com, then run `gh auth login`";
+    return `Install ${program}`;
+}
+/** The last lines of a command's output, for error messages that point at the cause. */
+export function tail(text, lines = 15) {
+    return text.trim().split("\n").slice(-lines).join("\n");
+}

package/dist/sandbox/sandbox.d.ts

@@ -0,0 +1,46 @@
+import type { Config } from "../config.js";
+import { type Mode } from "./compose.js";
+import { type Proc } from "./docker.js";
+export declare const PKG_DIR: string;
+export interface SandboxOptions {
+    /** Where the app's compose files resolve: the repository, or an exported pull request. */
+    sourceDir: string;
+    /** The e2e folder the tests and fakes come from. */
+    e2eDir: string;
+    /** Where the sandbox's own logs land: the run folder, or e2e/out for a dev sandbox. */
+    logDir: string;
+    config: Config;
+    mode: Mode;
+    /** Unique per sandbox; also the compose project name. */
+    project: string;
+    /** Shared by every run of this repository: the certificate authority. */
+    cacheDir: string;
+    env: NodeJS.ProcessEnv;
+    log: (line: string) => void;
+}
+export interface Sandbox {
+    project: string;
+    composeFile: string;
+    workDir: string;
+    notes: string[];
+}
+export declare function cacheDirFor(env: NodeJS.ProcessEnv, root: string): string;
+export declare function playwrightVersion(): string;
+export declare function runnerImage(): string;
+export declare function standinImage(): string;
+/** Install phase, with internet: images and builds. Nothing of the app runs yet. */
+export declare function installImages(proc: Proc, options: SandboxOptions, sandbox: Sandbox): Promise<void>;
+/** Writes the sealed compose file and everything the containers mount. */
+export declare function prepare(proc: Proc, options: SandboxOptions, attempt?: number): Promise<Sandbox>;
+/** Sealed phase: starts the app with no way out, waits for it, seeds the empty database. */
+export declare function start(proc: Proc, options: SandboxOptions, sandbox: Sandbox): Promise<Sandbox>;
+/** Runs Playwright in the runner container, answering exec() requests from the tests meanwhile. */
+export declare function runTests(proc: Proc, options: SandboxOptions, sandbox: Sandbox, playwrightArgs: string[]): Promise<number>;
+export declare function saveLogs(proc: Proc, sandbox: Sandbox, dir: string): Promise<void>;
+export declare function stop(proc: Proc, sandbox: Sandbox): Promise<void>;
+export declare function forwardedPorts(proc: Proc, sandbox: Sandbox): Promise<{
+    app?: string;
+    mail?: string;
+}>;
+export declare function hostUser(): string;
+export declare function compose(sandbox: Pick<Sandbox, "project" | "composeFile">): string[];