@davidrydberg/proofy 0.0.0-stage → 0.1.0

package/dist/style.js

@@ -0,0 +1,38 @@
+// Kept ASCII in source; these render as check mark, cross and bullet.
+export const GLYPH = { ok: "\u2713", fail: "\u2717", warn: "!", dot: "\u2022" };
+export function resolveStyle(env, isTTY, columns) {
+    let color = isTTY;
+    if (env.FORCE_COLOR !== undefined) {
+        color = env.FORCE_COLOR !== "0" && env.FORCE_COLOR !== "false";
+    }
+    if (env.NO_COLOR) {
+        color = false;
+    }
+    return { color, width: Math.max(60, columns || 100) };
+}
+export function painter(style) {
+    const wrap = (open, close) => (text) => style.color && text ? `\u001b[${open}m${text}\u001b[${close}m` : text;
+    return {
+        bold: wrap(1, 22),
+        dim: wrap(2, 22),
+        red: wrap(31, 39),
+        green: wrap(32, 39),
+        yellow: wrap(33, 39),
+        cyan: wrap(36, 39),
+    };
+}
+export function truncate(text, max) {
+    if (max <= 3)
+        return text.slice(0, Math.max(0, max));
+    return text.length > max ? `${text.slice(0, max - 3)}...` : text;
+}
+// Pads by visible length so colored text still lines up.
+export function padVisible(text, width, align = "left") {
+    const visible = stripAnsi(text).length;
+    const pad = " ".repeat(Math.max(0, width - visible));
+    return align === "left" ? text + pad : pad + text;
+}
+const ANSI = new RegExp(`${String.fromCharCode(27)}\\[\\d+m`, "g");
+export function stripAnsi(text) {
+    return text.replace(ANSI, "");
+}

package/dist/test/global-setup.d.ts

@@ -0,0 +1 @@
+export default function globalSetup(): Promise<void>;

package/dist/test/global-setup.js

@@ -0,0 +1,20 @@
+// Runs once in the runner container before any test: waits until the app answers, so a slow boot is reported
+// as an environment problem instead of as failed requirements.
+export default async function globalSetup() {
+    const url = process.env.PROOFY_BASE_URL ?? "";
+    const deadline = Date.now() + Number(process.env.PROOFY_START_MS ?? 600_000);
+    let last = "no answer";
+    while (Date.now() < deadline) {
+        try {
+            const response = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(5000) });
+            if (response.status < 500)
+                return;
+            last = `HTTP ${response.status}`;
+        }
+        catch (error) {
+            last = error instanceof Error ? (error.cause instanceof Error ? error.cause.message : error.message) : String(error);
+        }
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+    }
+    throw new Error(`proofy: the app at ${url} did not answer in time (last: ${last})`);
+}

package/dist/test/index.d.ts

@@ -0,0 +1,102 @@
+export { expect } from "@playwright/test";
+/** One line of e2e/out/<run>/flows/<flow>/claims.jsonl, read by the host to build the report. */
+export type Entry = {
+    type: "claim";
+    test: string;
+    spec: string;
+    did: string;
+    expected: string;
+    actual: string;
+    ok: boolean;
+    evidence?: unknown;
+    fakes: string[];
+} | {
+    type: "not-covered";
+    test: string;
+    spec: string;
+    why: string;
+} | {
+    type: "shot";
+    test: string;
+    file: string;
+    label: string;
+};
+export interface Claim {
+    /** The requirement number from the spec, such as "R2". */
+    spec: string;
+    /** What the test did to check it. */
+    did: string;
+    /** What should be true, in words. */
+    expected: string;
+    /** What was actually found, as read from the app, a fake or Mailpit. */
+    actual: unknown;
+    /** Whether the requirement held. Decide it from real values, never from text on the screen alone. */
+    ok: boolean;
+    /** Raw data behind the claim: a stored record, a request body. Shown in the report. */
+    evidence?: unknown;
+    /** Fakes this claim's evidence depends on. Hosts read with standin.requests() are added automatically. */
+    fakes?: string[];
+}
+export interface Mail {
+    id: string;
+    subject: string;
+    from: string;
+    to: string[];
+    cc: string[];
+    bcc: string[];
+    text: string;
+    html: string;
+}
+export interface RecordedRequest {
+    host: string;
+    method: string;
+    path: string;
+    query: Record<string, string>;
+    headers: Record<string, string>;
+    body: string;
+    handledAs: string;
+    status: number;
+}
+export declare const test: import("@playwright/test").TestType<import("@playwright/test").PlaywrightTestArgs & import("@playwright/test").PlaywrightTestOptions & {
+    proofy: void;
+}, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions>;
+/** Records one requirement's result. A failed claim fails the test at its end, after every claim is recorded. */
+export declare function claim(input: Claim): void;
+/** Marks a requirement the sandbox cannot show, with the reason. It appears in the report, never as a pass. */
+export declare function notCovered(spec: string, why: string): void;
+/** A numbered full-page screenshot for the report: shot('basket with one coffee'). */
+export declare function shot(label: string): Promise<void>;
+/** A value unique to this test, the same on every call: use it in names, emails and phone numbers. */
+export declare function tag(): string;
+/** Runs a command named in e2e/proofy.json exec, in its service, with extra arguments. Returns stdout. */
+export declare function exec(name: string, args?: string[]): Promise<string>;
+export interface MailOptions {
+    /** Wait until at least this many messages arrived. Default 1; 0 reads without waiting. */
+    atLeast?: number;
+    /** Give up waiting after this long. Default 15 seconds. */
+    timeoutMs?: number;
+    /** After that, keep reading until nothing new arrived for this long, to catch a late extra message. Default 0. */
+    settleMs?: number;
+}
+export declare const mail: {
+    /**
+     * Mail delivered to one address, oldest first. Waits until at least `atLeast` messages arrived or the time ran out.
+     * Use atLeast: 0 to read without waiting, for example to check that nothing was sent.
+     */
+    for(address: string, options?: MailOptions): Promise<Mail[]>;
+    /**
+     * Mail matching a Mailpit search, oldest first, such as an order number or `subject:"Your order"`.
+     * Every test shares the mailbox, so search for something only this test's data contains.
+     */
+    search(query: string, options?: MailOptions): Promise<Mail[]>;
+};
+export declare const standin: {
+    /**
+     * Requests the app sent to an outside host, oldest first, waiting until at least `atLeast` arrived.
+     * Reading a host marks the test's later claims as depending on its fake.
+     */
+    requests(host: string, options?: {
+        atLeast?: number;
+        timeoutMs?: number;
+    }): Promise<RecordedRequest[]>;
+};

package/dist/test/index.js

@@ -0,0 +1,184 @@
+import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { basename, join } from "node:path";
+import { test as base } from "@playwright/test";
+import { MAILPIT_API, MOUNT, STANDIN_ADMIN } from "../sandbox/contract.js";
+export { expect } from "@playwright/test";
+let current;
+export const test = base.extend({
+    proofy: [
+        async ({ page }, use, testInfo) => {
+            current = {
+                flow: basename(testInfo.file).replace(/\.spec\.[cm]?[jt]s$/, ""),
+                title: testInfo.title,
+                page,
+                tag: Math.random().toString(36).slice(2, 8),
+                shots: 0,
+                failed: [],
+                fakes: new Set(),
+            };
+            const mine = current;
+            await use();
+            current = undefined;
+            if (mine.failed.length > 0) {
+                throw new Error(`Requirements not met: ${mine.failed.join(", ")}. See the report for expected and actual values`);
+            }
+        },
+        { auto: true },
+    ],
+});
+/** Records one requirement's result. A failed claim fails the test at its end, after every claim is recorded. */
+export function claim(input) {
+    const state = active("claim");
+    const fakes = [...new Set([...(input.fakes ?? []), ...state.fakes])].sort();
+    write(state, {
+        type: "claim",
+        test: state.title,
+        spec: input.spec,
+        did: input.did,
+        expected: input.expected,
+        actual: typeof input.actual === "string" ? input.actual : JSON.stringify(input.actual),
+        ok: input.ok,
+        ...(input.evidence === undefined ? {} : { evidence: input.evidence }),
+        fakes,
+    });
+    if (!input.ok)
+        state.failed.push(input.spec);
+}
+/** Marks a requirement the sandbox cannot show, with the reason. It appears in the report, never as a pass. */
+export function notCovered(spec, why) {
+    const state = active("notCovered");
+    write(state, { type: "not-covered", test: state.title, spec, why });
+}
+/** A numbered full-page screenshot for the report: shot('basket with one coffee'). */
+export async function shot(label) {
+    const state = active("shot");
+    state.shots++;
+    const file = `${slug(state.title)}-${String(state.shots).padStart(2, "0")}-${slug(label)}.png`;
+    await state.page.screenshot({ path: join(flowDir(state), file), fullPage: true });
+    write(state, { type: "shot", test: state.title, file, label });
+}
+/** A value unique to this test, the same on every call: use it in names, emails and phone numbers. */
+export function tag() {
+    return active("tag").tag;
+}
+/** Runs a command named in e2e/proofy.json exec, in its service, with extra arguments. Returns stdout. */
+export async function exec(name, args = []) {
+    active("exec");
+    const id = `${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    mkdirSync(MOUNT.exec, { recursive: true });
+    writeFileSync(join(MOUNT.exec, `${id}.req.json`), JSON.stringify({ name, args }));
+    const response = join(MOUNT.exec, `${id}.res.json`);
+    const deadline = Date.now() + 120_000;
+    while (!existsSync(response)) {
+        if (Date.now() > deadline)
+            throw new Error(`exec("${name}") got no answer within 2 minutes`);
+        await sleep(100);
+    }
+    const result = JSON.parse(readFileSync(response, "utf8"));
+    rmSync(response, { force: true });
+    if (result.error)
+        throw new Error(result.error);
+    if (result.code !== 0)
+        throw new Error(`exec("${name}") exited with ${result.code}: ${result.stderr.trim() || result.stdout.trim()}`);
+    return result.stdout;
+}
+export const mail = {
+    /**
+     * Mail delivered to one address, oldest first. Waits until at least `atLeast` messages arrived or the time ran out.
+     * Use atLeast: 0 to read without waiting, for example to check that nothing was sent.
+     */
+    for(address, options = {}) {
+        active("mail.for");
+        return searchMail(`to:"${address}"`, options);
+    },
+    /**
+     * Mail matching a Mailpit search, oldest first, such as an order number or `subject:"Your order"`.
+     * Every test shares the mailbox, so search for something only this test's data contains.
+     */
+    search(query, options = {}) {
+        active("mail.search");
+        return searchMail(query, options);
+    },
+};
+async function searchMail(query, options) {
+    const atLeast = options.atLeast ?? 1;
+    const deadline = Date.now() + (options.timeoutMs ?? 15_000);
+    const ids = async () => {
+        const search = (await getJson(`${MAILPIT_API}/api/v1/search?query=${encodeURIComponent(query)}`));
+        return (search.messages ?? []).map((message) => message.ID);
+    };
+    let found = await ids();
+    while (found.length < atLeast && Date.now() < deadline) {
+        await sleep(500);
+        found = await ids();
+    }
+    if (options.settleMs) {
+        let stableSince = Date.now();
+        while (Date.now() - stableSince < options.settleMs) {
+            await sleep(Math.min(500, options.settleMs));
+            const again = await ids();
+            if (again.length !== found.length) {
+                found = again;
+                stableSince = Date.now();
+            }
+        }
+    }
+    const messages = await Promise.all(found.map((id) => getJson(`${MAILPIT_API}/api/v1/message/${id}`)));
+    return messages.map(toMail).reverse();
+}
+export const standin = {
+    /**
+     * Requests the app sent to an outside host, oldest first, waiting until at least `atLeast` arrived.
+     * Reading a host marks the test's later claims as depending on its fake.
+     */
+    async requests(host, options = {}) {
+        const state = active("standin.requests");
+        state.fakes.add(host);
+        const atLeast = options.atLeast ?? 0;
+        const deadline = Date.now() + (options.timeoutMs ?? 15_000);
+        for (;;) {
+            const requests = (await getJson(`${STANDIN_ADMIN}/requests?host=${encodeURIComponent(host)}`));
+            if (requests.length >= atLeast || Date.now() > deadline)
+                return requests;
+            await sleep(250);
+        }
+    },
+};
+function active(name) {
+    if (!current)
+        throw new Error(`${name}() must be called inside a test that imports test from "@davidrydberg/proofy/test"`);
+    return current;
+}
+function flowDir(state) {
+    const dir = join(MOUNT.out, "flows", state.flow);
+    mkdirSync(dir, { recursive: true });
+    return dir;
+}
+function write(state, entry) {
+    appendFileSync(join(flowDir(state), "claims.jsonl"), `${JSON.stringify(entry)}\n`);
+}
+async function getJson(url) {
+    const response = await fetch(url);
+    if (!response.ok)
+        throw new Error(`${url} answered ${response.status}`);
+    return response.json();
+}
+function toMail(raw) {
+    const message = raw;
+    return {
+        id: message.ID,
+        subject: message.Subject,
+        from: message.From?.Address ?? "",
+        to: (message.To ?? []).map((to) => to.Address),
+        cc: (message.Cc ?? []).map((cc) => cc.Address),
+        bcc: (message.Bcc ?? []).map((bcc) => bcc.Address),
+        text: message.Text,
+        html: message.HTML,
+    };
+}
+function slug(text) {
+    return text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 50) || "step";
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/docker/standin.Dockerfile

@@ -0,0 +1,3 @@
+# The fake-service container: Node for the server, openssl for the sandbox certificate authority.
+FROM node:24-alpine
+RUN apk add --no-cache openssl ca-certificates

package/package.json

@@ -1,6 +1,70 @@
 {
   "name": "@davidrydberg/proofy",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Sealed end-to-end tests that your coding agent writes and you approve: requirements in plain words, evidence and screenshots per requirement.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/davidrydberg/proofy.git"
+  },
+  "homepage": "https://github.com/davidrydberg/proofy#readme",
+  "bugs": {
+    "url": "https://github.com/davidrydberg/proofy/issues"
+  },
+  "author": "David Rydberg <david@davidrydberg.dev>",
+  "type": "module",
+  "bin": {
+    "proofy": "dist/bin/proofy.js"
+  },
+  "exports": {
+    "./test": {
+      "types": "./dist/test/index.d.ts",
+      "default": "./dist/test/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "docker",
+    "schema",
+    "templates"
+  ],
+  "engines": {
+    "node": ">=24"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "cli",
+    "agents",
+    "e2e",
+    "end-to-end",
+    "playwright",
+    "docker",
+    "testing"
+  ],
+  "scripts": {
+    "build": "rm -rf dist schema && tsc -p tsconfig.build.json && chmod +x dist/bin/proofy.js && node scripts/schema.mjs",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint .",
+    "test": "vitest run",
+    "prepublishOnly": "npm run typecheck && npm run lint && npm test && npm run build",
+    "test:docker": "npm run build && vitest run --config vitest.docker.config.ts"
+  },
+  "dependencies": {
+    "@playwright/test": "1.64.1",
+    "@toon-format/toon": "^4.4.0",
+    "yaml": "^2.9.1",
+    "zod": "^4.6.7"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@types/node": "^26.6.5",
+    "eslint": "^10.13.0",
+    "globals": "^17.13.0",
+    "typescript": "^6.0.3",
+    "typescript-eslint": "^8.71.2",
+    "vitest": "^5.0.5"
+  }
+}

package/schema/proofy.schema.json

@@ -0,0 +1,179 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "$schema": {
+      "type": "string"
+    },
+    "compose": {
+      "default": [],
+      "description": "The app's compose files, relative to the repository root",
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1
+      }
+    },
+    "app": {
+      "description": "Where the browser starts",
+      "type": "object",
+      "properties": {
+        "service": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Compose service the browser opens"
+        },
+        "port": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 65535,
+          "description": "Port inside the container, not a published port"
+        },
+        "path": {
+          "default": "/",
+          "type": "string",
+          "format": "starts_with",
+          "pattern": "^\\/.*"
+        }
+      },
+      "required": [
+        "service",
+        "port"
+      ],
+      "additionalProperties": false
+    },
+    "seed": {
+      "description": "Runs once on the empty database before the tests",
+      "type": "object",
+      "properties": {
+        "service": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Compose service to run it in"
+        },
+        "command": {
+          "minItems": 1,
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
+          "description": "Command and arguments, run without a shell"
+        }
+      },
+      "required": [
+        "service",
+        "command"
+      ],
+      "additionalProperties": false
+    },
+    "exec": {
+      "default": {},
+      "description": "Commands tests may run with exec(name), such as a scheduled job",
+      "type": "object",
+      "propertyNames": {
+        "type": "string",
+        "minLength": 1
+      },
+      "additionalProperties": {
+        "type": "object",
+        "properties": {
+          "service": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Compose service to run it in"
+          },
+          "command": {
+            "minItems": 1,
+            "type": "array",
+            "items": {
+              "type": "string",
+              "minLength": 1
+            },
+            "description": "Command and arguments, run without a shell"
+          }
+        },
+        "required": [
+          "service",
+          "command"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "hosts": {
+      "default": {},
+      "description": "Outside hosts: fake answers from e2e/standins/<host>.mjs, record accepts everything. Unlisted hosts are refused",
+      "type": "object",
+      "propertyNames": {
+        "type": "string",
+        "pattern": "^(?=.{1,253}$)([a-z0-9]([a-z0-9-]*[a-z0-9])?\\.)+[a-z]{2,63}$"
+      },
+      "additionalProperties": {
+        "type": "string",
+        "enum": [
+          "fake",
+          "record"
+        ]
+      }
+    },
+    "assets": {
+      "default": [],
+      "description": "Static third-party files downloaded at install and served in the sealed run",
+      "type": "array",
+      "items": {
+        "type": "string",
+        "format": "uri"
+      }
+    },
+    "env": {
+      "default": {},
+      "description": "Settings per compose service. The repository's .env is never read",
+      "type": "object",
+      "propertyNames": {
+        "type": "string",
+        "minLength": 1
+      },
+      "additionalProperties": {
+        "type": "object",
+        "propertyNames": {
+          "type": "string",
+          "minLength": 1
+        },
+        "additionalProperties": {
+          "type": "string"
+        }
+      }
+    },
+    "limits": {
+      "default": {},
+      "type": "object",
+      "properties": {
+        "startMinutes": {
+          "default": 10,
+          "type": "number",
+          "exclusiveMinimum": 0
+        },
+        "testMinutes": {
+          "default": 5,
+          "type": "number",
+          "exclusiveMinimum": 0
+        },
+        "maxRunMinutes": {
+          "type": "number",
+          "exclusiveMinimum": 0
+        },
+        "parallel": {
+          "default": 4,
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 9007199254740991
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false,
+  "description": "proofy configuration, e2e/proofy.json",
+  "$id": "https://unpkg.com/@davidrydberg/proofy/schema/proofy.schema.json",
+  "title": "proofy config"
+}