@davidrydberg/proofy 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +64 -2
  3. package/dist/args.d.ts +10 -0
  4. package/dist/args.js +36 -0
  5. package/dist/bin/proofy.d.ts +2 -0
  6. package/dist/bin/proofy.js +4 -0
  7. package/dist/cli.d.ts +9 -0
  8. package/dist/cli.js +140 -0
  9. package/dist/commands/approve.d.ts +4 -0
  10. package/dist/commands/approve.js +48 -0
  11. package/dist/commands/context.d.ts +16 -0
  12. package/dist/commands/context.js +27 -0
  13. package/dist/commands/dev.d.ts +4 -0
  14. package/dist/commands/dev.js +161 -0
  15. package/dist/commands/home.d.ts +3 -0
  16. package/dist/commands/home.js +122 -0
  17. package/dist/commands/init.d.ts +4 -0
  18. package/dist/commands/init.js +122 -0
  19. package/dist/commands/report.d.ts +4 -0
  20. package/dist/commands/report.js +26 -0
  21. package/dist/commands/run.d.ts +6 -0
  22. package/dist/commands/run.js +166 -0
  23. package/dist/commands/types.d.ts +8 -0
  24. package/dist/commands/types.js +1 -0
  25. package/dist/config.d.ts +34 -0
  26. package/dist/config.js +62 -0
  27. package/dist/deps.d.ts +10 -0
  28. package/dist/deps.js +1 -0
  29. package/dist/detect.d.ts +17 -0
  30. package/dist/detect.js +96 -0
  31. package/dist/errors.d.ts +6 -0
  32. package/dist/errors.js +21 -0
  33. package/dist/github.d.ts +16 -0
  34. package/dist/github.js +121 -0
  35. package/dist/human.d.ts +30 -0
  36. package/dist/human.js +160 -0
  37. package/dist/output.d.ts +8 -0
  38. package/dist/output.js +21 -0
  39. package/dist/project.d.ts +21 -0
  40. package/dist/project.js +71 -0
  41. package/dist/run/approvals.d.ts +15 -0
  42. package/dist/run/approvals.js +26 -0
  43. package/dist/run/engine.d.ts +48 -0
  44. package/dist/run/engine.js +168 -0
  45. package/dist/run/fingerprint.d.ts +11 -0
  46. package/dist/run/fingerprint.js +40 -0
  47. package/dist/run/history.d.ts +29 -0
  48. package/dist/run/history.js +85 -0
  49. package/dist/run/report.d.ts +14 -0
  50. package/dist/run/report.js +115 -0
  51. package/dist/run/results.d.ts +70 -0
  52. package/dist/run/results.js +129 -0
  53. package/dist/run/spec.d.ts +6 -0
  54. package/dist/run/spec.js +16 -0
  55. package/dist/sandbox/compose.d.ts +38 -0
  56. package/dist/sandbox/compose.js +132 -0
  57. package/dist/sandbox/contract.d.ts +19 -0
  58. package/dist/sandbox/contract.js +20 -0
  59. package/dist/sandbox/docker.d.ts +17 -0
  60. package/dist/sandbox/docker.js +48 -0
  61. package/dist/sandbox/sandbox.d.ts +46 -0
  62. package/dist/sandbox/sandbox.js +239 -0
  63. package/dist/standin/ca.d.ts +6 -0
  64. package/dist/standin/ca.js +40 -0
  65. package/dist/standin/dns.d.ts +10 -0
  66. package/dist/standin/dns.js +53 -0
  67. package/dist/standin/forward.d.ts +1 -0
  68. package/dist/standin/forward.js +17 -0
  69. package/dist/standin/handle.d.ts +49 -0
  70. package/dist/standin/handle.js +130 -0
  71. package/dist/standin/server.d.ts +1 -0
  72. package/dist/standin/server.js +88 -0
  73. package/dist/style.d.ts +23 -0
  74. package/dist/style.js +38 -0
  75. package/dist/test/global-setup.d.ts +1 -0
  76. package/dist/test/global-setup.js +20 -0
  77. package/dist/test/index.d.ts +102 -0
  78. package/dist/test/index.js +184 -0
  79. package/docker/standin.Dockerfile +3 -0
  80. package/package.json +68 -4
  81. package/schema/proofy.schema.json +179 -0
  82. package/templates/e2e/AGENTS.md +90 -0
  83. package/templates/e2e/templates/compose.yml +33 -0
  84. package/templates/e2e/templates/spec.md +20 -0
  85. package/templates/e2e/templates/standin.mjs +29 -0
  86. package/templates/e2e/templates/test.spec.ts +28 -0
@@ -0,0 +1,122 @@
1
+ import { basename, join } from "node:path";
2
+ import { loadConfig } from "../config.js";
3
+ import { CliError } from "../errors.js";
4
+ import { renderHome } from "../human.js";
5
+ import { exists, findRoot, flowNames, orphanTests, pathsFor } from "../project.js";
6
+ import { readApprovals } from "../run/approvals.js";
7
+ import { flowStatuses, recentReports } from "../run/history.js";
8
+ import { verdict } from "../run/report.js";
9
+ const SETUP_ASK = 'Ask your agent: "set up proofy". It follows the Setup section of e2e/AGENTS.md';
10
+ // Runs from session-start hooks: reads files only, never starts Docker, never exits non-zero.
11
+ export async function homeCommand(deps) {
12
+ const root = await findRoot(deps.cwd);
13
+ const paths = pathsFor(root);
14
+ const project = basename(root);
15
+ if (!(await exists(paths.config))) {
16
+ return `proofy: not set up in ${project}. Run \`proofy init\` to add end-to-end tests your agent writes and you approve`;
17
+ }
18
+ let config;
19
+ try {
20
+ config = await loadConfig(paths.config, "e2e/proofy.json");
21
+ }
22
+ catch (error) {
23
+ if (!(error instanceof CliError))
24
+ throw error;
25
+ return { data: { project, config: `invalid: ${error.message}` }, help: error.suggestions, pretty: () => `proofy: ${project}: ${error.message}` };
26
+ }
27
+ const checks = await setupChecks(config, root);
28
+ const reports = await recentReports(paths);
29
+ let flows;
30
+ try {
31
+ flows = await flowStatuses(paths, config, await flowNames(paths), await readApprovals(paths), reports);
32
+ }
33
+ catch (error) {
34
+ if (!(error instanceof CliError))
35
+ throw error;
36
+ return { data: { project, approvals: `invalid: ${error.message}` }, help: error.suggestions };
37
+ }
38
+ const orphans = await orphanTests(paths);
39
+ const steps = nextSteps(checks, flows, orphans);
40
+ const last = reports[0];
41
+ const lastRun = last ? `${last.id} ${last.kind}: ${verdict(last)}` : "none";
42
+ const data = { project, setup: checks.map((check) => ({ item: check.label, ok: check.ok ? "yes" : "no", value: check.value })) };
43
+ data.flows = flows.length > 0 ? flows.map((flow) => ({ flow: flow.name, state: flow.state, detail: flow.reason ?? "" })) : "none";
44
+ if (orphans.length > 0)
45
+ data.tests_without_requirements = orphans;
46
+ data.last_run = lastRun;
47
+ return {
48
+ data,
49
+ help: steps.map(([command, description]) => (description ? `Run \`${command}\` ${description}` : command)),
50
+ json: { project, setup: checks, flows: flows.map(({ name, state, runs, reason }) => ({ name, state, runs, ...(reason ? { reason } : {}) })), testsWithoutRequirements: orphans, lastRun: last?.id ?? null },
51
+ pretty: (style) => renderHome({ project, checks, flows, orphans, lastRun }, steps, style),
52
+ };
53
+ }
54
+ async function setupChecks(config, root) {
55
+ const missingCompose = [];
56
+ for (const file of config.compose) {
57
+ if (!(await exists(join(root, file))))
58
+ missingCompose.push(file);
59
+ }
60
+ const fakes = Object.values(config.hosts).filter((mode) => mode === "fake").length;
61
+ const recorded = Object.values(config.hosts).length - fakes;
62
+ return [
63
+ {
64
+ label: "compose",
65
+ ok: config.compose.length > 0 && missingCompose.length === 0,
66
+ value: config.compose.length === 0 ? "no compose file yet" : missingCompose.length > 0 ? `missing ${missingCompose.join(", ")}` : config.compose.join(", "),
67
+ },
68
+ {
69
+ label: "app",
70
+ ok: config.app !== undefined,
71
+ value: config.app ? `http://${config.app.service}:${config.app.port}${config.app.path}` : "which service and port the browser opens",
72
+ },
73
+ {
74
+ label: "seed",
75
+ ok: true,
76
+ value: config.seed ? `${config.seed.service}: ${config.seed.command.join(" ")}` : "none, fine if the app starts with the data it needs",
77
+ },
78
+ {
79
+ label: "hosts",
80
+ ok: true,
81
+ value: Object.keys(config.hosts).length === 0 ? "none listed, every outside call is refused" : `${fakes} fake, ${recorded} recorded`,
82
+ },
83
+ ];
84
+ }
85
+ // The order is the onboarding: finish setup, then describe flows, then write tests, then prove them.
86
+ function nextSteps(checks, flows, orphans) {
87
+ const steps = [];
88
+ const named = (state) => flows.filter((flow) => flow.state === state).map((flow) => flow.name);
89
+ const missing = checks.filter((check) => !check.ok).map((check) => check.label);
90
+ if (missing.length > 0) {
91
+ steps.push([`Setup is missing: ${missing.join(", ")}. ${SETUP_ASK}`, ""]);
92
+ }
93
+ if (flows.length === 0) {
94
+ steps.push(['Tell your agent what to test, in plain words: "add a proofy test: a guest can buy a product and gets a confirmation email"', ""]);
95
+ }
96
+ if (named("draft").length > 0) {
97
+ steps.push([`Ask your agent to write the tests for: ${named("draft").join(", ")} (step 4 in e2e/AGENTS.md)`, ""]);
98
+ }
99
+ for (const state of ["unproven", "unstable"]) {
100
+ const list = flows.filter((flow) => flow.state === state);
101
+ if (list.length > 0) {
102
+ const what = state === "unstable" ? "Fix the flaky tests in" : "Prove";
103
+ steps.push([`${what} ${list.map((flow) => `${flow.name} (${flow.reason ?? state})`).join(", ")}: \`proofy dev test <flow>\` until 3 runs match (step 5 in e2e/AGENTS.md)`, ""]);
104
+ }
105
+ }
106
+ const ready = [...named("ready"), ...flows.filter((flow) => flow.state === "out-of-date" && flow.runs.length === 3 && !flow.reason).map((flow) => flow.name)];
107
+ if (ready.length > 0) {
108
+ steps.push([`Ready for the human to approve: ${ready.map((name) => `\`proofy approve ${name}\``).join(", ")}. Agents never approve`, ""]);
109
+ }
110
+ const stale = flows.filter((flow) => flow.state === "out-of-date" && !ready.includes(flow.name)).map((flow) => flow.name);
111
+ if (stale.length > 0) {
112
+ steps.push([`Changed since approval: ${stale.join(", ")}. Prove them again with 3 matching runs, then the human approves again`, ""]);
113
+ }
114
+ if (orphans.length > 0) {
115
+ steps.push([`Tests without requirements: ${orphans.join(", ")}. Write e2e/specs/<flow>.md for each, or delete the test`, ""]);
116
+ }
117
+ if (named("approved").length > 0) {
118
+ steps.push(["proofy run --all", "to check every approved flow in a fresh sealed sandbox"]);
119
+ }
120
+ steps.push(["proofy --help", "for every command"]);
121
+ return steps;
122
+ }
@@ -0,0 +1,4 @@
1
+ import type { Deps } from "../deps.js";
2
+ import type { CommandResult } from "./types.js";
3
+ export declare const INIT_HELP = "usage: proofy init [--no-agents-md]\ndescription: Writes e2e/ with agent instructions, templates and a config filled from what it finds. Never overwrites a file\nflags[1]:\n --no-agents-md Do not add the pointer line to the repository's AGENTS.md or CLAUDE.md\nexamples:\n proofy init\n";
4
+ export declare function initCommand(args: string[], deps: Deps): Promise<CommandResult>;
@@ -0,0 +1,122 @@
1
+ import { appendFile, copyFile, mkdir, readdir, readFile, writeFile } from "node:fs/promises";
2
+ import { dirname, join, relative } from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+ import { parseArgs } from "../args.js";
5
+ import { SCHEMA_URL } from "../config.js";
6
+ import { renderInit } from "../human.js";
7
+ import { COMPOSE_NAMES, detectCompose, detectEnv } from "../detect.js";
8
+ import { exists, findRoot, pathsFor } from "../project.js";
9
+ export const INIT_HELP = `usage: proofy init [--no-agents-md]
10
+ description: Writes e2e/ with agent instructions, templates and a config filled from what it finds. Never overwrites a file
11
+ flags[1]:
12
+ --no-agents-md Do not add the pointer line to the repository's AGENTS.md or CLAUDE.md
13
+ examples:
14
+ proofy init
15
+ `;
16
+ const TEMPLATES = fileURLToPath(new URL("../../templates/", import.meta.url));
17
+ const POINTER = "End-to-end tests: read `e2e/AGENTS.md` before writing or changing anything under `e2e/`, and before changing a flow it covers.";
18
+ const IGNORE = "e2e/out/";
19
+ export async function initCommand(args, deps) {
20
+ const flags = parseArgs(args, { values: [], booleans: ["no-agents-md"] }, "init");
21
+ const root = await initRoot(deps.cwd);
22
+ const paths = pathsFor(root);
23
+ const written = [];
24
+ const skipped = [];
25
+ const found = [];
26
+ const compose = await detectCompose(root);
27
+ const env = await detectEnv(root);
28
+ if (compose) {
29
+ found.push(`${compose.file}: services ${compose.services.join(", ") || "none"}`);
30
+ found.push(compose.app ? `app: likely ${compose.app.service} on port ${compose.app.port}` : "app: no service with a port, your agent picks one");
31
+ }
32
+ else {
33
+ found.push("no compose file: your agent drafts one during setup");
34
+ }
35
+ if (env) {
36
+ found.push(`${env.file}: ${Object.keys(env.values).length} settings, ${env.replaced} secret-looking values replaced with fakes`);
37
+ }
38
+ const config = { $schema: SCHEMA_URL, compose: compose ? [compose.file] : [] };
39
+ if (compose?.app) {
40
+ config.app = { service: compose.app.service, port: compose.app.port, path: "/" };
41
+ if (env)
42
+ config.env = { [compose.app.service]: env.values };
43
+ }
44
+ config.hosts = {};
45
+ const write = async (file, content) => {
46
+ const label = relative(root, file);
47
+ if (await exists(file)) {
48
+ skipped.push(label);
49
+ return;
50
+ }
51
+ await mkdir(dirname(file), { recursive: true });
52
+ if (typeof content === "string")
53
+ await writeFile(file, content);
54
+ else
55
+ await content();
56
+ written.push(label);
57
+ };
58
+ await write(paths.config, `${JSON.stringify(config, null, 2)}\n`);
59
+ for (const file of await listFiles(TEMPLATES)) {
60
+ const source = join(TEMPLATES, file);
61
+ await write(join(root, file), () => copyFile(source, join(root, file)));
62
+ }
63
+ for (const dir of [paths.specs, paths.tests, paths.standins]) {
64
+ await mkdir(dir, { recursive: true });
65
+ }
66
+ if (await addLine(join(root, ".gitignore"), IGNORE))
67
+ written.push(".gitignore (e2e/out/)");
68
+ let pointer = "skipped (--no-agents-md)";
69
+ if (!flags.booleans.has("no-agents-md")) {
70
+ const target = (await exists(join(root, "AGENTS.md"))) || !(await exists(join(root, "CLAUDE.md"))) ? "AGENTS.md" : "CLAUDE.md";
71
+ pointer = (await addLine(join(root, target), POINTER)) ? `added to ${target}` : `already in ${target}`;
72
+ if (pointer.startsWith("added"))
73
+ written.push(`${target} (pointer to e2e/AGENTS.md)`);
74
+ }
75
+ const hints = [
76
+ ['Next, ask your agent: "set up proofy". It reads e2e/AGENTS.md, checks the app starts in Docker and fills in the rest of e2e/proofy.json', ""],
77
+ ['Then tell it what to test, in plain words: "add a proofy test: a guest can buy a product"', ""],
78
+ ["proofy", "see setup progress and the next step at any time"],
79
+ ];
80
+ return {
81
+ data: { root, found, written, kept: skipped, agents_md_pointer: pointer },
82
+ help: hints.map(([command, description]) => (description ? `Run \`${command}\` to ${description}` : command)),
83
+ pretty: (style) => renderInit({ root, found, written, skipped }, hints, style),
84
+ };
85
+ }
86
+ /** An existing proofy project, else the folder you are in when it has a compose file (an app in a monorepo), else the repository root. */
87
+ async function initRoot(cwd) {
88
+ const found = await findRoot(cwd);
89
+ if (await exists(pathsFor(found).config))
90
+ return found;
91
+ for (const name of COMPOSE_NAMES) {
92
+ if (await exists(join(cwd, name)))
93
+ return cwd;
94
+ }
95
+ return found;
96
+ }
97
+ /** Appends a line unless the file already has it. Returns whether it wrote. */
98
+ async function addLine(file, line) {
99
+ let current = "";
100
+ try {
101
+ current = await readFile(file, "utf8");
102
+ }
103
+ catch {
104
+ // A missing file is created.
105
+ }
106
+ if (current.split(/\r?\n/).some((existing) => existing.trim() === line))
107
+ return false;
108
+ const separator = current === "" || current.endsWith("\n") ? "" : "\n";
109
+ await appendFile(file, `${separator}${current === "" ? "" : "\n"}${line}\n`);
110
+ return true;
111
+ }
112
+ async function listFiles(dir, prefix = "") {
113
+ const out = [];
114
+ for (const entry of await readdir(join(dir, prefix), { withFileTypes: true })) {
115
+ const path = join(prefix, entry.name);
116
+ if (entry.isDirectory())
117
+ out.push(...(await listFiles(dir, path)));
118
+ else
119
+ out.push(path);
120
+ }
121
+ return out.sort();
122
+ }
@@ -0,0 +1,4 @@
1
+ import type { Deps } from "../deps.js";
2
+ import type { CommandResult } from "./types.js";
3
+ export declare const REPORT_HELP = "usage: proofy report [run]\ndescription: Shows a past run, the latest by default. The full report with evidence and screenshots is e2e/out/<run>/report.md\nexamples:\n proofy report\n proofy report 20261020-142233\n";
4
+ export declare function reportCommand(args: string[], deps: Deps): Promise<CommandResult>;
@@ -0,0 +1,26 @@
1
+ import { join } from "node:path";
2
+ import { parseArgs } from "../args.js";
3
+ import { CliError } from "../errors.js";
4
+ import { outDir, readReport, runIds } from "../run/history.js";
5
+ import { loadProject } from "./context.js";
6
+ import { runResult } from "./run.js";
7
+ export const REPORT_HELP = `usage: proofy report [run]
8
+ description: Shows a past run, the latest by default. The full report with evidence and screenshots is e2e/out/<run>/report.md
9
+ examples:
10
+ proofy report
11
+ proofy report 20261020-142233
12
+ `;
13
+ export async function reportCommand(args, deps) {
14
+ const { positionals } = parseArgs(args, { values: [], booleans: [] }, "report");
15
+ const project = await loadProject(deps);
16
+ const ids = await runIds(project.paths);
17
+ const id = positionals[0] ?? ids.at(-1);
18
+ if (!id)
19
+ throw new CliError("No runs yet", "VALIDATION_ERROR", ["Run `proofy run <flow>` or `proofy dev test <flow>`"]);
20
+ const report = await readReport(project.paths, id);
21
+ if (!report)
22
+ throw new CliError(`No run ${id}`, "VALIDATION_ERROR", [`Runs: ${ids.slice(-5).join(", ") || "none"}`]);
23
+ const result = runResult(report, join(outDir(project.paths), id), project.root);
24
+ // Showing a report is not running it: the verdict is in the output, the command itself succeeded.
25
+ return { ...result, exitCode: 0 };
26
+ }
@@ -0,0 +1,6 @@
1
+ import type { Deps } from "../deps.js";
2
+ import type { RunReport } from "../run/results.js";
3
+ import type { CommandResult } from "./types.js";
4
+ export declare const RUN_HELP = "usage: proofy run [flow ...] [--all] [--pr <number>] [--post] [--keep]\ndescription: One sealed run: a fresh sandbox with no way out, the tests, a report per requirement, teardown\nflags[4]:\n --all Every approved flow\n --pr <number> Test a pull request's code, exported to a cache folder; the working copy is never touched\n --post Post the report on the pull request (--pr, or the current branch's), screenshots on the proofy-reports branch\n --keep Leave the sandbox running afterwards\nexit_codes[4]:\n 0 Every requirement of every approved flow held\n 1 A requirement of an approved flow did not hold: a finding about the app\n 3 Environment failure: the sandbox did not work\n 4 Some flows that ran are not approved, so the result is not trusted\nexamples:\n proofy run guest-checkout\n proofy run --all\n proofy run --all --pr 42 --post\n";
5
+ export declare function runCommand(args: string[], deps: Deps): Promise<CommandResult>;
6
+ export declare function runResult(report: RunReport, runDir: string, root: string, posted?: string): CommandResult;
@@ -0,0 +1,166 @@
1
+ import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
2
+ import { join, relative } from "node:path";
3
+ import { parseArgs } from "../args.js";
4
+ import { loadConfig } from "../config.js";
5
+ import { CliError } from "../errors.js";
6
+ import { exportPullRequest, postReport, pullRequest } from "../github.js";
7
+ import { renderRun } from "../human.js";
8
+ import { exists, flowNames, gitRoot, pathsFor } from "../project.js";
9
+ import { planFlows, sealedRun } from "../run/engine.js";
10
+ import { outDir } from "../run/history.js";
11
+ import { EXIT_FOR, verdict } from "../run/report.js";
12
+ import { loadProject, projectName, workingTreeInfo } from "./context.js";
13
+ export const RUN_HELP = `usage: proofy run [flow ...] [--all] [--pr <number>] [--post] [--keep]
14
+ description: One sealed run: a fresh sandbox with no way out, the tests, a report per requirement, teardown
15
+ flags[4]:
16
+ --all Every approved flow
17
+ --pr <number> Test a pull request's code, exported to a cache folder; the working copy is never touched
18
+ --post Post the report on the pull request (--pr, or the current branch's), screenshots on the proofy-reports branch
19
+ --keep Leave the sandbox running afterwards
20
+ exit_codes[4]:
21
+ 0 Every requirement of every approved flow held
22
+ 1 A requirement of an approved flow did not hold: a finding about the app
23
+ 3 Environment failure: the sandbox did not work
24
+ 4 Some flows that ran are not approved, so the result is not trusted
25
+ examples:
26
+ proofy run guest-checkout
27
+ proofy run --all
28
+ proofy run --all --pr 42 --post
29
+ `;
30
+ export async function runCommand(args, deps) {
31
+ const flags = parseArgs(args, { values: ["pr"], booleans: ["all", "post", "keep"] }, "run");
32
+ const prNumber = flags.values.has("pr") ? prArg(flags.values.get("pr") ?? "") : undefined;
33
+ if (flags.positionals.length === 0 && !flags.booleans.has("all")) {
34
+ throw new CliError("Name the flows to run, or use --all for every approved flow", "VALIDATION_ERROR", ["Run `proofy` to see flows and their states"]);
35
+ }
36
+ const project = await loadProject(deps);
37
+ const { root, paths } = project;
38
+ let source = paths;
39
+ let config = project.config;
40
+ let pr;
41
+ let sourceInfo;
42
+ if (prNumber !== undefined) {
43
+ pr = await pullRequest(deps.proc, root, prNumber);
44
+ const dir = join(project.cacheDir, "pull-requests", String(pr.number));
45
+ deps.log(`exporting pull request #${pr.number} (${pr.sha.slice(0, 12)})`);
46
+ // The project can sit in a subfolder of the repository; the export holds the whole repository.
47
+ const top = (await gitRoot(root)) ?? root;
48
+ await exportPullRequest(deps.proc, top, pr, dir);
49
+ source = pathsFor(join(dir, relative(top, root)));
50
+ if (await exists(source.config))
51
+ config = await loadConfig(source.config, `e2e/proofy.json in PR #${pr.number}`);
52
+ sourceInfo = { commit: pr.sha, pr: pr.number };
53
+ }
54
+ else {
55
+ sourceInfo = await workingTreeInfo(deps, root);
56
+ }
57
+ const available = await flowNames(source);
58
+ const ctx = { proc: deps.proc, env: deps.env, now: deps.now, log: deps.log, base: paths, source, config, approvals: project.approvals, cacheDir: project.cacheDir, sourceInfo };
59
+ const names = flags.positionals;
60
+ for (const name of names) {
61
+ if (!available.includes(name)) {
62
+ throw new CliError(`No flow named ${name}: e2e/specs/${name}.md does not exist`, "VALIDATION_ERROR", [`Flows: ${available.join(", ") || "none yet"}`]);
63
+ }
64
+ if (!(await exists(join(source.tests, `${name}.spec.ts`)))) {
65
+ throw new CliError(`${name} has requirements but no test yet`, "VALIDATION_ERROR", ["Ask your agent to write it (step 4 in e2e/AGENTS.md)"]);
66
+ }
67
+ }
68
+ let planned = await planFlows(ctx, names);
69
+ if (flags.booleans.has("all")) {
70
+ const all = await planFlows(ctx, available.filter((name) => !names.includes(name)));
71
+ const approved = all.filter((flow) => flow.approved);
72
+ planned = [...planned, ...approved];
73
+ if (planned.length === 0) {
74
+ throw new CliError(pr ? `No flow in PR #${pr.number} matches its approval: none were approved, or the PR changed them` : "No flow is approved yet", "NOT_APPROVED", ["Run `proofy` to see each flow's state", "Run named flows to prove them: `proofy run <flow>`"]);
75
+ }
76
+ }
77
+ const lock = await takeLock(paths);
78
+ const ignore = () => deps.log("stopping: tearing down the sandbox first");
79
+ process.on("SIGINT", ignore);
80
+ let report;
81
+ try {
82
+ report = await sealedRun(ctx, planned, { keep: flags.booleans.has("keep"), project: projectName(root, `run-${deps.now().getTime().toString(36)}`) });
83
+ }
84
+ finally {
85
+ process.off("SIGINT", ignore);
86
+ await rm(lock, { force: true });
87
+ }
88
+ const runDir = join(outDir(paths), report.id);
89
+ let posted;
90
+ if (flags.booleans.has("post")) {
91
+ const target = pr ?? (await pullRequest(deps.proc, root));
92
+ deps.log(`posting the report on pull request #${target.number}`);
93
+ posted = await postReport(deps.proc, root, target, report, runDir, join(project.cacheDir, "post"));
94
+ }
95
+ return runResult(report, runDir, root, posted);
96
+ }
97
+ export function runResult(report, runDir, root, posted) {
98
+ const result = verdict(report);
99
+ const relative = runDir.startsWith(root) ? runDir.slice(root.length + 1) : runDir;
100
+ const failed = report.flows.flatMap((flow) => flow.requirements
101
+ .filter((requirement) => requirement.result === "fail" || requirement.result === "not-reached")
102
+ .map((requirement) => ({ flow: flow.name, req: requirement.id, expected: requirement.expected ?? "", actual: requirement.actual ?? requirement.result })));
103
+ const hints = [];
104
+ if (result === "environment")
105
+ hints.push([`Read ${relative}/sandbox.log and fix the sandbox setup; this run says nothing about the app`, ""]);
106
+ if (failed.length > 0)
107
+ hints.push([`Read ${relative}/report.md: decide per red requirement whether the app, the test, a fake or the requirement is wrong (e2e/AGENTS.md)`, ""]);
108
+ const untrusted = report.flows.filter((flow) => !flow.approved).map((flow) => flow.name);
109
+ if (untrusted.length > 0 && result !== "environment")
110
+ hints.push([`Not approved: ${untrusted.join(", ")}. Run \`proofy\` to see whether they are ready for the human to approve`, ""]);
111
+ hints.push([`proofy report ${report.id}`, "show this run again"]);
112
+ const data = {
113
+ run: report.id,
114
+ verdict: result,
115
+ report: `${relative}/report.md`,
116
+ flows: report.flows.map((flow) => ({ flow: flow.name, approved: flow.approved ? "yes" : "no", status: flow.status, failed: flow.requirements.filter((r) => r.result === "fail" || r.result === "not-reached").length })),
117
+ };
118
+ if (report.environmentError)
119
+ data.environment_error = report.environmentError.split("\n")[0];
120
+ if (failed.length > 0)
121
+ data.failed = failed;
122
+ const refused = report.outside.filter((call) => call.handledAs === "refused");
123
+ if (refused.length > 0)
124
+ data.refused_hosts = refused.map((call) => call.host);
125
+ if (report.notes.length > 0)
126
+ data.notes = report.notes;
127
+ if (posted)
128
+ data.posted = posted;
129
+ return {
130
+ data,
131
+ help: hints.map(([command, description]) => (description ? `Run \`${command}\` to ${description}` : command)),
132
+ json: { ...report, verdict: result, reportFile: `${relative}/report.md`, ...(posted ? { posted } : {}) },
133
+ pretty: (style) => renderRun(report, result, relative, hints, style, posted),
134
+ exitCode: EXIT_FOR[result],
135
+ };
136
+ }
137
+ async function takeLock(paths) {
138
+ const file = join(outDir(paths), ".run.lock");
139
+ await mkdir(outDir(paths), { recursive: true });
140
+ try {
141
+ await writeFile(file, String(process.pid), { flag: "wx" });
142
+ }
143
+ catch {
144
+ const pid = Number(await readFile(file, "utf8").catch(() => "0"));
145
+ if (pid > 0 && alive(pid)) {
146
+ throw new CliError(`Another proofy run is in progress (process ${pid})`, "ENVIRONMENT_ERROR", ["Wait for it to finish; only one run at a time"]);
147
+ }
148
+ await writeFile(file, String(process.pid));
149
+ }
150
+ return file;
151
+ }
152
+ function alive(pid) {
153
+ try {
154
+ process.kill(pid, 0);
155
+ return true;
156
+ }
157
+ catch {
158
+ return false;
159
+ }
160
+ }
161
+ function prArg(value) {
162
+ const number = Number(value);
163
+ if (!Number.isInteger(number) || number < 1)
164
+ throw new CliError("--pr needs a pull request number", "VALIDATION_ERROR", ["Example: proofy run --all --pr 42"]);
165
+ return number;
166
+ }
@@ -0,0 +1,8 @@
1
+ import type { Result } from "../output.js";
2
+ import type { Style } from "../style.js";
3
+ export interface CommandResult extends Result {
4
+ json?: unknown;
5
+ pretty?: (style: Style) => string;
6
+ /** Set by commands whose result is a verdict, such as run; 0 otherwise. */
7
+ exitCode?: number;
8
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,34 @@
1
+ import { z } from "zod";
2
+ export declare const SCHEMA_URL = "https://unpkg.com/@davidrydberg/proofy/schema/proofy.schema.json";
3
+ export declare const configSchema: z.ZodObject<{
4
+ $schema: z.ZodOptional<z.ZodString>;
5
+ compose: z.ZodDefault<z.ZodArray<z.ZodString>>;
6
+ app: z.ZodOptional<z.ZodObject<{
7
+ service: z.ZodString;
8
+ port: z.ZodNumber;
9
+ path: z.ZodDefault<z.ZodString>;
10
+ }, z.core.$strict>>;
11
+ seed: z.ZodOptional<z.ZodObject<{
12
+ service: z.ZodString;
13
+ command: z.ZodArray<z.ZodString>;
14
+ }, z.core.$strict>>;
15
+ exec: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
16
+ service: z.ZodString;
17
+ command: z.ZodArray<z.ZodString>;
18
+ }, z.core.$strict>>>;
19
+ hosts: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodEnum<{
20
+ record: "record";
21
+ fake: "fake";
22
+ }>>>;
23
+ assets: z.ZodDefault<z.ZodArray<z.ZodURL>>;
24
+ env: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodString>>>;
25
+ limits: z.ZodPrefault<z.ZodObject<{
26
+ startMinutes: z.ZodDefault<z.ZodNumber>;
27
+ testMinutes: z.ZodDefault<z.ZodNumber>;
28
+ maxRunMinutes: z.ZodOptional<z.ZodNumber>;
29
+ parallel: z.ZodDefault<z.ZodNumber>;
30
+ }, z.core.$strict>>;
31
+ }, z.core.$strict>;
32
+ export type Config = z.infer<typeof configSchema>;
33
+ export declare function parseConfig(raw: string, file: string): Config;
34
+ export declare function loadConfig(file: string, label: string): Promise<Config>;
package/dist/config.js ADDED
@@ -0,0 +1,62 @@
1
+ import { readFile } from "node:fs/promises";
2
+ import { z } from "zod";
3
+ import { CliError } from "./errors.js";
4
+ export const SCHEMA_URL = "https://unpkg.com/@davidrydberg/proofy/schema/proofy.schema.json";
5
+ const name = z.string().min(1);
6
+ const command = z.array(z.string().min(1)).min(1).describe("Command and arguments, run without a shell");
7
+ const hostname = z
8
+ .string()
9
+ .regex(/^(?=.{1,253}$)([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,63}$/, "must be a lowercase hostname like api.stripe.com");
10
+ const serviceCommand = z.strictObject({ service: name.describe("Compose service to run it in"), command });
11
+ export const configSchema = z
12
+ .strictObject({
13
+ $schema: z.string().optional(),
14
+ compose: z.array(name).default([]).describe("The app's compose files, relative to the repository root"),
15
+ app: z
16
+ .strictObject({
17
+ service: name.describe("Compose service the browser opens"),
18
+ port: z.number().int().min(1).max(65535).describe("Port inside the container, not a published port"),
19
+ path: z.string().startsWith("/").default("/"),
20
+ })
21
+ .optional()
22
+ .describe("Where the browser starts"),
23
+ seed: serviceCommand.optional().describe("Runs once on the empty database before the tests"),
24
+ exec: z.record(name, serviceCommand).default({}).describe("Commands tests may run with exec(name), such as a scheduled job"),
25
+ hosts: z
26
+ .record(hostname, z.enum(["fake", "record"]))
27
+ .default({})
28
+ .describe("Outside hosts: fake answers from e2e/standins/<host>.mjs, record accepts everything. Unlisted hosts are refused"),
29
+ assets: z.array(z.url()).default([]).describe("Static third-party files downloaded at install and served in the sealed run"),
30
+ env: z.record(name, z.record(name, z.string())).default({}).describe("Settings per compose service. The repository's .env is never read"),
31
+ limits: z
32
+ .strictObject({
33
+ startMinutes: z.number().positive().default(10),
34
+ testMinutes: z.number().positive().default(5),
35
+ maxRunMinutes: z.number().positive().optional(),
36
+ parallel: z.number().int().min(1).default(4),
37
+ })
38
+ .prefault({}),
39
+ })
40
+ .describe("proofy configuration, e2e/proofy.json");
41
+ export function parseConfig(raw, file) {
42
+ let json;
43
+ try {
44
+ json = JSON.parse(raw);
45
+ }
46
+ catch (error) {
47
+ throw invalid(`${file} is not valid JSON: ${error.message}`);
48
+ }
49
+ const result = configSchema.safeParse(json);
50
+ if (!result.success) {
51
+ const issue = result.error.issues[0];
52
+ const where = issue && issue.path.length > 0 ? issue.path.join(".") : "(root)";
53
+ throw invalid(`${file}: ${where}: ${issue?.message ?? "invalid"}`);
54
+ }
55
+ return result.data;
56
+ }
57
+ export async function loadConfig(file, label) {
58
+ return parseConfig(await readFile(file, "utf8"), label);
59
+ }
60
+ function invalid(message) {
61
+ return new CliError(message, "CONFIG_INVALID", [`Fix it, then run \`proofy\` to check. Every field is described in ${SCHEMA_URL}`]);
62
+ }
package/dist/deps.d.ts ADDED
@@ -0,0 +1,10 @@
1
+ import type { Proc } from "./sandbox/docker.js";
2
+ export interface Deps {
3
+ cwd: string;
4
+ env: NodeJS.ProcessEnv;
5
+ /** Runs docker, git and gh. Tests inject a fake. */
6
+ proc: Proc;
7
+ now: () => Date;
8
+ /** Progress lines for long commands. Goes to stderr, so stdout stays one clean result. */
9
+ log: (line: string) => void;
10
+ }
package/dist/deps.js ADDED
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,17 @@
1
+ export declare const COMPOSE_NAMES: string[];
2
+ export interface ComposeGuess {
3
+ file: string;
4
+ services: string[];
5
+ app?: {
6
+ service: string;
7
+ port: number;
8
+ };
9
+ }
10
+ export interface EnvGuess {
11
+ file: string;
12
+ values: Record<string, string>;
13
+ replaced: number;
14
+ }
15
+ export declare function detectCompose(root: string): Promise<ComposeGuess | undefined>;
16
+ /** Reads .env.example so the sandbox gets the same setting names; secret-looking values become obvious fakes. */
17
+ export declare function detectEnv(root: string): Promise<EnvGuess | undefined>;