@davidrydberg/proofy 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +64 -2
- package/dist/args.d.ts +10 -0
- package/dist/args.js +36 -0
- package/dist/bin/proofy.d.ts +2 -0
- package/dist/bin/proofy.js +4 -0
- package/dist/cli.d.ts +9 -0
- package/dist/cli.js +140 -0
- package/dist/commands/approve.d.ts +4 -0
- package/dist/commands/approve.js +48 -0
- package/dist/commands/context.d.ts +16 -0
- package/dist/commands/context.js +27 -0
- package/dist/commands/dev.d.ts +4 -0
- package/dist/commands/dev.js +161 -0
- package/dist/commands/home.d.ts +3 -0
- package/dist/commands/home.js +122 -0
- package/dist/commands/init.d.ts +4 -0
- package/dist/commands/init.js +122 -0
- package/dist/commands/report.d.ts +4 -0
- package/dist/commands/report.js +26 -0
- package/dist/commands/run.d.ts +6 -0
- package/dist/commands/run.js +166 -0
- package/dist/commands/types.d.ts +8 -0
- package/dist/commands/types.js +1 -0
- package/dist/config.d.ts +34 -0
- package/dist/config.js +62 -0
- package/dist/deps.d.ts +10 -0
- package/dist/deps.js +1 -0
- package/dist/detect.d.ts +17 -0
- package/dist/detect.js +96 -0
- package/dist/errors.d.ts +6 -0
- package/dist/errors.js +21 -0
- package/dist/github.d.ts +16 -0
- package/dist/github.js +121 -0
- package/dist/human.d.ts +30 -0
- package/dist/human.js +160 -0
- package/dist/output.d.ts +8 -0
- package/dist/output.js +21 -0
- package/dist/project.d.ts +21 -0
- package/dist/project.js +71 -0
- package/dist/run/approvals.d.ts +15 -0
- package/dist/run/approvals.js +26 -0
- package/dist/run/engine.d.ts +48 -0
- package/dist/run/engine.js +168 -0
- package/dist/run/fingerprint.d.ts +11 -0
- package/dist/run/fingerprint.js +40 -0
- package/dist/run/history.d.ts +29 -0
- package/dist/run/history.js +85 -0
- package/dist/run/report.d.ts +14 -0
- package/dist/run/report.js +115 -0
- package/dist/run/results.d.ts +70 -0
- package/dist/run/results.js +129 -0
- package/dist/run/spec.d.ts +6 -0
- package/dist/run/spec.js +16 -0
- package/dist/sandbox/compose.d.ts +38 -0
- package/dist/sandbox/compose.js +132 -0
- package/dist/sandbox/contract.d.ts +19 -0
- package/dist/sandbox/contract.js +20 -0
- package/dist/sandbox/docker.d.ts +17 -0
- package/dist/sandbox/docker.js +48 -0
- package/dist/sandbox/sandbox.d.ts +46 -0
- package/dist/sandbox/sandbox.js +239 -0
- package/dist/standin/ca.d.ts +6 -0
- package/dist/standin/ca.js +40 -0
- package/dist/standin/dns.d.ts +10 -0
- package/dist/standin/dns.js +53 -0
- package/dist/standin/forward.d.ts +1 -0
- package/dist/standin/forward.js +17 -0
- package/dist/standin/handle.d.ts +49 -0
- package/dist/standin/handle.js +130 -0
- package/dist/standin/server.d.ts +1 -0
- package/dist/standin/server.js +88 -0
- package/dist/style.d.ts +23 -0
- package/dist/style.js +38 -0
- package/dist/test/global-setup.d.ts +1 -0
- package/dist/test/global-setup.js +20 -0
- package/dist/test/index.d.ts +102 -0
- package/dist/test/index.js +184 -0
- package/docker/standin.Dockerfile +3 -0
- package/package.json +68 -4
- package/schema/proofy.schema.json +179 -0
- package/templates/e2e/AGENTS.md +90 -0
- package/templates/e2e/templates/compose.yml +33 -0
- package/templates/e2e/templates/spec.md +20 -0
- package/templates/e2e/templates/standin.mjs +29 -0
- package/templates/e2e/templates/test.spec.ts +28 -0
package/dist/detect.js
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { parse } from "yaml";
|
|
4
|
+
export const COMPOSE_NAMES = ["compose.yaml", "compose.yml", "docker-compose.yaml", "docker-compose.yml"];
|
|
5
|
+
const INFRA = /(^|\/)(mysql|mariadb|postgres|postgis|redis|valkey|memcached|mongo|mailpit|mailhog|elasticsearch|opensearch|rabbitmq|minio|adminer|phpmyadmin|nginx-proxy|traefik)(:|$)/;
|
|
6
|
+
const SECRET = /(KEY|SECRET|TOKEN|PASSWORD|PASS|PRIVATE|CREDENTIAL|DSN)/i;
|
|
7
|
+
export async function detectCompose(root) {
|
|
8
|
+
for (const file of COMPOSE_NAMES) {
|
|
9
|
+
let raw;
|
|
10
|
+
try {
|
|
11
|
+
raw = await readFile(join(root, file), "utf8");
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
continue;
|
|
15
|
+
}
|
|
16
|
+
let doc;
|
|
17
|
+
try {
|
|
18
|
+
doc = parse(raw);
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return { file, services: [] };
|
|
22
|
+
}
|
|
23
|
+
const services = isRecord(doc) && isRecord(doc.services) ? doc.services : {};
|
|
24
|
+
const app = guessApp(services);
|
|
25
|
+
return { file, services: Object.keys(services), ...(app ? { app } : {}) };
|
|
26
|
+
}
|
|
27
|
+
return undefined;
|
|
28
|
+
}
|
|
29
|
+
/** Reads .env.example so the sandbox gets the same setting names; secret-looking values become obvious fakes. */
|
|
30
|
+
export async function detectEnv(root) {
|
|
31
|
+
const file = ".env.example";
|
|
32
|
+
let raw;
|
|
33
|
+
try {
|
|
34
|
+
raw = await readFile(join(root, file), "utf8");
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
const values = {};
|
|
40
|
+
let replaced = 0;
|
|
41
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
42
|
+
const match = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
|
|
43
|
+
if (!match)
|
|
44
|
+
continue;
|
|
45
|
+
const [, key = "", rawValue = ""] = match;
|
|
46
|
+
if (SECRET.test(key)) {
|
|
47
|
+
values[key] = `proofy-fake-${key.toLowerCase().replace(/_/g, "-")}`;
|
|
48
|
+
replaced++;
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
values[key] = unquote(rawValue.replace(/\s+#.*$/, "").trim());
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return { file, values, replaced };
|
|
55
|
+
}
|
|
56
|
+
function guessApp(services) {
|
|
57
|
+
const candidates = [];
|
|
58
|
+
for (const [service, definition] of Object.entries(services)) {
|
|
59
|
+
if (!isRecord(definition))
|
|
60
|
+
continue;
|
|
61
|
+
const image = typeof definition.image === "string" ? definition.image : "";
|
|
62
|
+
if (INFRA.test(image) || INFRA.test(service))
|
|
63
|
+
continue;
|
|
64
|
+
const port = firstPort(definition.ports) ?? firstPort(definition.expose);
|
|
65
|
+
if (port !== undefined)
|
|
66
|
+
candidates.push({ service, port, built: definition.build !== undefined });
|
|
67
|
+
}
|
|
68
|
+
// A service built from the repository is the app more often than an image pulled from elsewhere.
|
|
69
|
+
const pick = candidates.find((c) => c.built) ?? candidates[0];
|
|
70
|
+
return pick ? { service: pick.service, port: pick.port } : undefined;
|
|
71
|
+
}
|
|
72
|
+
// The browser reaches the app inside the sandbox network, so the container side of a mapping is what counts.
|
|
73
|
+
function firstPort(ports) {
|
|
74
|
+
if (!Array.isArray(ports))
|
|
75
|
+
return undefined;
|
|
76
|
+
for (const entry of ports) {
|
|
77
|
+
if (isRecord(entry) && typeof entry.target === "number")
|
|
78
|
+
return entry.target;
|
|
79
|
+
if (typeof entry === "number")
|
|
80
|
+
return entry;
|
|
81
|
+
if (typeof entry === "string") {
|
|
82
|
+
const container = entry.split("/")[0]?.split(":").at(-1) ?? "";
|
|
83
|
+
const port = Number(container.split("-")[0]);
|
|
84
|
+
if (Number.isInteger(port) && port > 0)
|
|
85
|
+
return port;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return undefined;
|
|
89
|
+
}
|
|
90
|
+
function unquote(value) {
|
|
91
|
+
const quoted = /^(['"])(.*)\1$/.exec(value);
|
|
92
|
+
return quoted ? (quoted[2] ?? "") : value;
|
|
93
|
+
}
|
|
94
|
+
function isRecord(value) {
|
|
95
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
96
|
+
}
|
package/dist/errors.d.ts
ADDED
package/dist/errors.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export class CliError extends Error {
|
|
2
|
+
code;
|
|
3
|
+
suggestions;
|
|
4
|
+
constructor(message, code, suggestions = []) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = "CliError";
|
|
7
|
+
this.code = code;
|
|
8
|
+
this.suggestions = suggestions;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
// 1 is reserved for a failed requirement, so an unexpected crash must not look like a finding about the app.
|
|
12
|
+
const EXIT_CODES = {
|
|
13
|
+
VALIDATION_ERROR: 2,
|
|
14
|
+
CONFIG_INVALID: 2,
|
|
15
|
+
NOT_SET_UP: 2,
|
|
16
|
+
ENVIRONMENT_ERROR: 3,
|
|
17
|
+
NOT_APPROVED: 4,
|
|
18
|
+
};
|
|
19
|
+
export function exitCodeFor(error) {
|
|
20
|
+
return error instanceof CliError ? (EXIT_CODES[error.code] ?? 3) : 3;
|
|
21
|
+
}
|
package/dist/github.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { RunReport } from "./run/results.js";
|
|
2
|
+
import { type Proc } from "./sandbox/docker.js";
|
|
3
|
+
export declare const REPORTS_BRANCH = "proofy-reports";
|
|
4
|
+
export interface PullRequest {
|
|
5
|
+
number: number;
|
|
6
|
+
sha: string;
|
|
7
|
+
url: string;
|
|
8
|
+
}
|
|
9
|
+
export declare function pullRequest(proc: Proc, root: string, number?: number): Promise<PullRequest>;
|
|
10
|
+
/** The pull request's files in a folder of their own. The working copy is never touched. `root` is the repository's top folder. */
|
|
11
|
+
export declare function exportPullRequest(proc: Proc, root: string, pr: PullRequest, dir: string): Promise<void>;
|
|
12
|
+
/**
|
|
13
|
+
* Pushes the run's screenshots to the reports branch, replaced on every push so the repository never grows,
|
|
14
|
+
* then writes or updates the one proofy comment on the pull request.
|
|
15
|
+
*/
|
|
16
|
+
export declare function postReport(proc: Proc, root: string, pr: PullRequest, report: RunReport, runDir: string, workDir: string): Promise<string>;
|
package/dist/github.js
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { cp, mkdir, readdir, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { CliError } from "./errors.js";
|
|
4
|
+
import { renderMarkdown, verdict } from "./run/report.js";
|
|
5
|
+
import { tail } from "./sandbox/docker.js";
|
|
6
|
+
export const REPORTS_BRANCH = "proofy-reports";
|
|
7
|
+
const MARKER = "<!-- proofy -->";
|
|
8
|
+
const KEEP = 20;
|
|
9
|
+
const HISTORY = 10;
|
|
10
|
+
async function gh(proc, args, cwd) {
|
|
11
|
+
const result = await proc("gh", args, { cwd, timeoutMs: 60_000 });
|
|
12
|
+
if (result.code !== 0) {
|
|
13
|
+
const auth = /auth login|not logged/i.test(result.stderr);
|
|
14
|
+
throw new CliError(`gh ${args.slice(0, 2).join(" ")} failed: ${tail(result.stderr, 3)}`, "ENVIRONMENT_ERROR", [
|
|
15
|
+
auth ? "Run `gh auth login`" : "Check that this repository is on GitHub and gh can reach it",
|
|
16
|
+
]);
|
|
17
|
+
}
|
|
18
|
+
return result.stdout;
|
|
19
|
+
}
|
|
20
|
+
export async function pullRequest(proc, root, number) {
|
|
21
|
+
const args = ["pr", "view", ...(number === undefined ? [] : [String(number)]), "--json", "number,headRefOid,url"];
|
|
22
|
+
let out;
|
|
23
|
+
try {
|
|
24
|
+
out = await gh(proc, args, root);
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
if (number === undefined && error instanceof CliError) {
|
|
28
|
+
throw new CliError("The current branch has no pull request to post on", "VALIDATION_ERROR", ["Open one first, or name it with --pr <number>"]);
|
|
29
|
+
}
|
|
30
|
+
throw error;
|
|
31
|
+
}
|
|
32
|
+
const parsed = JSON.parse(out);
|
|
33
|
+
return { number: parsed.number, sha: parsed.headRefOid, url: parsed.url };
|
|
34
|
+
}
|
|
35
|
+
/** The pull request's files in a folder of their own. The working copy is never touched. `root` is the repository's top folder. */
|
|
36
|
+
export async function exportPullRequest(proc, root, pr, dir) {
|
|
37
|
+
const fetch = await proc("git", ["-C", root, "fetch", "--quiet", "origin", `refs/pull/${pr.number}/head`], { timeoutMs: 300_000 });
|
|
38
|
+
if (fetch.code !== 0) {
|
|
39
|
+
throw new CliError(`Fetching pull request #${pr.number} failed`, "ENVIRONMENT_ERROR", [tail(fetch.stderr, 5)]);
|
|
40
|
+
}
|
|
41
|
+
await rm(dir, { recursive: true, force: true });
|
|
42
|
+
await mkdir(dir, { recursive: true });
|
|
43
|
+
const archive = join(dir, "..", `pr-${pr.number}.tar`);
|
|
44
|
+
const packed = await proc("git", ["-C", root, "archive", "--format=tar", "-o", archive, pr.sha]);
|
|
45
|
+
if (packed.code !== 0)
|
|
46
|
+
throw new CliError(`Exporting ${pr.sha.slice(0, 12)} failed`, "ENVIRONMENT_ERROR", [tail(packed.stderr, 5)]);
|
|
47
|
+
const unpacked = await proc("tar", ["-xf", archive, "-C", dir]);
|
|
48
|
+
await rm(archive, { force: true });
|
|
49
|
+
if (unpacked.code !== 0)
|
|
50
|
+
throw new CliError("Unpacking the pull request failed", "ENVIRONMENT_ERROR", [tail(unpacked.stderr, 5)]);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Pushes the run's screenshots to the reports branch, replaced on every push so the repository never grows,
|
|
54
|
+
* then writes or updates the one proofy comment on the pull request.
|
|
55
|
+
*/
|
|
56
|
+
export async function postReport(proc, root, pr, report, runDir, workDir) {
|
|
57
|
+
const slug = (await gh(proc, ["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"], root)).trim();
|
|
58
|
+
await publishShots(proc, slug, report, runDir, workDir);
|
|
59
|
+
const imageBase = `https://github.com/${slug}/blob/${REPORTS_BRANCH}/runs/${report.id}`;
|
|
60
|
+
const comments = JSON.parse(await gh(proc, ["api", `repos/${slug}/issues/${pr.number}/comments`, "--paginate", "--slurp"], root));
|
|
61
|
+
const existing = comments.flat().find((comment) => comment.body.startsWith(MARKER));
|
|
62
|
+
const history = [historyLine(report), ...(existing ? previousHistory(existing.body) : [])].slice(0, HISTORY);
|
|
63
|
+
const body = [MARKER, renderMarkdown(report, (path) => `${imageBase}/${path}?raw=true`, { maxShots: 6 }), "### Run history", "", ...history.map((line) => `- ${line}`), ""].join("\n");
|
|
64
|
+
const payload = join(workDir, "comment.json");
|
|
65
|
+
await writeFile(payload, JSON.stringify({ body }));
|
|
66
|
+
if (existing) {
|
|
67
|
+
await gh(proc, ["api", "-X", "PATCH", `repos/${slug}/issues/comments/${existing.id}`, "--input", payload], root);
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
await gh(proc, ["api", "-X", "POST", `repos/${slug}/issues/${pr.number}/comments`, "--input", payload], root);
|
|
71
|
+
}
|
|
72
|
+
return pr.url;
|
|
73
|
+
}
|
|
74
|
+
async function publishShots(proc, slug, report, runDir, workDir) {
|
|
75
|
+
const dir = join(workDir, "reports-branch");
|
|
76
|
+
await rm(dir, { recursive: true, force: true });
|
|
77
|
+
await mkdir(dir, { recursive: true });
|
|
78
|
+
const git = (args) => proc("git", ["-C", dir, "-c", "credential.helper=", "-c", "credential.helper=!gh auth git-credential", "-c", "user.name=proofy", "-c", "user.email=proofy@users.noreply.github.com", ...args], {
|
|
79
|
+
timeoutMs: 300_000,
|
|
80
|
+
});
|
|
81
|
+
await git(["init", "--quiet"]);
|
|
82
|
+
await git(["remote", "add", "origin", `https://github.com/${slug}.git`]);
|
|
83
|
+
if ((await git(["fetch", "--quiet", "--depth", "1", "origin", REPORTS_BRANCH])).code === 0) {
|
|
84
|
+
await git(["checkout", "--quiet", "FETCH_HEAD"]);
|
|
85
|
+
}
|
|
86
|
+
const target = join(dir, "runs", report.id);
|
|
87
|
+
await mkdir(target, { recursive: true });
|
|
88
|
+
for (const flow of report.flows) {
|
|
89
|
+
try {
|
|
90
|
+
await cp(join(runDir, "flows", flow.name), join(target, "flows", flow.name), { recursive: true, filter: (source) => !source.endsWith(".jsonl") });
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
// A flow that never reached a screenshot.
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
await cp(join(runDir, "report.md"), join(target, "report.md"));
|
|
97
|
+
const runs = (await readdir(join(dir, "runs"))).sort();
|
|
98
|
+
for (const old of runs.slice(0, Math.max(0, runs.length - KEEP))) {
|
|
99
|
+
await rm(join(dir, "runs", old), { recursive: true, force: true });
|
|
100
|
+
}
|
|
101
|
+
await writeFile(join(dir, "README.md"), `Screenshots for proofy reports. Replaced on every run and keeps the last ${KEEP} runs.\n`);
|
|
102
|
+
await git(["checkout", "--quiet", "--orphan", `proofy-${report.id}`]);
|
|
103
|
+
await git(["add", "-A"]);
|
|
104
|
+
const commit = await git(["commit", "--quiet", "-m", `proofy run ${report.id}`]);
|
|
105
|
+
if (commit.code !== 0)
|
|
106
|
+
throw new CliError("Committing screenshots failed", "ENVIRONMENT_ERROR", [tail(commit.stderr, 5)]);
|
|
107
|
+
const push = await git(["push", "--quiet", "--force", "origin", `HEAD:refs/heads/${REPORTS_BRANCH}`]);
|
|
108
|
+
if (push.code !== 0)
|
|
109
|
+
throw new CliError(`Pushing screenshots to ${REPORTS_BRANCH} failed`, "ENVIRONMENT_ERROR", [tail(push.stderr, 5), "Check that gh is logged in with push access"]);
|
|
110
|
+
}
|
|
111
|
+
function historyLine(report) {
|
|
112
|
+
const counts = report.flows.map((flow) => `${flow.name} ${flow.status}`).join(", ");
|
|
113
|
+
return `${report.id}: ${verdict(report)}${report.source.commit ? ` on ${report.source.commit.slice(0, 7)}` : ""}${counts ? ` (${counts})` : ""}`;
|
|
114
|
+
}
|
|
115
|
+
function previousHistory(body) {
|
|
116
|
+
const section = body.split("### Run history")[1] ?? "";
|
|
117
|
+
return section
|
|
118
|
+
.split("\n")
|
|
119
|
+
.filter((line) => line.startsWith("- "))
|
|
120
|
+
.map((line) => line.slice(2));
|
|
121
|
+
}
|
package/dist/human.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { Verdict } from "./run/report.js";
|
|
2
|
+
import type { RunReport } from "./run/results.js";
|
|
3
|
+
import { type Style } from "./style.js";
|
|
4
|
+
export type Hint = [command: string, description: string];
|
|
5
|
+
export interface Check {
|
|
6
|
+
label: string;
|
|
7
|
+
ok: boolean;
|
|
8
|
+
value: string;
|
|
9
|
+
}
|
|
10
|
+
export interface HomeView {
|
|
11
|
+
project: string;
|
|
12
|
+
checks: Check[];
|
|
13
|
+
flows: Array<{
|
|
14
|
+
name: string;
|
|
15
|
+
state: string;
|
|
16
|
+
reason?: string;
|
|
17
|
+
}>;
|
|
18
|
+
orphans: string[];
|
|
19
|
+
lastRun: string;
|
|
20
|
+
}
|
|
21
|
+
export interface InitView {
|
|
22
|
+
root: string;
|
|
23
|
+
found: string[];
|
|
24
|
+
written: string[];
|
|
25
|
+
skipped: string[];
|
|
26
|
+
}
|
|
27
|
+
export declare function renderHome(view: HomeView, hints: Hint[], style: Style): string;
|
|
28
|
+
export declare function renderInit(view: InitView, hints: Hint[], style: Style): string;
|
|
29
|
+
export declare function renderRun(report: RunReport, result: Verdict, reportPath: string, hints: Hint[], style: Style, posted?: string): string;
|
|
30
|
+
export declare function renderErrorHuman(message: string, code: string, suggestions: string[], style: Style): string;
|
package/dist/human.js
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { GLYPH, padVisible, painter, stripAnsi, truncate } from "./style.js";
|
|
2
|
+
export function renderHome(view, hints, style) {
|
|
3
|
+
const p = painter(style);
|
|
4
|
+
const ready = view.checks.every((check) => check.ok);
|
|
5
|
+
const lines = [`${p.bold(view.project)} ${ready ? p.green("set up") : p.yellow("setup incomplete")}`, "", p.bold("SETUP")];
|
|
6
|
+
lines.push(...checkRows(view.checks, p, style));
|
|
7
|
+
lines.push("", p.bold(`FLOWS (${view.flows.length})`));
|
|
8
|
+
if (view.flows.length === 0) {
|
|
9
|
+
lines.push(p.dim(" none yet"));
|
|
10
|
+
}
|
|
11
|
+
const width = Math.max(0, ...view.flows.map((flow) => flow.name.length));
|
|
12
|
+
for (const flow of view.flows) {
|
|
13
|
+
lines.push(` ${padVisible(flow.name, width)} ${stateText(flow.state, p)}${flow.reason ? p.dim(` ${flow.reason}`) : ""}`);
|
|
14
|
+
}
|
|
15
|
+
for (const orphan of view.orphans) {
|
|
16
|
+
lines.push(` ${padVisible(orphan, width)} ${p.red(`${GLYPH.fail} test without requirements`)}`);
|
|
17
|
+
}
|
|
18
|
+
lines.push("", `${p.bold("LAST RUN")} ${view.lastRun}`);
|
|
19
|
+
return [lines.join("\n"), nextSection(hints, p, style)].join("\n\n");
|
|
20
|
+
}
|
|
21
|
+
export function renderInit(view, hints, style) {
|
|
22
|
+
const p = painter(style);
|
|
23
|
+
const lines = [p.bold(`proofy init ${view.root}`)];
|
|
24
|
+
if (view.found.length > 0) {
|
|
25
|
+
lines.push("", p.bold("FOUND"), ...view.found.map((line) => ` ${truncate(line, style.width - 2)}`));
|
|
26
|
+
}
|
|
27
|
+
lines.push("", p.bold("WROTE"));
|
|
28
|
+
lines.push(...(view.written.length > 0 ? view.written.map((file) => ` ${p.green(GLYPH.ok)} ${file}`) : [p.dim(" nothing, everything was there")]));
|
|
29
|
+
if (view.skipped.length > 0) {
|
|
30
|
+
lines.push("", p.bold("KEPT"), ...view.skipped.map((file) => p.dim(` ${file} already exists`)));
|
|
31
|
+
}
|
|
32
|
+
return [lines.join("\n"), nextSection(hints, p, style)].join("\n\n");
|
|
33
|
+
}
|
|
34
|
+
const VERDICT_TEXT = {
|
|
35
|
+
pass: "PASS every requirement of every approved flow held",
|
|
36
|
+
fail: "FAIL an approved flow has a requirement that did not hold",
|
|
37
|
+
untrusted: "NOT TRUSTED some flows are not approved, so this proves nothing yet",
|
|
38
|
+
environment: "ENVIRONMENT FAILURE the sandbox did not work; this says nothing about the app",
|
|
39
|
+
};
|
|
40
|
+
export function renderRun(report, result, reportPath, hints, style, posted) {
|
|
41
|
+
const p = painter(style);
|
|
42
|
+
const color = result === "pass" ? p.green : result === "untrusted" ? p.yellow : p.red;
|
|
43
|
+
const lines = [`${color(p.bold(VERDICT_TEXT[result]))}`, p.dim(`run ${report.id} ${reportPath}`)];
|
|
44
|
+
if (report.environmentError) {
|
|
45
|
+
lines.push("", ...report.environmentError.split("\n").slice(0, 8).map((line) => ` ${p.red(truncate(line, style.width - 2))}`));
|
|
46
|
+
}
|
|
47
|
+
for (const flow of report.flows) {
|
|
48
|
+
const trust = flow.approved ? "" : p.yellow(" not approved");
|
|
49
|
+
lines.push("", `${p.bold(flow.name)} ${flow.status === "passed" ? p.green("passed") : p.red("failed")}${trust}`);
|
|
50
|
+
for (const requirement of flow.requirements) {
|
|
51
|
+
const detail = requirement.result === "pass"
|
|
52
|
+
? requirement.text
|
|
53
|
+
: requirement.result === "not-covered"
|
|
54
|
+
? `${requirement.text} ${p.dim(`(not covered: ${requirement.why ?? ""})`)}`
|
|
55
|
+
: requirement.result === "not-checked"
|
|
56
|
+
? `${requirement.text} ${p.yellow("(no claim checks it)")}`
|
|
57
|
+
: requirement.result === "not-reached"
|
|
58
|
+
? `${requirement.text} ${p.red("(not reached: the test stopped before checking it)")}`
|
|
59
|
+
: `${requirement.text} ${p.red(`expected ${requirement.expected ?? "?"}, got ${requirement.actual ?? "?"}`)}`;
|
|
60
|
+
lines.push(` ${outcomeMark(requirement.result, p)} ${padVisible(requirement.id, 4)} ${truncateVisible(detail, style.width - 9)}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const broken = report.flows.flatMap((flow) => flow.tests.filter((test) => test.status !== "passed" && test.error && !/Requirements not met/.test(test.error)).map((test) => ({ flow: flow.name, test })));
|
|
64
|
+
if (broken.length > 0) {
|
|
65
|
+
lines.push("", p.bold("TEST ERRORS"));
|
|
66
|
+
for (const { flow, test } of broken) {
|
|
67
|
+
const first = (test.error ?? "").split("\n").filter((line) => line.trim()).slice(0, 3);
|
|
68
|
+
lines.push(` ${p.bold(flow)}: ${test.title} (${test.status})`, ...first.map((line) => ` ${p.red(truncate(line.trim(), style.width - 4))}`));
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
const refused = report.outside.filter((call) => call.handledAs === "refused").map((call) => call.host);
|
|
72
|
+
if (report.outside.length > 0) {
|
|
73
|
+
const handled = report.outside.filter((call) => call.handledAs !== "refused").map((call) => `${call.host} ${call.handledAs} x${call.count}`);
|
|
74
|
+
lines.push("", p.bold("OUTSIDE CALLS"));
|
|
75
|
+
if (handled.length > 0)
|
|
76
|
+
lines.push(` ${truncate(handled.join(", "), style.width - 2)}`);
|
|
77
|
+
if (refused.length > 0)
|
|
78
|
+
lines.push(` ${p.yellow(truncate(`refused: ${refused.join(", ")}`, style.width - 2))}`);
|
|
79
|
+
}
|
|
80
|
+
if (posted)
|
|
81
|
+
lines.push("", `posted on ${posted}`);
|
|
82
|
+
return [lines.join("\n"), nextSection(hints, p, style)].join("\n\n");
|
|
83
|
+
}
|
|
84
|
+
function outcomeMark(result, p) {
|
|
85
|
+
switch (result) {
|
|
86
|
+
case "pass":
|
|
87
|
+
return p.green(GLYPH.ok);
|
|
88
|
+
case "not-covered":
|
|
89
|
+
return p.dim("-");
|
|
90
|
+
case "not-checked":
|
|
91
|
+
return p.yellow(GLYPH.warn);
|
|
92
|
+
default:
|
|
93
|
+
return p.red(GLYPH.fail);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// Keeps colors when the line fits and drops them when it has to be cut.
|
|
97
|
+
function truncateVisible(text, max) {
|
|
98
|
+
const plain = stripAnsi(text);
|
|
99
|
+
return plain.length > max ? truncate(plain, max) : text;
|
|
100
|
+
}
|
|
101
|
+
export function renderErrorHuman(message, code, suggestions, style) {
|
|
102
|
+
const p = painter(style);
|
|
103
|
+
const lines = [`${p.red(`${GLYPH.fail} ${message}`)} ${p.dim(`(${code})`)}`];
|
|
104
|
+
for (const suggestion of suggestions) {
|
|
105
|
+
lines.push(` ${suggestion}`);
|
|
106
|
+
}
|
|
107
|
+
return lines.join("\n");
|
|
108
|
+
}
|
|
109
|
+
function checkRows(checks, p, style) {
|
|
110
|
+
const width = Math.max(...checks.map((check) => check.label.length));
|
|
111
|
+
return checks.map((check) => {
|
|
112
|
+
const mark = check.ok ? p.green(GLYPH.ok) : p.yellow(GLYPH.warn);
|
|
113
|
+
const value = truncate(check.value, Math.max(10, style.width - width - 7));
|
|
114
|
+
return ` ${mark} ${padVisible(check.label, width)} ${check.ok ? value : p.yellow(value)}`;
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
function stateText(state, p) {
|
|
118
|
+
switch (state) {
|
|
119
|
+
case "approved":
|
|
120
|
+
return p.green(`${GLYPH.ok} approved`);
|
|
121
|
+
case "ready":
|
|
122
|
+
return p.cyan("ready, waiting for the human to approve");
|
|
123
|
+
case "draft":
|
|
124
|
+
return p.yellow("draft, no test yet");
|
|
125
|
+
case "unproven":
|
|
126
|
+
return p.yellow("unproven");
|
|
127
|
+
case "unstable":
|
|
128
|
+
return p.red("unstable, runs differ");
|
|
129
|
+
case "out-of-date":
|
|
130
|
+
return p.yellow("out of date, changed since approval");
|
|
131
|
+
default:
|
|
132
|
+
return state;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
function nextSection(hints, p, style) {
|
|
136
|
+
const width = Math.max(0, ...hints.filter(([, description]) => description).map(([command]) => command.length));
|
|
137
|
+
return [
|
|
138
|
+
p.bold("NEXT"),
|
|
139
|
+
...hints.map(([command, description]) => description
|
|
140
|
+
? ` ${padVisible(p.cyan(command), width)} ${p.dim(truncate(description, Math.max(10, style.width - width - 4)))}`
|
|
141
|
+
: wrap(command, style.width, " ", " ")),
|
|
142
|
+
].join("\n");
|
|
143
|
+
}
|
|
144
|
+
// Onboarding sentences are the instructions; they wrap instead of being cut off.
|
|
145
|
+
function wrap(text, width, first, rest) {
|
|
146
|
+
const lines = [];
|
|
147
|
+
let line = first;
|
|
148
|
+
for (const word of text.split(" ")) {
|
|
149
|
+
const indent = lines.length === 0 ? first : rest;
|
|
150
|
+
if (line.length > indent.length && line.length + 1 + word.length > width) {
|
|
151
|
+
lines.push(line);
|
|
152
|
+
line = rest + word;
|
|
153
|
+
}
|
|
154
|
+
else {
|
|
155
|
+
line = line.length > indent.length ? `${line} ${word}` : line + word;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
lines.push(line);
|
|
159
|
+
return lines.join("\n");
|
|
160
|
+
}
|
package/dist/output.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export type Data = Record<string, unknown>;
|
|
2
|
+
export interface Result {
|
|
3
|
+
data: Data;
|
|
4
|
+
help: string[];
|
|
5
|
+
}
|
|
6
|
+
export declare function renderResult(result: Result): string;
|
|
7
|
+
export declare function renderJson(value: unknown): string;
|
|
8
|
+
export declare function renderError(message: string, code: string, suggestions: string[]): string;
|
package/dist/output.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { encode } from "@toon-format/toon";
|
|
2
|
+
// Help is rendered one hint per line, which TOON's inline primitive arrays would not do.
|
|
3
|
+
export function renderResult(result) {
|
|
4
|
+
const parts = [];
|
|
5
|
+
if (Object.keys(result.data).length > 0) {
|
|
6
|
+
parts.push(encode(result.data));
|
|
7
|
+
}
|
|
8
|
+
if (result.help.length > 0) {
|
|
9
|
+
parts.push(renderHelp(result.help));
|
|
10
|
+
}
|
|
11
|
+
return parts.join("\n");
|
|
12
|
+
}
|
|
13
|
+
export function renderJson(value) {
|
|
14
|
+
return JSON.stringify(value, null, 2);
|
|
15
|
+
}
|
|
16
|
+
export function renderError(message, code, suggestions) {
|
|
17
|
+
return renderResult({ data: { error: message, code }, help: suggestions });
|
|
18
|
+
}
|
|
19
|
+
function renderHelp(help) {
|
|
20
|
+
return [`help[${help.length}]:`, ...help.map((line) => ` ${line}`)].join("\n");
|
|
21
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface Paths {
|
|
2
|
+
root: string;
|
|
3
|
+
e2e: string;
|
|
4
|
+
config: string;
|
|
5
|
+
specs: string;
|
|
6
|
+
tests: string;
|
|
7
|
+
standins: string;
|
|
8
|
+
}
|
|
9
|
+
/** The git repository's top folder, if there is one. */
|
|
10
|
+
export declare function gitRoot(cwd: string): Promise<string | undefined>;
|
|
11
|
+
/**
|
|
12
|
+
* The project proofy works on: the nearest folder from cwd upwards, within the repository, that has
|
|
13
|
+
* e2e/proofy.json. Otherwise the repository root, or cwd outside git. Apps in a subfolder of a monorepo work too.
|
|
14
|
+
*/
|
|
15
|
+
export declare function findRoot(cwd: string): Promise<string>;
|
|
16
|
+
export declare function pathsFor(root: string): Paths;
|
|
17
|
+
/** Flow names: one per requirements file in e2e/specs. */
|
|
18
|
+
export declare function flowNames(paths: Paths): Promise<string[]>;
|
|
19
|
+
/** Tests without a requirements file: they cannot be traced to anything the user asked for. */
|
|
20
|
+
export declare function orphanTests(paths: Paths): Promise<string[]>;
|
|
21
|
+
export declare function exists(path: string): Promise<boolean>;
|
package/dist/project.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { access, readdir, realpath } from "node:fs/promises";
|
|
3
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
4
|
+
import { promisify } from "node:util";
|
|
5
|
+
const run = promisify(execFile);
|
|
6
|
+
/** The git repository's top folder, if there is one. */
|
|
7
|
+
export async function gitRoot(cwd) {
|
|
8
|
+
try {
|
|
9
|
+
const { stdout } = await run("git", ["rev-parse", "--show-toplevel"], { cwd, timeout: 2000 });
|
|
10
|
+
return stdout.trim() || undefined;
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The project proofy works on: the nearest folder from cwd upwards, within the repository, that has
|
|
18
|
+
* e2e/proofy.json. Otherwise the repository root, or cwd outside git. Apps in a subfolder of a monorepo work too.
|
|
19
|
+
*/
|
|
20
|
+
export async function findRoot(cwd) {
|
|
21
|
+
const top = await gitRoot(cwd);
|
|
22
|
+
// git reports real paths, so compare against the real path of cwd, not a symlinked one such as /var on macOS.
|
|
23
|
+
let dir = await realpath(cwd).catch(() => resolve(cwd));
|
|
24
|
+
for (;;) {
|
|
25
|
+
if (await exists(join(dir, "e2e", "proofy.json")))
|
|
26
|
+
return dir;
|
|
27
|
+
const parent = dirname(dir);
|
|
28
|
+
if (dir === top || parent === dir)
|
|
29
|
+
break;
|
|
30
|
+
dir = parent;
|
|
31
|
+
}
|
|
32
|
+
return top ?? cwd;
|
|
33
|
+
}
|
|
34
|
+
export function pathsFor(root) {
|
|
35
|
+
const e2e = join(root, "e2e");
|
|
36
|
+
return {
|
|
37
|
+
root,
|
|
38
|
+
e2e,
|
|
39
|
+
config: join(e2e, "proofy.json"),
|
|
40
|
+
specs: join(e2e, "specs"),
|
|
41
|
+
tests: join(e2e, "tests"),
|
|
42
|
+
standins: join(e2e, "standins"),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/** Flow names: one per requirements file in e2e/specs. */
|
|
46
|
+
export async function flowNames(paths) {
|
|
47
|
+
return (await stems(paths.specs, ".md")).sort();
|
|
48
|
+
}
|
|
49
|
+
/** Tests without a requirements file: they cannot be traced to anything the user asked for. */
|
|
50
|
+
export async function orphanTests(paths) {
|
|
51
|
+
const specs = new Set(await stems(paths.specs, ".md"));
|
|
52
|
+
return (await stems(paths.tests, ".spec.ts")).filter((name) => !specs.has(name)).sort();
|
|
53
|
+
}
|
|
54
|
+
async function stems(dir, suffix) {
|
|
55
|
+
try {
|
|
56
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
57
|
+
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(suffix)).map((entry) => basename(entry.name, suffix));
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return [];
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
export async function exists(path) {
|
|
64
|
+
try {
|
|
65
|
+
await access(path);
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { Paths } from "../project.js";
|
|
2
|
+
import type { Fingerprint } from "./fingerprint.js";
|
|
3
|
+
export interface Approval extends Fingerprint {
|
|
4
|
+
approvedAt: string;
|
|
5
|
+
/** The runs whose identical results the approval rests on. */
|
|
6
|
+
runs: string[];
|
|
7
|
+
requirements: number;
|
|
8
|
+
}
|
|
9
|
+
export interface Approvals {
|
|
10
|
+
version: 1;
|
|
11
|
+
flows: Record<string, Approval>;
|
|
12
|
+
}
|
|
13
|
+
export declare function approvalsFile(paths: Paths): string;
|
|
14
|
+
export declare function readApprovals(paths: Paths): Promise<Approvals>;
|
|
15
|
+
export declare function writeApprovals(paths: Paths, approvals: Approvals): Promise<void>;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { CliError } from "../errors.js";
|
|
4
|
+
export function approvalsFile(paths) {
|
|
5
|
+
return join(paths.e2e, "proofy.lock.json");
|
|
6
|
+
}
|
|
7
|
+
export async function readApprovals(paths) {
|
|
8
|
+
let raw;
|
|
9
|
+
try {
|
|
10
|
+
raw = await readFile(approvalsFile(paths), "utf8");
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return { version: 1, flows: {} };
|
|
14
|
+
}
|
|
15
|
+
try {
|
|
16
|
+
const parsed = JSON.parse(raw);
|
|
17
|
+
return { version: 1, flows: parsed.flows ?? {} };
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
throw new CliError("e2e/proofy.lock.json is not valid JSON", "CONFIG_INVALID", ["Restore it from git; it records approvals"]);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export async function writeApprovals(paths, approvals) {
|
|
24
|
+
const flows = Object.fromEntries(Object.entries(approvals.flows).sort(([a], [b]) => a.localeCompare(b)));
|
|
25
|
+
await writeFile(approvalsFile(paths), `${JSON.stringify({ version: 1, flows }, null, 2)}\n`);
|
|
26
|
+
}
|