@unframework.dev/testing 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +536 -2
- package/dist/browser/axe.js +34 -0
- package/dist/browser/behaviour.js +59 -0
- package/dist/browser/capabilities.js +58 -0
- package/dist/browser/clock.d.ts +17 -0
- package/dist/browser/clock.js +69 -0
- package/dist/browser/console.d.ts +27 -0
- package/dist/browser/console.js +142 -0
- package/dist/browser/determinism.js +108 -0
- package/dist/browser/element-timeout.js +28 -0
- package/dist/browser/events.js +68 -0
- package/dist/browser/geometry.js +135 -0
- package/dist/browser/keys.js +62 -0
- package/dist/browser/matchers.d.ts +39 -0
- package/dist/browser/matchers.js +78 -0
- package/dist/browser/navigation.js +38 -0
- package/dist/browser/parity.d.ts +10 -0
- package/dist/browser/parity.js +93 -0
- package/dist/browser/spec.d.ts +25 -0
- package/dist/browser/spec.js +38 -0
- package/dist/browser/target.d.ts +16 -0
- package/dist/browser/target.js +27 -0
- package/dist/browser/text-runs.js +68 -0
- package/dist/browser/thrown.js +71 -0
- package/dist/browser/trace.d.ts +21 -0
- package/dist/browser/trace.js +73 -0
- package/dist/browser/user.d.ts +45 -0
- package/dist/browser/user.js +184 -0
- package/dist/browser/view.d.ts +114 -0
- package/dist/browser/view.js +282 -0
- package/dist/browser/visual.d.ts +20 -0
- package/dist/browser/visual.js +36 -0
- package/dist/cli.d.ts +23 -0
- package/dist/cli.js +90 -0
- package/dist/commands.d.ts +107 -0
- package/dist/config/broadcast.js +29 -0
- package/dist/config/cases.d.ts +28 -0
- package/dist/config/cases.js +94 -0
- package/dist/config/index.d.ts +82 -0
- package/dist/config/index.js +295 -0
- package/dist/config/project-vite.d.ts +40 -0
- package/dist/config/project-vite.js +152 -0
- package/dist/diff.d.ts +14 -0
- package/dist/diff.js +92 -0
- package/dist/dom/serialize.js +236 -0
- package/dist/errors.d.ts +13 -0
- package/dist/errors.js +38 -0
- package/dist/harness.d.ts +189 -0
- package/dist/harness.js +53 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +12 -0
- package/dist/internal.d.ts +8 -0
- package/dist/internal.js +6 -0
- package/dist/layers.d.ts +183 -0
- package/dist/layers.js +241 -0
- package/dist/node/browser.d.ts +33 -0
- package/dist/node/browser.js +64 -0
- package/dist/node/commands.d.ts +137 -0
- package/dist/node/commands.js +139 -0
- package/dist/node/context.d.ts +28 -0
- package/dist/node/context.js +55 -0
- package/dist/node/events.d.ts +25 -0
- package/dist/node/events.js +50 -0
- package/dist/node/finished.d.ts +12 -0
- package/dist/node/finished.js +28 -0
- package/dist/node/index.d.ts +21 -0
- package/dist/node/index.js +20 -0
- package/dist/node/matrix.d.ts +219 -0
- package/dist/node/matrix.js +309 -0
- package/dist/node/mode.d.ts +103 -0
- package/dist/node/mode.js +204 -0
- package/dist/node/names.d.ts +4 -0
- package/dist/node/names.js +5 -0
- package/dist/node/policy.d.ts +44 -0
- package/dist/node/policy.js +140 -0
- package/dist/node/reporter.d.ts +101 -0
- package/dist/node/reporter.js +251 -0
- package/dist/node/summary.d.ts +99 -0
- package/dist/node/summary.js +352 -0
- package/dist/node/thrown.js +78 -0
- package/dist/node/viewport.d.ts +7 -0
- package/dist/node/viewport.js +8 -0
- package/dist/node/visual.d.ts +85 -0
- package/dist/node/visual.js +373 -0
- package/dist/node_modules/.pnpm/playwright-core@1.63.0/node_modules/playwright-core/types/protocol.d.ts +23930 -0
- package/dist/node_modules/.pnpm/playwright-core@1.63.0/node_modules/playwright-core/types/structs.d.ts +22 -0
- package/dist/node_modules/.pnpm/playwright-core@1.63.0/node_modules/playwright-core/types/types.d.ts +22221 -0
- package/dist/normalize/display.js +309 -0
- package/dist/normalize/index.d.ts +34 -0
- package/dist/normalize/index.js +38 -0
- package/dist/normalize/pipeline.d.ts +19 -0
- package/dist/normalize/pipeline.js +45 -0
- package/dist/normalize/print.js +103 -0
- package/dist/normalize/rules/angular-hosts.js +53 -0
- package/dist/normalize/rules/aria-hosts.js +48 -0
- package/dist/normalize/rules/comments.js +16 -0
- package/dist/normalize/rules/control-values.js +65 -0
- package/dist/normalize/rules/engine-aliases.js +34 -0
- package/dist/normalize/rules/framework-attributes.js +85 -0
- package/dist/normalize/rules/generated-ids.js +64 -0
- package/dist/normalize/rules/ordering.js +39 -0
- package/dist/normalize/rules/qwik-styles.js +21 -0
- package/dist/normalize/rules/qwik-templates.js +20 -0
- package/dist/normalize/rules/values.js +125 -0
- package/dist/normalize/rules/whitespace.js +381 -0
- package/dist/normalize/style.js +121 -0
- package/dist/normalize/targets.d.ts +8 -0
- package/dist/normalize/targets.js +17 -0
- package/dist/normalize/tree.d.ts +1 -0
- package/dist/normalize/tree.js +151 -0
- package/dist/setup.d.ts +1 -0
- package/dist/setup.js +149 -0
- package/dist/ssr/index.d.ts +10 -0
- package/dist/ssr/index.js +103 -0
- package/dist/ssr/spec.d.ts +1 -0
- package/dist/ssr/spec.js +7 -0
- package/dist/targets/angular.d.ts +1 -0
- package/dist/targets/angular.js +5 -0
- package/dist/targets/astro.d.ts +1 -0
- package/dist/targets/astro.js +5 -0
- package/dist/targets/qwik.d.ts +1 -0
- package/dist/targets/qwik.js +5 -0
- package/dist/targets/react.d.ts +1 -0
- package/dist/targets/react.js +5 -0
- package/dist/targets/solid.d.ts +1 -0
- package/dist/targets/solid.js +5 -0
- package/dist/targets/svelte.d.ts +1 -0
- package/dist/targets/svelte.js +5 -0
- package/dist/targets/vue.d.ts +1 -0
- package/dist/targets/vue.js +5 -0
- package/dist/visual-types.d.ts +92 -0
- package/dist/visual-types.js +24 -0
- package/package.json +146 -3
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import { BROWSER_ENGINES } from "../harness.js";
|
|
2
|
+
import { existsSync, readdirSync, rmSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
//#region src/node/mode.ts
|
|
5
|
+
/**
|
|
6
|
+
* The image and platform the committed Linux baselines (`*-chromium-linux.png`, geometry) come
|
|
7
|
+
* from: CI's browser jobs run in it, and `pnpm test:baselines` exports it as
|
|
8
|
+
* `UF_BASELINE_ENVIRONMENT`. Only a run in it writes them. The image follows the playwright
|
|
9
|
+
* version of the catalog, which ships its browsers.
|
|
10
|
+
*/
|
|
11
|
+
const BASELINE_ENVIRONMENT = "mcr.microsoft.com/playwright:v1.63.0-noble linux/amd64";
|
|
12
|
+
/** Resolves the run's mode, refusing combinations that would write the wrong thing. */
|
|
13
|
+
function resolveHarnessMode(input) {
|
|
14
|
+
const { env } = input;
|
|
15
|
+
const ci = isCI(env);
|
|
16
|
+
if (input.argv.some((arg) => arg === "-u" || arg === "--update" || arg.startsWith("--update="))) throw new Error(`[uf] Vitest's -u does not update the harness's artefacts, and would let every project write the shared ones. Run ${input.project ? "`unframework test --update`" : "`pnpm test:update` (UF_UPDATE=1)"} instead.`);
|
|
17
|
+
const update = flag(env, "UF_UPDATE");
|
|
18
|
+
if (update && ci) throw new Error("[uf] CI never writes artefacts: refusing UF_UPDATE while CI is set.");
|
|
19
|
+
const canary = env.UF_CANARY?.trim() || null;
|
|
20
|
+
if (canary && update) throw new Error(`[uf] A canary run never writes artefacts: refusing UF_UPDATE with UF_CANARY=${canary}.`);
|
|
21
|
+
const baselineEnvironment = env.UF_BASELINE_ENVIRONMENT?.trim() || null;
|
|
22
|
+
const here = `${input.platform}/${input.arch ?? process.arch}`;
|
|
23
|
+
if (baselineEnvironment === "mcr.microsoft.com/playwright:v1.63.0-noble linux/amd64" && here !== "linux/x64") throw new Error(`[uf] UF_BASELINE_ENVIRONMENT says ${baselineEnvironment}, but this run is on ${here}.`);
|
|
24
|
+
const forced = env.UF_PIXELS?.trim();
|
|
25
|
+
if (forced && forced !== "baseline" && forced !== "live") throw new Error(`[uf] UF_PIXELS must be "baseline" or "live", not "${forced}".`);
|
|
26
|
+
const browser = resolveBrowser(env);
|
|
27
|
+
if (input.liveOnly) {
|
|
28
|
+
if (forced) throw new Error(`[uf] This run has no committed baselines: UF_PIXELS=${forced} cannot apply.`);
|
|
29
|
+
if (browser !== "chromium") throw new Error(`[uf] This run has no committed baselines and runs in Chromium alone: UF_BROWSER=${browser} cannot apply.`);
|
|
30
|
+
return {
|
|
31
|
+
update,
|
|
32
|
+
pixels: "live",
|
|
33
|
+
canary,
|
|
34
|
+
ci,
|
|
35
|
+
baselineEnvironment,
|
|
36
|
+
browser
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
if (browser !== "chromium") {
|
|
40
|
+
if (update) throw new Error(`[uf] Every committed artefact is Chromium's: refusing UF_UPDATE with UF_BROWSER=${browser}.`);
|
|
41
|
+
if (forced === "baseline") throw new Error(`[uf] The committed baselines are Chromium's: ${browser} compares live, with the reference's capture in the same run. Unset UF_PIXELS.`);
|
|
42
|
+
return {
|
|
43
|
+
update,
|
|
44
|
+
pixels: "live",
|
|
45
|
+
canary,
|
|
46
|
+
ci,
|
|
47
|
+
baselineEnvironment,
|
|
48
|
+
browser
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
const pixels = forced ?? input.project?.pixels ?? (ci || baselineEnvironment ? "baseline" : "live");
|
|
52
|
+
if (ci && pixels === "live" && !input.project) throw new Error("[uf] CI compares pixels against the committed Linux baselines, never live.");
|
|
53
|
+
if (ci && pixels === "baseline" && input.platform !== "linux") throw new Error(`[uf] CI compares pixels against the committed Linux baselines, so it runs on Linux, not ${input.platform}.`);
|
|
54
|
+
if (update && pixels === "baseline" && baselineEnvironment !== "mcr.microsoft.com/playwright:v1.63.0-noble linux/amd64") {
|
|
55
|
+
if (input.project) throw new Error(`[uf] \`unframework test --update\` with \`pixels: "baseline"\` would write the committed Linux baselines, which only ${BASELINE_ENVIRONMENT} writes; this run is ${baselineEnvironment ?? `on ${here}, outside it`}. Run \`UF_PIXELS=live unframework test --update\` to update every other expectation.`);
|
|
56
|
+
throw new Error(`[uf] UF_UPDATE with baseline pixels would write the committed Linux baselines, which only ${BASELINE_ENVIRONMENT} writes (\`pnpm test:baselines\`); this run is ${baselineEnvironment ?? `on ${here}, outside it`}. Unset UF_PIXELS (or set it to live) to update everything else.`);
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
update,
|
|
60
|
+
pixels,
|
|
61
|
+
canary,
|
|
62
|
+
ci,
|
|
63
|
+
baselineEnvironment,
|
|
64
|
+
browser
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** The engine `UF_BROWSER` names, Chromium when it is unset. */
|
|
68
|
+
function resolveBrowser(env) {
|
|
69
|
+
const value = env.UF_BROWSER?.trim() || "chromium";
|
|
70
|
+
const browser = BROWSER_ENGINES.find((engine) => engine === value);
|
|
71
|
+
if (!browser) throw new Error(`[uf] UF_BROWSER must be chromium, firefox or webkit, not "${value}".`);
|
|
72
|
+
return browser;
|
|
73
|
+
}
|
|
74
|
+
/** Whether `CI` is set to anything but `0` or `false`, as CI providers set it. */
|
|
75
|
+
function isCI(env) {
|
|
76
|
+
return flag(env, "CI", { allowAny: true });
|
|
77
|
+
}
|
|
78
|
+
function flag(env, name, options = {}) {
|
|
79
|
+
const value = env[name]?.trim();
|
|
80
|
+
if (!value || value === "0" || value === "false") return false;
|
|
81
|
+
if (value === "1" || value === "true" || options.allowAny) return true;
|
|
82
|
+
throw new Error(`[uf] ${name} must be 1 or 0, not "${value}".`);
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* A project's `sequence.groupOrder`. Groups run one after another, so a project that reads what
|
|
86
|
+
* another writes in the same run must sit in a later group:
|
|
87
|
+
*
|
|
88
|
+
* - update: compile (and the harness's own tests, and the budgets) 0, then the toolchains and the reference's
|
|
89
|
+
* ssr and browser projects 1, then every follower 2; where a case names its own reference
|
|
90
|
+
* (`caseReferences`, ADR-0057), that target's projects run at 2 alone, as they follow the
|
|
91
|
+
* reference everywhere else, and the other followers at 3;
|
|
92
|
+
* - check with live pixels: the reference browser project 0, the other browser projects 1; the
|
|
93
|
+
* browser projects of the targets some case names at 1 alone, and the others at 2;
|
|
94
|
+
* - check with live pixels in Firefox or WebKit (ADR-0081): every browser project alone, the
|
|
95
|
+
* reference at 1, then the targets some case names, then each other target in the order of
|
|
96
|
+
* `targets`. Five followers at once in one engine ran the nightly's WebKit job out of memory.
|
|
97
|
+
* Each of those projects also runs one file at a time (`fileParallelism: false`), so the
|
|
98
|
+
* reference is not at 0: Vitest moves a project at 0 that runs one file at a time after every
|
|
99
|
+
* other group, and refuses one there beside projects with another worker count;
|
|
100
|
+
* - check with baselines (CI): everything 0, in parallel.
|
|
101
|
+
*/
|
|
102
|
+
function groupOrder(kind, target, mode, reference, caseReferences = [], targets = []) {
|
|
103
|
+
const isReference = target === reference;
|
|
104
|
+
const named = caseReferences.length ? 1 : 0;
|
|
105
|
+
const namedByCase = target !== void 0 && caseReferences.includes(target);
|
|
106
|
+
if (mode.update) {
|
|
107
|
+
if (kind === "compile" || kind === "harness" || kind === "bench") return 0;
|
|
108
|
+
if (kind === "toolchain" || isReference) return 1;
|
|
109
|
+
return namedByCase ? 2 : 2 + named;
|
|
110
|
+
}
|
|
111
|
+
if (mode.pixels === "live" && kind === "browser") {
|
|
112
|
+
if (mode.browser !== void 0 && mode.browser !== "chromium") {
|
|
113
|
+
const order = [
|
|
114
|
+
reference,
|
|
115
|
+
...caseReferences,
|
|
116
|
+
...targets.filter((name) => name !== reference && !caseReferences.includes(name))
|
|
117
|
+
];
|
|
118
|
+
const index = target === void 0 ? -1 : order.indexOf(target);
|
|
119
|
+
if (index === -1) throw new Error(`[uf] groupOrder: the browser project of ${target ?? "no target"} is not among the run's targets (${order.join(", ")}).`);
|
|
120
|
+
return 1 + index;
|
|
121
|
+
}
|
|
122
|
+
return isReference ? 0 : namedByCase ? 1 : 1 + named;
|
|
123
|
+
}
|
|
124
|
+
return 0;
|
|
125
|
+
}
|
|
126
|
+
const RUN = Symbol.for("unframework.harness.run");
|
|
127
|
+
/**
|
|
128
|
+
* Creates the run's context, once per process: Vitest evaluates the config file more than once
|
|
129
|
+
* (the root, then each project), and every evaluation must agree on the run id. Each run's
|
|
130
|
+
* scratch output (the ledger, live captures and diffs) is its own directory, named by the run
|
|
131
|
+
* id; the first call removes the directories of runs that have ended, and never those of a run
|
|
132
|
+
* still going in another process (`pnpm test:canaries` next to `pnpm test`).
|
|
133
|
+
*/
|
|
134
|
+
function createHarnessRun(options) {
|
|
135
|
+
const store = globalThis;
|
|
136
|
+
const existing = store[RUN];
|
|
137
|
+
const quarantine = applicableQuarantine(options.quarantine, options.mode.pixels);
|
|
138
|
+
if (existing && existing.casesDir === options.casesDir) {
|
|
139
|
+
existing.cases = options.cases;
|
|
140
|
+
existing.quarantine = quarantine;
|
|
141
|
+
if (options.liveFonts === void 0) delete existing.liveFonts;
|
|
142
|
+
else existing.liveFonts = options.liveFonts;
|
|
143
|
+
return existing;
|
|
144
|
+
}
|
|
145
|
+
const runId = `${Date.now().toString(36)}-${process.pid}`;
|
|
146
|
+
const scratch = {
|
|
147
|
+
ledger: join(options.root, ".reports", "ledger"),
|
|
148
|
+
live: join(options.root, ".live"),
|
|
149
|
+
diffs: join(options.root, ".reports", "diffs")
|
|
150
|
+
};
|
|
151
|
+
for (const directory of Object.values(scratch)) removeEndedRuns(directory);
|
|
152
|
+
const run = {
|
|
153
|
+
runId,
|
|
154
|
+
casesDir: options.casesDir,
|
|
155
|
+
reference: options.reference,
|
|
156
|
+
update: options.mode.update,
|
|
157
|
+
pixels: options.mode.pixels,
|
|
158
|
+
canary: options.mode.canary,
|
|
159
|
+
baselineEnvironment: options.mode.baselineEnvironment ?? null,
|
|
160
|
+
browser: options.mode.browser ?? "chromium",
|
|
161
|
+
ledgerDir: join(scratch.ledger, runId),
|
|
162
|
+
liveDir: join(scratch.live, runId),
|
|
163
|
+
diffsDir: join(scratch.diffs, runId),
|
|
164
|
+
quarantine,
|
|
165
|
+
cases: options.cases,
|
|
166
|
+
...options.updateCommand === void 0 ? {} : { updateCommand: options.updateCommand },
|
|
167
|
+
...options.liveFonts === void 0 ? {} : { liveFonts: options.liveFonts }
|
|
168
|
+
};
|
|
169
|
+
store[RUN] = run;
|
|
170
|
+
return run;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* The entries a run in this pixel mode applies: every entry, but an L10 entry that names the
|
|
174
|
+
* other mode. Throws on an entry that names a pixel mode for a layer that has none.
|
|
175
|
+
*/
|
|
176
|
+
function applicableQuarantine(entries, pixels) {
|
|
177
|
+
for (const entry of entries) {
|
|
178
|
+
if (entry.pixels === void 0) continue;
|
|
179
|
+
if (entry.layer !== "L10" || entry.pixels !== "baseline" && entry.pixels !== "live") throw new Error(`[uf] The quarantine entry ${entry.case} › ${entry.target} › ${entry.layer} names pixels: ${JSON.stringify(entry.pixels)}: only L10 depends on the pixel mode, which is "baseline" or "live".`);
|
|
180
|
+
}
|
|
181
|
+
return entries.filter((entry) => entry.pixels === void 0 || entry.pixels === pixels);
|
|
182
|
+
}
|
|
183
|
+
/** Removes the run directories under `directory` whose process has ended (`<time>-<pid>`). */
|
|
184
|
+
function removeEndedRuns(directory) {
|
|
185
|
+
if (!existsSync(directory)) return;
|
|
186
|
+
for (const name of readdirSync(directory)) {
|
|
187
|
+
const pid = /^[0-9a-z]+-(\d+)$/.exec(name)?.[1];
|
|
188
|
+
if (pid !== void 0 && isRunning(Number(pid))) continue;
|
|
189
|
+
rmSync(join(directory, name), {
|
|
190
|
+
recursive: true,
|
|
191
|
+
force: true
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
function isRunning(pid) {
|
|
196
|
+
try {
|
|
197
|
+
process.kill(pid, 0);
|
|
198
|
+
return true;
|
|
199
|
+
} catch (error) {
|
|
200
|
+
return error.code === "EPERM";
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
//#endregion
|
|
204
|
+
export { BASELINE_ENVIRONMENT, createHarnessRun, groupOrder, isCI, resolveBrowser, resolveHarnessMode };
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
//#region src/node/policy.d.ts
|
|
2
|
+
/** Who may write an artefact. */
|
|
3
|
+
export type ArtefactRole = "owner" | "reference" | "follower";
|
|
4
|
+
/** How an artefact was settled. */
|
|
5
|
+
export type ArtefactStatus = "matched" | "written" | "missing" | "missing-reference" | "mismatch" | "stale";
|
|
6
|
+
/** The result of settling an artefact. `message` is empty when it passed. */
|
|
7
|
+
export interface ArtefactOutcome {
|
|
8
|
+
pass: boolean;
|
|
9
|
+
status: ArtefactStatus;
|
|
10
|
+
message: string;
|
|
11
|
+
}
|
|
12
|
+
/** Where and how an artefact is settled. */
|
|
13
|
+
export interface ArtefactContext {
|
|
14
|
+
role: ArtefactRole;
|
|
15
|
+
update: boolean;
|
|
16
|
+
/** Paths in messages are relative to this directory. */
|
|
17
|
+
root: string;
|
|
18
|
+
/** Update mode, for the reference and its followers: the run's ledger directory. */
|
|
19
|
+
ledgerDir?: string;
|
|
20
|
+
/** The reference target's name, for messages. */
|
|
21
|
+
reference?: string;
|
|
22
|
+
/**
|
|
23
|
+
* The command that writes the artefacts, for messages: `pnpm test:update` in the corpus, and
|
|
24
|
+
* `unframework test --update` in a project's tests (ADR-0086).
|
|
25
|
+
*/
|
|
26
|
+
updateCommand?: string;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Settles one artefact. `actual` is what this run produced; `undefined` means the artefact must
|
|
30
|
+
* not exist (an IR snapshot for a case with compile errors, a stale diagnostics.txt).
|
|
31
|
+
*/
|
|
32
|
+
export declare function settleArtefact(path: string, actual: string | undefined, context: ArtefactContext): ArtefactOutcome;
|
|
33
|
+
/**
|
|
34
|
+
* Settles a directory that holds exactly a set of files (a target's golden outputs): no missing
|
|
35
|
+
* file, no stale file, every file equal. In update mode it writes the files and deletes the
|
|
36
|
+
* stale ones. Owners only: shared expectations are single files.
|
|
37
|
+
*/
|
|
38
|
+
export declare function settleArtefactDirectory(directory: string, files: ReadonlyMap<string, string>, context: {
|
|
39
|
+
update: boolean;
|
|
40
|
+
root: string;
|
|
41
|
+
}): ArtefactOutcome;
|
|
42
|
+
/** Writes a file only when its contents change, so update runs leave mtimes alone. */
|
|
43
|
+
export declare function writeIfChanged(path: string, contents: string | Uint8Array): boolean;
|
|
44
|
+
//#endregion
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { diffLines } from "../diff.js";
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname, isAbsolute, join, relative, sep } from "node:path";
|
|
4
|
+
//#region src/node/policy.ts
|
|
5
|
+
/** The corpus's command that writes the artefacts. */
|
|
6
|
+
const CORPUS_UPDATE = "pnpm test:update";
|
|
7
|
+
const updateHint = (command = CORPUS_UPDATE) => `Run \`${command}\` and review the diff if the change is intended.`;
|
|
8
|
+
/**
|
|
9
|
+
* Settles one artefact. `actual` is what this run produced; `undefined` means the artefact must
|
|
10
|
+
* not exist (an IR snapshot for a case with compile errors, a stale diagnostics.txt).
|
|
11
|
+
*/
|
|
12
|
+
function settleArtefact(path, actual, context) {
|
|
13
|
+
const rel = display(path, context.root);
|
|
14
|
+
if (!isAbsolute(path)) throw new Error(`An artefact path must be absolute: ${path}`);
|
|
15
|
+
if (context.update && context.role !== "follower") {
|
|
16
|
+
if (actual === void 0) {
|
|
17
|
+
if (existsSync(path)) rmSync(path);
|
|
18
|
+
} else writeIfChanged(path, actual);
|
|
19
|
+
if (context.role === "reference") recordInLedger(path, context);
|
|
20
|
+
return {
|
|
21
|
+
pass: true,
|
|
22
|
+
status: "written",
|
|
23
|
+
message: ""
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
if (context.update && !ledgerHas(path, context)) return {
|
|
27
|
+
pass: false,
|
|
28
|
+
status: "missing-reference",
|
|
29
|
+
message: `The reference target (${context.reference ?? "unknown"}) did not write ${rel} earlier in this run. In update mode the reference runs first (sequence.groupOrder) and must not be filtered out with --project.`
|
|
30
|
+
};
|
|
31
|
+
const exists = existsSync(path);
|
|
32
|
+
if (actual === void 0) return exists ? {
|
|
33
|
+
pass: false,
|
|
34
|
+
status: "stale",
|
|
35
|
+
message: `Stale artefact ${rel}: this run no longer produces it. Run \`${context.updateCommand ?? CORPUS_UPDATE}\` to delete it.`
|
|
36
|
+
} : {
|
|
37
|
+
pass: true,
|
|
38
|
+
status: "matched",
|
|
39
|
+
message: ""
|
|
40
|
+
};
|
|
41
|
+
if (!exists) {
|
|
42
|
+
const writer = context.role === "follower" ? ` (the reference target, ${context.reference ?? "unknown"}, writes it)` : "";
|
|
43
|
+
return {
|
|
44
|
+
pass: false,
|
|
45
|
+
status: "missing",
|
|
46
|
+
message: `Missing artefact ${rel}. Run \`${context.updateCommand ?? CORPUS_UPDATE}\` to write it${writer}.`
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
const expected = readFileSync(path, "utf8");
|
|
50
|
+
if (expected === actual) return {
|
|
51
|
+
pass: true,
|
|
52
|
+
status: "matched",
|
|
53
|
+
message: ""
|
|
54
|
+
};
|
|
55
|
+
return {
|
|
56
|
+
pass: false,
|
|
57
|
+
status: "mismatch",
|
|
58
|
+
message: `${context.update ? `${rel}, as the reference wrote it in this run,` : rel} differs from this run's output:\n${diffLines(expected, actual)}\n${context.update ? "" : updateHint(context.updateCommand)}`.trimEnd()
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Settles a directory that holds exactly a set of files (a target's golden outputs): no missing
|
|
63
|
+
* file, no stale file, every file equal. In update mode it writes the files and deletes the
|
|
64
|
+
* stale ones. Owners only: shared expectations are single files.
|
|
65
|
+
*/
|
|
66
|
+
function settleArtefactDirectory(directory, files, context) {
|
|
67
|
+
const stale = (existsSync(directory) ? listFiles(directory) : []).filter((file) => !files.has(file));
|
|
68
|
+
if (context.update) {
|
|
69
|
+
for (const file of stale) rmSync(join(directory, file));
|
|
70
|
+
for (const [file, contents] of files) writeIfChanged(join(directory, file), contents);
|
|
71
|
+
removeEmptyDirectories(directory);
|
|
72
|
+
return {
|
|
73
|
+
pass: true,
|
|
74
|
+
status: "written",
|
|
75
|
+
message: ""
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
const problems = [];
|
|
79
|
+
for (const file of stale) problems.push(`Stale artefact ${display(join(directory, file), context.root)}: this run no longer produces it.`);
|
|
80
|
+
for (const [file, contents] of [...files].sort(([a], [b]) => a.localeCompare(b))) {
|
|
81
|
+
const outcome = settleArtefact(join(directory, file), contents, {
|
|
82
|
+
role: "owner",
|
|
83
|
+
update: false,
|
|
84
|
+
root: context.root
|
|
85
|
+
});
|
|
86
|
+
if (!outcome.pass) problems.push(outcome.message);
|
|
87
|
+
}
|
|
88
|
+
if (!problems.length) return {
|
|
89
|
+
pass: true,
|
|
90
|
+
status: "matched",
|
|
91
|
+
message: ""
|
|
92
|
+
};
|
|
93
|
+
return {
|
|
94
|
+
pass: false,
|
|
95
|
+
status: problems.every((problem) => problem.startsWith("Missing")) ? "missing" : "mismatch",
|
|
96
|
+
message: `${problems.join("\n\n")}\n${updateHint()}`
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
/** Writes a file only when its contents change, so update runs leave mtimes alone. */
|
|
100
|
+
function writeIfChanged(path, contents) {
|
|
101
|
+
if (existsSync(path)) {
|
|
102
|
+
const current = readFileSync(path);
|
|
103
|
+
const next = typeof contents === "string" ? Buffer.from(contents) : Buffer.from(contents);
|
|
104
|
+
if (current.equals(next)) return false;
|
|
105
|
+
}
|
|
106
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
107
|
+
writeFileSync(path, contents);
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
function ledgerEntry(path, context) {
|
|
111
|
+
if (!context.ledgerDir) throw new Error("Update mode needs the run's ledger directory to coordinate the reference and its followers.");
|
|
112
|
+
const rel = relative(context.root, path);
|
|
113
|
+
if (rel.startsWith("..") || isAbsolute(rel)) throw new Error(`The artefact ${path} is outside the harness root ${context.root}.`);
|
|
114
|
+
return join(context.ledgerDir, rel);
|
|
115
|
+
}
|
|
116
|
+
function recordInLedger(path, context) {
|
|
117
|
+
const entry = ledgerEntry(path, context);
|
|
118
|
+
mkdirSync(dirname(entry), { recursive: true });
|
|
119
|
+
writeFileSync(entry, "");
|
|
120
|
+
}
|
|
121
|
+
function ledgerHas(path, context) {
|
|
122
|
+
return existsSync(ledgerEntry(path, context));
|
|
123
|
+
}
|
|
124
|
+
/** Every file under a directory, as relative paths with forward slashes, sorted. */
|
|
125
|
+
function listFiles(directory) {
|
|
126
|
+
return readdirSync(directory, { recursive: true }).filter((entry) => statSync(join(directory, entry)).isFile()).map((entry) => entry.split(sep).join("/")).sort();
|
|
127
|
+
}
|
|
128
|
+
function removeEmptyDirectories(directory) {
|
|
129
|
+
if (!existsSync(directory)) return;
|
|
130
|
+
for (const entry of readdirSync(directory)) {
|
|
131
|
+
const path = join(directory, entry);
|
|
132
|
+
if (statSync(path).isDirectory()) removeEmptyDirectories(path);
|
|
133
|
+
}
|
|
134
|
+
if (!readdirSync(directory).length) rmdirSync(directory);
|
|
135
|
+
}
|
|
136
|
+
function display(path, root) {
|
|
137
|
+
return relative(root, path).split(sep).join("/");
|
|
138
|
+
}
|
|
139
|
+
//#endregion
|
|
140
|
+
export { settleArtefact, settleArtefactDirectory, writeIfChanged };
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { UfLayerMeta } from "../layers.js";
|
|
2
|
+
import { HarnessContext } from "../harness.js";
|
|
3
|
+
import { PartialMatrix } from "./matrix.js";
|
|
4
|
+
//#region src/node/reporter.d.ts
|
|
5
|
+
/** The part of a Vitest test case the reporter reads. */
|
|
6
|
+
interface ReportedTest {
|
|
7
|
+
project: {
|
|
8
|
+
name: string;
|
|
9
|
+
};
|
|
10
|
+
fullName: string;
|
|
11
|
+
meta(): {
|
|
12
|
+
uf?: UfLayerMeta;
|
|
13
|
+
};
|
|
14
|
+
result(): {
|
|
15
|
+
state: "passed" | "failed" | "skipped" | "pending";
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
/** The part of a Vitest test module the reporter reads. */
|
|
19
|
+
interface ReportedModule {
|
|
20
|
+
project: {
|
|
21
|
+
name: string;
|
|
22
|
+
};
|
|
23
|
+
meta(): {
|
|
24
|
+
uf?: UfLayerMeta;
|
|
25
|
+
};
|
|
26
|
+
children: {
|
|
27
|
+
allTests(): Iterable<ReportedTest>;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
/** The part of a Vitest project the reporter reads. */
|
|
31
|
+
interface ReportedProject {
|
|
32
|
+
name: string;
|
|
33
|
+
config: {
|
|
34
|
+
root: string;
|
|
35
|
+
include: string[];
|
|
36
|
+
};
|
|
37
|
+
getProvidedContext(): {
|
|
38
|
+
ufHarness?: HarnessContext;
|
|
39
|
+
ufOnly?: string[];
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
/** The part of the Vitest instance the reporter reads. */
|
|
43
|
+
interface ReportedVitest {
|
|
44
|
+
projects: readonly ReportedProject[];
|
|
45
|
+
config: {
|
|
46
|
+
testNamePattern?: RegExp | undefined;
|
|
47
|
+
shard?: {
|
|
48
|
+
index: number;
|
|
49
|
+
count: number;
|
|
50
|
+
} | undefined;
|
|
51
|
+
watch?: boolean;
|
|
52
|
+
/** `--changed [since]`. */
|
|
53
|
+
changed?: boolean | string | undefined;
|
|
54
|
+
/** `vitest related <files>`, and the files `--changed` found. */
|
|
55
|
+
related?: readonly string[] | undefined;
|
|
56
|
+
/** `--tags-filter`. */
|
|
57
|
+
tagsFilter?: readonly string[] | undefined;
|
|
58
|
+
};
|
|
59
|
+
logger: {
|
|
60
|
+
log(...messages: unknown[]): void;
|
|
61
|
+
error(...messages: unknown[]): void;
|
|
62
|
+
};
|
|
63
|
+
/** Internal: the CLI's file filters. */
|
|
64
|
+
filenamePattern?: readonly string[] | undefined;
|
|
65
|
+
}
|
|
66
|
+
/** Options for {@link ParityReporter}. */
|
|
67
|
+
export interface ParityReporterOptions {
|
|
68
|
+
/** Where partial matrices go: `parity-matrix.<run>.json`. */
|
|
69
|
+
reportsDir: string;
|
|
70
|
+
/** Every project the config defines; a run of all of them is named `all`, and is complete. */
|
|
71
|
+
allProjects?: readonly string[];
|
|
72
|
+
/**
|
|
73
|
+
* A canary run's directory: its matrix goes to `<canaryDir>/parity-matrix.json`, where the
|
|
74
|
+
* canary runner reads it, and it supersedes no partial matrix.
|
|
75
|
+
*/
|
|
76
|
+
canaryDir?: string;
|
|
77
|
+
}
|
|
78
|
+
/** The parity reporter. Add it to `test.reporters` next to Vitest's own. */
|
|
79
|
+
export declare class ParityReporter {
|
|
80
|
+
#private;
|
|
81
|
+
/** The matrix of the last finished run, for tooling and tests. */
|
|
82
|
+
matrix: PartialMatrix | undefined;
|
|
83
|
+
constructor(options: ParityReporterOptions);
|
|
84
|
+
onInit(vitest: ReportedVitest): void;
|
|
85
|
+
onTestRunEnd(testModules: readonly ReportedModule[], errors: readonly unknown[], reason: string): void;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* A test's name without the target `describeTargets` puts in its suite's name (`state/counter
|
|
89
|
+
* [vue] > increments` → `state/counter > increments`): the same on every target.
|
|
90
|
+
*/
|
|
91
|
+
export declare function testKey(fullName: string, target: string): string;
|
|
92
|
+
/** A file-name-safe name for the selected projects: `all`, or the project names. */
|
|
93
|
+
export declare function runName(projects: readonly string[], allProjects?: readonly string[]): string;
|
|
94
|
+
/**
|
|
95
|
+
* Whether a file name is a partial matrix's, `parity-matrix.<run>.json`. An engine's merged
|
|
96
|
+
* matrix, `parity-matrix.<engine>.json` (ADR-0066), is not: no run is named after an engine.
|
|
97
|
+
*/
|
|
98
|
+
export declare function isPartialMatrixFile(name: string): boolean;
|
|
99
|
+
/** The partial matrices in a directory, by file. Throws on one that is not a current matrix. */
|
|
100
|
+
export declare function readPartialMatrices(reportsDir: string): Map<string, PartialMatrix>;
|
|
101
|
+
//#endregion
|