@unframework.dev/testing 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +511 -2
  2. package/dist/browser/axe.js +34 -0
  3. package/dist/browser/behaviour.js +59 -0
  4. package/dist/browser/capabilities.js +58 -0
  5. package/dist/browser/clock.d.ts +17 -0
  6. package/dist/browser/clock.js +69 -0
  7. package/dist/browser/console.d.ts +27 -0
  8. package/dist/browser/console.js +142 -0
  9. package/dist/browser/determinism.js +108 -0
  10. package/dist/browser/element-timeout.js +28 -0
  11. package/dist/browser/events.js +68 -0
  12. package/dist/browser/geometry.js +135 -0
  13. package/dist/browser/keys.js +62 -0
  14. package/dist/browser/matchers.d.ts +39 -0
  15. package/dist/browser/matchers.js +78 -0
  16. package/dist/browser/navigation.js +38 -0
  17. package/dist/browser/parity.d.ts +10 -0
  18. package/dist/browser/parity.js +93 -0
  19. package/dist/browser/spec.d.ts +25 -0
  20. package/dist/browser/spec.js +38 -0
  21. package/dist/browser/target.d.ts +16 -0
  22. package/dist/browser/target.js +27 -0
  23. package/dist/browser/text-runs.js +68 -0
  24. package/dist/browser/thrown.js +71 -0
  25. package/dist/browser/trace.d.ts +21 -0
  26. package/dist/browser/trace.js +73 -0
  27. package/dist/browser/user.d.ts +45 -0
  28. package/dist/browser/user.js +184 -0
  29. package/dist/browser/view.d.ts +114 -0
  30. package/dist/browser/view.js +282 -0
  31. package/dist/browser/visual.d.ts +20 -0
  32. package/dist/browser/visual.js +36 -0
  33. package/dist/cli.d.ts +23 -0
  34. package/dist/cli.js +90 -0
  35. package/dist/commands.d.ts +107 -0
  36. package/dist/config/broadcast.js +29 -0
  37. package/dist/config/cases.d.ts +28 -0
  38. package/dist/config/cases.js +94 -0
  39. package/dist/config/index.d.ts +81 -0
  40. package/dist/config/index.js +275 -0
  41. package/dist/diff.d.ts +14 -0
  42. package/dist/diff.js +92 -0
  43. package/dist/dom/serialize.js +236 -0
  44. package/dist/errors.d.ts +13 -0
  45. package/dist/errors.js +38 -0
  46. package/dist/harness.d.ts +184 -0
  47. package/dist/harness.js +53 -0
  48. package/dist/index.d.ts +16 -0
  49. package/dist/index.js +12 -0
  50. package/dist/internal.d.ts +8 -0
  51. package/dist/internal.js +6 -0
  52. package/dist/layers.d.ts +183 -0
  53. package/dist/layers.js +241 -0
  54. package/dist/node/browser.d.ts +33 -0
  55. package/dist/node/browser.js +64 -0
  56. package/dist/node/commands.d.ts +137 -0
  57. package/dist/node/commands.js +139 -0
  58. package/dist/node/context.d.ts +28 -0
  59. package/dist/node/context.js +55 -0
  60. package/dist/node/events.d.ts +25 -0
  61. package/dist/node/events.js +50 -0
  62. package/dist/node/finished.d.ts +12 -0
  63. package/dist/node/finished.js +28 -0
  64. package/dist/node/index.d.ts +21 -0
  65. package/dist/node/index.js +20 -0
  66. package/dist/node/matrix.d.ts +219 -0
  67. package/dist/node/matrix.js +309 -0
  68. package/dist/node/mode.d.ts +101 -0
  69. package/dist/node/mode.js +201 -0
  70. package/dist/node/names.d.ts +4 -0
  71. package/dist/node/names.js +5 -0
  72. package/dist/node/policy.d.ts +44 -0
  73. package/dist/node/policy.js +140 -0
  74. package/dist/node/reporter.d.ts +101 -0
  75. package/dist/node/reporter.js +251 -0
  76. package/dist/node/summary.d.ts +99 -0
  77. package/dist/node/summary.js +352 -0
  78. package/dist/node/thrown.js +78 -0
  79. package/dist/node/viewport.d.ts +7 -0
  80. package/dist/node/viewport.js +8 -0
  81. package/dist/node/visual.d.ts +85 -0
  82. package/dist/node/visual.js +370 -0
  83. package/dist/node_modules/.pnpm/playwright-core@1.63.0/node_modules/playwright-core/types/protocol.d.ts +23930 -0
  84. package/dist/node_modules/.pnpm/playwright-core@1.63.0/node_modules/playwright-core/types/structs.d.ts +22 -0
  85. package/dist/node_modules/.pnpm/playwright-core@1.63.0/node_modules/playwright-core/types/types.d.ts +22221 -0
  86. package/dist/normalize/display.js +309 -0
  87. package/dist/normalize/index.d.ts +34 -0
  88. package/dist/normalize/index.js +38 -0
  89. package/dist/normalize/pipeline.d.ts +19 -0
  90. package/dist/normalize/pipeline.js +45 -0
  91. package/dist/normalize/print.js +103 -0
  92. package/dist/normalize/rules/angular-hosts.js +53 -0
  93. package/dist/normalize/rules/aria-hosts.js +48 -0
  94. package/dist/normalize/rules/comments.js +16 -0
  95. package/dist/normalize/rules/control-values.js +65 -0
  96. package/dist/normalize/rules/engine-aliases.js +34 -0
  97. package/dist/normalize/rules/framework-attributes.js +85 -0
  98. package/dist/normalize/rules/generated-ids.js +64 -0
  99. package/dist/normalize/rules/ordering.js +39 -0
  100. package/dist/normalize/rules/qwik-styles.js +21 -0
  101. package/dist/normalize/rules/qwik-templates.js +20 -0
  102. package/dist/normalize/rules/values.js +125 -0
  103. package/dist/normalize/rules/whitespace.js +381 -0
  104. package/dist/normalize/style.js +121 -0
  105. package/dist/normalize/targets.d.ts +8 -0
  106. package/dist/normalize/targets.js +17 -0
  107. package/dist/normalize/tree.d.ts +1 -0
  108. package/dist/normalize/tree.js +151 -0
  109. package/dist/setup.d.ts +1 -0
  110. package/dist/setup.js +149 -0
  111. package/dist/ssr/index.d.ts +10 -0
  112. package/dist/ssr/index.js +103 -0
  113. package/dist/ssr/spec.d.ts +1 -0
  114. package/dist/ssr/spec.js +7 -0
  115. package/dist/targets/angular.d.ts +1 -0
  116. package/dist/targets/angular.js +5 -0
  117. package/dist/targets/astro.d.ts +1 -0
  118. package/dist/targets/astro.js +5 -0
  119. package/dist/targets/qwik.d.ts +1 -0
  120. package/dist/targets/qwik.js +5 -0
  121. package/dist/targets/react.d.ts +1 -0
  122. package/dist/targets/react.js +5 -0
  123. package/dist/targets/solid.d.ts +1 -0
  124. package/dist/targets/solid.js +5 -0
  125. package/dist/targets/svelte.d.ts +1 -0
  126. package/dist/targets/svelte.js +5 -0
  127. package/dist/targets/vue.d.ts +1 -0
  128. package/dist/targets/vue.js +5 -0
  129. package/dist/visual-types.d.ts +92 -0
  130. package/dist/visual-types.js +24 -0
  131. package/package.json +135 -3
@@ -0,0 +1,309 @@
1
+ import { LAYERS, mergeOutcomes, quarantineFor, settleOutcome } from "../layers.js";
2
+ //#region src/node/matrix.ts
3
+ /** The version of the matrix files; a file of another version is never merged. */
4
+ const MATRIX_VERSION = 5;
5
+ /** Why a run covered only part of its projects' tests, its shard included, or `null`. */
6
+ function narrowing(run) {
7
+ const reasons = [...run.filtered === null ? [] : [run.filtered], ...run.shard === null ? [] : [`shard ${run.shard.index}/${run.shard.count}`]];
8
+ return reasons.length ? reasons.join("; ") : null;
9
+ }
10
+ /**
11
+ * An outcome as a matrix cell. A failure keeps its whole message, every sub-check's included
12
+ * (both the DOM and the ARIA diff of L7, the server's and the page's console of L13), so a
13
+ * canary can require the evidence of each.
14
+ */
15
+ function cellOf(outcome) {
16
+ if (outcome.status === "skip") return `skip(${outcome.reason})`;
17
+ if (outcome.status === "quarantined") return `quarantined(${outcome.issue})`;
18
+ if (outcome.status === "pass") return "pass";
19
+ return `fail: ${outcome.message.trim() || "failed"}`;
20
+ }
21
+ /** A matrix cell as an outcome. Throws on text that is not a cell, so a corrupt matrix is loud. */
22
+ function outcomeOfCell(cell) {
23
+ if (cell === "pass") return { status: "pass" };
24
+ if (cell.startsWith("fail: ")) return {
25
+ status: "fail",
26
+ message: cell.slice(6)
27
+ };
28
+ const match = /^(skip|quarantined)\((.*)\)$/s.exec(cell);
29
+ if (match?.[1] === "skip") return {
30
+ status: "skip",
31
+ reason: match[2]
32
+ };
33
+ if (match?.[1] === "quarantined") return {
34
+ status: "quarantined",
35
+ issue: match[2]
36
+ };
37
+ throw new Error(`Not a parity matrix cell: ${JSON.stringify(cell)}`);
38
+ }
39
+ /** The first line of a cell, for a one-line summary; the matrix file has the rest. */
40
+ function cellHeadline(cell) {
41
+ const lines = cell.split("\n");
42
+ const more = lines.length - 1;
43
+ return more ? `${lines[0]} (+${more} more line(s))` : cell;
44
+ }
45
+ /** Builds cells from test records, merging every record of a cell (fail beats pass). */
46
+ function buildCells(records) {
47
+ const outcomes = /* @__PURE__ */ new Map();
48
+ for (const record of records) {
49
+ let targets = outcomes.get(record.case);
50
+ if (!targets) outcomes.set(record.case, targets = /* @__PURE__ */ new Map());
51
+ let layers = targets.get(record.target);
52
+ if (!layers) targets.set(record.target, layers = /* @__PURE__ */ new Map());
53
+ for (const layer of LAYERS) {
54
+ const outcome = record.layers[layer];
55
+ if (outcome) layers.set(layer, [...layers.get(layer) ?? [], outcome]);
56
+ }
57
+ }
58
+ const cells = {};
59
+ for (const caseId of [...outcomes.keys()].sort()) {
60
+ const targets = outcomes.get(caseId);
61
+ cells[caseId] = {};
62
+ for (const target of [...targets.keys()].sort()) {
63
+ const layers = targets.get(target);
64
+ cells[caseId][target] = Object.fromEntries(LAYERS.filter((layer) => layers.has(layer)).map((layer) => [layer, cellOf(mergeOutcomes(...layers.get(layer)))]));
65
+ }
66
+ }
67
+ return cells;
68
+ }
69
+ /** The scenarios of test records: each (case, target)'s names, once each, sorted. */
70
+ function buildScenarios(records) {
71
+ return mergeScenarios(records.flatMap((record) => record.scenarios?.length ? [{ [record.case]: { [record.target]: [...record.scenarios] } }] : []));
72
+ }
73
+ /** Merges sets of scenarios: the union of each (case, target)'s names, sorted. */
74
+ function mergeScenarios(sets) {
75
+ const names = /* @__PURE__ */ new Map();
76
+ for (const scenarios of sets) for (const [caseId, targets] of Object.entries(scenarios)) {
77
+ let byTarget = names.get(caseId);
78
+ if (!byTarget) names.set(caseId, byTarget = /* @__PURE__ */ new Map());
79
+ for (const [target, list] of Object.entries(targets)) {
80
+ const set = byTarget.get(target) ?? /* @__PURE__ */ new Set();
81
+ for (const name of list) set.add(name);
82
+ byTarget.set(target, set);
83
+ }
84
+ }
85
+ const merged = {};
86
+ for (const caseId of [...names.keys()].sort()) {
87
+ const byTarget = names.get(caseId);
88
+ merged[caseId] = Object.fromEntries([...byTarget.keys()].sort().map((target) => [target, [...byTarget.get(target)].sort()]));
89
+ }
90
+ return merged;
91
+ }
92
+ /**
93
+ * The tests of test records: each named test's sorted scenarios, capability skip and stand-in.
94
+ * Records without a test name (a file's own) and tests that checked nothing and were neither
95
+ * skipped nor run against a stand-in are left out: they have nothing to compare.
96
+ */
97
+ function buildTests(records) {
98
+ return mergeTests(records.flatMap((record) => {
99
+ if (record.test === void 0 || !record.scenarios?.length && record.skipped === void 0 && record.standIn === void 0) return [];
100
+ const test = {
101
+ scenarios: [...new Set(record.scenarios ?? [])].sort(),
102
+ ...record.skipped === void 0 ? {} : { skipped: record.skipped },
103
+ ...record.standIn === void 0 ? {} : { standIn: record.standIn }
104
+ };
105
+ return [{ [record.case]: { [record.target]: { [record.test]: test } } }];
106
+ }));
107
+ }
108
+ /**
109
+ * Merges sets of tests, a later set's record of a test replacing an earlier one's (a newer run
110
+ * of the test). Within one set a test is recorded once: two tests of a case never share a name.
111
+ */
112
+ function mergeTests(sets) {
113
+ const merged = {};
114
+ for (const tests of sets) for (const [caseId, targets] of Object.entries(tests)) for (const [target, byName] of Object.entries(targets)) Object.assign((merged[caseId] ??= {})[target] ??= {}, byName);
115
+ return Object.fromEntries(Object.keys(merged).sort().map((caseId) => [caseId, Object.fromEntries(Object.keys(merged[caseId]).sort().map((target) => {
116
+ const byName = merged[caseId][target];
117
+ return [target, Object.fromEntries(Object.keys(byName).sort().map((name) => [name, byName[name]]))];
118
+ }))]));
119
+ }
120
+ /** Merges several projects' cells into one set, cell by cell (fail beats pass). */
121
+ function mergeCells(sets) {
122
+ return buildCells(sets.flatMap((cells) => Object.entries(cells).flatMap(([caseId, targets]) => Object.entries(targets).map(([target, layers]) => ({
123
+ case: caseId,
124
+ target,
125
+ layers: Object.fromEntries(Object.entries(layers).map(([layer, cell]) => [layer, outcomeOfCell(cell)]))
126
+ })))));
127
+ }
128
+ /** Builds a run's partial matrix from its test records. */
129
+ function buildPartialMatrix(input) {
130
+ const projects = [...new Set(input.projects)].sort();
131
+ const byProject = {};
132
+ const scenariosByProject = {};
133
+ const testsByProject = {};
134
+ for (const project of projects) {
135
+ const records = input.records.filter((entry) => entry.project === project).map((entry) => entry.record);
136
+ byProject[project] = buildCells(records);
137
+ scenariosByProject[project] = buildScenarios(records);
138
+ testsByProject[project] = buildTests(records);
139
+ }
140
+ return {
141
+ version: 5,
142
+ run: input.run,
143
+ finishedAt: input.finishedAt,
144
+ mode: input.mode,
145
+ filtered: input.filtered,
146
+ shard: input.shard,
147
+ projects,
148
+ empty: [...new Set(input.empty)].sort(),
149
+ quarantine: [...input.quarantine],
150
+ reference: input.reference ?? null,
151
+ ...input.references && Object.keys(input.references).length ? { references: { ...input.references } } : {},
152
+ ...input.errors?.length ? { errors: [...input.errors] } : {},
153
+ cases: mergeCells(Object.values(byProject)),
154
+ scenarios: mergeScenarios(Object.values(scenariosByProject)),
155
+ tests: mergeTests(Object.values(testsByProject)),
156
+ byProject,
157
+ scenariosByProject,
158
+ testsByProject
159
+ };
160
+ }
161
+ /**
162
+ * A partial matrix without some projects' records (a newer run has them), or none if empty. A
163
+ * project with a scope loses only the cells inside it, and goes once it has none left. A matrix
164
+ * that loses nothing is returned as it is, and a project that loses nothing keeps its records.
165
+ */
166
+ function withoutProjects(matrix, projects, scopes = /* @__PURE__ */ new Map()) {
167
+ const byProject = {};
168
+ const scenariosByProject = {};
169
+ const testsByProject = {};
170
+ let dropped = false;
171
+ for (const project of matrix.projects) {
172
+ const scope = projects.has(project) ? scopes.get(project) ?? {} : void 0;
173
+ const before = [
174
+ matrix.byProject[project] ?? {},
175
+ matrix.scenariosByProject[project] ?? {},
176
+ matrix.testsByProject[project] ?? {}
177
+ ];
178
+ const cells = outsideScope(before[0], scope);
179
+ const scenarios = outsideScope(before[1], scope);
180
+ const tests = outsideScope(before[2], scope);
181
+ if (cells !== before[0] || scenarios !== before[1] || tests !== before[2]) dropped = true;
182
+ if (scope && ![
183
+ cells,
184
+ scenarios,
185
+ tests
186
+ ].some((set) => Object.keys(set).length)) {
187
+ dropped = true;
188
+ continue;
189
+ }
190
+ byProject[project] = cells;
191
+ scenariosByProject[project] = scenarios;
192
+ testsByProject[project] = tests;
193
+ }
194
+ if (!dropped) return matrix;
195
+ const kept = matrix.projects.filter((project) => project in byProject);
196
+ if (!kept.length) return void 0;
197
+ return {
198
+ ...matrix,
199
+ projects: kept,
200
+ empty: matrix.empty.filter((project) => kept.includes(project)),
201
+ byProject,
202
+ scenariosByProject,
203
+ testsByProject,
204
+ cases: mergeCells(Object.values(byProject)),
205
+ scenarios: mergeScenarios(Object.values(scenariosByProject)),
206
+ tests: mergeTests(Object.values(testsByProject))
207
+ };
208
+ }
209
+ /**
210
+ * Case → target → what is recorded, without what a scope covers; all of it without a scope. The
211
+ * same object when the scope covers none of it.
212
+ */
213
+ function outsideScope(byCase, scope) {
214
+ if (!scope) return byCase;
215
+ const kept = {};
216
+ let dropped = false;
217
+ for (const [caseId, targets] of Object.entries(byCase)) for (const [target, value] of Object.entries(targets)) {
218
+ if ((scope.cases?.has(caseId) ?? true) && (scope.targets?.has(target) ?? true)) {
219
+ dropped = true;
220
+ continue;
221
+ }
222
+ (kept[caseId] ??= {})[target] = value;
223
+ }
224
+ return dropped ? kept : byCase;
225
+ }
226
+ /**
227
+ * Merges partial matrices (CI's jobs, local runs of a subset) into one. For each project the
228
+ * newest record of each cell wins, so a re-run replaces what an earlier run recorded; then the
229
+ * projects merge cell by cell (fail beats pass). Scenarios add up, as cells another run did not
230
+ * record are kept: a run of a project replaces its older records of the targets and cases it
231
+ * re-judged (the reporter's `supersede`). The quarantine and the reference are the newest run's:
232
+ * an entry removed since an older run must neither hide that run's failures nor go stale.
233
+ */
234
+ function mergeMatrices(partials) {
235
+ for (const partial of partials) if (partial.version !== 5) throw new Error(`Unsupported parity matrix version ${String(partial.version)} (run ${partial.run}): rerun the tests to replace it.`);
236
+ const ordered = [...partials].sort((a, b) => a.finishedAt.localeCompare(b.finishedAt) || a.run.localeCompare(b.run));
237
+ const latest = /* @__PURE__ */ new Map();
238
+ const scenarios = [];
239
+ const tests = [];
240
+ for (const partial of ordered) {
241
+ scenarios.push(...Object.values(partial.scenariosByProject));
242
+ tests.push(...Object.values(partial.testsByProject));
243
+ for (const [project, cells] of Object.entries(partial.byProject)) {
244
+ const merged = latest.get(project) ?? {};
245
+ for (const [caseId, targets] of Object.entries(cells)) for (const [target, layers] of Object.entries(targets)) Object.assign((merged[caseId] ??= {})[target] ??= {}, layers);
246
+ latest.set(project, merged);
247
+ }
248
+ }
249
+ return {
250
+ matrix: {
251
+ version: 5,
252
+ run: "merged",
253
+ projects: [...new Set(ordered.flatMap((partial) => partial.projects))].sort(),
254
+ cases: mergeCells([...latest.values()]),
255
+ scenarios: mergeScenarios(scenarios),
256
+ tests: mergeTests(tests),
257
+ runs: ordered.map(({ run, finishedAt, mode, filtered, shard, projects, empty }) => ({
258
+ run,
259
+ finishedAt,
260
+ mode,
261
+ filtered,
262
+ shard,
263
+ projects,
264
+ empty
265
+ }))
266
+ },
267
+ quarantine: [...ordered.at(-1)?.quarantine ?? []],
268
+ reference: ordered.findLast((partial) => partial.reference !== null)?.reference ?? null,
269
+ references: { ...ordered.findLast((partial) => partial.reference !== null)?.references }
270
+ };
271
+ }
272
+ /**
273
+ * Judges cells by a quarantine, which may be newer than the runs that recorded them (see
274
+ * `mergeMatrices`): a failure under an entry is quarantined, and a cell an older run recorded as
275
+ * quarantined, under an entry since removed, fails (its message is not recorded: rerun its
276
+ * project to see it). With `stale`, for cells every record has reached (a complete run, or a
277
+ * merge of complete runs), a cell under an entry that passes, or is skipped, becomes a
278
+ * stale-entry failure (plan §7.7). Returns the failures it made.
279
+ */
280
+ function settleQuarantine(cells, quarantine, options) {
281
+ const problems = [];
282
+ for (const [caseId, targets] of Object.entries(cells)) for (const [target, layers] of Object.entries(targets)) for (const layer of LAYERS) {
283
+ const cell = layers[layer];
284
+ if (cell === void 0) continue;
285
+ const outcome = outcomeOfCell(cell);
286
+ const entry = quarantineFor(quarantine, {
287
+ case: caseId,
288
+ target
289
+ }, layer);
290
+ let settled;
291
+ if (!entry) {
292
+ if (outcome.status !== "quarantined") continue;
293
+ settled = {
294
+ status: "fail",
295
+ message: `quarantined for ${outcome.issue} by an older run, but no entry quarantines ${caseId} › ${target} › ${layer} any more: rerun its project to see the failure.`
296
+ };
297
+ } else if (outcome.status === "fail" || options.stale) settled = settleOutcome(outcome, entry);
298
+ else continue;
299
+ layers[layer] = cellOf(settled);
300
+ if (settled.status === "fail") problems.push(settled.message);
301
+ }
302
+ return problems;
303
+ }
304
+ /** Serialises a matrix with a trailing newline; equal matrices are equal bytes. */
305
+ function stringifyMatrix(matrix) {
306
+ return `${JSON.stringify(matrix, null, 2)}\n`;
307
+ }
308
+ //#endregion
309
+ export { MATRIX_VERSION, buildCells, buildPartialMatrix, buildScenarios, buildTests, cellHeadline, cellOf, mergeCells, mergeMatrices, mergeScenarios, mergeTests, narrowing, outcomeOfCell, settleQuarantine, stringifyMatrix, withoutProjects };
@@ -0,0 +1,101 @@
1
+ import { QuarantineEntry } from "../layers.js";
2
+ import { BrowserEngine, CaseConfig, HarnessContext, PixelMode } from "../harness.js";
3
+ //#region src/node/mode.d.ts
4
+ /**
5
+ * The image and platform the committed Linux baselines (`*-chromium-linux.png`, geometry) come
6
+ * from: CI's browser jobs run in it, and `pnpm test:baselines` exports it as
7
+ * `UF_BASELINE_ENVIRONMENT`. Only a run in it writes them. The image follows the playwright
8
+ * version of the catalog, which ships its browsers.
9
+ */
10
+ export declare const BASELINE_ENVIRONMENT = "mcr.microsoft.com/playwright:v1.63.0-noble linux/amd64";
11
+ /** How this run treats artefacts. */
12
+ export interface HarnessMode {
13
+ /** `UF_UPDATE=1`: owners and the reference write. */
14
+ update: boolean;
15
+ /**
16
+ * `baseline` in CI and in the baseline environment, `live` elsewhere; `UF_PIXELS` overrides.
17
+ * Another host's Chromium and system libraries render differently from the image's, so a
18
+ * plain Linux host compares live too.
19
+ */
20
+ pixels: PixelMode;
21
+ /** `UF_CANARY=<id>`: the canary compiler plugin to install. */
22
+ canary: string | null;
23
+ /** `CI` is set: check mode only, against committed baselines. */
24
+ ci: boolean;
25
+ /**
26
+ * `UF_BASELINE_ENVIRONMENT`: the image and platform the run says it is in, if any. Only
27
+ * {@link BASELINE_ENVIRONMENT} may write the committed visual baselines.
28
+ */
29
+ baselineEnvironment?: string | null;
30
+ /** `UF_BROWSER`: the engine the browser projects run in; Chromium when absent. */
31
+ browser?: BrowserEngine;
32
+ }
33
+ /** What {@link resolveHarnessMode} reads. */
34
+ export interface ModeInput {
35
+ env: Readonly<Record<string, string | undefined>>;
36
+ platform: NodeJS.Platform;
37
+ /** The CPU architecture, `process.arch` by default: the baseline environment is x64. */
38
+ arch?: NodeJS.Architecture;
39
+ argv: readonly string[];
40
+ /**
41
+ * The run has no committed visual baselines (the testing package's own fixture, whose one
42
+ * target is its own reference): pixels are always live, in CI too, and `UF_PIXELS` is refused.
43
+ * It runs in Chromium alone, so another `UF_BROWSER` is refused too.
44
+ */
45
+ liveOnly?: boolean;
46
+ /**
47
+ * A project's own tests (`defineTestConfig`, ADR-0086), not the corpus. Pixels compare as the
48
+ * project chose, live by default, in CI too: a project has no committed baselines unless it
49
+ * opts into them with `baseline`, and then only {@link BASELINE_ENVIRONMENT} writes them.
50
+ * `UF_PIXELS` still overrides.
51
+ */
52
+ project?: {
53
+ pixels: PixelMode;
54
+ };
55
+ }
56
+ /** Resolves the run's mode, refusing combinations that would write the wrong thing. */
57
+ export declare function resolveHarnessMode(input: ModeInput): HarnessMode;
58
+ /** The engine `UF_BROWSER` names, Chromium when it is unset. */
59
+ export declare function resolveBrowser(env: Readonly<Record<string, string | undefined>>): BrowserEngine;
60
+ /** The kinds of project the harness runs (plan §7.3). */
61
+ export type ProjectKind = "compile" | "harness" | "toolchain" | "ssr" | "browser" | "bench";
62
+ /**
63
+ * A project's `sequence.groupOrder`. Groups run one after another, so a project that reads what
64
+ * another writes in the same run must sit in a later group:
65
+ *
66
+ * - update: compile (and the harness's own tests, and the budgets) 0, then the toolchains and the reference's
67
+ * ssr and browser projects 1, then every follower 2; where a case names its own reference
68
+ * (`caseReferences`, ADR-0057), that target's projects run at 2 alone, as they follow the
69
+ * reference everywhere else, and the other followers at 3;
70
+ * - check with live pixels: the reference browser project 0, the other browser projects 1; the
71
+ * browser projects of the targets some case names at 1 alone, and the others at 2;
72
+ * - check with live pixels in Firefox or WebKit (ADR-0081): every browser project alone, the
73
+ * reference at 1, then the targets some case names, then each other target in the order of
74
+ * `targets`. Five followers at once in one engine ran the nightly's WebKit job out of memory.
75
+ * Each of those projects also runs one file at a time (`fileParallelism: false`), so the
76
+ * reference is not at 0: Vitest moves a project at 0 that runs one file at a time after every
77
+ * other group, and refuses one there beside projects with another worker count;
78
+ * - check with baselines (CI): everything 0, in parallel.
79
+ */
80
+ export declare function groupOrder(kind: ProjectKind, target: string | undefined, mode: Pick<HarnessMode, "update" | "pixels" | "browser">, reference: string, caseReferences?: readonly string[], targets?: readonly string[]): number;
81
+ /** What a run is made of, besides the mode. */
82
+ export interface HarnessRunOptions {
83
+ /** The integration package's directory: scratch output goes under it. */
84
+ root: string;
85
+ casesDir: string;
86
+ reference: string;
87
+ mode: HarnessMode;
88
+ quarantine: QuarantineEntry[];
89
+ cases: Record<string, CaseConfig>;
90
+ /** The command a failure names to write the artefacts (`HarnessContext.updateCommand`). */
91
+ updateCommand?: string;
92
+ }
93
+ /**
94
+ * Creates the run's context, once per process: Vitest evaluates the config file more than once
95
+ * (the root, then each project), and every evaluation must agree on the run id. Each run's
96
+ * scratch output (the ledger, live captures and diffs) is its own directory, named by the run
97
+ * id; the first call removes the directories of runs that have ended, and never those of a run
98
+ * still going in another process (`pnpm test:canaries` next to `pnpm test`).
99
+ */
100
+ export declare function createHarnessRun(options: HarnessRunOptions): HarnessContext;
101
+ //#endregion
@@ -0,0 +1,201 @@
1
+ import { BROWSER_ENGINES } from "../harness.js";
2
+ import { existsSync, readdirSync, rmSync } from "node:fs";
3
+ import { join } from "node:path";
4
+ //#region src/node/mode.ts
5
+ /**
6
+ * The image and platform the committed Linux baselines (`*-chromium-linux.png`, geometry) come
7
+ * from: CI's browser jobs run in it, and `pnpm test:baselines` exports it as
8
+ * `UF_BASELINE_ENVIRONMENT`. Only a run in it writes them. The image follows the playwright
9
+ * version of the catalog, which ships its browsers.
10
+ */
11
+ const BASELINE_ENVIRONMENT = "mcr.microsoft.com/playwright:v1.63.0-noble linux/amd64";
12
+ /** Resolves the run's mode, refusing combinations that would write the wrong thing. */
13
+ function resolveHarnessMode(input) {
14
+ const { env } = input;
15
+ const ci = isCI(env);
16
+ if (input.argv.some((arg) => arg === "-u" || arg === "--update" || arg.startsWith("--update="))) throw new Error(`[uf] Vitest's -u does not update the harness's artefacts, and would let every project write the shared ones. Run ${input.project ? "`unframework test --update`" : "`pnpm test:update` (UF_UPDATE=1)"} instead.`);
17
+ const update = flag(env, "UF_UPDATE");
18
+ if (update && ci) throw new Error("[uf] CI never writes artefacts: refusing UF_UPDATE while CI is set.");
19
+ const canary = env.UF_CANARY?.trim() || null;
20
+ if (canary && update) throw new Error(`[uf] A canary run never writes artefacts: refusing UF_UPDATE with UF_CANARY=${canary}.`);
21
+ const baselineEnvironment = env.UF_BASELINE_ENVIRONMENT?.trim() || null;
22
+ const here = `${input.platform}/${input.arch ?? process.arch}`;
23
+ if (baselineEnvironment === "mcr.microsoft.com/playwright:v1.63.0-noble linux/amd64" && here !== "linux/x64") throw new Error(`[uf] UF_BASELINE_ENVIRONMENT says ${baselineEnvironment}, but this run is on ${here}.`);
24
+ const forced = env.UF_PIXELS?.trim();
25
+ if (forced && forced !== "baseline" && forced !== "live") throw new Error(`[uf] UF_PIXELS must be "baseline" or "live", not "${forced}".`);
26
+ const browser = resolveBrowser(env);
27
+ if (input.liveOnly) {
28
+ if (forced) throw new Error(`[uf] This run has no committed baselines: UF_PIXELS=${forced} cannot apply.`);
29
+ if (browser !== "chromium") throw new Error(`[uf] This run has no committed baselines and runs in Chromium alone: UF_BROWSER=${browser} cannot apply.`);
30
+ return {
31
+ update,
32
+ pixels: "live",
33
+ canary,
34
+ ci,
35
+ baselineEnvironment,
36
+ browser
37
+ };
38
+ }
39
+ if (browser !== "chromium") {
40
+ if (update) throw new Error(`[uf] Every committed artefact is Chromium's: refusing UF_UPDATE with UF_BROWSER=${browser}.`);
41
+ if (forced === "baseline") throw new Error(`[uf] The committed baselines are Chromium's: ${browser} compares live, with the reference's capture in the same run. Unset UF_PIXELS.`);
42
+ return {
43
+ update,
44
+ pixels: "live",
45
+ canary,
46
+ ci,
47
+ baselineEnvironment,
48
+ browser
49
+ };
50
+ }
51
+ const pixels = forced ?? input.project?.pixels ?? (ci || baselineEnvironment ? "baseline" : "live");
52
+ if (ci && pixels === "live" && !input.project) throw new Error("[uf] CI compares pixels against the committed Linux baselines, never live.");
53
+ if (ci && pixels === "baseline" && input.platform !== "linux") throw new Error(`[uf] CI compares pixels against the committed Linux baselines, so it runs on Linux, not ${input.platform}.`);
54
+ if (update && pixels === "baseline" && baselineEnvironment !== "mcr.microsoft.com/playwright:v1.63.0-noble linux/amd64") {
55
+ if (input.project) throw new Error(`[uf] \`unframework test --update\` with \`pixels: "baseline"\` would write the committed Linux baselines, which only ${BASELINE_ENVIRONMENT} writes; this run is ${baselineEnvironment ?? `on ${here}, outside it`}. Run \`UF_PIXELS=live unframework test --update\` to update every other expectation.`);
56
+ throw new Error(`[uf] UF_UPDATE with baseline pixels would write the committed Linux baselines, which only ${BASELINE_ENVIRONMENT} writes (\`pnpm test:baselines\`); this run is ${baselineEnvironment ?? `on ${here}, outside it`}. Unset UF_PIXELS (or set it to live) to update everything else.`);
57
+ }
58
+ return {
59
+ update,
60
+ pixels,
61
+ canary,
62
+ ci,
63
+ baselineEnvironment,
64
+ browser
65
+ };
66
+ }
67
+ /** The engine `UF_BROWSER` names, Chromium when it is unset. */
68
+ function resolveBrowser(env) {
69
+ const value = env.UF_BROWSER?.trim() || "chromium";
70
+ const browser = BROWSER_ENGINES.find((engine) => engine === value);
71
+ if (!browser) throw new Error(`[uf] UF_BROWSER must be chromium, firefox or webkit, not "${value}".`);
72
+ return browser;
73
+ }
74
+ /** Whether `CI` is set to anything but `0` or `false`, as CI providers set it. */
75
+ function isCI(env) {
76
+ return flag(env, "CI", { allowAny: true });
77
+ }
78
+ function flag(env, name, options = {}) {
79
+ const value = env[name]?.trim();
80
+ if (!value || value === "0" || value === "false") return false;
81
+ if (value === "1" || value === "true" || options.allowAny) return true;
82
+ throw new Error(`[uf] ${name} must be 1 or 0, not "${value}".`);
83
+ }
84
+ /**
85
+ * A project's `sequence.groupOrder`. Groups run one after another, so a project that reads what
86
+ * another writes in the same run must sit in a later group:
87
+ *
88
+ * - update: compile (and the harness's own tests, and the budgets) 0, then the toolchains and the reference's
89
+ * ssr and browser projects 1, then every follower 2; where a case names its own reference
90
+ * (`caseReferences`, ADR-0057), that target's projects run at 2 alone, as they follow the
91
+ * reference everywhere else, and the other followers at 3;
92
+ * - check with live pixels: the reference browser project 0, the other browser projects 1; the
93
+ * browser projects of the targets some case names at 1 alone, and the others at 2;
94
+ * - check with live pixels in Firefox or WebKit (ADR-0081): every browser project alone, the
95
+ * reference at 1, then the targets some case names, then each other target in the order of
96
+ * `targets`. Five followers at once in one engine ran the nightly's WebKit job out of memory.
97
+ * Each of those projects also runs one file at a time (`fileParallelism: false`), so the
98
+ * reference is not at 0: Vitest moves a project at 0 that runs one file at a time after every
99
+ * other group, and refuses one there beside projects with another worker count;
100
+ * - check with baselines (CI): everything 0, in parallel.
101
+ */
102
+ function groupOrder(kind, target, mode, reference, caseReferences = [], targets = []) {
103
+ const isReference = target === reference;
104
+ const named = caseReferences.length ? 1 : 0;
105
+ const namedByCase = target !== void 0 && caseReferences.includes(target);
106
+ if (mode.update) {
107
+ if (kind === "compile" || kind === "harness" || kind === "bench") return 0;
108
+ if (kind === "toolchain" || isReference) return 1;
109
+ return namedByCase ? 2 : 2 + named;
110
+ }
111
+ if (mode.pixels === "live" && kind === "browser") {
112
+ if (mode.browser !== void 0 && mode.browser !== "chromium") {
113
+ const order = [
114
+ reference,
115
+ ...caseReferences,
116
+ ...targets.filter((name) => name !== reference && !caseReferences.includes(name))
117
+ ];
118
+ const index = target === void 0 ? -1 : order.indexOf(target);
119
+ if (index === -1) throw new Error(`[uf] groupOrder: the browser project of ${target ?? "no target"} is not among the run's targets (${order.join(", ")}).`);
120
+ return 1 + index;
121
+ }
122
+ return isReference ? 0 : namedByCase ? 1 : 1 + named;
123
+ }
124
+ return 0;
125
+ }
126
+ const RUN = Symbol.for("unframework.harness.run");
127
+ /**
128
+ * Creates the run's context, once per process: Vitest evaluates the config file more than once
129
+ * (the root, then each project), and every evaluation must agree on the run id. Each run's
130
+ * scratch output (the ledger, live captures and diffs) is its own directory, named by the run
131
+ * id; the first call removes the directories of runs that have ended, and never those of a run
132
+ * still going in another process (`pnpm test:canaries` next to `pnpm test`).
133
+ */
134
+ function createHarnessRun(options) {
135
+ const store = globalThis;
136
+ const existing = store[RUN];
137
+ const quarantine = applicableQuarantine(options.quarantine, options.mode.pixels);
138
+ if (existing && existing.casesDir === options.casesDir) {
139
+ existing.cases = options.cases;
140
+ existing.quarantine = quarantine;
141
+ return existing;
142
+ }
143
+ const runId = `${Date.now().toString(36)}-${process.pid}`;
144
+ const scratch = {
145
+ ledger: join(options.root, ".reports", "ledger"),
146
+ live: join(options.root, ".live"),
147
+ diffs: join(options.root, ".reports", "diffs")
148
+ };
149
+ for (const directory of Object.values(scratch)) removeEndedRuns(directory);
150
+ const run = {
151
+ runId,
152
+ casesDir: options.casesDir,
153
+ reference: options.reference,
154
+ update: options.mode.update,
155
+ pixels: options.mode.pixels,
156
+ canary: options.mode.canary,
157
+ baselineEnvironment: options.mode.baselineEnvironment ?? null,
158
+ browser: options.mode.browser ?? "chromium",
159
+ ledgerDir: join(scratch.ledger, runId),
160
+ liveDir: join(scratch.live, runId),
161
+ diffsDir: join(scratch.diffs, runId),
162
+ quarantine,
163
+ cases: options.cases,
164
+ ...options.updateCommand === void 0 ? {} : { updateCommand: options.updateCommand }
165
+ };
166
+ store[RUN] = run;
167
+ return run;
168
+ }
169
+ /**
170
+ * The entries a run in this pixel mode applies: every entry, but an L10 entry that names the
171
+ * other mode. Throws on an entry that names a pixel mode for a layer that has none.
172
+ */
173
+ function applicableQuarantine(entries, pixels) {
174
+ for (const entry of entries) {
175
+ if (entry.pixels === void 0) continue;
176
+ if (entry.layer !== "L10" || entry.pixels !== "baseline" && entry.pixels !== "live") throw new Error(`[uf] The quarantine entry ${entry.case} › ${entry.target} › ${entry.layer} names pixels: ${JSON.stringify(entry.pixels)}: only L10 depends on the pixel mode, which is "baseline" or "live".`);
177
+ }
178
+ return entries.filter((entry) => entry.pixels === void 0 || entry.pixels === pixels);
179
+ }
180
+ /** Removes the run directories under `directory` whose process has ended (`<time>-<pid>`). */
181
+ function removeEndedRuns(directory) {
182
+ if (!existsSync(directory)) return;
183
+ for (const name of readdirSync(directory)) {
184
+ const pid = /^[0-9a-z]+-(\d+)$/.exec(name)?.[1];
185
+ if (pid !== void 0 && isRunning(Number(pid))) continue;
186
+ rmSync(join(directory, name), {
187
+ recursive: true,
188
+ force: true
189
+ });
190
+ }
191
+ }
192
+ function isRunning(pid) {
193
+ try {
194
+ process.kill(pid, 0);
195
+ return true;
196
+ } catch (error) {
197
+ return error.code === "EPERM";
198
+ }
199
+ }
200
+ //#endregion
201
+ export { BASELINE_ENVIRONMENT, createHarnessRun, groupOrder, isCI, resolveBrowser, resolveHarnessMode };
@@ -0,0 +1,4 @@
1
+ //#region src/node/names.d.ts
2
+ /** Lower-case letters and digits in words joined by single hyphens, such as `after-click`. */
3
+ export declare const KEBAB_CASE: RegExp;
4
+ //#endregion
@@ -0,0 +1,5 @@
1
+ //#region src/node/names.ts
2
+ /** Lower-case letters and digits in words joined by single hyphens, such as `after-click`. */
3
+ const KEBAB_CASE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
4
+ //#endregion
5
+ export { KEBAB_CASE };
@@ -0,0 +1,44 @@
1
+ //#region src/node/policy.d.ts
2
+ /** Who may write an artefact. */
3
+ export type ArtefactRole = "owner" | "reference" | "follower";
4
+ /** How an artefact was settled. */
5
+ export type ArtefactStatus = "matched" | "written" | "missing" | "missing-reference" | "mismatch" | "stale";
6
+ /** The result of settling an artefact. `message` is empty when it passed. */
7
+ export interface ArtefactOutcome {
8
+ pass: boolean;
9
+ status: ArtefactStatus;
10
+ message: string;
11
+ }
12
+ /** Where and how an artefact is settled. */
13
+ export interface ArtefactContext {
14
+ role: ArtefactRole;
15
+ update: boolean;
16
+ /** Paths in messages are relative to this directory. */
17
+ root: string;
18
+ /** Update mode, for the reference and its followers: the run's ledger directory. */
19
+ ledgerDir?: string;
20
+ /** The reference target's name, for messages. */
21
+ reference?: string;
22
+ /**
23
+ * The command that writes the artefacts, for messages: `pnpm test:update` in the corpus, and
24
+ * `unframework test --update` in a project's tests (ADR-0086).
25
+ */
26
+ updateCommand?: string;
27
+ }
28
+ /**
29
+ * Settles one artefact. `actual` is what this run produced; `undefined` means the artefact must
30
+ * not exist (an IR snapshot for a case with compile errors, a stale diagnostics.txt).
31
+ */
32
+ export declare function settleArtefact(path: string, actual: string | undefined, context: ArtefactContext): ArtefactOutcome;
33
+ /**
34
+ * Settles a directory that holds exactly a set of files (a target's golden outputs): no missing
35
+ * file, no stale file, every file equal. In update mode it writes the files and deletes the
36
+ * stale ones. Owners only: shared expectations are single files.
37
+ */
38
+ export declare function settleArtefactDirectory(directory: string, files: ReadonlyMap<string, string>, context: {
39
+ update: boolean;
40
+ root: string;
41
+ }): ArtefactOutcome;
42
+ /** Writes a file only when its contents change, so update runs leave mtimes alone. */
43
+ export declare function writeIfChanged(path: string, contents: string | Uint8Array): boolean;
44
+ //#endregion