@unframework.dev/testing 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +536 -2
- package/dist/browser/axe.js +34 -0
- package/dist/browser/behaviour.js +59 -0
- package/dist/browser/capabilities.js +58 -0
- package/dist/browser/clock.d.ts +17 -0
- package/dist/browser/clock.js +69 -0
- package/dist/browser/console.d.ts +27 -0
- package/dist/browser/console.js +142 -0
- package/dist/browser/determinism.js +108 -0
- package/dist/browser/element-timeout.js +28 -0
- package/dist/browser/events.js +68 -0
- package/dist/browser/geometry.js +135 -0
- package/dist/browser/keys.js +62 -0
- package/dist/browser/matchers.d.ts +39 -0
- package/dist/browser/matchers.js +78 -0
- package/dist/browser/navigation.js +38 -0
- package/dist/browser/parity.d.ts +10 -0
- package/dist/browser/parity.js +93 -0
- package/dist/browser/spec.d.ts +25 -0
- package/dist/browser/spec.js +38 -0
- package/dist/browser/target.d.ts +16 -0
- package/dist/browser/target.js +27 -0
- package/dist/browser/text-runs.js +68 -0
- package/dist/browser/thrown.js +71 -0
- package/dist/browser/trace.d.ts +21 -0
- package/dist/browser/trace.js +73 -0
- package/dist/browser/user.d.ts +45 -0
- package/dist/browser/user.js +184 -0
- package/dist/browser/view.d.ts +114 -0
- package/dist/browser/view.js +282 -0
- package/dist/browser/visual.d.ts +20 -0
- package/dist/browser/visual.js +36 -0
- package/dist/cli.d.ts +23 -0
- package/dist/cli.js +90 -0
- package/dist/commands.d.ts +107 -0
- package/dist/config/broadcast.js +29 -0
- package/dist/config/cases.d.ts +28 -0
- package/dist/config/cases.js +94 -0
- package/dist/config/index.d.ts +82 -0
- package/dist/config/index.js +295 -0
- package/dist/config/project-vite.d.ts +40 -0
- package/dist/config/project-vite.js +152 -0
- package/dist/diff.d.ts +14 -0
- package/dist/diff.js +92 -0
- package/dist/dom/serialize.js +236 -0
- package/dist/errors.d.ts +13 -0
- package/dist/errors.js +38 -0
- package/dist/harness.d.ts +189 -0
- package/dist/harness.js +53 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +12 -0
- package/dist/internal.d.ts +8 -0
- package/dist/internal.js +6 -0
- package/dist/layers.d.ts +183 -0
- package/dist/layers.js +241 -0
- package/dist/node/browser.d.ts +33 -0
- package/dist/node/browser.js +64 -0
- package/dist/node/commands.d.ts +137 -0
- package/dist/node/commands.js +139 -0
- package/dist/node/context.d.ts +28 -0
- package/dist/node/context.js +55 -0
- package/dist/node/events.d.ts +25 -0
- package/dist/node/events.js +50 -0
- package/dist/node/finished.d.ts +12 -0
- package/dist/node/finished.js +28 -0
- package/dist/node/index.d.ts +21 -0
- package/dist/node/index.js +20 -0
- package/dist/node/matrix.d.ts +219 -0
- package/dist/node/matrix.js +309 -0
- package/dist/node/mode.d.ts +103 -0
- package/dist/node/mode.js +204 -0
- package/dist/node/names.d.ts +4 -0
- package/dist/node/names.js +5 -0
- package/dist/node/policy.d.ts +44 -0
- package/dist/node/policy.js +140 -0
- package/dist/node/reporter.d.ts +101 -0
- package/dist/node/reporter.js +251 -0
- package/dist/node/summary.d.ts +99 -0
- package/dist/node/summary.js +352 -0
- package/dist/node/thrown.js +78 -0
- package/dist/node/viewport.d.ts +7 -0
- package/dist/node/viewport.js +8 -0
- package/dist/node/visual.d.ts +85 -0
- package/dist/node/visual.js +373 -0
- package/dist/node_modules/.pnpm/playwright-core@1.63.0/node_modules/playwright-core/types/protocol.d.ts +23930 -0
- package/dist/node_modules/.pnpm/playwright-core@1.63.0/node_modules/playwright-core/types/structs.d.ts +22 -0
- package/dist/node_modules/.pnpm/playwright-core@1.63.0/node_modules/playwright-core/types/types.d.ts +22221 -0
- package/dist/normalize/display.js +309 -0
- package/dist/normalize/index.d.ts +34 -0
- package/dist/normalize/index.js +38 -0
- package/dist/normalize/pipeline.d.ts +19 -0
- package/dist/normalize/pipeline.js +45 -0
- package/dist/normalize/print.js +103 -0
- package/dist/normalize/rules/angular-hosts.js +53 -0
- package/dist/normalize/rules/aria-hosts.js +48 -0
- package/dist/normalize/rules/comments.js +16 -0
- package/dist/normalize/rules/control-values.js +65 -0
- package/dist/normalize/rules/engine-aliases.js +34 -0
- package/dist/normalize/rules/framework-attributes.js +85 -0
- package/dist/normalize/rules/generated-ids.js +64 -0
- package/dist/normalize/rules/ordering.js +39 -0
- package/dist/normalize/rules/qwik-styles.js +21 -0
- package/dist/normalize/rules/qwik-templates.js +20 -0
- package/dist/normalize/rules/values.js +125 -0
- package/dist/normalize/rules/whitespace.js +381 -0
- package/dist/normalize/style.js +121 -0
- package/dist/normalize/targets.d.ts +8 -0
- package/dist/normalize/targets.js +17 -0
- package/dist/normalize/tree.d.ts +1 -0
- package/dist/normalize/tree.js +151 -0
- package/dist/setup.d.ts +1 -0
- package/dist/setup.js +149 -0
- package/dist/ssr/index.d.ts +10 -0
- package/dist/ssr/index.js +103 -0
- package/dist/ssr/spec.d.ts +1 -0
- package/dist/ssr/spec.js +7 -0
- package/dist/targets/angular.d.ts +1 -0
- package/dist/targets/angular.js +5 -0
- package/dist/targets/astro.d.ts +1 -0
- package/dist/targets/astro.js +5 -0
- package/dist/targets/qwik.d.ts +1 -0
- package/dist/targets/qwik.js +5 -0
- package/dist/targets/react.d.ts +1 -0
- package/dist/targets/react.js +5 -0
- package/dist/targets/solid.d.ts +1 -0
- package/dist/targets/solid.js +5 -0
- package/dist/targets/svelte.d.ts +1 -0
- package/dist/targets/svelte.js +5 -0
- package/dist/targets/vue.d.ts +1 -0
- package/dist/targets/vue.js +5 -0
- package/dist/visual-types.d.ts +92 -0
- package/dist/visual-types.js +24 -0
- package/package.json +146 -3
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import { LayerName, LayerOutcome, QuarantineEntry, UfLayerMeta } from "../layers.js";
|
|
2
|
+
import { PixelMode } from "../harness.js";
|
|
3
|
+
//#region src/node/matrix.d.ts
|
|
4
|
+
/** Case → target → layer → cell. */
|
|
5
|
+
export type MatrixCells = Record<string, Record<string, Partial<Record<LayerName, string>>>>;
|
|
6
|
+
/** Case → target → the parity scenarios its tests checked (`expectParity` names), sorted. */
|
|
7
|
+
export type MatrixScenarios = Record<string, Record<string, string[]>>;
|
|
8
|
+
/**
|
|
9
|
+
* What the matrix keeps of one test: the scenarios it checked, or why it was skipped or ran
|
|
10
|
+
* against a stand-in.
|
|
11
|
+
*/
|
|
12
|
+
export interface MatrixTest {
|
|
13
|
+
/** The parity scenarios the test checked, sorted. */
|
|
14
|
+
scenarios: string[];
|
|
15
|
+
/**
|
|
16
|
+
* Why the test was skipped on this target: a capability it requires is unsupported, or the
|
|
17
|
+
* target has no output for the case.
|
|
18
|
+
*/
|
|
19
|
+
skipped?: string;
|
|
20
|
+
/** Why the test ran against a stand-in on this target: it cannot compile the case yet. */
|
|
21
|
+
standIn?: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Case → target → test (its name without the target, `UfLayerMeta.test`) → what it checked. The
|
|
25
|
+
* summary compares each target's tests of a case with the reference's, test by test, and
|
|
26
|
+
* excuses a test that a target skipped, alone.
|
|
27
|
+
*/
|
|
28
|
+
export type MatrixTests = Record<string, Record<string, Record<string, MatrixTest>>>;
|
|
29
|
+
/** The version of the matrix files; a file of another version is never merged. */
|
|
30
|
+
export declare const MATRIX_VERSION = 5;
|
|
31
|
+
/** One run's (or a merged) parity matrix. */
|
|
32
|
+
export interface ParityMatrix {
|
|
33
|
+
version: typeof MATRIX_VERSION;
|
|
34
|
+
/** The run that wrote it: `all`, a project list, or `merged`. */
|
|
35
|
+
run: string;
|
|
36
|
+
/** The projects whose tests fed the matrix, sorted. */
|
|
37
|
+
projects: string[];
|
|
38
|
+
/** Case → target → layer → cell, merged across projects. */
|
|
39
|
+
cases: MatrixCells;
|
|
40
|
+
/**
|
|
41
|
+
* Case → target → the parity scenarios checked, merged across projects. Every target runs the
|
|
42
|
+
* same spec, so each must check the reference's scenarios (see `summarise`).
|
|
43
|
+
*/
|
|
44
|
+
scenarios: MatrixScenarios;
|
|
45
|
+
/**
|
|
46
|
+
* Case → target → test → the scenarios it checked, or why it was skipped by capability: the
|
|
47
|
+
* skips the cells cannot show (a skipped test's cells merge with its case's other tests'), and
|
|
48
|
+
* what the summary compares test by test.
|
|
49
|
+
*/
|
|
50
|
+
tests: MatrixTests;
|
|
51
|
+
}
|
|
52
|
+
/** How a run treated artefacts (the harness mode), for the reader of a matrix. */
|
|
53
|
+
export interface RunMode {
|
|
54
|
+
update: boolean;
|
|
55
|
+
pixels: PixelMode;
|
|
56
|
+
canary: string | null;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* One part of a sharded run (`--shard <index>/<count>`): Vitest splits the test files of the
|
|
60
|
+
* selected projects into `count` parts and runs part `index` (from 1).
|
|
61
|
+
*/
|
|
62
|
+
export interface Shard {
|
|
63
|
+
index: number;
|
|
64
|
+
count: number;
|
|
65
|
+
}
|
|
66
|
+
/** When and how a partial matrix's run ran. */
|
|
67
|
+
export interface RunInfo {
|
|
68
|
+
run: string;
|
|
69
|
+
/** When the run ended (ISO 8601). Of two records of one project's cell, the newer counts. */
|
|
70
|
+
finishedAt: string;
|
|
71
|
+
/** The harness mode, or `null` for a run outside the harness. */
|
|
72
|
+
mode: RunMode | null;
|
|
73
|
+
/**
|
|
74
|
+
* Why the run covered only part of its projects' tests besides a shard (a file, test-name,
|
|
75
|
+
* tags or changed-files filter, or watch mode), or `null` when nothing else narrowed it.
|
|
76
|
+
*/
|
|
77
|
+
filtered: string | null;
|
|
78
|
+
/**
|
|
79
|
+
* The run's shard, or `null`. Kept apart from `filtered` because shards add up: shards 1 to
|
|
80
|
+
* `count` of one selection, with no other filter, ran every test of its projects.
|
|
81
|
+
*/
|
|
82
|
+
shard: Shard | null;
|
|
83
|
+
projects: string[];
|
|
84
|
+
/**
|
|
85
|
+
* The selected projects that collected no tests. A run nothing narrowed fails on one itself;
|
|
86
|
+
* a shard may hold none of a project's files, so the summary judges them once every shard ran.
|
|
87
|
+
*/
|
|
88
|
+
empty: string[];
|
|
89
|
+
}
|
|
90
|
+
/** Why a run covered only part of its projects' tests, its shard included, or `null`. */
|
|
91
|
+
export declare function narrowing(run: Pick<RunInfo, "filtered" | "shard">): string | null;
|
|
92
|
+
/** What one Vitest run writes: `parity-matrix.<run>.json`. */
|
|
93
|
+
export interface PartialMatrix extends ParityMatrix, RunInfo {
|
|
94
|
+
/**
|
|
95
|
+
* The quarantine the run applied. A merge judges every record by the newest run's, the one
|
|
96
|
+
* `harness/quarantine.ts` held last.
|
|
97
|
+
*/
|
|
98
|
+
quarantine: QuarantineEntry[];
|
|
99
|
+
/**
|
|
100
|
+
* The run's reference target (D10), whose parity scenarios every target's must equal, or
|
|
101
|
+
* `null` for a run outside the harness. A merge takes the newest run's.
|
|
102
|
+
*/
|
|
103
|
+
reference: string | null;
|
|
104
|
+
/**
|
|
105
|
+
* The cases that name their own reference (ADR-0057), with it: their scenarios are judged
|
|
106
|
+
* against that target's. A merge takes the newest run's.
|
|
107
|
+
*/
|
|
108
|
+
references?: Record<string, string>;
|
|
109
|
+
/**
|
|
110
|
+
* The errors Vitest reported outside every test in this run, each as its message and its
|
|
111
|
+
* causes' on one line, such as the closed browser connection that stopped the reference's
|
|
112
|
+
* project (ADR-0091). Absent when there were none.
|
|
113
|
+
*/
|
|
114
|
+
errors?: string[];
|
|
115
|
+
/** Project → its own cells, so a later run of a project replaces exactly its cells. */
|
|
116
|
+
byProject: Record<string, MatrixCells>;
|
|
117
|
+
/** Project → its own scenarios, kept and replaced with its cells. */
|
|
118
|
+
scenariosByProject: Record<string, MatrixScenarios>;
|
|
119
|
+
/** Project → its own tests, kept and replaced with its cells. */
|
|
120
|
+
testsByProject: Record<string, MatrixTests>;
|
|
121
|
+
}
|
|
122
|
+
/** A merge of partial matrices (`parity-matrix.json`), with what it was merged from. */
|
|
123
|
+
export interface MergedMatrix extends ParityMatrix {
|
|
124
|
+
/** The partial matrices merged, oldest first. */
|
|
125
|
+
runs: RunInfo[];
|
|
126
|
+
}
|
|
127
|
+
/** A test's record and the project it ran in. */
|
|
128
|
+
export interface ProjectRecord {
|
|
129
|
+
project: string;
|
|
130
|
+
record: UfLayerMeta;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* An outcome as a matrix cell. A failure keeps its whole message, every sub-check's included
|
|
134
|
+
* (both the DOM and the ARIA diff of L7, the server's and the page's console of L13), so a
|
|
135
|
+
* canary can require the evidence of each.
|
|
136
|
+
*/
|
|
137
|
+
export declare function cellOf(outcome: LayerOutcome): string;
|
|
138
|
+
/** A matrix cell as an outcome. Throws on text that is not a cell, so a corrupt matrix is loud. */
|
|
139
|
+
export declare function outcomeOfCell(cell: string): LayerOutcome;
|
|
140
|
+
/** The first line of a cell, for a one-line summary; the matrix file has the rest. */
|
|
141
|
+
export declare function cellHeadline(cell: string): string;
|
|
142
|
+
/** Builds cells from test records, merging every record of a cell (fail beats pass). */
|
|
143
|
+
export declare function buildCells(records: readonly UfLayerMeta[]): MatrixCells;
|
|
144
|
+
/**
|
|
145
|
+
* The tests of test records: each named test's sorted scenarios, capability skip and stand-in.
|
|
146
|
+
* Records without a test name (a file's own) and tests that checked nothing and were neither
|
|
147
|
+
* skipped nor run against a stand-in are left out: they have nothing to compare.
|
|
148
|
+
*/
|
|
149
|
+
export declare function buildTests(records: readonly UfLayerMeta[]): MatrixTests;
|
|
150
|
+
/**
|
|
151
|
+
* Merges sets of tests, a later set's record of a test replacing an earlier one's (a newer run
|
|
152
|
+
* of the test). Within one set a test is recorded once: two tests of a case never share a name.
|
|
153
|
+
*/
|
|
154
|
+
export declare function mergeTests(sets: readonly MatrixTests[]): MatrixTests;
|
|
155
|
+
/** Merges several projects' cells into one set, cell by cell (fail beats pass). */
|
|
156
|
+
export declare function mergeCells(sets: readonly MatrixCells[]): MatrixCells;
|
|
157
|
+
/** Builds a run's partial matrix from its test records. */
|
|
158
|
+
export declare function buildPartialMatrix(input: {
|
|
159
|
+
run: string;
|
|
160
|
+
/** Every project the run selected, also those that recorded nothing. */
|
|
161
|
+
projects: readonly string[];
|
|
162
|
+
records: readonly ProjectRecord[];
|
|
163
|
+
finishedAt: string;
|
|
164
|
+
mode: RunMode | null;
|
|
165
|
+
filtered: string | null;
|
|
166
|
+
shard: Shard | null;
|
|
167
|
+
/** The selected projects that collected no tests. */
|
|
168
|
+
empty: readonly string[];
|
|
169
|
+
quarantine: readonly QuarantineEntry[];
|
|
170
|
+
/** The run's reference target; none outside the harness. */
|
|
171
|
+
reference?: string | null;
|
|
172
|
+
/** The cases that name their own reference, with it. */
|
|
173
|
+
references?: Record<string, string>;
|
|
174
|
+
/** The errors Vitest reported outside every test, one line each. */
|
|
175
|
+
errors?: readonly string[];
|
|
176
|
+
}): PartialMatrix;
|
|
177
|
+
/**
|
|
178
|
+
* The part of a project a newer run re-judged: only some targets' cells (a run of some targets,
|
|
179
|
+
* `UF_TARGETS`), or only some cases' (a target the run adds for the cases that name it as their
|
|
180
|
+
* own reference, ADR-0057). What a scope leaves out is every target, or every case.
|
|
181
|
+
*/
|
|
182
|
+
export interface ProjectScope {
|
|
183
|
+
targets?: ReadonlySet<string>;
|
|
184
|
+
cases?: ReadonlySet<string>;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* A partial matrix without some projects' records (a newer run has them), or none if empty. A
|
|
188
|
+
* project with a scope loses only the cells inside it, and goes once it has none left. A matrix
|
|
189
|
+
* that loses nothing is returned as it is, and a project that loses nothing keeps its records.
|
|
190
|
+
*/
|
|
191
|
+
export declare function withoutProjects(matrix: PartialMatrix, projects: ReadonlySet<string>, scopes?: ReadonlyMap<string, ProjectScope>): PartialMatrix | undefined;
|
|
192
|
+
/**
|
|
193
|
+
* Merges partial matrices (CI's jobs, local runs of a subset) into one. For each project the
|
|
194
|
+
* newest record of each cell wins, so a re-run replaces what an earlier run recorded; then the
|
|
195
|
+
* projects merge cell by cell (fail beats pass). Scenarios add up, as cells another run did not
|
|
196
|
+
* record are kept: a run of a project replaces its older records of the targets and cases it
|
|
197
|
+
* re-judged (the reporter's `supersede`). The quarantine and the reference are the newest run's:
|
|
198
|
+
* an entry removed since an older run must neither hide that run's failures nor go stale.
|
|
199
|
+
*/
|
|
200
|
+
export declare function mergeMatrices(partials: readonly PartialMatrix[]): {
|
|
201
|
+
matrix: MergedMatrix;
|
|
202
|
+
quarantine: QuarantineEntry[];
|
|
203
|
+
reference: string | null;
|
|
204
|
+
references: Record<string, string>;
|
|
205
|
+
};
|
|
206
|
+
/**
|
|
207
|
+
* Judges cells by a quarantine, which may be newer than the runs that recorded them (see
|
|
208
|
+
* `mergeMatrices`): a failure under an entry is quarantined, and a cell an older run recorded as
|
|
209
|
+
* quarantined, under an entry since removed, fails (its message is not recorded: rerun its
|
|
210
|
+
* project to see it). With `stale`, for cells every record has reached (a complete run, or a
|
|
211
|
+
* merge of complete runs), a cell under an entry that passes, or is skipped, becomes a
|
|
212
|
+
* stale-entry failure (plan §7.7). Returns the failures it made.
|
|
213
|
+
*/
|
|
214
|
+
export declare function settleQuarantine(cells: MatrixCells, quarantine: readonly QuarantineEntry[], options: {
|
|
215
|
+
stale: boolean;
|
|
216
|
+
}): string[];
|
|
217
|
+
/** Serialises a matrix with a trailing newline; equal matrices are equal bytes. */
|
|
218
|
+
export declare function stringifyMatrix(matrix: ParityMatrix): string;
|
|
219
|
+
//#endregion
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
import { LAYERS, mergeOutcomes, quarantineFor, settleOutcome } from "../layers.js";
|
|
2
|
+
//#region src/node/matrix.ts
|
|
3
|
+
/** The version of the matrix files; a file of another version is never merged. */
|
|
4
|
+
const MATRIX_VERSION = 5;
|
|
5
|
+
/** Why a run covered only part of its projects' tests, its shard included, or `null`. */
|
|
6
|
+
function narrowing(run) {
|
|
7
|
+
const reasons = [...run.filtered === null ? [] : [run.filtered], ...run.shard === null ? [] : [`shard ${run.shard.index}/${run.shard.count}`]];
|
|
8
|
+
return reasons.length ? reasons.join("; ") : null;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* An outcome as a matrix cell. A failure keeps its whole message, every sub-check's included
|
|
12
|
+
* (both the DOM and the ARIA diff of L7, the server's and the page's console of L13), so a
|
|
13
|
+
* canary can require the evidence of each.
|
|
14
|
+
*/
|
|
15
|
+
function cellOf(outcome) {
|
|
16
|
+
if (outcome.status === "skip") return `skip(${outcome.reason})`;
|
|
17
|
+
if (outcome.status === "quarantined") return `quarantined(${outcome.issue})`;
|
|
18
|
+
if (outcome.status === "pass") return "pass";
|
|
19
|
+
return `fail: ${outcome.message.trim() || "failed"}`;
|
|
20
|
+
}
|
|
21
|
+
/** A matrix cell as an outcome. Throws on text that is not a cell, so a corrupt matrix is loud. */
|
|
22
|
+
function outcomeOfCell(cell) {
|
|
23
|
+
if (cell === "pass") return { status: "pass" };
|
|
24
|
+
if (cell.startsWith("fail: ")) return {
|
|
25
|
+
status: "fail",
|
|
26
|
+
message: cell.slice(6)
|
|
27
|
+
};
|
|
28
|
+
const match = /^(skip|quarantined)\((.*)\)$/s.exec(cell);
|
|
29
|
+
if (match?.[1] === "skip") return {
|
|
30
|
+
status: "skip",
|
|
31
|
+
reason: match[2]
|
|
32
|
+
};
|
|
33
|
+
if (match?.[1] === "quarantined") return {
|
|
34
|
+
status: "quarantined",
|
|
35
|
+
issue: match[2]
|
|
36
|
+
};
|
|
37
|
+
throw new Error(`Not a parity matrix cell: ${JSON.stringify(cell)}`);
|
|
38
|
+
}
|
|
39
|
+
/** The first line of a cell, for a one-line summary; the matrix file has the rest. */
|
|
40
|
+
function cellHeadline(cell) {
|
|
41
|
+
const lines = cell.split("\n");
|
|
42
|
+
const more = lines.length - 1;
|
|
43
|
+
return more ? `${lines[0]} (+${more} more line(s))` : cell;
|
|
44
|
+
}
|
|
45
|
+
/** Builds cells from test records, merging every record of a cell (fail beats pass). */
|
|
46
|
+
function buildCells(records) {
|
|
47
|
+
const outcomes = /* @__PURE__ */ new Map();
|
|
48
|
+
for (const record of records) {
|
|
49
|
+
let targets = outcomes.get(record.case);
|
|
50
|
+
if (!targets) outcomes.set(record.case, targets = /* @__PURE__ */ new Map());
|
|
51
|
+
let layers = targets.get(record.target);
|
|
52
|
+
if (!layers) targets.set(record.target, layers = /* @__PURE__ */ new Map());
|
|
53
|
+
for (const layer of LAYERS) {
|
|
54
|
+
const outcome = record.layers[layer];
|
|
55
|
+
if (outcome) layers.set(layer, [...layers.get(layer) ?? [], outcome]);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
const cells = {};
|
|
59
|
+
for (const caseId of [...outcomes.keys()].sort()) {
|
|
60
|
+
const targets = outcomes.get(caseId);
|
|
61
|
+
cells[caseId] = {};
|
|
62
|
+
for (const target of [...targets.keys()].sort()) {
|
|
63
|
+
const layers = targets.get(target);
|
|
64
|
+
cells[caseId][target] = Object.fromEntries(LAYERS.filter((layer) => layers.has(layer)).map((layer) => [layer, cellOf(mergeOutcomes(...layers.get(layer)))]));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return cells;
|
|
68
|
+
}
|
|
69
|
+
/** The scenarios of test records: each (case, target)'s names, once each, sorted. */
|
|
70
|
+
function buildScenarios(records) {
|
|
71
|
+
return mergeScenarios(records.flatMap((record) => record.scenarios?.length ? [{ [record.case]: { [record.target]: [...record.scenarios] } }] : []));
|
|
72
|
+
}
|
|
73
|
+
/** Merges sets of scenarios: the union of each (case, target)'s names, sorted. */
|
|
74
|
+
function mergeScenarios(sets) {
|
|
75
|
+
const names = /* @__PURE__ */ new Map();
|
|
76
|
+
for (const scenarios of sets) for (const [caseId, targets] of Object.entries(scenarios)) {
|
|
77
|
+
let byTarget = names.get(caseId);
|
|
78
|
+
if (!byTarget) names.set(caseId, byTarget = /* @__PURE__ */ new Map());
|
|
79
|
+
for (const [target, list] of Object.entries(targets)) {
|
|
80
|
+
const set = byTarget.get(target) ?? /* @__PURE__ */ new Set();
|
|
81
|
+
for (const name of list) set.add(name);
|
|
82
|
+
byTarget.set(target, set);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const merged = {};
|
|
86
|
+
for (const caseId of [...names.keys()].sort()) {
|
|
87
|
+
const byTarget = names.get(caseId);
|
|
88
|
+
merged[caseId] = Object.fromEntries([...byTarget.keys()].sort().map((target) => [target, [...byTarget.get(target)].sort()]));
|
|
89
|
+
}
|
|
90
|
+
return merged;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* The tests of test records: each named test's sorted scenarios, capability skip and stand-in.
|
|
94
|
+
* Records without a test name (a file's own) and tests that checked nothing and were neither
|
|
95
|
+
* skipped nor run against a stand-in are left out: they have nothing to compare.
|
|
96
|
+
*/
|
|
97
|
+
function buildTests(records) {
|
|
98
|
+
return mergeTests(records.flatMap((record) => {
|
|
99
|
+
if (record.test === void 0 || !record.scenarios?.length && record.skipped === void 0 && record.standIn === void 0) return [];
|
|
100
|
+
const test = {
|
|
101
|
+
scenarios: [...new Set(record.scenarios ?? [])].sort(),
|
|
102
|
+
...record.skipped === void 0 ? {} : { skipped: record.skipped },
|
|
103
|
+
...record.standIn === void 0 ? {} : { standIn: record.standIn }
|
|
104
|
+
};
|
|
105
|
+
return [{ [record.case]: { [record.target]: { [record.test]: test } } }];
|
|
106
|
+
}));
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Merges sets of tests, a later set's record of a test replacing an earlier one's (a newer run
|
|
110
|
+
* of the test). Within one set a test is recorded once: two tests of a case never share a name.
|
|
111
|
+
*/
|
|
112
|
+
function mergeTests(sets) {
|
|
113
|
+
const merged = {};
|
|
114
|
+
for (const tests of sets) for (const [caseId, targets] of Object.entries(tests)) for (const [target, byName] of Object.entries(targets)) Object.assign((merged[caseId] ??= {})[target] ??= {}, byName);
|
|
115
|
+
return Object.fromEntries(Object.keys(merged).sort().map((caseId) => [caseId, Object.fromEntries(Object.keys(merged[caseId]).sort().map((target) => {
|
|
116
|
+
const byName = merged[caseId][target];
|
|
117
|
+
return [target, Object.fromEntries(Object.keys(byName).sort().map((name) => [name, byName[name]]))];
|
|
118
|
+
}))]));
|
|
119
|
+
}
|
|
120
|
+
/** Merges several projects' cells into one set, cell by cell (fail beats pass). */
|
|
121
|
+
function mergeCells(sets) {
|
|
122
|
+
return buildCells(sets.flatMap((cells) => Object.entries(cells).flatMap(([caseId, targets]) => Object.entries(targets).map(([target, layers]) => ({
|
|
123
|
+
case: caseId,
|
|
124
|
+
target,
|
|
125
|
+
layers: Object.fromEntries(Object.entries(layers).map(([layer, cell]) => [layer, outcomeOfCell(cell)]))
|
|
126
|
+
})))));
|
|
127
|
+
}
|
|
128
|
+
/** Builds a run's partial matrix from its test records. */
|
|
129
|
+
function buildPartialMatrix(input) {
|
|
130
|
+
const projects = [...new Set(input.projects)].sort();
|
|
131
|
+
const byProject = {};
|
|
132
|
+
const scenariosByProject = {};
|
|
133
|
+
const testsByProject = {};
|
|
134
|
+
for (const project of projects) {
|
|
135
|
+
const records = input.records.filter((entry) => entry.project === project).map((entry) => entry.record);
|
|
136
|
+
byProject[project] = buildCells(records);
|
|
137
|
+
scenariosByProject[project] = buildScenarios(records);
|
|
138
|
+
testsByProject[project] = buildTests(records);
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
version: 5,
|
|
142
|
+
run: input.run,
|
|
143
|
+
finishedAt: input.finishedAt,
|
|
144
|
+
mode: input.mode,
|
|
145
|
+
filtered: input.filtered,
|
|
146
|
+
shard: input.shard,
|
|
147
|
+
projects,
|
|
148
|
+
empty: [...new Set(input.empty)].sort(),
|
|
149
|
+
quarantine: [...input.quarantine],
|
|
150
|
+
reference: input.reference ?? null,
|
|
151
|
+
...input.references && Object.keys(input.references).length ? { references: { ...input.references } } : {},
|
|
152
|
+
...input.errors?.length ? { errors: [...input.errors] } : {},
|
|
153
|
+
cases: mergeCells(Object.values(byProject)),
|
|
154
|
+
scenarios: mergeScenarios(Object.values(scenariosByProject)),
|
|
155
|
+
tests: mergeTests(Object.values(testsByProject)),
|
|
156
|
+
byProject,
|
|
157
|
+
scenariosByProject,
|
|
158
|
+
testsByProject
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* A partial matrix without some projects' records (a newer run has them), or none if empty. A
|
|
163
|
+
* project with a scope loses only the cells inside it, and goes once it has none left. A matrix
|
|
164
|
+
* that loses nothing is returned as it is, and a project that loses nothing keeps its records.
|
|
165
|
+
*/
|
|
166
|
+
function withoutProjects(matrix, projects, scopes = /* @__PURE__ */ new Map()) {
|
|
167
|
+
const byProject = {};
|
|
168
|
+
const scenariosByProject = {};
|
|
169
|
+
const testsByProject = {};
|
|
170
|
+
let dropped = false;
|
|
171
|
+
for (const project of matrix.projects) {
|
|
172
|
+
const scope = projects.has(project) ? scopes.get(project) ?? {} : void 0;
|
|
173
|
+
const before = [
|
|
174
|
+
matrix.byProject[project] ?? {},
|
|
175
|
+
matrix.scenariosByProject[project] ?? {},
|
|
176
|
+
matrix.testsByProject[project] ?? {}
|
|
177
|
+
];
|
|
178
|
+
const cells = outsideScope(before[0], scope);
|
|
179
|
+
const scenarios = outsideScope(before[1], scope);
|
|
180
|
+
const tests = outsideScope(before[2], scope);
|
|
181
|
+
if (cells !== before[0] || scenarios !== before[1] || tests !== before[2]) dropped = true;
|
|
182
|
+
if (scope && ![
|
|
183
|
+
cells,
|
|
184
|
+
scenarios,
|
|
185
|
+
tests
|
|
186
|
+
].some((set) => Object.keys(set).length)) {
|
|
187
|
+
dropped = true;
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
byProject[project] = cells;
|
|
191
|
+
scenariosByProject[project] = scenarios;
|
|
192
|
+
testsByProject[project] = tests;
|
|
193
|
+
}
|
|
194
|
+
if (!dropped) return matrix;
|
|
195
|
+
const kept = matrix.projects.filter((project) => project in byProject);
|
|
196
|
+
if (!kept.length) return void 0;
|
|
197
|
+
return {
|
|
198
|
+
...matrix,
|
|
199
|
+
projects: kept,
|
|
200
|
+
empty: matrix.empty.filter((project) => kept.includes(project)),
|
|
201
|
+
byProject,
|
|
202
|
+
scenariosByProject,
|
|
203
|
+
testsByProject,
|
|
204
|
+
cases: mergeCells(Object.values(byProject)),
|
|
205
|
+
scenarios: mergeScenarios(Object.values(scenariosByProject)),
|
|
206
|
+
tests: mergeTests(Object.values(testsByProject))
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Case → target → what is recorded, without what a scope covers; all of it without a scope. The
|
|
211
|
+
* same object when the scope covers none of it.
|
|
212
|
+
*/
|
|
213
|
+
function outsideScope(byCase, scope) {
|
|
214
|
+
if (!scope) return byCase;
|
|
215
|
+
const kept = {};
|
|
216
|
+
let dropped = false;
|
|
217
|
+
for (const [caseId, targets] of Object.entries(byCase)) for (const [target, value] of Object.entries(targets)) {
|
|
218
|
+
if ((scope.cases?.has(caseId) ?? true) && (scope.targets?.has(target) ?? true)) {
|
|
219
|
+
dropped = true;
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
(kept[caseId] ??= {})[target] = value;
|
|
223
|
+
}
|
|
224
|
+
return dropped ? kept : byCase;
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Merges partial matrices (CI's jobs, local runs of a subset) into one. For each project the
|
|
228
|
+
* newest record of each cell wins, so a re-run replaces what an earlier run recorded; then the
|
|
229
|
+
* projects merge cell by cell (fail beats pass). Scenarios add up, as cells another run did not
|
|
230
|
+
* record are kept: a run of a project replaces its older records of the targets and cases it
|
|
231
|
+
* re-judged (the reporter's `supersede`). The quarantine and the reference are the newest run's:
|
|
232
|
+
* an entry removed since an older run must neither hide that run's failures nor go stale.
|
|
233
|
+
*/
|
|
234
|
+
function mergeMatrices(partials) {
|
|
235
|
+
for (const partial of partials) if (partial.version !== 5) throw new Error(`Unsupported parity matrix version ${String(partial.version)} (run ${partial.run}): rerun the tests to replace it.`);
|
|
236
|
+
const ordered = [...partials].sort((a, b) => a.finishedAt.localeCompare(b.finishedAt) || a.run.localeCompare(b.run));
|
|
237
|
+
const latest = /* @__PURE__ */ new Map();
|
|
238
|
+
const scenarios = [];
|
|
239
|
+
const tests = [];
|
|
240
|
+
for (const partial of ordered) {
|
|
241
|
+
scenarios.push(...Object.values(partial.scenariosByProject));
|
|
242
|
+
tests.push(...Object.values(partial.testsByProject));
|
|
243
|
+
for (const [project, cells] of Object.entries(partial.byProject)) {
|
|
244
|
+
const merged = latest.get(project) ?? {};
|
|
245
|
+
for (const [caseId, targets] of Object.entries(cells)) for (const [target, layers] of Object.entries(targets)) Object.assign((merged[caseId] ??= {})[target] ??= {}, layers);
|
|
246
|
+
latest.set(project, merged);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return {
|
|
250
|
+
matrix: {
|
|
251
|
+
version: 5,
|
|
252
|
+
run: "merged",
|
|
253
|
+
projects: [...new Set(ordered.flatMap((partial) => partial.projects))].sort(),
|
|
254
|
+
cases: mergeCells([...latest.values()]),
|
|
255
|
+
scenarios: mergeScenarios(scenarios),
|
|
256
|
+
tests: mergeTests(tests),
|
|
257
|
+
runs: ordered.map(({ run, finishedAt, mode, filtered, shard, projects, empty }) => ({
|
|
258
|
+
run,
|
|
259
|
+
finishedAt,
|
|
260
|
+
mode,
|
|
261
|
+
filtered,
|
|
262
|
+
shard,
|
|
263
|
+
projects,
|
|
264
|
+
empty
|
|
265
|
+
}))
|
|
266
|
+
},
|
|
267
|
+
quarantine: [...ordered.at(-1)?.quarantine ?? []],
|
|
268
|
+
reference: ordered.findLast((partial) => partial.reference !== null)?.reference ?? null,
|
|
269
|
+
references: { ...ordered.findLast((partial) => partial.reference !== null)?.references }
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Judges cells by a quarantine, which may be newer than the runs that recorded them (see
|
|
274
|
+
* `mergeMatrices`): a failure under an entry is quarantined, and a cell an older run recorded as
|
|
275
|
+
* quarantined, under an entry since removed, fails (its message is not recorded: rerun its
|
|
276
|
+
* project to see it). With `stale`, for cells every record has reached (a complete run, or a
|
|
277
|
+
* merge of complete runs), a cell under an entry that passes, or is skipped, becomes a
|
|
278
|
+
* stale-entry failure (plan §7.7). Returns the failures it made.
|
|
279
|
+
*/
|
|
280
|
+
function settleQuarantine(cells, quarantine, options) {
|
|
281
|
+
const problems = [];
|
|
282
|
+
for (const [caseId, targets] of Object.entries(cells)) for (const [target, layers] of Object.entries(targets)) for (const layer of LAYERS) {
|
|
283
|
+
const cell = layers[layer];
|
|
284
|
+
if (cell === void 0) continue;
|
|
285
|
+
const outcome = outcomeOfCell(cell);
|
|
286
|
+
const entry = quarantineFor(quarantine, {
|
|
287
|
+
case: caseId,
|
|
288
|
+
target
|
|
289
|
+
}, layer);
|
|
290
|
+
let settled;
|
|
291
|
+
if (!entry) {
|
|
292
|
+
if (outcome.status !== "quarantined") continue;
|
|
293
|
+
settled = {
|
|
294
|
+
status: "fail",
|
|
295
|
+
message: `quarantined for ${outcome.issue} by an older run, but no entry quarantines ${caseId} › ${target} › ${layer} any more: rerun its project to see the failure.`
|
|
296
|
+
};
|
|
297
|
+
} else if (outcome.status === "fail" || options.stale) settled = settleOutcome(outcome, entry);
|
|
298
|
+
else continue;
|
|
299
|
+
layers[layer] = cellOf(settled);
|
|
300
|
+
if (settled.status === "fail") problems.push(settled.message);
|
|
301
|
+
}
|
|
302
|
+
return problems;
|
|
303
|
+
}
|
|
304
|
+
/** Serialises a matrix with a trailing newline; equal matrices are equal bytes. */
|
|
305
|
+
function stringifyMatrix(matrix) {
|
|
306
|
+
return `${JSON.stringify(matrix, null, 2)}\n`;
|
|
307
|
+
}
|
|
308
|
+
//#endregion
|
|
309
|
+
export { MATRIX_VERSION, buildCells, buildPartialMatrix, buildScenarios, buildTests, cellHeadline, cellOf, mergeCells, mergeMatrices, mergeScenarios, mergeTests, narrowing, outcomeOfCell, settleQuarantine, stringifyMatrix, withoutProjects };
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { QuarantineEntry } from "../layers.js";
|
|
2
|
+
import { BrowserEngine, CaseConfig, HarnessContext, PixelMode } from "../harness.js";
|
|
3
|
+
//#region src/node/mode.d.ts
|
|
4
|
+
/**
|
|
5
|
+
* The image and platform the committed Linux baselines (`*-chromium-linux.png`, geometry) come
|
|
6
|
+
* from: CI's browser jobs run in it, and `pnpm test:baselines` exports it as
|
|
7
|
+
* `UF_BASELINE_ENVIRONMENT`. Only a run in it writes them. The image follows the playwright
|
|
8
|
+
* version of the catalog, which ships its browsers.
|
|
9
|
+
*/
|
|
10
|
+
export declare const BASELINE_ENVIRONMENT = "mcr.microsoft.com/playwright:v1.63.0-noble linux/amd64";
|
|
11
|
+
/** How this run treats artefacts. */
|
|
12
|
+
export interface HarnessMode {
|
|
13
|
+
/** `UF_UPDATE=1`: owners and the reference write. */
|
|
14
|
+
update: boolean;
|
|
15
|
+
/**
|
|
16
|
+
* `baseline` in CI and in the baseline environment, `live` elsewhere; `UF_PIXELS` overrides.
|
|
17
|
+
* Another host's Chromium and system libraries render differently from the image's, so a
|
|
18
|
+
* plain Linux host compares live too.
|
|
19
|
+
*/
|
|
20
|
+
pixels: PixelMode;
|
|
21
|
+
/** `UF_CANARY=<id>`: the canary compiler plugin to install. */
|
|
22
|
+
canary: string | null;
|
|
23
|
+
/** `CI` is set: check mode only, against committed baselines. */
|
|
24
|
+
ci: boolean;
|
|
25
|
+
/**
|
|
26
|
+
* `UF_BASELINE_ENVIRONMENT`: the image and platform the run says it is in, if any. Only
|
|
27
|
+
* {@link BASELINE_ENVIRONMENT} may write the committed visual baselines.
|
|
28
|
+
*/
|
|
29
|
+
baselineEnvironment?: string | null;
|
|
30
|
+
/** `UF_BROWSER`: the engine the browser projects run in; Chromium when absent. */
|
|
31
|
+
browser?: BrowserEngine;
|
|
32
|
+
}
|
|
33
|
+
/** What {@link resolveHarnessMode} reads. */
|
|
34
|
+
export interface ModeInput {
|
|
35
|
+
env: Readonly<Record<string, string | undefined>>;
|
|
36
|
+
platform: NodeJS.Platform;
|
|
37
|
+
/** The CPU architecture, `process.arch` by default: the baseline environment is x64. */
|
|
38
|
+
arch?: NodeJS.Architecture;
|
|
39
|
+
argv: readonly string[];
|
|
40
|
+
/**
|
|
41
|
+
* The run has no committed visual baselines (the testing package's own fixture, whose one
|
|
42
|
+
* target is its own reference): pixels are always live, in CI too, and `UF_PIXELS` is refused.
|
|
43
|
+
* It runs in Chromium alone, so another `UF_BROWSER` is refused too.
|
|
44
|
+
*/
|
|
45
|
+
liveOnly?: boolean;
|
|
46
|
+
/**
|
|
47
|
+
* A project's own tests (`defineTestConfig`, ADR-0086), not the corpus. Pixels compare as the
|
|
48
|
+
* project chose, live by default, in CI too: a project has no committed baselines unless it
|
|
49
|
+
* opts into them with `baseline`, and then only {@link BASELINE_ENVIRONMENT} writes them.
|
|
50
|
+
* `UF_PIXELS` still overrides.
|
|
51
|
+
*/
|
|
52
|
+
project?: {
|
|
53
|
+
pixels: PixelMode;
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/** Resolves the run's mode, refusing combinations that would write the wrong thing. */
|
|
57
|
+
export declare function resolveHarnessMode(input: ModeInput): HarnessMode;
|
|
58
|
+
/** The engine `UF_BROWSER` names, Chromium when it is unset. */
|
|
59
|
+
export declare function resolveBrowser(env: Readonly<Record<string, string | undefined>>): BrowserEngine;
|
|
60
|
+
/** The kinds of project the harness runs (plan §7.3). */
|
|
61
|
+
export type ProjectKind = "compile" | "harness" | "toolchain" | "ssr" | "browser" | "bench";
|
|
62
|
+
/**
|
|
63
|
+
* A project's `sequence.groupOrder`. Groups run one after another, so a project that reads what
|
|
64
|
+
* another writes in the same run must sit in a later group:
|
|
65
|
+
*
|
|
66
|
+
* - update: compile (and the harness's own tests, and the budgets) 0, then the toolchains and the reference's
|
|
67
|
+
* ssr and browser projects 1, then every follower 2; where a case names its own reference
|
|
68
|
+
* (`caseReferences`, ADR-0057), that target's projects run at 2 alone, as they follow the
|
|
69
|
+
* reference everywhere else, and the other followers at 3;
|
|
70
|
+
* - check with live pixels: the reference browser project 0, the other browser projects 1; the
|
|
71
|
+
* browser projects of the targets some case names at 1 alone, and the others at 2;
|
|
72
|
+
* - check with live pixels in Firefox or WebKit (ADR-0081): every browser project alone, the
|
|
73
|
+
* reference at 1, then the targets some case names, then each other target in the order of
|
|
74
|
+
* `targets`. Five followers at once in one engine ran the nightly's WebKit job out of memory.
|
|
75
|
+
* Each of those projects also runs one file at a time (`fileParallelism: false`), so the
|
|
76
|
+
* reference is not at 0: Vitest moves a project at 0 that runs one file at a time after every
|
|
77
|
+
* other group, and refuses one there beside projects with another worker count;
|
|
78
|
+
* - check with baselines (CI): everything 0, in parallel.
|
|
79
|
+
*/
|
|
80
|
+
export declare function groupOrder(kind: ProjectKind, target: string | undefined, mode: Pick<HarnessMode, "update" | "pixels" | "browser">, reference: string, caseReferences?: readonly string[], targets?: readonly string[]): number;
|
|
81
|
+
/** What a run is made of, besides the mode. */
|
|
82
|
+
export interface HarnessRunOptions {
|
|
83
|
+
/** The integration package's directory: scratch output goes under it. */
|
|
84
|
+
root: string;
|
|
85
|
+
casesDir: string;
|
|
86
|
+
reference: string;
|
|
87
|
+
mode: HarnessMode;
|
|
88
|
+
quarantine: QuarantineEntry[];
|
|
89
|
+
cases: Record<string, CaseConfig>;
|
|
90
|
+
/** The command a failure names to write the artefacts (`HarnessContext.updateCommand`). */
|
|
91
|
+
updateCommand?: string;
|
|
92
|
+
/** The font families a live capture may take (`HarnessContext.liveFonts`, ADR-0105). */
|
|
93
|
+
liveFonts?: readonly string[];
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Creates the run's context, once per process: Vitest evaluates the config file more than once
|
|
97
|
+
* (the root, then each project), and every evaluation must agree on the run id. Each run's
|
|
98
|
+
* scratch output (the ledger, live captures and diffs) is its own directory, named by the run
|
|
99
|
+
* id; the first call removes the directories of runs that have ended, and never those of a run
|
|
100
|
+
* still going in another process (`pnpm test:canaries` next to `pnpm test`).
|
|
101
|
+
*/
|
|
102
|
+
export declare function createHarnessRun(options: HarnessRunOptions): HarnessContext;
|
|
103
|
+
//#endregion
|