@dcaldeira/tooling 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -34
- package/checks/a11y.mjs +206 -206
- package/checks/architecture.mjs +137 -137
- package/checks/audit.mjs +121 -121
- package/checks/bundle.mjs +128 -126
- package/checks/dependencies.mjs +277 -277
- package/checks/duplication.mjs +135 -0
- package/checks/lint-scss.mjs +363 -349
- package/checks/package.mjs +30 -30
- package/checks/playwright-summary.mjs +53 -53
- package/checks/sass.mjs +67 -67
- package/checks/scan-secrets.mjs +58 -58
- package/checks/shared.mjs +86 -86
- package/checks/translations.mjs +55 -55
- package/checks/visual-compare.mjs +148 -148
- package/checks/visual-report-html.mjs +470 -470
- package/checks/visual-report.mjs +13 -13
- package/config.schema.json +999 -927
- package/configs/eslint.mjs +27 -27
- package/configs/knip.json +11 -11
- package/configs/playwright.mjs +15 -15
- package/configs/vitest.mjs +16 -16
- package/package.json +105 -105
- package/release/finalize.mjs +65 -65
- package/release/github.mjs +64 -64
- package/release/prepare.mjs +30 -30
- package/runner/browsers.mjs +73 -73
- package/runner/cache.mjs +121 -121
- package/runner/ci-plan.mjs +54 -54
- package/runner/cli.mjs +250 -248
- package/runner/config.mjs +148 -148
- package/runner/docs-install.mjs +93 -93
- package/runner/execute.mjs +298 -286
- package/runner/generate.mjs +122 -122
- package/runner/git.mjs +122 -122
- package/runner/graph.mjs +250 -243
- package/runner/insights.mjs +313 -313
- package/runner/provision.mjs +53 -53
- package/runner/repo-standard.mjs +104 -0
- package/runner/report.mjs +35 -35
- package/runner/serve-static.mjs +76 -76
- package/runner/setup.mjs +241 -237
- package/runner/summary.mjs +112 -112
- package/runner/watch.mjs +82 -82
- package/templates/ci.yml +25 -25
- package/templates/dependabot-merge.yml +13 -13
- package/templates/dependabot.yml +26 -26
- package/templates/pre-push +13 -13
- package/templates/release.yml +23 -23
package/runner/insights.mjs
CHANGED
|
@@ -1,313 +1,313 @@
|
|
|
1
|
-
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
-
import {
|
|
3
|
-
mkdirSync,
|
|
4
|
-
openSync,
|
|
5
|
-
closeSync,
|
|
6
|
-
readFileSync,
|
|
7
|
-
statSync,
|
|
8
|
-
writeFileSync,
|
|
9
|
-
renameSync,
|
|
10
|
-
unlinkSync,
|
|
11
|
-
} from 'node:fs';
|
|
12
|
-
import { join, resolve } from 'node:path';
|
|
13
|
-
import { inside } from './config.mjs';
|
|
14
|
-
|
|
15
|
-
const finite = (value) => typeof value === 'number' && Number.isFinite(value) && value >= 0;
|
|
16
|
-
const median = (values) => {
|
|
17
|
-
const sorted = [...values].sort((a, b) => a - b);
|
|
18
|
-
if (!sorted.length) return null;
|
|
19
|
-
const middle = Math.floor(sorted.length / 2);
|
|
20
|
-
return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
|
|
21
|
-
};
|
|
22
|
-
const readFresh = (path, since) => {
|
|
23
|
-
try {
|
|
24
|
-
const stat = statSync(path);
|
|
25
|
-
if (stat.size > 20 * 1024 * 1024 || stat.mtimeMs < since - 1000) return null;
|
|
26
|
-
return JSON.parse(readFileSync(path, 'utf8'));
|
|
27
|
-
} catch (error) {
|
|
28
|
-
if (error.code === 'ENOENT') return null;
|
|
29
|
-
return null;
|
|
30
|
-
}
|
|
31
|
-
};
|
|
32
|
-
|
|
33
|
-
export function collectMetrics(root, output, report, settings = {}) {
|
|
34
|
-
const since = Date.parse(report.startedAt) || Date.now();
|
|
35
|
-
const metrics = { slowTests: [] };
|
|
36
|
-
const threshold = settings.slowTestMilliseconds ?? 500;
|
|
37
|
-
const seen = new Set();
|
|
38
|
-
const read = (path) => {
|
|
39
|
-
const full = resolve(path);
|
|
40
|
-
if (seen.has(full)) return null;
|
|
41
|
-
seen.add(full);
|
|
42
|
-
return readFresh(full, since);
|
|
43
|
-
};
|
|
44
|
-
const totals = {};
|
|
45
|
-
function coverage(data) {
|
|
46
|
-
if (!data?.total) return;
|
|
47
|
-
for (const key of ['lines', 'branches', 'functions', 'statements']) {
|
|
48
|
-
const value = data.total[key];
|
|
49
|
-
if (!value) continue;
|
|
50
|
-
if (finite(value.total) && finite(value.covered)) {
|
|
51
|
-
const sum = totals[key] ?? { total: 0, covered: 0 };
|
|
52
|
-
sum.total += value.total;
|
|
53
|
-
sum.covered += value.covered;
|
|
54
|
-
totals[key] = sum;
|
|
55
|
-
metrics.coverage ??= {};
|
|
56
|
-
metrics.coverage[key] = sum.total ? (100 * sum.covered) / sum.total : 100;
|
|
57
|
-
} else if (finite(value.pct) && !totals[key]) {
|
|
58
|
-
metrics.coverage ??= {};
|
|
59
|
-
metrics.coverage[key] = value.pct;
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
function unit(data) {
|
|
64
|
-
for (const file of data.testResults ?? [])
|
|
65
|
-
for (const result of file.assertionResults ?? []) {
|
|
66
|
-
if (finite(result.duration) && result.duration >= threshold)
|
|
67
|
-
metrics.slowTests.push({
|
|
68
|
-
name: [
|
|
69
|
-
...(result.ancestorTitles ?? []),
|
|
70
|
-
result.title ?? result.fullName ?? file.name,
|
|
71
|
-
].join(' > '),
|
|
72
|
-
milliseconds: result.duration,
|
|
73
|
-
});
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
function playwright(data) {
|
|
77
|
-
const stats = data.stats ?? {};
|
|
78
|
-
metrics.browser ??= { passed: 0, failed: 0, flaky: 0, skipped: 0 };
|
|
79
|
-
for (const [key, field] of Object.entries({
|
|
80
|
-
passed: 'expected',
|
|
81
|
-
failed: 'unexpected',
|
|
82
|
-
flaky: 'flaky',
|
|
83
|
-
skipped: 'skipped',
|
|
84
|
-
}))
|
|
85
|
-
if (finite(stats[field])) metrics.browser[key] += stats[field];
|
|
86
|
-
function visit(suite, parents = []) {
|
|
87
|
-
for (const spec of suite.specs ?? [])
|
|
88
|
-
for (const test of spec.tests ?? []) {
|
|
89
|
-
const duration = Math.max(
|
|
90
|
-
0,
|
|
91
|
-
...(test.results ?? []).map((result) => result.duration ?? 0),
|
|
92
|
-
);
|
|
93
|
-
if (finite(duration) && duration >= threshold)
|
|
94
|
-
metrics.slowTests.push({
|
|
95
|
-
name: [...parents, spec.title].join(' > '),
|
|
96
|
-
milliseconds: duration,
|
|
97
|
-
});
|
|
98
|
-
}
|
|
99
|
-
for (const child of suite.suites ?? []) visit(child, [...parents, child.title]);
|
|
100
|
-
}
|
|
101
|
-
for (const suite of data.suites ?? []) visit(suite, [suite.title]);
|
|
102
|
-
}
|
|
103
|
-
for (const check of report.results) {
|
|
104
|
-
if (check.cached || ['excluded', 'unavailable'].includes(check.status)) continue;
|
|
105
|
-
const kind = check.kind ?? check.id;
|
|
106
|
-
const details = read(join(output, `${check.id}.details.json`));
|
|
107
|
-
if (details?.bundle) metrics.bundle = details.bundle;
|
|
108
|
-
if (details?.a11y)
|
|
109
|
-
metrics.a11y = {
|
|
110
|
-
pages: details.a11y.pages?.length ?? 0,
|
|
111
|
-
violations: details.a11y.failures ?? 0,
|
|
112
|
-
errors: details.a11y.errors?.length ?? 0,
|
|
113
|
-
};
|
|
114
|
-
if (details?.architecture)
|
|
115
|
-
metrics.architecture = {
|
|
116
|
-
files: details.architecture.files,
|
|
117
|
-
issues: details.architecture.issues?.length ?? 0,
|
|
118
|
-
};
|
|
119
|
-
if (details?.dependencies)
|
|
120
|
-
metrics.dependencies = {
|
|
121
|
-
packages: details.dependencies.packages,
|
|
122
|
-
issues: details.dependencies.issues?.length ?? 0,
|
|
123
|
-
duplicates: details.dependencies.warnings?.length ?? 0,
|
|
124
|
-
};
|
|
125
|
-
if (kind === 'unit') {
|
|
126
|
-
const data = read(join(output, `${check.id}-results/vitest.json`));
|
|
127
|
-
if (data) unit(data);
|
|
128
|
-
coverage(read(join(output, `${check.id}-results/coverage/coverage-summary.json`)));
|
|
129
|
-
}
|
|
130
|
-
if (['e2e', 'visual'].includes(kind)) {
|
|
131
|
-
const data = read(join(output, `${check.id}-results/results.json`));
|
|
132
|
-
if (data) playwright(data);
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
for (const path of settings.coverage ?? []) {
|
|
136
|
-
if (
|
|
137
|
-
!report.results.some(
|
|
138
|
-
(check) =>
|
|
139
|
-
(check.kind ?? check.id) === 'unit' &&
|
|
140
|
-
!check.cached &&
|
|
141
|
-
!['excluded', 'unavailable'].includes(check.status),
|
|
142
|
-
)
|
|
143
|
-
)
|
|
144
|
-
break;
|
|
145
|
-
const data = read(inside(root, path));
|
|
146
|
-
coverage(data);
|
|
147
|
-
}
|
|
148
|
-
for (const path of settings.unit ?? []) {
|
|
149
|
-
if (
|
|
150
|
-
report.results.some(
|
|
151
|
-
(check) =>
|
|
152
|
-
(check.kind ?? check.id) === 'unit' &&
|
|
153
|
-
!check.cached &&
|
|
154
|
-
!['excluded', 'unavailable'].includes(check.status),
|
|
155
|
-
)
|
|
156
|
-
) {
|
|
157
|
-
const data = read(inside(root, path));
|
|
158
|
-
if (data) unit(data);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
for (const path of settings.playwright ?? []) {
|
|
162
|
-
if (
|
|
163
|
-
report.results.some(
|
|
164
|
-
(check) =>
|
|
165
|
-
['e2e', 'visual'].includes(check.kind ?? check.id) &&
|
|
166
|
-
!['excluded', 'unavailable'].includes(check.status),
|
|
167
|
-
)
|
|
168
|
-
) {
|
|
169
|
-
const data = read(inside(root, path));
|
|
170
|
-
if (data) playwright(data);
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
metrics.slowTests.sort((a, b) => b.milliseconds - a.milliseconds);
|
|
174
|
-
metrics.slowTests = metrics.slowTests.slice(0, 20);
|
|
175
|
-
return metrics;
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
export function readHistory(root) {
|
|
179
|
-
try {
|
|
180
|
-
const path = join(root, '.tooling/history/runs.json');
|
|
181
|
-
if (statSync(path).size > 10 * 1024 * 1024) return [];
|
|
182
|
-
const data = JSON.parse(readFileSync(path, 'utf8'));
|
|
183
|
-
if (data.schemaVersion !== 1 || !Array.isArray(data.runs)) return [];
|
|
184
|
-
return data.runs
|
|
185
|
-
.filter(
|
|
186
|
-
(run) =>
|
|
187
|
-
typeof run.context === 'string' &&
|
|
188
|
-
typeof run.finishedAt === 'string' &&
|
|
189
|
-
typeof run.profile === 'string' &&
|
|
190
|
-
typeof run.commit === 'string' &&
|
|
191
|
-
typeof run.complete === 'boolean' &&
|
|
192
|
-
finite(run.wallMilliseconds) &&
|
|
193
|
-
Array.isArray(run.checks) &&
|
|
194
|
-
run.checks.every((check) => typeof check.id === 'string' && finite(check.milliseconds)),
|
|
195
|
-
)
|
|
196
|
-
.slice(-1000);
|
|
197
|
-
} catch {
|
|
198
|
-
return [];
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
export function performanceHistory(root, report, settings = {}) {
|
|
203
|
-
const runs = readHistory(root);
|
|
204
|
-
const context = createHash('sha256')
|
|
205
|
-
.update(
|
|
206
|
-
JSON.stringify({
|
|
207
|
-
profile: report.profile,
|
|
208
|
-
groups: [...report.groups].sort(),
|
|
209
|
-
checks: [...report.required].sort(),
|
|
210
|
-
node: report.node,
|
|
211
|
-
platform: report.platform,
|
|
212
|
-
configuration: report.configuration,
|
|
213
|
-
}),
|
|
214
|
-
)
|
|
215
|
-
.digest('hex');
|
|
216
|
-
const checks = report.results.map((result) => ({
|
|
217
|
-
id: result.id,
|
|
218
|
-
status: result.status,
|
|
219
|
-
cached: Boolean(result.cached),
|
|
220
|
-
milliseconds: finite(result.milliseconds) ? result.milliseconds : 0,
|
|
221
|
-
}));
|
|
222
|
-
const prior = runs.filter((run) => run.context === context && run.complete).slice(-10);
|
|
223
|
-
const comparable = prior.filter(
|
|
224
|
-
(run) =>
|
|
225
|
-
run.checks.map((check) => `${check.id}:${check.cached}:${check.status}`).join() ===
|
|
226
|
-
checks.map((check) => `${check.id}:${check.cached}:${check.status}`).join(),
|
|
227
|
-
);
|
|
228
|
-
const previousMs = median(comparable.map((run) => run.wallMilliseconds));
|
|
229
|
-
const regressions = checks
|
|
230
|
-
.filter((check) => check.status === 'passed' && !check.cached)
|
|
231
|
-
.flatMap((check) => {
|
|
232
|
-
const samples = prior.flatMap((run) =>
|
|
233
|
-
run.checks
|
|
234
|
-
.filter(
|
|
235
|
-
(previous) =>
|
|
236
|
-
previous.id === check.id && previous.status === 'passed' && !previous.cached,
|
|
237
|
-
)
|
|
238
|
-
.map((previous) => previous.milliseconds),
|
|
239
|
-
);
|
|
240
|
-
const baseline = median(samples);
|
|
241
|
-
return samples.length >= 3 &&
|
|
242
|
-
baseline > 0 &&
|
|
243
|
-
check.milliseconds > baseline * (1 + (settings.regressionPercent ?? 25) / 100)
|
|
244
|
-
? [{ id: check.id, previousMs: baseline, currentMs: check.milliseconds }]
|
|
245
|
-
: [];
|
|
246
|
-
});
|
|
247
|
-
const result = {
|
|
248
|
-
runs: Math.min(runs.length + 1, settings.limit ?? 200),
|
|
249
|
-
cacheHits: checks.filter((check) => check.cached).length,
|
|
250
|
-
previousMs,
|
|
251
|
-
deltaMs: previousMs === null ? null : report.wallMilliseconds - previousMs,
|
|
252
|
-
regressions,
|
|
253
|
-
bundleDeltaKB:
|
|
254
|
-
finite(report.metrics?.bundle?.initialKB) && finite(prior.at(-1)?.bundleKB)
|
|
255
|
-
? report.metrics.bundle.initialKB - prior.at(-1).bundleKB
|
|
256
|
-
: null,
|
|
257
|
-
};
|
|
258
|
-
if (settings.enabled === false) return { ...result, runs: 0 };
|
|
259
|
-
const record = {
|
|
260
|
-
context,
|
|
261
|
-
finishedAt: report.finishedAt,
|
|
262
|
-
commit: report.commit,
|
|
263
|
-
profile: report.profile,
|
|
264
|
-
complete: report.complete,
|
|
265
|
-
wallMilliseconds: report.wallMilliseconds,
|
|
266
|
-
checks,
|
|
267
|
-
bundleKB: report.metrics?.bundle?.initialKB,
|
|
268
|
-
coverage: report.metrics?.coverage,
|
|
269
|
-
slowTests: report.metrics?.slowTests ?? [],
|
|
270
|
-
};
|
|
271
|
-
const directory = join(root, '.tooling/history');
|
|
272
|
-
mkdirSync(directory, { recursive: true });
|
|
273
|
-
const lockPath = join(directory, 'write.lock');
|
|
274
|
-
let lock;
|
|
275
|
-
try {
|
|
276
|
-
try {
|
|
277
|
-
if (Date.now() - statSync(lockPath).mtimeMs > 120000) unlinkSync(lockPath);
|
|
278
|
-
} catch (error) {
|
|
279
|
-
if (error.code !== 'ENOENT') throw error;
|
|
280
|
-
}
|
|
281
|
-
lock = openSync(lockPath, 'wx');
|
|
282
|
-
const target = join(directory, 'runs.json'),
|
|
283
|
-
temporary = `${target}.${randomUUID()}.tmp`;
|
|
284
|
-
writeFileSync(
|
|
285
|
-
temporary,
|
|
286
|
-
JSON.stringify({
|
|
287
|
-
schemaVersion: 1,
|
|
288
|
-
runs: [...readHistory(root), record].slice(-(settings.limit ?? 200)),
|
|
289
|
-
}),
|
|
290
|
-
);
|
|
291
|
-
renameSync(temporary, target);
|
|
292
|
-
} catch (error) {
|
|
293
|
-
if (error.code !== 'EEXIST') throw error;
|
|
294
|
-
} finally {
|
|
295
|
-
if (lock !== undefined) {
|
|
296
|
-
closeSync(lock);
|
|
297
|
-
unlinkSync(lockPath);
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
return result;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
export function historyText(root, limit = 20) {
|
|
304
|
-
const runs = readHistory(root).slice(-limit);
|
|
305
|
-
return runs.length
|
|
306
|
-
? runs
|
|
307
|
-
.map(
|
|
308
|
-
(run) =>
|
|
309
|
-
`${run.finishedAt} ${run.profile.padEnd(4)} ${run.complete ? 'passed' : 'failed'} ${(run.wallMilliseconds / 1000).toFixed(2)}s ${run.checks.filter((check) => check.cached).length}/${run.checks.length} cached ${run.commit?.slice(0, 8) ?? ''}`,
|
|
310
|
-
)
|
|
311
|
-
.join('\n')
|
|
312
|
-
: 'No validation history yet.';
|
|
313
|
-
}
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import {
|
|
3
|
+
mkdirSync,
|
|
4
|
+
openSync,
|
|
5
|
+
closeSync,
|
|
6
|
+
readFileSync,
|
|
7
|
+
statSync,
|
|
8
|
+
writeFileSync,
|
|
9
|
+
renameSync,
|
|
10
|
+
unlinkSync,
|
|
11
|
+
} from 'node:fs';
|
|
12
|
+
import { join, resolve } from 'node:path';
|
|
13
|
+
import { inside } from './config.mjs';
|
|
14
|
+
|
|
15
|
+
const finite = (value) => typeof value === 'number' && Number.isFinite(value) && value >= 0;
|
|
16
|
+
const median = (values) => {
|
|
17
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
18
|
+
if (!sorted.length) return null;
|
|
19
|
+
const middle = Math.floor(sorted.length / 2);
|
|
20
|
+
return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
|
|
21
|
+
};
|
|
22
|
+
const readFresh = (path, since) => {
|
|
23
|
+
try {
|
|
24
|
+
const stat = statSync(path);
|
|
25
|
+
if (stat.size > 20 * 1024 * 1024 || stat.mtimeMs < since - 1000) return null;
|
|
26
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
27
|
+
} catch (error) {
|
|
28
|
+
if (error.code === 'ENOENT') return null;
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export function collectMetrics(root, output, report, settings = {}) {
|
|
34
|
+
const since = Date.parse(report.startedAt) || Date.now();
|
|
35
|
+
const metrics = { slowTests: [] };
|
|
36
|
+
const threshold = settings.slowTestMilliseconds ?? 500;
|
|
37
|
+
const seen = new Set();
|
|
38
|
+
const read = (path) => {
|
|
39
|
+
const full = resolve(path);
|
|
40
|
+
if (seen.has(full)) return null;
|
|
41
|
+
seen.add(full);
|
|
42
|
+
return readFresh(full, since);
|
|
43
|
+
};
|
|
44
|
+
const totals = {};
|
|
45
|
+
function coverage(data) {
|
|
46
|
+
if (!data?.total) return;
|
|
47
|
+
for (const key of ['lines', 'branches', 'functions', 'statements']) {
|
|
48
|
+
const value = data.total[key];
|
|
49
|
+
if (!value) continue;
|
|
50
|
+
if (finite(value.total) && finite(value.covered)) {
|
|
51
|
+
const sum = totals[key] ?? { total: 0, covered: 0 };
|
|
52
|
+
sum.total += value.total;
|
|
53
|
+
sum.covered += value.covered;
|
|
54
|
+
totals[key] = sum;
|
|
55
|
+
metrics.coverage ??= {};
|
|
56
|
+
metrics.coverage[key] = sum.total ? (100 * sum.covered) / sum.total : 100;
|
|
57
|
+
} else if (finite(value.pct) && !totals[key]) {
|
|
58
|
+
metrics.coverage ??= {};
|
|
59
|
+
metrics.coverage[key] = value.pct;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function unit(data) {
|
|
64
|
+
for (const file of data.testResults ?? [])
|
|
65
|
+
for (const result of file.assertionResults ?? []) {
|
|
66
|
+
if (finite(result.duration) && result.duration >= threshold)
|
|
67
|
+
metrics.slowTests.push({
|
|
68
|
+
name: [
|
|
69
|
+
...(result.ancestorTitles ?? []),
|
|
70
|
+
result.title ?? result.fullName ?? file.name,
|
|
71
|
+
].join(' > '),
|
|
72
|
+
milliseconds: result.duration,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
function playwright(data) {
|
|
77
|
+
const stats = data.stats ?? {};
|
|
78
|
+
metrics.browser ??= { passed: 0, failed: 0, flaky: 0, skipped: 0 };
|
|
79
|
+
for (const [key, field] of Object.entries({
|
|
80
|
+
passed: 'expected',
|
|
81
|
+
failed: 'unexpected',
|
|
82
|
+
flaky: 'flaky',
|
|
83
|
+
skipped: 'skipped',
|
|
84
|
+
}))
|
|
85
|
+
if (finite(stats[field])) metrics.browser[key] += stats[field];
|
|
86
|
+
function visit(suite, parents = []) {
|
|
87
|
+
for (const spec of suite.specs ?? [])
|
|
88
|
+
for (const test of spec.tests ?? []) {
|
|
89
|
+
const duration = Math.max(
|
|
90
|
+
0,
|
|
91
|
+
...(test.results ?? []).map((result) => result.duration ?? 0),
|
|
92
|
+
);
|
|
93
|
+
if (finite(duration) && duration >= threshold)
|
|
94
|
+
metrics.slowTests.push({
|
|
95
|
+
name: [...parents, spec.title].join(' > '),
|
|
96
|
+
milliseconds: duration,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
for (const child of suite.suites ?? []) visit(child, [...parents, child.title]);
|
|
100
|
+
}
|
|
101
|
+
for (const suite of data.suites ?? []) visit(suite, [suite.title]);
|
|
102
|
+
}
|
|
103
|
+
for (const check of report.results) {
|
|
104
|
+
if (check.cached || ['excluded', 'unavailable'].includes(check.status)) continue;
|
|
105
|
+
const kind = check.kind ?? check.id;
|
|
106
|
+
const details = read(join(output, `${check.id}.details.json`));
|
|
107
|
+
if (details?.bundle) metrics.bundle = details.bundle;
|
|
108
|
+
if (details?.a11y)
|
|
109
|
+
metrics.a11y = {
|
|
110
|
+
pages: details.a11y.pages?.length ?? 0,
|
|
111
|
+
violations: details.a11y.failures ?? 0,
|
|
112
|
+
errors: details.a11y.errors?.length ?? 0,
|
|
113
|
+
};
|
|
114
|
+
if (details?.architecture)
|
|
115
|
+
metrics.architecture = {
|
|
116
|
+
files: details.architecture.files,
|
|
117
|
+
issues: details.architecture.issues?.length ?? 0,
|
|
118
|
+
};
|
|
119
|
+
if (details?.dependencies)
|
|
120
|
+
metrics.dependencies = {
|
|
121
|
+
packages: details.dependencies.packages,
|
|
122
|
+
issues: details.dependencies.issues?.length ?? 0,
|
|
123
|
+
duplicates: details.dependencies.warnings?.length ?? 0,
|
|
124
|
+
};
|
|
125
|
+
if (kind === 'unit') {
|
|
126
|
+
const data = read(join(output, `${check.id}-results/vitest.json`));
|
|
127
|
+
if (data) unit(data);
|
|
128
|
+
coverage(read(join(output, `${check.id}-results/coverage/coverage-summary.json`)));
|
|
129
|
+
}
|
|
130
|
+
if (['e2e', 'visual'].includes(kind)) {
|
|
131
|
+
const data = read(join(output, `${check.id}-results/results.json`));
|
|
132
|
+
if (data) playwright(data);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
for (const path of settings.coverage ?? []) {
|
|
136
|
+
if (
|
|
137
|
+
!report.results.some(
|
|
138
|
+
(check) =>
|
|
139
|
+
(check.kind ?? check.id) === 'unit' &&
|
|
140
|
+
!check.cached &&
|
|
141
|
+
!['excluded', 'unavailable'].includes(check.status),
|
|
142
|
+
)
|
|
143
|
+
)
|
|
144
|
+
break;
|
|
145
|
+
const data = read(inside(root, path));
|
|
146
|
+
coverage(data);
|
|
147
|
+
}
|
|
148
|
+
for (const path of settings.unit ?? []) {
|
|
149
|
+
if (
|
|
150
|
+
report.results.some(
|
|
151
|
+
(check) =>
|
|
152
|
+
(check.kind ?? check.id) === 'unit' &&
|
|
153
|
+
!check.cached &&
|
|
154
|
+
!['excluded', 'unavailable'].includes(check.status),
|
|
155
|
+
)
|
|
156
|
+
) {
|
|
157
|
+
const data = read(inside(root, path));
|
|
158
|
+
if (data) unit(data);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
for (const path of settings.playwright ?? []) {
|
|
162
|
+
if (
|
|
163
|
+
report.results.some(
|
|
164
|
+
(check) =>
|
|
165
|
+
['e2e', 'visual'].includes(check.kind ?? check.id) &&
|
|
166
|
+
!['excluded', 'unavailable'].includes(check.status),
|
|
167
|
+
)
|
|
168
|
+
) {
|
|
169
|
+
const data = read(inside(root, path));
|
|
170
|
+
if (data) playwright(data);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
metrics.slowTests.sort((a, b) => b.milliseconds - a.milliseconds);
|
|
174
|
+
metrics.slowTests = metrics.slowTests.slice(0, 20);
|
|
175
|
+
return metrics;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
export function readHistory(root) {
|
|
179
|
+
try {
|
|
180
|
+
const path = join(root, '.tooling/history/runs.json');
|
|
181
|
+
if (statSync(path).size > 10 * 1024 * 1024) return [];
|
|
182
|
+
const data = JSON.parse(readFileSync(path, 'utf8'));
|
|
183
|
+
if (data.schemaVersion !== 1 || !Array.isArray(data.runs)) return [];
|
|
184
|
+
return data.runs
|
|
185
|
+
.filter(
|
|
186
|
+
(run) =>
|
|
187
|
+
typeof run.context === 'string' &&
|
|
188
|
+
typeof run.finishedAt === 'string' &&
|
|
189
|
+
typeof run.profile === 'string' &&
|
|
190
|
+
typeof run.commit === 'string' &&
|
|
191
|
+
typeof run.complete === 'boolean' &&
|
|
192
|
+
finite(run.wallMilliseconds) &&
|
|
193
|
+
Array.isArray(run.checks) &&
|
|
194
|
+
run.checks.every((check) => typeof check.id === 'string' && finite(check.milliseconds)),
|
|
195
|
+
)
|
|
196
|
+
.slice(-1000);
|
|
197
|
+
} catch {
|
|
198
|
+
return [];
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export function performanceHistory(root, report, settings = {}) {
|
|
203
|
+
const runs = readHistory(root);
|
|
204
|
+
const context = createHash('sha256')
|
|
205
|
+
.update(
|
|
206
|
+
JSON.stringify({
|
|
207
|
+
profile: report.profile,
|
|
208
|
+
groups: [...report.groups].sort(),
|
|
209
|
+
checks: [...report.required].sort(),
|
|
210
|
+
node: report.node,
|
|
211
|
+
platform: report.platform,
|
|
212
|
+
configuration: report.configuration,
|
|
213
|
+
}),
|
|
214
|
+
)
|
|
215
|
+
.digest('hex');
|
|
216
|
+
const checks = report.results.map((result) => ({
|
|
217
|
+
id: result.id,
|
|
218
|
+
status: result.status,
|
|
219
|
+
cached: Boolean(result.cached),
|
|
220
|
+
milliseconds: finite(result.milliseconds) ? result.milliseconds : 0,
|
|
221
|
+
}));
|
|
222
|
+
const prior = runs.filter((run) => run.context === context && run.complete).slice(-10);
|
|
223
|
+
const comparable = prior.filter(
|
|
224
|
+
(run) =>
|
|
225
|
+
run.checks.map((check) => `${check.id}:${check.cached}:${check.status}`).join() ===
|
|
226
|
+
checks.map((check) => `${check.id}:${check.cached}:${check.status}`).join(),
|
|
227
|
+
);
|
|
228
|
+
const previousMs = median(comparable.map((run) => run.wallMilliseconds));
|
|
229
|
+
const regressions = checks
|
|
230
|
+
.filter((check) => check.status === 'passed' && !check.cached)
|
|
231
|
+
.flatMap((check) => {
|
|
232
|
+
const samples = prior.flatMap((run) =>
|
|
233
|
+
run.checks
|
|
234
|
+
.filter(
|
|
235
|
+
(previous) =>
|
|
236
|
+
previous.id === check.id && previous.status === 'passed' && !previous.cached,
|
|
237
|
+
)
|
|
238
|
+
.map((previous) => previous.milliseconds),
|
|
239
|
+
);
|
|
240
|
+
const baseline = median(samples);
|
|
241
|
+
return samples.length >= 3 &&
|
|
242
|
+
baseline > 0 &&
|
|
243
|
+
check.milliseconds > baseline * (1 + (settings.regressionPercent ?? 25) / 100)
|
|
244
|
+
? [{ id: check.id, previousMs: baseline, currentMs: check.milliseconds }]
|
|
245
|
+
: [];
|
|
246
|
+
});
|
|
247
|
+
const result = {
|
|
248
|
+
runs: Math.min(runs.length + 1, settings.limit ?? 200),
|
|
249
|
+
cacheHits: checks.filter((check) => check.cached).length,
|
|
250
|
+
previousMs,
|
|
251
|
+
deltaMs: previousMs === null ? null : report.wallMilliseconds - previousMs,
|
|
252
|
+
regressions,
|
|
253
|
+
bundleDeltaKB:
|
|
254
|
+
finite(report.metrics?.bundle?.initialKB) && finite(prior.at(-1)?.bundleKB)
|
|
255
|
+
? report.metrics.bundle.initialKB - prior.at(-1).bundleKB
|
|
256
|
+
: null,
|
|
257
|
+
};
|
|
258
|
+
if (settings.enabled === false) return { ...result, runs: 0 };
|
|
259
|
+
const record = {
|
|
260
|
+
context,
|
|
261
|
+
finishedAt: report.finishedAt,
|
|
262
|
+
commit: report.commit,
|
|
263
|
+
profile: report.profile,
|
|
264
|
+
complete: report.complete,
|
|
265
|
+
wallMilliseconds: report.wallMilliseconds,
|
|
266
|
+
checks,
|
|
267
|
+
bundleKB: report.metrics?.bundle?.initialKB,
|
|
268
|
+
coverage: report.metrics?.coverage,
|
|
269
|
+
slowTests: report.metrics?.slowTests ?? [],
|
|
270
|
+
};
|
|
271
|
+
const directory = join(root, '.tooling/history');
|
|
272
|
+
mkdirSync(directory, { recursive: true });
|
|
273
|
+
const lockPath = join(directory, 'write.lock');
|
|
274
|
+
let lock;
|
|
275
|
+
try {
|
|
276
|
+
try {
|
|
277
|
+
if (Date.now() - statSync(lockPath).mtimeMs > 120000) unlinkSync(lockPath);
|
|
278
|
+
} catch (error) {
|
|
279
|
+
if (error.code !== 'ENOENT') throw error;
|
|
280
|
+
}
|
|
281
|
+
lock = openSync(lockPath, 'wx');
|
|
282
|
+
const target = join(directory, 'runs.json'),
|
|
283
|
+
temporary = `${target}.${randomUUID()}.tmp`;
|
|
284
|
+
writeFileSync(
|
|
285
|
+
temporary,
|
|
286
|
+
JSON.stringify({
|
|
287
|
+
schemaVersion: 1,
|
|
288
|
+
runs: [...readHistory(root), record].slice(-(settings.limit ?? 200)),
|
|
289
|
+
}),
|
|
290
|
+
);
|
|
291
|
+
renameSync(temporary, target);
|
|
292
|
+
} catch (error) {
|
|
293
|
+
if (error.code !== 'EEXIST') throw error;
|
|
294
|
+
} finally {
|
|
295
|
+
if (lock !== undefined) {
|
|
296
|
+
closeSync(lock);
|
|
297
|
+
unlinkSync(lockPath);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
return result;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
export function historyText(root, limit = 20) {
|
|
304
|
+
const runs = readHistory(root).slice(-limit);
|
|
305
|
+
return runs.length
|
|
306
|
+
? runs
|
|
307
|
+
.map(
|
|
308
|
+
(run) =>
|
|
309
|
+
`${run.finishedAt} ${run.profile.padEnd(4)} ${run.complete ? 'passed' : 'failed'} ${(run.wallMilliseconds / 1000).toFixed(2)}s ${run.checks.filter((check) => check.cached).length}/${run.checks.length} cached ${run.commit?.slice(0, 8) ?? ''}`,
|
|
310
|
+
)
|
|
311
|
+
.join('\n')
|
|
312
|
+
: 'No validation history yet.';
|
|
313
|
+
}
|