@flighthq/tool-capture 0.3.0 → 0.3.1-next.1041.35b950c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/baselineStore.d.ts +12 -1
- package/dist/baselineStore.d.ts.map +1 -1
- package/dist/baselineStore.js +83 -6
- package/dist/baselineStore.js.map +1 -1
- package/dist/bin.js +80 -6
- package/dist/bin.js.map +1 -1
- package/dist/captureBaselineCoverageManifest.d.ts +80 -0
- package/dist/captureBaselineCoverageManifest.d.ts.map +1 -0
- package/dist/captureBaselineCoverageManifest.js +178 -0
- package/dist/captureBaselineCoverageManifest.js.map +1 -0
- package/dist/captureBaselineSanity.d.ts +9 -0
- package/dist/captureBaselineSanity.d.ts.map +1 -0
- package/dist/captureBaselineSanity.js +32 -0
- package/dist/captureBaselineSanity.js.map +1 -0
- package/dist/captureBenchmark.d.ts.map +1 -1
- package/dist/captureBenchmark.js +5 -1
- package/dist/captureBenchmark.js.map +1 -1
- package/dist/captureDomReadback.d.ts +4 -0
- package/dist/captureDomReadback.d.ts.map +1 -0
- package/dist/captureDomReadback.js +51 -0
- package/dist/captureDomReadback.js.map +1 -0
- package/dist/captureEntry.d.ts +12 -3
- package/dist/captureEntry.d.ts.map +1 -1
- package/dist/captureEntry.js +87 -36
- package/dist/captureEntry.js.map +1 -1
- package/dist/captureEntryFilter.d.ts +5 -0
- package/dist/captureEntryFilter.d.ts.map +1 -0
- package/dist/captureEntryFilter.js +23 -0
- package/dist/captureEntryFilter.js.map +1 -0
- package/dist/captureFlightPreset.js +10 -4
- package/dist/captureFlightPreset.js.map +1 -1
- package/dist/captureProtocol.d.ts +3 -0
- package/dist/captureProtocol.d.ts.map +1 -1
- package/dist/captureProtocol.js.map +1 -1
- package/dist/captureResourceFailure.d.ts +4 -0
- package/dist/captureResourceFailure.d.ts.map +1 -0
- package/dist/captureResourceFailure.js +31 -0
- package/dist/captureResourceFailure.js.map +1 -0
- package/dist/captureScreenshotHash.d.ts +30 -0
- package/dist/captureScreenshotHash.d.ts.map +1 -0
- package/dist/captureScreenshotHash.js +108 -0
- package/dist/captureScreenshotHash.js.map +1 -0
- package/dist/captureServer.d.ts +8 -0
- package/dist/captureServer.d.ts.map +1 -1
- package/dist/captureServer.js +126 -22
- package/dist/captureServer.js.map +1 -1
- package/dist/captureSuite.d.ts +11 -0
- package/dist/captureSuite.d.ts.map +1 -1
- package/dist/captureSuite.js +6 -4
- package/dist/captureSuite.js.map +1 -1
- package/dist/captureValidation.d.ts +20 -3
- package/dist/captureValidation.d.ts.map +1 -1
- package/dist/captureValidation.js +222 -70
- package/dist/captureValidation.js.map +1 -1
- package/dist/captureWorkflow.d.ts.map +1 -1
- package/dist/captureWorkflow.js +3 -0
- package/dist/captureWorkflow.js.map +1 -1
- package/dist/functionalScene3Ds.d.ts.map +1 -1
- package/dist/functionalScene3Ds.js +5 -3
- package/dist/functionalScene3Ds.js.map +1 -1
- package/dist/functionalVerify.d.ts.map +1 -1
- package/dist/functionalVerify.js +57 -16
- package/dist/functionalVerify.js.map +1 -1
- package/package.json +5 -5
- package/src/baselineStore.test.ts +210 -7
- package/src/bin.test.ts +32 -0
- package/src/captureBaselineCoverageManifest.test.ts +368 -0
- package/src/captureBaselineSanity.test.ts +38 -0
- package/src/captureBenchmark.e2e.test.ts +29 -1
- package/src/captureDomReadback.test.ts +35 -0
- package/src/captureEntry.e2e.test.ts +108 -0
- package/src/captureEntryFilter.test.ts +45 -0
- package/src/captureEyes.e2e.test.ts +43 -0
- package/src/captureFlightPreset.test.ts +10 -0
- package/src/capturePage.test.ts +23 -3
- package/src/captureResourceFailure.test.ts +60 -0
- package/src/captureScreenshotHash.test.ts +142 -0
- package/src/captureServer.test.ts +103 -3
- package/src/captureValidation.test.ts +217 -36
- package/src/functionalScene3Ds.test.ts +7 -0
- package/src/functionalVerify.test.ts +33 -3
|
@@ -22,12 +22,17 @@ import { resolve } from 'node:path';
|
|
|
22
22
|
import { CAPTURE_PARITY_TOLERANCE, CAPTURE_REGRESSION_TOLERANCE, compareCaptureFingerprints, evaluateCaptureParity, evaluateCaptureRegression, } from '@flighthq/capture/contract';
|
|
23
23
|
import pc from 'picocolors';
|
|
24
24
|
import { getBaselineField, setBaselineField } from './baselineStore.js';
|
|
25
|
+
import { diffCaptureBaselineCoverage, formatCaptureBaselineCoverageIdentity, isCaptureBaselineCoverageFailure, readCaptureBaselineCoverageManifest, writeCaptureBaselineCoverageManifest, } from './captureBaselineCoverageManifest.js';
|
|
26
|
+
import { isUniformCaptureFingerprint } from './captureBaselineSanity.js';
|
|
25
27
|
import { launchBrowser } from './captureBrowser.js';
|
|
28
|
+
import { provideCaptureDomRenderPixels } from './captureDomReadback.js';
|
|
26
29
|
import { BACKEND_UNAVAILABLE, getCaptureEntryRoute, rendererMatchesFilter } from './captureEntries.js';
|
|
30
|
+
import { selectCaptureEntriesByName } from './captureEntryFilter.js';
|
|
27
31
|
import { formatDetailLine, formatStatusLine, formatSummaryCount, formatSummaryLine } from './captureFormat.js';
|
|
28
32
|
import { installAbortHandler, isBrowserClosedError } from './captureInterrupt.js';
|
|
29
33
|
import { CAPTURE_PROTOCOL_VERSION } from './captureProtocol.js';
|
|
30
34
|
import { writeCaptureReport } from './captureReport.js';
|
|
35
|
+
import { formatCaptureConsoleMessage, listenForCaptureResourceFailures } from './captureResourceFailure.js';
|
|
31
36
|
import { getCaptureSceneSourceHash } from './captureSourceHash.js';
|
|
32
37
|
import { getCaptureTimeoutMs } from './captureTimeout.js';
|
|
33
38
|
function fingerprintKey(entry, renderer) {
|
|
@@ -53,12 +58,19 @@ function addPair(pairs, fingerprints, a, b, group, tolerance) {
|
|
|
53
58
|
// Why an entry produced no parity pair. Each case has a different remedy, and a bare "0 comparisons"
|
|
54
59
|
// sends a reader to the wrong one: an ineligible backend needs a parity group or a committed baseline,
|
|
55
60
|
// while a single eligible backend has nothing to disagree with and needs a second one in scope.
|
|
56
|
-
export function explainCaptureParityUncovered(eligibleCount, hasGroups) {
|
|
61
|
+
export function explainCaptureParityUncovered(eligibleCount, hasGroups, unavailableReferences = []) {
|
|
57
62
|
if (eligibleCount === 0) {
|
|
58
63
|
return hasGroups
|
|
59
64
|
? 'no renderer in any parity group is eligible'
|
|
60
65
|
: 'no renderer is parity-eligible — declare a parity group, or commit a fingerprint baseline';
|
|
61
66
|
}
|
|
67
|
+
// Checked before the counts because it is a DIFFERENT remedy: with a reference missing, adding an
|
|
68
|
+
// eligible backend or shortening the skip list changes nothing until the reference itself is back.
|
|
69
|
+
// Collapsing it into the skip message would send the reader to the wrong list, and the reference can
|
|
70
|
+
// be absent for a reason that has nothing to do with skips (never parity-eligible).
|
|
71
|
+
if (unavailableReferences.length > 0) {
|
|
72
|
+
return `parity group reference not present (${unavailableReferences.join(', ')}) — a reference group compares against that backend only, so it yields NO pairs rather than falling back to all-pairs`;
|
|
73
|
+
}
|
|
62
74
|
if (eligibleCount === 1)
|
|
63
75
|
return 'only one parity-eligible renderer — nothing to compare it against';
|
|
64
76
|
return 'every eligible pair is excluded by a parity skip';
|
|
@@ -71,6 +83,7 @@ async function loadFingerprint(context, baseUrl, entry, renderer, subject) {
|
|
|
71
83
|
// collect those too. Without this, a thrown verifier or a renamed/missing export reads as the
|
|
72
84
|
// uninformative "verifier did not run" instead of the actual message.
|
|
73
85
|
let pageError = '';
|
|
86
|
+
let resourceError = '';
|
|
74
87
|
let page = null;
|
|
75
88
|
try {
|
|
76
89
|
// newPage is inside the try: once an interrupt has closed the browser it throws, and that must read
|
|
@@ -79,8 +92,9 @@ async function loadFingerprint(context, baseUrl, entry, renderer, subject) {
|
|
|
79
92
|
page.on('pageerror', (e) => (pageError ||= e.message));
|
|
80
93
|
page.on('console', (m) => {
|
|
81
94
|
if (m.type() === 'error')
|
|
82
|
-
pageError ||= m
|
|
95
|
+
pageError ||= formatCaptureConsoleMessage(m);
|
|
83
96
|
});
|
|
97
|
+
listenForCaptureResourceFailures(page, (message) => (resourceError ||= message));
|
|
84
98
|
const route = getCaptureEntryRoute(entry, renderer, subject);
|
|
85
99
|
await page.goto(`${baseUrl}/${route}`, {
|
|
86
100
|
waitUntil: 'domcontentloaded',
|
|
@@ -93,12 +107,27 @@ async function loadFingerprint(context, baseUrl, entry, renderer, subject) {
|
|
|
93
107
|
// race because their readback is synchronous). So wait until the fingerprint is populated OR an error
|
|
94
108
|
// overlay appears, then read the result. Poll on a timer (the capture harness halts rAF).
|
|
95
109
|
const waitStartedAt = performance.now();
|
|
96
|
-
await page
|
|
110
|
+
const reachedReadbackOrTerminal = await page
|
|
97
111
|
.waitForFunction(() => {
|
|
98
|
-
const
|
|
99
|
-
|
|
112
|
+
const w = window;
|
|
113
|
+
const verification = w.__ftVerification;
|
|
114
|
+
return (verification?.state === 'passed' ||
|
|
115
|
+
verification?.state === 'failed' ||
|
|
116
|
+
(verification?.render === 'dom' && typeof w.__ftProvideDomRenderPixels === 'function') ||
|
|
117
|
+
document.getElementById('ft-error') !== null);
|
|
100
118
|
}, null, { timeout: getCaptureTimeoutMs(), polling: 100 })
|
|
101
|
-
.
|
|
119
|
+
.then(() => true)
|
|
120
|
+
.catch(() => false);
|
|
121
|
+
if (reachedReadbackOrTerminal && (await provideCaptureDomRenderPixels(page))) {
|
|
122
|
+
await page
|
|
123
|
+
.waitForFunction(() => {
|
|
124
|
+
const verification = window.__ftVerification;
|
|
125
|
+
return (verification?.state === 'passed' ||
|
|
126
|
+
verification?.state === 'failed' ||
|
|
127
|
+
document.getElementById('ft-error') !== null);
|
|
128
|
+
}, null, { timeout: getCaptureTimeoutMs(), polling: 100 })
|
|
129
|
+
.catch(() => { });
|
|
130
|
+
}
|
|
102
131
|
const waitedMs = Math.round(performance.now() - waitStartedAt);
|
|
103
132
|
const verification = await page
|
|
104
133
|
.evaluate(() => window.__ftVerification ?? null)
|
|
@@ -122,7 +151,7 @@ async function loadFingerprint(context, baseUrl, entry, renderer, subject) {
|
|
|
122
151
|
// The functional entry paints any error into #ft-error (covering window.error AND unhandledrejection);
|
|
123
152
|
// read it as the most reliable real reason when neither a fingerprint nor a pageerror surfaced.
|
|
124
153
|
const overlay = await page.$eval('#ft-error', (el) => el.textContent ?? '').catch(() => '');
|
|
125
|
-
const detail = verification?.error || pageError || overlay;
|
|
154
|
+
const detail = verification?.error || resourceError || pageError || overlay;
|
|
126
155
|
if (BACKEND_UNAVAILABLE.test(detail))
|
|
127
156
|
return { fingerprint: null, reason: `backend unavailable (${detail})`, unavailable: true, aborted: false };
|
|
128
157
|
return {
|
|
@@ -234,11 +263,35 @@ export function explainCaptureVerificationStall(verification, waitedMs) {
|
|
|
234
263
|
}
|
|
235
264
|
// Registered and still non-terminal: it started and never finished, which is a stall rather than a
|
|
236
265
|
// scene that is merely expensive. The STAGE is the actionable half — 'awaitingFrame' means a presented
|
|
237
|
-
// frame never arrived (page/scheduler), 'readingBack' means
|
|
266
|
+
// frame never arrived (page/scheduler), while 'readingBack' means a GPU readback or the DOM runner bridge
|
|
267
|
+
// never resolved.
|
|
238
268
|
const stage = verification.stage;
|
|
239
269
|
const where = stage === undefined ? '' : ` at stage "${stage}"`;
|
|
240
270
|
return `verifier registered but stalled${where} in state "${verification.state}" (${budget}); it started and never reached a terminal state`;
|
|
241
271
|
}
|
|
272
|
+
/**
|
|
273
|
+
* Ranks measured parity distances, widest disagreement first.
|
|
274
|
+
*
|
|
275
|
+
* Returns null when nothing was compared, so a run that measured nothing prints no ranking rather
|
|
276
|
+
* than an empty one that would read as agreement.
|
|
277
|
+
*/
|
|
278
|
+
export function formatCaptureParityRanking(checks, limit = 10) {
|
|
279
|
+
const measured = checks
|
|
280
|
+
.filter((check) => check.kind === 'parity' && typeof check.distance === 'number')
|
|
281
|
+
.map((check) => ({ distance: check.distance, entry: check.entry, renderers: check.renderers ?? [] }))
|
|
282
|
+
.sort((a, b) => b.distance - a.distance || a.entry.localeCompare(b.entry));
|
|
283
|
+
if (measured.length === 0)
|
|
284
|
+
return null;
|
|
285
|
+
const shown = measured.slice(0, limit);
|
|
286
|
+
const median = measured[measured.length >> 1].distance;
|
|
287
|
+
const lines = shown.map((row) => ` ${row.distance.toFixed(2)} ${row.entry} ${row.renderers.join('·')}`);
|
|
288
|
+
const omitted = measured.length - shown.length;
|
|
289
|
+
return [
|
|
290
|
+
` widest parity distances (${measured.length} compared, median ${median.toFixed(2)}):`,
|
|
291
|
+
...lines,
|
|
292
|
+
...(omitted > 0 ? [` … ${omitted} more not shown`] : []),
|
|
293
|
+
].join('\n');
|
|
294
|
+
}
|
|
242
295
|
export function isCaptureParityCoverageFailure(run) {
|
|
243
296
|
if (!run.gateParity || run.interrupted)
|
|
244
297
|
return false;
|
|
@@ -259,29 +312,11 @@ export function isCaptureRegressionCoverageFailure(run) {
|
|
|
259
312
|
return false;
|
|
260
313
|
return run.regressionComparisons === 0 && run.regressionUncovered > 0;
|
|
261
314
|
}
|
|
262
|
-
// Loads a single test/renderer page and returns its render fingerprint, or null with a reason and a
|
|
263
|
-
// flag marking whether the cause is a genuinely-unavailable backend (skippable) versus a real error.
|
|
264
|
-
// True when every cell of a coarse fingerprint carries the same value — a blank or flat-filled frame.
|
|
265
|
-
//
|
|
266
|
-
// This is a WRITE-SIDE gate, not a render check: a uniform frame may be a legitimate render (a solid
|
|
267
|
-
// background scene), but it is never worth BLESSING as a regression baseline, because it cannot
|
|
268
|
-
// distinguish a working scene from a broken one. Refusing it costs a real flat scene its regression
|
|
269
|
-
// coverage and costs a broken one nothing — the asymmetry is the point.
|
|
270
|
-
//
|
|
271
|
-
// Format: "<cellSize>:<hex cells>", so the payload after the colon is split into fixed-width cells.
|
|
272
|
-
export function isUniformCaptureFingerprint(fingerprint) {
|
|
273
|
-
const payload = fingerprint.slice(fingerprint.indexOf(':') + 1);
|
|
274
|
-
if (payload.length <= CAPTURE_FINGERPRINT_CELL_CHARS)
|
|
275
|
-
return true;
|
|
276
|
-
const first = payload.slice(0, CAPTURE_FINGERPRINT_CELL_CHARS);
|
|
277
|
-
for (let i = CAPTURE_FINGERPRINT_CELL_CHARS; i < payload.length; i += CAPTURE_FINGERPRINT_CELL_CHARS) {
|
|
278
|
-
if (payload.slice(i, i + CAPTURE_FINGERPRINT_CELL_CHARS) !== first)
|
|
279
|
-
return false;
|
|
280
|
-
}
|
|
281
|
-
return true;
|
|
282
|
-
}
|
|
283
315
|
async function processEntry(entry, entryIndex, totalEntries, isAborted, options, samples) {
|
|
284
316
|
const result = {
|
|
317
|
+
visited: [],
|
|
318
|
+
undetermined: [],
|
|
319
|
+
covered: [],
|
|
285
320
|
regressionFailures: 0,
|
|
286
321
|
regressionUncovered: 0,
|
|
287
322
|
parityFailures: 0,
|
|
@@ -313,6 +348,7 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
313
348
|
const statusLine = (tone, label, message) => formatStatusLine(tone, label, labelWidth, message);
|
|
314
349
|
const detailLine = (glyph, label, message, paint) => formatDetailLine(glyph, label, labelWidth, message, paint);
|
|
315
350
|
if (options.fingerprintSkip.has(entry.name)) {
|
|
351
|
+
result.undetermined.push(...renderers.map((renderer) => formatCaptureBaselineCoverageIdentity(entry.name, renderer)));
|
|
316
352
|
result.skipped += renderers.length;
|
|
317
353
|
result.checks.push({
|
|
318
354
|
entry: entry.name,
|
|
@@ -358,9 +394,16 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
358
394
|
message: first.reason,
|
|
359
395
|
});
|
|
360
396
|
}
|
|
397
|
+
// A renderer that never produced a fingerprint tells us NOTHING about its coverage. Recording it as
|
|
398
|
+
// visited-but-uncovered would report a load failure a second time as a coverage loss; recording it
|
|
399
|
+
// as never-visited would report it as vanished. It is neither — it is unknown, and an unknown is
|
|
400
|
+
// carried forward on acceptance rather than retired.
|
|
401
|
+
result.undetermined.push(formatCaptureBaselineCoverageIdentity(entry.name, renderer));
|
|
361
402
|
continue;
|
|
362
403
|
}
|
|
363
404
|
const fingerprint = first.fingerprint;
|
|
405
|
+
// Coverage for this identity is now DETERMINABLE: the baseline lookup below settles it either way.
|
|
406
|
+
result.visited.push(formatCaptureBaselineCoverageIdentity(entry.name, renderer));
|
|
364
407
|
// Explicit groups are same-run comparisons and do not require a committed regression baseline.
|
|
365
408
|
// Legacy all-pairs parity retains its prior proven-stable/baselined eligibility policy.
|
|
366
409
|
if (Object.keys(options.parityGroups).length > 0)
|
|
@@ -405,7 +448,13 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
405
448
|
});
|
|
406
449
|
continue;
|
|
407
450
|
}
|
|
408
|
-
|
|
451
|
+
// ★ STAMP ONLY WHAT THIS PASS PRODUCED. The provenance is looked up by the SAME (entry, renderer)
|
|
452
|
+
// key the fingerprint came from, and is passed only when this pass actually captured that
|
|
453
|
+
// fingerprint. A fingerprint supplied from elsewhere, or already on disk, gets NO provenance:
|
|
454
|
+
// attaching today's conditions to a value produced by an earlier capture would manufacture an
|
|
455
|
+
// agreement nobody observed, and a wrong provenance is worse than the absent one it replaces.
|
|
456
|
+
const provenance = options.fingerprintProvenance[entry.name]?.[renderer];
|
|
457
|
+
setBaselineField(options.root, options.subject, entry.name, renderer, 'fingerprint', fingerprint, provenance);
|
|
409
458
|
const sourceHash = getCaptureSceneSourceHash(options.root, options.subject, entry, renderer);
|
|
410
459
|
if (sourceHash !== null) {
|
|
411
460
|
setBaselineField(options.root, options.subject, entry.name, renderer, 'sourceHash', sourceHash);
|
|
@@ -419,6 +468,7 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
419
468
|
status: 'passed',
|
|
420
469
|
message: 'baseline written',
|
|
421
470
|
});
|
|
471
|
+
result.covered.push(formatCaptureBaselineCoverageIdentity(entry.name, renderer));
|
|
422
472
|
eligible.set(renderer, fingerprint);
|
|
423
473
|
continue;
|
|
424
474
|
}
|
|
@@ -436,6 +486,10 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
436
486
|
});
|
|
437
487
|
continue;
|
|
438
488
|
}
|
|
489
|
+
// Coverage is recorded HERE, where it is a fact, not inferred later from the shape of the checks:
|
|
490
|
+
// a smoke leg gates neither regression nor report and so emits no regression check at all, and
|
|
491
|
+
// deriving coverage by subtracting skips would call every one of those targets lost.
|
|
492
|
+
result.covered.push(formatCaptureBaselineCoverageIdentity(entry.name, renderer));
|
|
439
493
|
const dist = distance(fingerprint, committed);
|
|
440
494
|
eligible.set(renderer, fingerprint);
|
|
441
495
|
if (dist === null) {
|
|
@@ -506,6 +560,9 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
506
560
|
const allowed = (renderer) => skip !== 'all' && !skip?.includes(renderer);
|
|
507
561
|
const pairs = [];
|
|
508
562
|
const groups = Object.entries(options.parityGroups);
|
|
563
|
+
// Groups whose declared reference is not present, as "<group> → <reference>". Carried to the uncovered
|
|
564
|
+
// explanation so the reason names the missing reference instead of collapsing into the skip message.
|
|
565
|
+
const unavailableReferences = [];
|
|
509
566
|
if (groups.length === 0) {
|
|
510
567
|
const present = [...eligible.keys()].filter(allowed);
|
|
511
568
|
for (let i = 0; i < present.length; i++) {
|
|
@@ -517,18 +574,48 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
517
574
|
else {
|
|
518
575
|
for (const [groupName, group] of groups) {
|
|
519
576
|
const present = group.targets.filter((renderer) => eligible.has(renderer) && allowed(renderer));
|
|
520
|
-
|
|
577
|
+
// A group that DECLARES a reference is claiming agreement WITH THAT BACKEND. If the reference is
|
|
578
|
+
// not present — removed by a skip, or never parity-eligible — that claim cannot be checked, and
|
|
579
|
+
// comparing the remaining targets to each other substitutes a DIFFERENT, weaker claim under the
|
|
580
|
+
// same group name. Yield no pairs instead, so the scene reports itself uncovered by name.
|
|
581
|
+
if (group.reference !== undefined) {
|
|
582
|
+
// Only a SKIP-removed reference kills the group. A skip is a deliberate statement that this
|
|
583
|
+
// backend is not to be trusted for this scene, so comparing the survivors to each other
|
|
584
|
+
// substitutes a different, weaker claim under the same group name — yield nothing and let the
|
|
585
|
+
// scene report itself uncovered.
|
|
586
|
+
//
|
|
587
|
+
// A reference that was never there is NOT the same thing and must keep the all-pairs branch:
|
|
588
|
+
// the built-in group declares `reference: 'canvas'` once for every scene, and 83 scenes have no
|
|
589
|
+
// canvas column at all. Treating those as reference-removed silently deletes 85 real
|
|
590
|
+
// cross-backend comparisons — measured, not estimated: 253 → 168 on the functional suite.
|
|
591
|
+
if (eligible.has(group.reference) && !allowed(group.reference)) {
|
|
592
|
+
unavailableReferences.push(`${groupName} → ${group.reference}`);
|
|
593
|
+
continue;
|
|
594
|
+
}
|
|
595
|
+
if (!present.includes(group.reference)) {
|
|
596
|
+
// The comparison is FINE here and the CLAIM was false: nobody asked for the reference to go,
|
|
597
|
+
// it simply is not a column for this scene, and all-pairs among the columns that DO exist is
|
|
598
|
+
// real coverage worth keeping. What was wrong is reporting it under a group that asserts a
|
|
599
|
+
// reference it never used, so the label says so rather than the pairs disappearing.
|
|
600
|
+
const label = `${groupName} (all-pairs, no ${group.reference} column)`;
|
|
601
|
+
for (let i = 0; i < present.length; i++) {
|
|
602
|
+
for (let j = i + 1; j < present.length; j++) {
|
|
603
|
+
addPair(pairs, eligible, present[i], present[j], label, group.tolerance ?? options.parityTolerance);
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
continue;
|
|
607
|
+
}
|
|
521
608
|
for (const renderer of present) {
|
|
522
609
|
if (renderer !== group.reference) {
|
|
523
610
|
addPair(pairs, eligible, group.reference, renderer, groupName, group.tolerance ?? options.parityTolerance);
|
|
524
611
|
}
|
|
525
612
|
}
|
|
613
|
+
continue;
|
|
526
614
|
}
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
}
|
|
615
|
+
// No reference declared at all: all-pairs is what this group means, not a fallback for a lost one.
|
|
616
|
+
for (let i = 0; i < present.length; i++) {
|
|
617
|
+
for (let j = i + 1; j < present.length; j++) {
|
|
618
|
+
addPair(pairs, eligible, present[i], present[j], groupName, group.tolerance ?? options.parityTolerance);
|
|
532
619
|
}
|
|
533
620
|
}
|
|
534
621
|
}
|
|
@@ -544,7 +631,7 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
544
631
|
renderers: [...eligible.keys()],
|
|
545
632
|
kind: 'parity',
|
|
546
633
|
status: 'skipped',
|
|
547
|
-
message: explainCaptureParityUncovered(eligible.size, groups.length > 0),
|
|
634
|
+
message: explainCaptureParityUncovered(eligible.size, groups.length > 0, unavailableReferences),
|
|
548
635
|
});
|
|
549
636
|
}
|
|
550
637
|
if (pairs.length > 0) {
|
|
@@ -557,7 +644,7 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
557
644
|
renderers: [pair.a, pair.b],
|
|
558
645
|
kind: 'parity',
|
|
559
646
|
status: 'reported',
|
|
560
|
-
message: `parity distance ${pair.dist.toFixed(2)}`,
|
|
647
|
+
message: `parity ${pair.label} distance ${pair.dist.toFixed(2)}`,
|
|
561
648
|
distance: pair.dist,
|
|
562
649
|
threshold: pair.tolerance,
|
|
563
650
|
});
|
|
@@ -576,7 +663,7 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
576
663
|
renderers: [p.a, p.b],
|
|
577
664
|
kind: 'parity',
|
|
578
665
|
status: 'failed',
|
|
579
|
-
message: `parity ${p.dist.toFixed(2)} > ${p.tolerance}`,
|
|
666
|
+
message: `parity ${p.label} ${p.dist.toFixed(2)} > ${p.tolerance}`,
|
|
580
667
|
distance: p.dist,
|
|
581
668
|
threshold: p.tolerance,
|
|
582
669
|
});
|
|
@@ -588,7 +675,7 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
588
675
|
renderers: [p.a, p.b],
|
|
589
676
|
kind: 'parity',
|
|
590
677
|
status: 'passed',
|
|
591
|
-
message: `parity ${p.dist.toFixed(2)} ≤ ${p.tolerance}`,
|
|
678
|
+
message: `parity ${p.label} ${p.dist.toFixed(2)} ≤ ${p.tolerance}`,
|
|
592
679
|
distance: p.dist,
|
|
593
680
|
threshold: p.tolerance,
|
|
594
681
|
});
|
|
@@ -602,13 +689,41 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
602
689
|
}
|
|
603
690
|
return result;
|
|
604
691
|
}
|
|
692
|
+
// Loads a single test/renderer page and returns its render fingerprint, or null with a reason and a
|
|
693
|
+
// flag marking whether the cause is a genuinely-unavailable backend (skippable) versus a real error.
|
|
694
|
+
export { isUniformCaptureFingerprint } from './captureBaselineSanity.js';
|
|
695
|
+
function classifyCaptureBaselineFreshness(recordedSourceHash, currentSourceHash) {
|
|
696
|
+
if (recordedSourceHash === null) {
|
|
697
|
+
return {
|
|
698
|
+
message: 'scene-source freshness unavailable (baseline has no recorded source hash)',
|
|
699
|
+
status: 'unavailable',
|
|
700
|
+
};
|
|
701
|
+
}
|
|
702
|
+
if (currentSourceHash === null) {
|
|
703
|
+
return {
|
|
704
|
+
message: 'scene-source freshness unavailable (current scene source could not be read)',
|
|
705
|
+
status: 'unavailable',
|
|
706
|
+
};
|
|
707
|
+
}
|
|
708
|
+
if (recordedSourceHash !== currentSourceHash) {
|
|
709
|
+
return {
|
|
710
|
+
message: 'scene source changed since baseline — recapture owed by the scene owner',
|
|
711
|
+
status: 'changed',
|
|
712
|
+
};
|
|
713
|
+
}
|
|
714
|
+
return {
|
|
715
|
+
message: 'scene source unchanged since baseline — environment drift; never rebaseline',
|
|
716
|
+
status: 'unchanged',
|
|
717
|
+
};
|
|
718
|
+
}
|
|
605
719
|
export async function runCaptureValidation(input) {
|
|
606
720
|
const startedAt = performance.now();
|
|
607
|
-
const entries = input.filter
|
|
608
|
-
? input.entries.filter((entry) => entry.name.includes(input.filter))
|
|
609
|
-
: [...input.entries];
|
|
721
|
+
const entries = selectCaptureEntriesByName(input.entries, input.filter, input.filterExact);
|
|
610
722
|
if (entries.length === 0)
|
|
611
723
|
throw new Error(`No validation entries found subject=${input.subject}`);
|
|
724
|
+
// A filtered run cannot tell "this pinned target vanished" from "I excluded it", so it must not claim
|
|
725
|
+
// an absence. It can still report a LOSS, because that is about a target it actually ran.
|
|
726
|
+
const entryFiltered = (input.filter !== undefined && input.filter !== '') || input.filterExact !== undefined;
|
|
612
727
|
const options = {
|
|
613
728
|
subject: input.subject,
|
|
614
729
|
root: resolve(input.root ?? process.cwd()),
|
|
@@ -617,6 +732,7 @@ export async function runCaptureValidation(input) {
|
|
|
617
732
|
report: input.report ?? false,
|
|
618
733
|
quiet: input.quiet ?? false,
|
|
619
734
|
updateFingerprints: input.updateFingerprints ?? false,
|
|
735
|
+
updateCoverage: input.updateCoverage ?? false,
|
|
620
736
|
gateRegression: input.gateRegression ?? true,
|
|
621
737
|
gateParity: input.gateParity ?? true,
|
|
622
738
|
stabilityEpsilon: input.stabilityEpsilon ?? 4,
|
|
@@ -626,6 +742,7 @@ export async function runCaptureValidation(input) {
|
|
|
626
742
|
paritySkip: input.paritySkip ?? {},
|
|
627
743
|
parityGroups: input.parityGroups ?? {},
|
|
628
744
|
fingerprints: input.fingerprints ?? {},
|
|
745
|
+
fingerprintProvenance: input.fingerprintProvenance ?? {},
|
|
629
746
|
};
|
|
630
747
|
const ownsBrowser = input.browserSession === undefined;
|
|
631
748
|
const launched = input.browserSession ??
|
|
@@ -645,6 +762,9 @@ export async function runCaptureValidation(input) {
|
|
|
645
762
|
let updated = 0;
|
|
646
763
|
let skipped = 0;
|
|
647
764
|
const checks = [];
|
|
765
|
+
const coveredIdentities = [];
|
|
766
|
+
const visitedIdentities = [];
|
|
767
|
+
const undeterminedIdentities = [];
|
|
648
768
|
try {
|
|
649
769
|
// Balance pages, not entries: a four-renderer entry must not monopolize one worker while another
|
|
650
770
|
// worker gets a one-renderer entry. The same flattened queue also makes capture-provided
|
|
@@ -696,6 +816,12 @@ export async function runCaptureValidation(input) {
|
|
|
696
816
|
updated += result.updated;
|
|
697
817
|
skipped += result.skipped;
|
|
698
818
|
checks.push(...result.checks);
|
|
819
|
+
coveredIdentities.push(...result.covered);
|
|
820
|
+
visitedIdentities.push(...result.visited);
|
|
821
|
+
undeterminedIdentities.push(...result.undetermined);
|
|
822
|
+
undeterminedIdentities.push(...result.undetermined);
|
|
823
|
+
coveredIdentities.push(...result.covered);
|
|
824
|
+
visitedIdentities.push(...result.visited);
|
|
699
825
|
if (!options.quiet)
|
|
700
826
|
for (const line of result.output)
|
|
701
827
|
console.log(line);
|
|
@@ -772,7 +898,41 @@ export async function runCaptureValidation(input) {
|
|
|
772
898
|
parityUncovered,
|
|
773
899
|
rendererFilterCount: options.rendererFilter.length,
|
|
774
900
|
});
|
|
775
|
-
|
|
901
|
+
// The zero-floor above asks only whether ANY comparison ran. This asks WHICH ones did: a pinned
|
|
902
|
+
// identity that ran and no longer has a baseline is named individually, instead of being absorbed into
|
|
903
|
+
// an "uncovered" count that still satisfies the floor as long as one other target compared.
|
|
904
|
+
const coverageDiff = options.gateRegression && !options.updateCoverage
|
|
905
|
+
? diffCaptureBaselineCoverage(readCaptureBaselineCoverageManifest(options.root), options.subject, Object.fromEntries([...new Set(coveredIdentities)].map((identity) => [identity, ['fingerprint']])), [...new Set(visitedIdentities)], {
|
|
906
|
+
entryFiltered,
|
|
907
|
+
activeRenderers: options.rendererFilter.length > 0 ? [...options.rendererFilter] : null,
|
|
908
|
+
undetermined: [...new Set(undeterminedIdentities)],
|
|
909
|
+
// Validation observes the FINGERPRINT column and nothing else. Without this the screenshot
|
|
910
|
+
// and oracle pins would read as lost on every validate run — absence of observation reported
|
|
911
|
+
// as absence of evidence, which is the defect this manifest exists to prevent.
|
|
912
|
+
kinds: ['fingerprint'],
|
|
913
|
+
})
|
|
914
|
+
: { gained: [], lost: [], absent: [] };
|
|
915
|
+
const coverageFailed = isCaptureBaselineCoverageFailure(coverageDiff);
|
|
916
|
+
// ★ Accepting new coverage does NOT excuse the run's own verdict. Writing the manifest from a branch
|
|
917
|
+
// that returned early would report exit 0 over a leg with real regression failures — an acceptance path
|
|
918
|
+
// that reports green is the same defect class this manifest exists to close, one level up. So the write
|
|
919
|
+
// happens here, after the verdict is computed, and only the COVERAGE component of that verdict is
|
|
920
|
+
// suppressed: a target with a failing comparison still has baseline evidence, so it is still covered.
|
|
921
|
+
if (options.updateCoverage) {
|
|
922
|
+
// An interrupted run's own summary is not trustworthy, so it never writes. A run with load failures
|
|
923
|
+
// does write: the write retires only identities this run positively determined to be uncovered, so a
|
|
924
|
+
// target that never loaded keeps its pin instead of being silently retired by its own flakiness.
|
|
925
|
+
if (interrupted) {
|
|
926
|
+
console.error(pc.red('\nRefusing to update the capture baseline coverage manifest from an interrupted run.'));
|
|
927
|
+
return createResult(true);
|
|
928
|
+
}
|
|
929
|
+
const identities = [...new Set(coveredIdentities)];
|
|
930
|
+
writeCaptureBaselineCoverageManifest(options.root, options.subject, Object.fromEntries(identities.map((identity) => [identity, ['fingerprint']])), options.rendererFilter.length > 0 ? [...options.rendererFilter] : null, [...new Set(visitedIdentities)], ['fingerprint']);
|
|
931
|
+
if (!options.quiet)
|
|
932
|
+
console.log(`\ncapture baseline coverage manifest updated — ${identities.length} identit${identities.length === 1 ? 'y' : 'ies'} pinned for ${options.subject}`);
|
|
933
|
+
}
|
|
934
|
+
const failed = coverageFailed ||
|
|
935
|
+
regressionFailures > 0 ||
|
|
776
936
|
parityFailures > 0 ||
|
|
777
937
|
loadFailures > 0 ||
|
|
778
938
|
parityCoverageFailed ||
|
|
@@ -790,6 +950,24 @@ export async function runCaptureValidation(input) {
|
|
|
790
950
|
formatSummaryCount(loadFailures, 'load failures', 'fail'),
|
|
791
951
|
]) +
|
|
792
952
|
note);
|
|
953
|
+
// A parity PASS spans zero to the tolerance, so the verdict alone cannot distinguish a scene that
|
|
954
|
+
// matched exactly from one that came within a hair of failing. Rank the measured distances so the
|
|
955
|
+
// reader sees what the comparison actually found rather than only that nothing crossed the line.
|
|
956
|
+
const ranking = formatCaptureParityRanking(checks);
|
|
957
|
+
if (ranking !== null && !options.quiet)
|
|
958
|
+
console.log(ranking);
|
|
959
|
+
for (const identity of coverageDiff.lost) {
|
|
960
|
+
console.error(pc.red(` - ${identity} (pinned, ran, no comparable baseline)`));
|
|
961
|
+
}
|
|
962
|
+
for (const identity of coverageDiff.absent) {
|
|
963
|
+
console.error(pc.red(` - ${identity} (pinned, never reached by this run)`));
|
|
964
|
+
}
|
|
965
|
+
for (const identity of coverageDiff.gained) {
|
|
966
|
+
console.error(pc.red(` + ${identity} (newly covered, not yet pinned)`));
|
|
967
|
+
}
|
|
968
|
+
if (coverageFailed) {
|
|
969
|
+
console.error(pc.red(`\nCapture baseline coverage does not match scripts/capture-baseline-coverage-manifest.json — the manifest is an exact set, so a gain counts too. Repair a missing baseline, or accept the change deliberately with --update-coverage (no --filter).`));
|
|
970
|
+
}
|
|
793
971
|
// Name the entries behind an "N uncovered" count: a bare number tells a reader something is missing but
|
|
794
972
|
// not what to go look at, and the entries differ in why.
|
|
795
973
|
for (const [label, uncovered, failedTier] of [
|
|
@@ -844,30 +1022,4 @@ export async function runCaptureValidation(input) {
|
|
|
844
1022
|
return result;
|
|
845
1023
|
}
|
|
846
1024
|
}
|
|
847
|
-
function classifyCaptureBaselineFreshness(recordedSourceHash, currentSourceHash) {
|
|
848
|
-
if (recordedSourceHash === null) {
|
|
849
|
-
return {
|
|
850
|
-
message: 'scene-source freshness unavailable (baseline has no recorded source hash)',
|
|
851
|
-
status: 'unavailable',
|
|
852
|
-
};
|
|
853
|
-
}
|
|
854
|
-
if (currentSourceHash === null) {
|
|
855
|
-
return {
|
|
856
|
-
message: 'scene-source freshness unavailable (current scene source could not be read)',
|
|
857
|
-
status: 'unavailable',
|
|
858
|
-
};
|
|
859
|
-
}
|
|
860
|
-
if (recordedSourceHash !== currentSourceHash) {
|
|
861
|
-
return {
|
|
862
|
-
message: 'scene source changed since baseline — recapture owed by the scene owner',
|
|
863
|
-
status: 'changed',
|
|
864
|
-
};
|
|
865
|
-
}
|
|
866
|
-
return {
|
|
867
|
-
message: 'scene source unchanged since baseline — environment drift; never rebaseline',
|
|
868
|
-
status: 'unchanged',
|
|
869
|
-
};
|
|
870
|
-
}
|
|
871
|
-
// Hex characters per fingerprint cell (one RGB triplet).
|
|
872
|
-
const CAPTURE_FINGERPRINT_CELL_CHARS = 6;
|
|
873
1025
|
//# sourceMappingURL=captureValidation.js.map
|