@flighthq/tool-capture 0.3.0-next.906.07cea63 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/baselineStore.d.ts +1 -1
- package/dist/baselineStore.d.ts.map +1 -1
- package/dist/bin.js +19 -3
- package/dist/bin.js.map +1 -1
- package/dist/captureBrowser.d.ts +1 -0
- package/dist/captureBrowser.d.ts.map +1 -1
- package/dist/captureBrowser.js +20 -4
- package/dist/captureBrowser.js.map +1 -1
- package/dist/captureEntry.d.ts +3 -1
- package/dist/captureEntry.d.ts.map +1 -1
- package/dist/captureEntry.js +72 -23
- package/dist/captureEntry.js.map +1 -1
- package/dist/captureFlightPreset.js +12 -2
- package/dist/captureFlightPreset.js.map +1 -1
- package/dist/captureFormat.d.ts +1 -1
- package/dist/captureFormat.d.ts.map +1 -1
- package/dist/captureFormat.js +2 -1
- package/dist/captureFormat.js.map +1 -1
- package/dist/capturePage.d.ts +4 -2
- package/dist/capturePage.d.ts.map +1 -1
- package/dist/capturePage.js +14 -0
- package/dist/capturePage.js.map +1 -1
- package/dist/captureProtocol.d.ts +2 -0
- package/dist/captureProtocol.d.ts.map +1 -1
- package/dist/captureProtocol.js.map +1 -1
- package/dist/captureRegistryMiss.d.ts +8 -0
- package/dist/captureRegistryMiss.d.ts.map +1 -0
- package/dist/captureRegistryMiss.js +41 -0
- package/dist/captureRegistryMiss.js.map +1 -0
- package/dist/captureServer.d.ts.map +1 -1
- package/dist/captureServer.js +13 -19
- package/dist/captureServer.js.map +1 -1
- package/dist/captureSourceHash.d.ts +4 -0
- package/dist/captureSourceHash.d.ts.map +1 -0
- package/dist/captureSourceHash.js +22 -0
- package/dist/captureSourceHash.js.map +1 -0
- package/dist/captureSuite.d.ts.map +1 -1
- package/dist/captureSuite.js +6 -2
- package/dist/captureSuite.js.map +1 -1
- package/dist/captureTimeout.d.ts +4 -0
- package/dist/captureTimeout.d.ts.map +1 -0
- package/dist/captureTimeout.js +39 -0
- package/dist/captureTimeout.js.map +1 -0
- package/dist/captureValidation.d.ts +39 -0
- package/dist/captureValidation.d.ts.map +1 -1
- package/dist/captureValidation.js +274 -32
- package/dist/captureValidation.js.map +1 -1
- package/dist/captureWorkerCount.d.ts +2 -0
- package/dist/captureWorkerCount.d.ts.map +1 -0
- package/dist/captureWorkerCount.js +20 -0
- package/dist/captureWorkerCount.js.map +1 -0
- package/dist/captureWorkflow.d.ts +8 -0
- package/dist/captureWorkflow.d.ts.map +1 -1
- package/dist/captureWorkflow.js +15 -5
- package/dist/captureWorkflow.js.map +1 -1
- package/dist/contract.d.ts +1 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +1 -0
- package/dist/contract.js.map +1 -1
- package/dist/functionalVerify.d.ts +3 -3
- package/dist/functionalVerify.d.ts.map +1 -1
- package/dist/functionalVerify.js +93 -33
- package/dist/functionalVerify.js.map +1 -1
- package/package.json +7 -5
- package/src/captureEntry.test.ts +36 -0
- package/src/captureEyes.e2e.test.ts +32 -1
- package/src/captureFlightPreset.test.ts +12 -0
- package/src/captureFormat.test.ts +22 -0
- package/src/capturePage.test.ts +25 -2
- package/src/captureRegistryMiss.test.ts +106 -0
- package/src/captureServer.test.ts +31 -5
- package/src/captureSourceHash.test.ts +62 -0
- package/src/captureTimeout.test.ts +77 -0
- package/src/captureValidation.test.ts +221 -1
- package/src/captureWorkerCount.test.ts +28 -0
- package/src/captureWorkflow.test.ts +8 -1
- package/src/functionalVerify.test.ts +73 -3
package/dist/captureSuite.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"captureSuite.js","sourceRoot":"","sources":["../src/captureSuite.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,oGAAoG;AACpG,2EAA2E;AAE3E,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,MAAM,YAAY,CAAC;AAE5B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"captureSuite.js","sourceRoot":"","sources":["../src/captureSuite.ts"],"names":[],"mappings":"AAAA,gGAAgG;AAChG,oGAAoG;AACpG,2EAA2E;AAE3E,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,MAAM,YAAY,CAAC;AAE5B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE3E,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AA6CxD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAAsC;IAC1E,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,IAAI,YAAY,CAAC,CAAC;IAC/D,MAAM,cAAc,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM;QAC5B,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAO,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACzB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAElG,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,CAAC,CAAC;IACjD,oGAAoG;IACpG,oGAAoG;IACpG,mGAAmG;IACnG,mEAAmE;IACnE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACxE,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,KAAK,SAAS,CAAC;IACzD,MAAM,QAAQ,GACZ,OAAO,CAAC,cAAc;QACtB,CAAC,MAAM,aAAa,CAAC,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;YACjG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACtB,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IACtC,MAAM,SAAS,GAAG,mBAAmB,EAAE,CAAC;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAO,GAA0B,EAAE,CAAC;IACxC,MAAM,YAAY,GAA0B,EAAE,CAAC;IAC/C,MAAM,qBAAqB,GAAG,CAAC,KAAa,EAAE,QAAgB,EAAE,WAAmB,EAAQ,EAAE;QAC3F,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,WAAW,CAAC;IACvD,CAAC,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC;YACnC,OAAO;YACP,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC;YACrB,cAAc;YACd,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG;YAC3B,IAAI,EAAE,OAAO,CAAC,OAAO;YACrB,OAAO;YACP,IAAI;YACJ,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,aAAa;YACb,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,MAAM;YACN,SAAS;YACT,WAAW,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;YAChE,qBAAqB;YACrB,UAAU,EAAE,OAAO,CAAC,UAAU;SAC/B,CAAC,CAAC;QACH,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC3B,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QACzB,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,IAAI,WAAW;YAAE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACvD,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,EAAE,CAAC;IAC5B,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,gCAAgC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxE,OAAO,CAAC,GAAG,CACT,IAAI;QACF,iBAAiB,CAAC,UAAU,EAAE;YAC5B,kBAAkB,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC;YAChD,kBAAkB,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC;YAC9C,kBAAkB,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC;SAC7C,CAAC;QACF,IAAI,CACP,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC;IACxD,MAAM,UAAU,GACd,OAAO,CAAC,UAAU,KAAK,KAAK;QAC1B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS;YAChC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,aAAa,CAAC;YAClD,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,MAAM,MAAM,GAAuB;QACjC,OAAO;QACP,QAAQ;QACR,OAAO;QACP,MAAM;QACN,UAAU;QACV,YAAY;QACZ,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QACnE,OAAO;QACP,UAAU;KACX,CAAC;IACF,IAAI,UAAU,KAAK,IAAI;QAAE,kBAAkB,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3E,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare function getCaptureTimeoutMs(): number;
|
|
2
|
+
export declare function resolveCaptureTimeoutMs(timeoutFlag: string | undefined, environmentOverride: string | undefined): number;
|
|
3
|
+
export declare function setCaptureTimeoutMs(timeoutMs: number | null): void;
|
|
4
|
+
//# sourceMappingURL=captureTimeout.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"captureTimeout.d.ts","sourceRoot":"","sources":["../src/captureTimeout.ts"],"names":[],"mappings":"AAUA,wBAAgB,mBAAmB,IAAI,MAAM,CAE5C;AAQD,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,mBAAmB,EAAE,MAAM,GAAG,SAAS,GACtC,MAAM,CAER;AAKD,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,CAElE"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// The wall-clock budget for a single capture wait — a page load, a frame-reached poll, or a wait for
|
|
2
|
+
// the in-page verifier to reach a terminal state. One number governs all of them on purpose: they are
|
|
3
|
+
// the same question ("how long may one page take before we call it stalled?"), and a run that has to
|
|
4
|
+
// be given more room on a slow or contended machine needs every one of them to move together.
|
|
5
|
+
//
|
|
6
|
+
// It is resolved from a flag or the environment rather than edited in source because the value is a
|
|
7
|
+
// property of the MACHINE, not of the code: the same budget that is generous on a developer laptop is
|
|
8
|
+
// tight on a CI runner sharing one software adapter between concurrent workers. A constant that has to
|
|
9
|
+
// be edited to run somewhere else is a constant that gets edited and committed for one environment,
|
|
10
|
+
// then silently governs every other.
|
|
11
|
+
export function getCaptureTimeoutMs() {
|
|
12
|
+
return captureTimeoutMs ?? resolveCaptureTimeoutMs(undefined, process.env['FLIGHT_CAPTURE_TIMEOUT_MS']);
|
|
13
|
+
}
|
|
14
|
+
// The flag wins over the environment, which wins over the default — the same precedence
|
|
15
|
+
// resolveCaptureWorkerCount uses, so the two knobs are configured the same way.
|
|
16
|
+
//
|
|
17
|
+
// A value that is not a positive finite number is ignored rather than honored: a zero or negative
|
|
18
|
+
// budget would make every wait expire immediately and report every page as stalled, which reads as a
|
|
19
|
+
// fleet-wide rendering failure rather than as the typo it is.
|
|
20
|
+
export function resolveCaptureTimeoutMs(timeoutFlag, environmentOverride) {
|
|
21
|
+
return parseCaptureTimeoutMs(timeoutFlag) ?? parseCaptureTimeoutMs(environmentOverride) ?? DEFAULT_CAPTURE_TIMEOUT_MS;
|
|
22
|
+
}
|
|
23
|
+
// Pins the budget for the rest of the process, so a CLI flag reaches the capture and validation waits
|
|
24
|
+
// without threading a parameter through every call. Passing null restores environment/default
|
|
25
|
+
// resolution, which is what a test that pinned a budget uses to put it back.
|
|
26
|
+
export function setCaptureTimeoutMs(timeoutMs) {
|
|
27
|
+
captureTimeoutMs = timeoutMs;
|
|
28
|
+
}
|
|
29
|
+
function parseCaptureTimeoutMs(value) {
|
|
30
|
+
if (value === undefined)
|
|
31
|
+
return null;
|
|
32
|
+
const parsed = Number.parseInt(value, 10);
|
|
33
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
|
|
34
|
+
}
|
|
35
|
+
let captureTimeoutMs = null;
|
|
36
|
+
// Unchanged from the constant this replaced, so a run that sets neither flag nor environment behaves
|
|
37
|
+
// exactly as before.
|
|
38
|
+
const DEFAULT_CAPTURE_TIMEOUT_MS = 15_000;
|
|
39
|
+
//# sourceMappingURL=captureTimeout.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"captureTimeout.js","sourceRoot":"","sources":["../src/captureTimeout.ts"],"names":[],"mappings":"AAAA,qGAAqG;AACrG,sGAAsG;AACtG,qGAAqG;AACrG,8FAA8F;AAC9F,EAAE;AACF,oGAAoG;AACpG,sGAAsG;AACtG,uGAAuG;AACvG,oGAAoG;AACpG,qCAAqC;AACrC,MAAM,UAAU,mBAAmB;IACjC,OAAO,gBAAgB,IAAI,uBAAuB,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC,CAAC;AAC1G,CAAC;AAED,wFAAwF;AACxF,gFAAgF;AAChF,EAAE;AACF,kGAAkG;AAClG,qGAAqG;AACrG,8DAA8D;AAC9D,MAAM,UAAU,uBAAuB,CACrC,WAA+B,EAC/B,mBAAuC;IAEvC,OAAO,qBAAqB,CAAC,WAAW,CAAC,IAAI,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,0BAA0B,CAAC;AACxH,CAAC;AAED,sGAAsG;AACtG,8FAA8F;AAC9F,6EAA6E;AAC7E,MAAM,UAAU,mBAAmB,CAAC,SAAwB;IAC1D,gBAAgB,GAAG,SAAS,CAAC;AAC/B,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACtD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AAC/D,CAAC;AAED,IAAI,gBAAgB,GAAkB,IAAI,CAAC;AAE3C,qGAAqG;AACrG,qBAAqB;AACrB,MAAM,0BAA0B,GAAG,MAAM,CAAC"}
|
|
@@ -22,6 +22,18 @@ export interface CaptureValidationOptions {
|
|
|
22
22
|
workerCount?: number;
|
|
23
23
|
fingerprintSkip?: Readonly<string[]>;
|
|
24
24
|
paritySkip?: Readonly<Record<string, 'all' | Readonly<string[]>>>;
|
|
25
|
+
/**
|
|
26
|
+
* Suppress the human-facing progress and summary output. The returned `CaptureValidationResult`
|
|
27
|
+
* carries every fact those lines report, so a programmatic caller loses nothing by silencing them.
|
|
28
|
+
*
|
|
29
|
+
* This exists because a caller that is *asserting how a failure is classified* still renders a real
|
|
30
|
+
* failure summary — a `✗ FAILED … 1 regression failed` block printed by a test that passes. In a CI
|
|
31
|
+
* log those lines are indistinguishable from an actual failure, which is worse than merely untidy:
|
|
32
|
+
* it trains a reader to scroll past the exact string they should stop at. Printing is a side effect
|
|
33
|
+
* the caller should ask for rather than one it has to work around, which is why this is an explicit
|
|
34
|
+
* option and not a check of the ambient environment.
|
|
35
|
+
*/
|
|
36
|
+
quiet?: boolean;
|
|
25
37
|
/** Explicit visual comparison groups. Unlike legacy parity, these may include DOM and arbitrary targets. */
|
|
26
38
|
parityGroups?: Readonly<Record<string, Readonly<CaptureParityGroup>>>;
|
|
27
39
|
/** Assertion-passed fingerprints from a preceding capture pass. Avoids reloading those pages. */
|
|
@@ -36,7 +48,9 @@ export interface CaptureValidationResult {
|
|
|
36
48
|
loadFailures: number;
|
|
37
49
|
parityFailures: number;
|
|
38
50
|
parityPasses: number;
|
|
51
|
+
parityUncovered: number;
|
|
39
52
|
regressionFailures: number;
|
|
53
|
+
regressionUncovered: number;
|
|
40
54
|
regressionPasses: number;
|
|
41
55
|
shouldFail: boolean;
|
|
42
56
|
skipped: number;
|
|
@@ -45,6 +59,19 @@ export interface CaptureValidationResult {
|
|
|
45
59
|
reportPath: string | null;
|
|
46
60
|
checks: CaptureValidationCheck[];
|
|
47
61
|
}
|
|
62
|
+
export interface CaptureParityCoverage {
|
|
63
|
+
gateParity: boolean;
|
|
64
|
+
interrupted: boolean;
|
|
65
|
+
parityComparisons: number;
|
|
66
|
+
parityUncovered: number;
|
|
67
|
+
rendererFilterCount: number;
|
|
68
|
+
}
|
|
69
|
+
export interface CaptureRegressionCoverage {
|
|
70
|
+
gateRegression: boolean;
|
|
71
|
+
interrupted: boolean;
|
|
72
|
+
regressionComparisons: number;
|
|
73
|
+
regressionUncovered: number;
|
|
74
|
+
}
|
|
48
75
|
export interface CaptureValidationCheck {
|
|
49
76
|
entry: string;
|
|
50
77
|
renderers: string[];
|
|
@@ -53,6 +80,18 @@ export interface CaptureValidationCheck {
|
|
|
53
80
|
message: string;
|
|
54
81
|
distance?: number;
|
|
55
82
|
threshold?: number;
|
|
83
|
+
/** Classification present only on a fingerprint mismatch that the regression leg already failed. */
|
|
84
|
+
sourceHashStatus?: 'changed' | 'unchanged' | 'unavailable';
|
|
85
|
+
recordedSourceHash?: string | null;
|
|
86
|
+
currentSourceHash?: string | null;
|
|
56
87
|
}
|
|
88
|
+
export declare function explainCaptureParityUncovered(eligibleCount: number, hasGroups: boolean): string;
|
|
89
|
+
export declare function explainCaptureVerificationStall(verification: Readonly<{
|
|
90
|
+
fingerprint?: string | null;
|
|
91
|
+
state?: string;
|
|
92
|
+
}> | null, waitedMs: number): string;
|
|
93
|
+
export declare function isCaptureParityCoverageFailure(run: Readonly<CaptureParityCoverage>): boolean;
|
|
94
|
+
export declare function isCaptureRegressionCoverageFailure(run: Readonly<CaptureRegressionCoverage>): boolean;
|
|
95
|
+
export declare function isUniformCaptureFingerprint(fingerprint: string): boolean;
|
|
57
96
|
export declare function runCaptureValidation(input: Readonly<CaptureValidationOptions>): Promise<CaptureValidationResult>;
|
|
58
97
|
//# sourceMappingURL=captureValidation.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"captureValidation.d.ts","sourceRoot":"","sources":["../src/captureValidation.ts"],"names":[],"mappings":"AAmCA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAKjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAG/D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"captureValidation.d.ts","sourceRoot":"","sources":["../src/captureValidation.ts"],"names":[],"mappings":"AAmCA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAKjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAG/D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAG/D,MAAM,WAAW,wBAAwB;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACpC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACrC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IAClE;;;;;;;;;;OAUG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,4GAA4G;IAC5G,YAAY,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;IACtE,iGAAiG;IACjG,YAAY,CAAC,EAAE,QAAQ,CAAC,qBAAqB,CAAC,CAAC;IAC/C,kFAAkF;IAClF,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,gGAAgG;IAChG,UAAU,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;CAC7B;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC;AAID,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,OAAO,CAAC;IACpB,WAAW,EAAE,OAAO,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAKD,MAAM,WAAW,yBAAyB;IACxC,cAAc,EAAE,OAAO,CAAC;IACxB,WAAW,EAAE,OAAO,CAAC;IACrB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,YAAY,GAAG,QAAQ,CAAC;IAClE,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,CAAC;IACrD,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oGAAoG;IACpG,gBAAgB,CAAC,EAAE,SAAS,GAAG,WAAW,GAAG,aAAa,CAAC;IAC3D,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAsED,wBAAgB,6BAA6B,CAAC,aAAa,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,MAAM,CAQ/F;AAyKD,wBAAgB,+BAA+B,CAC7C,YAAY,EAAE,QAAQ,CAAC;IAAE,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,IAAI,EAC9E,QAAQ,EAAE,MAAM,GACf,MAAM,CAiBR;AAED,wBAAgB,8BAA8B,CAAC,GAAG,EAAE,QAAQ,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAI5F;AAUD,wBAAgB,kCAAkC,CAAC,GAAG,EAAE,QAAQ,CAAC,yBAAyB,CAAC,GAAG,OAAO,CAGpG;AAYD,wBAAgB,2BAA2B,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAQxE;AAgWD,wBAAsB,oBAAoB,CACxC,KAAK,EAAE,QAAQ,CAAC,wBAAwB,CAAC,GACxC,OAAO,CAAC,uBAAuB,CAAC,CAsQlC"}
|
|
@@ -28,6 +28,8 @@ import { formatDetailLine, formatStatusLine, formatSummaryCount, formatSummaryLi
|
|
|
28
28
|
import { installAbortHandler, isBrowserClosedError } from './captureInterrupt.js';
|
|
29
29
|
import { CAPTURE_PROTOCOL_VERSION } from './captureProtocol.js';
|
|
30
30
|
import { writeCaptureReport } from './captureReport.js';
|
|
31
|
+
import { getCaptureSceneSourceHash } from './captureSourceHash.js';
|
|
32
|
+
import { getCaptureTimeoutMs } from './captureTimeout.js';
|
|
31
33
|
function fingerprintKey(entry, renderer) {
|
|
32
34
|
return `${entry}\0${renderer}`;
|
|
33
35
|
}
|
|
@@ -48,8 +50,19 @@ function addPair(pairs, fingerprints, a, b, group, tolerance) {
|
|
|
48
50
|
if (dist !== null)
|
|
49
51
|
pairs.push({ a, b, label: `${group === '' ? '' : `${group}:`}${a}·${b}`, dist, tolerance });
|
|
50
52
|
}
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
+
// Why an entry produced no parity pair. Each case has a different remedy, and a bare "0 comparisons"
|
|
54
|
+
// sends a reader to the wrong one: an ineligible backend needs a parity group or a committed baseline,
|
|
55
|
+
// while a single eligible backend has nothing to disagree with and needs a second one in scope.
|
|
56
|
+
export function explainCaptureParityUncovered(eligibleCount, hasGroups) {
|
|
57
|
+
if (eligibleCount === 0) {
|
|
58
|
+
return hasGroups
|
|
59
|
+
? 'no renderer in any parity group is eligible'
|
|
60
|
+
: 'no renderer is parity-eligible — declare a parity group, or commit a fingerprint baseline';
|
|
61
|
+
}
|
|
62
|
+
if (eligibleCount === 1)
|
|
63
|
+
return 'only one parity-eligible renderer — nothing to compare it against';
|
|
64
|
+
return 'every eligible pair is excluded by a parity skip';
|
|
65
|
+
}
|
|
53
66
|
async function loadFingerprint(context, baseUrl, entry, renderer, subject) {
|
|
54
67
|
// The real failure reason can arrive three ways, and Playwright's pageerror only catches the first:
|
|
55
68
|
// a synchronous uncaught exception (pageerror), an unhandled promise rejection (the verifier is an
|
|
@@ -71,7 +84,7 @@ async function loadFingerprint(context, baseUrl, entry, renderer, subject) {
|
|
|
71
84
|
const route = getCaptureEntryRoute(entry, renderer, subject);
|
|
72
85
|
await page.goto(`${baseUrl}/${route}`, {
|
|
73
86
|
waitUntil: 'domcontentloaded',
|
|
74
|
-
timeout:
|
|
87
|
+
timeout: getCaptureTimeoutMs(),
|
|
75
88
|
});
|
|
76
89
|
// Wait for a TERMINAL state, not merely for __ftVerification to exist. runRenderVerification sets
|
|
77
90
|
// that object up front with fingerprint:null, then fills the fingerprint only AFTER an async step —
|
|
@@ -79,12 +92,14 @@ async function loadFingerprint(context, baseUrl, entry, renderer, subject) {
|
|
|
79
92
|
// null mid-readback and misreport every webgpu test as "verifier did not run" (canvas/WebGL win the
|
|
80
93
|
// race because their readback is synchronous). So wait until the fingerprint is populated OR an error
|
|
81
94
|
// overlay appears, then read the result. Poll on a timer (the capture harness halts rAF).
|
|
95
|
+
const waitStartedAt = performance.now();
|
|
82
96
|
await page
|
|
83
97
|
.waitForFunction(() => {
|
|
84
98
|
const v = window.__ftVerification;
|
|
85
99
|
return v?.state === 'passed' || v?.state === 'failed' || document.getElementById('ft-error') !== null;
|
|
86
|
-
}, null, { timeout:
|
|
100
|
+
}, null, { timeout: getCaptureTimeoutMs(), polling: 100 })
|
|
87
101
|
.catch(() => { });
|
|
102
|
+
const waitedMs = Math.round(performance.now() - waitStartedAt);
|
|
88
103
|
const verification = await page
|
|
89
104
|
.evaluate(() => window.__ftVerification ?? null)
|
|
90
105
|
.catch(() => null);
|
|
@@ -110,7 +125,12 @@ async function loadFingerprint(context, baseUrl, entry, renderer, subject) {
|
|
|
110
125
|
const detail = verification?.error || pageError || overlay;
|
|
111
126
|
if (BACKEND_UNAVAILABLE.test(detail))
|
|
112
127
|
return { fingerprint: null, reason: `backend unavailable (${detail})`, unavailable: true, aborted: false };
|
|
113
|
-
return {
|
|
128
|
+
return {
|
|
129
|
+
fingerprint: null,
|
|
130
|
+
reason: detail || explainCaptureVerificationStall(verification, waitedMs),
|
|
131
|
+
unavailable: false,
|
|
132
|
+
aborted: false,
|
|
133
|
+
};
|
|
114
134
|
}
|
|
115
135
|
catch (err) {
|
|
116
136
|
// A closed browser/page is the interrupt tearing things down — report it as an abort, not a failure.
|
|
@@ -187,10 +207,85 @@ async function captureDomFingerprint(page) {
|
|
|
187
207
|
await handle?.dispose();
|
|
188
208
|
}
|
|
189
209
|
}
|
|
210
|
+
// Why a verification wait ended without a fingerprint, as a reason a reader can act on. The bare
|
|
211
|
+
// "verifier did not run" it replaces named a symptom shared by causes with opposite remedies, and left
|
|
212
|
+
// the one number that decides between them — how long it actually waited — unrecorded.
|
|
213
|
+
//
|
|
214
|
+
// This exists because a gate that silently fails to run is the same inert-gate class as a check that
|
|
215
|
+
// silently passes: the leg reports a failure nobody can diagnose, so it gets re-run rather than fixed.
|
|
216
|
+
// Measured for context: the heaviest example scene verifies in ~4.1-6.2s across 6 to 16 workers, even
|
|
217
|
+
// with a full monorepo check and test suite running alongside — about 42% of the budget at worst. So a
|
|
218
|
+
// stall at or near the full budget is NOT the scene being slow, and the reason says so rather than
|
|
219
|
+
// leaving the next reader to assume it and raise the timeout.
|
|
220
|
+
//
|
|
221
|
+
// The budget is read from the same seam the wait reads, not passed in, so the two can never disagree
|
|
222
|
+
// about what it was. Passing it would let a caller name a budget the wait did not use, which is the
|
|
223
|
+
// one thing this message must never do — it exists to be trusted about how long the wait actually had.
|
|
224
|
+
export function explainCaptureVerificationStall(verification, waitedMs) {
|
|
225
|
+
const budget = `waited ${waitedMs}ms of ${getCaptureTimeoutMs()}ms`;
|
|
226
|
+
if (verification === null || verification === undefined) {
|
|
227
|
+
return `verifier never registered — no __ftVerification on the page (${budget}); the page's module likely failed to run`;
|
|
228
|
+
}
|
|
229
|
+
if (verification.state === undefined) {
|
|
230
|
+
return `verifier object present but carries no state (${budget}); suspect a capture-protocol mismatch`;
|
|
231
|
+
}
|
|
232
|
+
if (verification.state === 'passed') {
|
|
233
|
+
return `verifier passed but produced no fingerprint (${budget}); the readback completed empty`;
|
|
234
|
+
}
|
|
235
|
+
// Registered and still non-terminal: it started and never finished, which is a stall rather than a
|
|
236
|
+
// scene that is merely expensive. The STAGE is the actionable half — 'awaitingFrame' means a presented
|
|
237
|
+
// frame never arrived (page/scheduler), 'readingBack' means the GPU readback never resolved (driver).
|
|
238
|
+
const stage = verification.stage;
|
|
239
|
+
const where = stage === undefined ? '' : ` at stage "${stage}"`;
|
|
240
|
+
return `verifier registered but stalled${where} in state "${verification.state}" (${budget}); it started and never reached a terminal state`;
|
|
241
|
+
}
|
|
242
|
+
export function isCaptureParityCoverageFailure(run) {
|
|
243
|
+
if (!run.gateParity || run.interrupted)
|
|
244
|
+
return false;
|
|
245
|
+
if (run.rendererFilterCount === 1)
|
|
246
|
+
return false;
|
|
247
|
+
return run.parityComparisons === 0 && run.parityUncovered > 0;
|
|
248
|
+
}
|
|
249
|
+
// A tier either has what it needs and gates hard, or it says so loudly — there is no silent
|
|
250
|
+
// degrade-to-success. A gated parity run that compared NOTHING is UNCONFIGURED, not clean, and the
|
|
251
|
+
// green tick it used to print is worse than an absent leg, because an absent leg is visibly absent.
|
|
252
|
+
//
|
|
253
|
+
// Two runs genuinely cannot compare, and are exempt rather than excused: one narrowed to a single
|
|
254
|
+
// renderer (the operator disabled parity themselves, so reporting it back at them is noise), and an
|
|
255
|
+
// interrupted run, whose remaining entries never ran at all. A run with nothing uncovered — every
|
|
256
|
+
// entry skipped by policy before parity was reached — is not this failure either.
|
|
257
|
+
export function isCaptureRegressionCoverageFailure(run) {
|
|
258
|
+
if (!run.gateRegression || run.interrupted)
|
|
259
|
+
return false;
|
|
260
|
+
return run.regressionComparisons === 0 && run.regressionUncovered > 0;
|
|
261
|
+
}
|
|
262
|
+
// Loads a single test/renderer page and returns its render fingerprint, or null with a reason and a
|
|
263
|
+
// flag marking whether the cause is a genuinely-unavailable backend (skippable) versus a real error.
|
|
264
|
+
// True when every cell of a coarse fingerprint carries the same value — a blank or flat-filled frame.
|
|
265
|
+
//
|
|
266
|
+
// This is a WRITE-SIDE gate, not a render check: a uniform frame may be a legitimate render (a solid
|
|
267
|
+
// background scene), but it is never worth BLESSING as a regression baseline, because it cannot
|
|
268
|
+
// distinguish a working scene from a broken one. Refusing it costs a real flat scene its regression
|
|
269
|
+
// coverage and costs a broken one nothing — the asymmetry is the point.
|
|
270
|
+
//
|
|
271
|
+
// Format: "<cellSize>:<hex cells>", so the payload after the colon is split into fixed-width cells.
|
|
272
|
+
export function isUniformCaptureFingerprint(fingerprint) {
|
|
273
|
+
const payload = fingerprint.slice(fingerprint.indexOf(':') + 1);
|
|
274
|
+
if (payload.length <= CAPTURE_FINGERPRINT_CELL_CHARS)
|
|
275
|
+
return true;
|
|
276
|
+
const first = payload.slice(0, CAPTURE_FINGERPRINT_CELL_CHARS);
|
|
277
|
+
for (let i = CAPTURE_FINGERPRINT_CELL_CHARS; i < payload.length; i += CAPTURE_FINGERPRINT_CELL_CHARS) {
|
|
278
|
+
if (payload.slice(i, i + CAPTURE_FINGERPRINT_CELL_CHARS) !== first)
|
|
279
|
+
return false;
|
|
280
|
+
}
|
|
281
|
+
return true;
|
|
282
|
+
}
|
|
190
283
|
async function processEntry(entry, entryIndex, totalEntries, isAborted, options, samples) {
|
|
191
284
|
const result = {
|
|
192
285
|
regressionFailures: 0,
|
|
286
|
+
regressionUncovered: 0,
|
|
193
287
|
parityFailures: 0,
|
|
288
|
+
parityUncovered: 0,
|
|
194
289
|
regressionPasses: 0,
|
|
195
290
|
parityPasses: 0,
|
|
196
291
|
loadFailures: 0,
|
|
@@ -201,6 +296,19 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
201
296
|
};
|
|
202
297
|
result.output.push(`${pc.dim(`[${entryIndex + 1}/${totalEntries}]`)} ${pc.bold(entry.name)}`);
|
|
203
298
|
const renderers = validationRenderers(entry, options.rendererFilter, options.parityGroups);
|
|
299
|
+
// An entry whose renderers are all filtered out otherwise emits NOTHING — no skip line, no parity line,
|
|
300
|
+
// no failure — so a reader cannot tell "covered and silent" from "never ran". Say it out loud.
|
|
301
|
+
if (renderers.length === 0) {
|
|
302
|
+
result.output.push(` ${pc.dim('·')} ${pc.dim(`no renderer in scope for this leg (entry declares: ${entry.renderers.join(', ') || 'none'})`)}`);
|
|
303
|
+
result.checks.push({
|
|
304
|
+
entry: entry.name,
|
|
305
|
+
renderers: [],
|
|
306
|
+
kind: 'load',
|
|
307
|
+
status: 'skipped',
|
|
308
|
+
message: `no renderer in scope; entry declares ${entry.renderers.join(', ') || 'none'}`,
|
|
309
|
+
});
|
|
310
|
+
return result;
|
|
311
|
+
}
|
|
204
312
|
const labelWidth = Math.max(6, ...renderers.map((r) => r.length));
|
|
205
313
|
const statusLine = (tone, label, message) => formatStatusLine(tone, label, labelWidth, message);
|
|
206
314
|
const detailLine = (glyph, label, message, paint) => formatDetailLine(glyph, label, labelWidth, message, paint);
|
|
@@ -278,7 +386,30 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
278
386
|
});
|
|
279
387
|
continue;
|
|
280
388
|
}
|
|
389
|
+
// A uniform fingerprint is every cell identical — a blank or flat-filled frame. The stability check
|
|
390
|
+
// above CANNOT catch it: a frame that renders blank every time has a self-distance of zero, the most
|
|
391
|
+
// stable result possible, so it sails through and gets blessed as ground truth. That has already
|
|
392
|
+
// happened once in this repo (a WebGPU capture on a software adapter that cannot present to the
|
|
393
|
+
// swapchain), and a blank baseline is worse than no baseline, because every later run then compares
|
|
394
|
+
// against the blank and passes.
|
|
395
|
+
if (isUniformCaptureFingerprint(fingerprint)) {
|
|
396
|
+
const note = 'not baselined — uniform frame (every cell identical); a blank capture must not become ground truth';
|
|
397
|
+
result.output.push(statusLine('skip', renderer, note));
|
|
398
|
+
result.skipped++;
|
|
399
|
+
result.checks.push({
|
|
400
|
+
entry: entry.name,
|
|
401
|
+
renderers: [renderer],
|
|
402
|
+
kind: 'baseline',
|
|
403
|
+
status: 'skipped',
|
|
404
|
+
message: note,
|
|
405
|
+
});
|
|
406
|
+
continue;
|
|
407
|
+
}
|
|
281
408
|
setBaselineField(options.root, options.subject, entry.name, renderer, 'fingerprint', fingerprint);
|
|
409
|
+
const sourceHash = getCaptureSceneSourceHash(options.root, options.subject, entry, renderer);
|
|
410
|
+
if (sourceHash !== null) {
|
|
411
|
+
setBaselineField(options.root, options.subject, entry.name, renderer, 'sourceHash', sourceHash);
|
|
412
|
+
}
|
|
282
413
|
result.output.push(statusLine('pass', renderer, 'baseline written'));
|
|
283
414
|
result.updated++;
|
|
284
415
|
result.checks.push({
|
|
@@ -293,6 +424,7 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
293
424
|
}
|
|
294
425
|
const committed = getBaselineField(options.root, options.subject, entry.name, renderer, 'fingerprint');
|
|
295
426
|
if (committed === null) {
|
|
427
|
+
result.regressionUncovered++;
|
|
296
428
|
result.output.push(statusLine('muted', renderer, 'no fingerprint baseline — skipped'));
|
|
297
429
|
result.skipped++;
|
|
298
430
|
result.checks.push({
|
|
@@ -332,16 +464,23 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
332
464
|
else if (options.gateRegression) {
|
|
333
465
|
const check = evaluateCaptureRegression(fingerprint, committed, options.regressionTolerance);
|
|
334
466
|
if (!check.pass) {
|
|
335
|
-
|
|
467
|
+
const recordedSourceHash = getBaselineField(options.root, options.subject, entry.name, renderer, 'sourceHash');
|
|
468
|
+
const currentSourceHash = getCaptureSceneSourceHash(options.root, options.subject, entry, renderer);
|
|
469
|
+
const freshness = classifyCaptureBaselineFreshness(recordedSourceHash, currentSourceHash);
|
|
470
|
+
const message = `regression ${dist.toFixed(2)} > ${options.regressionTolerance} — ${freshness.message}`;
|
|
471
|
+
result.output.push(statusLine('fail', renderer, message));
|
|
336
472
|
result.regressionFailures++;
|
|
337
473
|
result.checks.push({
|
|
338
474
|
entry: entry.name,
|
|
339
475
|
renderers: [renderer],
|
|
340
476
|
kind: 'regression',
|
|
341
477
|
status: 'failed',
|
|
342
|
-
message
|
|
478
|
+
message,
|
|
343
479
|
distance: dist,
|
|
344
480
|
threshold: options.regressionTolerance,
|
|
481
|
+
sourceHashStatus: freshness.status,
|
|
482
|
+
recordedSourceHash,
|
|
483
|
+
currentSourceHash,
|
|
345
484
|
});
|
|
346
485
|
}
|
|
347
486
|
else {
|
|
@@ -394,6 +533,20 @@ async function processEntry(entry, entryIndex, totalEntries, isAborted, options,
|
|
|
394
533
|
}
|
|
395
534
|
}
|
|
396
535
|
}
|
|
536
|
+
// A gated parity run that compared NOTHING is the failure mode this whole tier was silently missing:
|
|
537
|
+
// with no eligible renderers there are no pairs, so every branch below is skipped and the entry reports
|
|
538
|
+
// success having checked nothing. Record it as uncovered so the run-level verdict can say so out loud.
|
|
539
|
+
// Report and baseline-write modes are excluded because neither claims to gate anything.
|
|
540
|
+
if (pairs.length === 0 && options.gateParity && !options.report && !options.updateFingerprints) {
|
|
541
|
+
result.parityUncovered++;
|
|
542
|
+
result.checks.push({
|
|
543
|
+
entry: entry.name,
|
|
544
|
+
renderers: [...eligible.keys()],
|
|
545
|
+
kind: 'parity',
|
|
546
|
+
status: 'skipped',
|
|
547
|
+
message: explainCaptureParityUncovered(eligible.size, groups.length > 0),
|
|
548
|
+
});
|
|
549
|
+
}
|
|
397
550
|
if (pairs.length > 0) {
|
|
398
551
|
if (options.report) {
|
|
399
552
|
const segments = pairs.map((p) => `${p.label} ${p.dist.toFixed(2)}`).join(' ');
|
|
@@ -462,6 +615,7 @@ export async function runCaptureValidation(input) {
|
|
|
462
615
|
rendererFilter: input.rendererFilter ?? [],
|
|
463
616
|
captureFrames: Math.max(1, input.captureFrames ?? 1),
|
|
464
617
|
report: input.report ?? false,
|
|
618
|
+
quiet: input.quiet ?? false,
|
|
465
619
|
updateFingerprints: input.updateFingerprints ?? false,
|
|
466
620
|
gateRegression: input.gateRegression ?? true,
|
|
467
621
|
gateParity: input.gateParity ?? true,
|
|
@@ -483,6 +637,8 @@ export async function runCaptureValidation(input) {
|
|
|
483
637
|
const isAborted = installAbortHandler();
|
|
484
638
|
let regressionFailures = 0;
|
|
485
639
|
let parityFailures = 0;
|
|
640
|
+
let parityUncovered = 0;
|
|
641
|
+
let regressionUncovered = 0;
|
|
486
642
|
let regressionPasses = 0;
|
|
487
643
|
let parityPasses = 0;
|
|
488
644
|
let loadFailures = 0;
|
|
@@ -532,14 +688,17 @@ export async function runCaptureValidation(input) {
|
|
|
532
688
|
const result = await processEntry(job.entry, job.index, entries.length, isAborted, options, samples);
|
|
533
689
|
regressionFailures += result.regressionFailures;
|
|
534
690
|
parityFailures += result.parityFailures;
|
|
691
|
+
parityUncovered += result.parityUncovered;
|
|
692
|
+
regressionUncovered += result.regressionUncovered;
|
|
535
693
|
regressionPasses += result.regressionPasses;
|
|
536
694
|
parityPasses += result.parityPasses;
|
|
537
695
|
loadFailures += result.loadFailures;
|
|
538
696
|
updated += result.updated;
|
|
539
697
|
skipped += result.skipped;
|
|
540
698
|
checks.push(...result.checks);
|
|
541
|
-
|
|
542
|
-
|
|
699
|
+
if (!options.quiet)
|
|
700
|
+
for (const line of result.output)
|
|
701
|
+
console.log(line);
|
|
543
702
|
}
|
|
544
703
|
});
|
|
545
704
|
await Promise.all(workers);
|
|
@@ -551,14 +710,17 @@ export async function runCaptureValidation(input) {
|
|
|
551
710
|
const result = await processEntry(entries[entryIndex], entryIndex, entries.length, isAborted, options, samples);
|
|
552
711
|
regressionFailures += result.regressionFailures;
|
|
553
712
|
parityFailures += result.parityFailures;
|
|
713
|
+
parityUncovered += result.parityUncovered;
|
|
714
|
+
regressionUncovered += result.regressionUncovered;
|
|
554
715
|
regressionPasses += result.regressionPasses;
|
|
555
716
|
parityPasses += result.parityPasses;
|
|
556
717
|
loadFailures += result.loadFailures;
|
|
557
718
|
updated += result.updated;
|
|
558
719
|
skipped += result.skipped;
|
|
559
720
|
checks.push(...result.checks);
|
|
560
|
-
|
|
561
|
-
|
|
721
|
+
if (!options.quiet)
|
|
722
|
+
for (const line of result.output)
|
|
723
|
+
console.log(line);
|
|
562
724
|
}
|
|
563
725
|
}
|
|
564
726
|
}
|
|
@@ -570,13 +732,14 @@ export async function runCaptureValidation(input) {
|
|
|
570
732
|
const interrupted = isAborted();
|
|
571
733
|
const note = interrupted ? pc.yellow(' — interrupted (partial run)') : '';
|
|
572
734
|
if (options.updateFingerprints) {
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
735
|
+
if (!options.quiet)
|
|
736
|
+
console.log('\n' +
|
|
737
|
+
formatSummaryLine(loadFailures > 0, [
|
|
738
|
+
formatSummaryCount(updated, 'baselines written', 'pass'),
|
|
739
|
+
formatSummaryCount(skipped, 'skipped', 'warn'),
|
|
740
|
+
formatSummaryCount(loadFailures, 'load failures', 'fail'),
|
|
741
|
+
]) +
|
|
742
|
+
note);
|
|
580
743
|
if (loadFailures > 0) {
|
|
581
744
|
console.error(pc.red(`${loadFailures} test(s) failed to load/verify — not a clean baseline run.`));
|
|
582
745
|
return createResult(true);
|
|
@@ -584,26 +747,77 @@ export async function runCaptureValidation(input) {
|
|
|
584
747
|
return createResult(loadFailures > 0);
|
|
585
748
|
}
|
|
586
749
|
if (options.report) {
|
|
750
|
+
if (!options.quiet)
|
|
751
|
+
console.log('\n' +
|
|
752
|
+
formatSummaryLine(loadFailures > 0, [
|
|
753
|
+
formatSummaryCount(skipped, 'skipped', 'warn'),
|
|
754
|
+
formatSummaryCount(loadFailures, 'load failures', 'fail'),
|
|
755
|
+
]) +
|
|
756
|
+
pc.dim(' (report only — nothing gated)') +
|
|
757
|
+
note);
|
|
758
|
+
return createResult(loadFailures > 0);
|
|
759
|
+
}
|
|
760
|
+
// Same doctrine, other tier: a gated regression leg that compared NOTHING is unconfigured, not clean.
|
|
761
|
+
// Without this the tier reads as a green pass with zero coverage the moment nothing else is red.
|
|
762
|
+
const regressionCoverageFailed = isCaptureRegressionCoverageFailure({
|
|
763
|
+
gateRegression: options.gateRegression,
|
|
764
|
+
interrupted,
|
|
765
|
+
regressionComparisons: regressionPasses + regressionFailures,
|
|
766
|
+
regressionUncovered,
|
|
767
|
+
});
|
|
768
|
+
const parityCoverageFailed = isCaptureParityCoverageFailure({
|
|
769
|
+
gateParity: options.gateParity,
|
|
770
|
+
interrupted,
|
|
771
|
+
parityComparisons: parityPasses + parityFailures,
|
|
772
|
+
parityUncovered,
|
|
773
|
+
rendererFilterCount: options.rendererFilter.length,
|
|
774
|
+
});
|
|
775
|
+
const failed = regressionFailures > 0 ||
|
|
776
|
+
parityFailures > 0 ||
|
|
777
|
+
loadFailures > 0 ||
|
|
778
|
+
parityCoverageFailed ||
|
|
779
|
+
regressionCoverageFailed;
|
|
780
|
+
if (!options.quiet)
|
|
587
781
|
console.log('\n' +
|
|
588
|
-
formatSummaryLine(
|
|
782
|
+
formatSummaryLine(failed, [
|
|
783
|
+
formatSummaryCount(regressionPasses, 'regression passed', 'pass'),
|
|
784
|
+
formatSummaryCount(regressionFailures, 'regression failed', 'fail'),
|
|
785
|
+
formatSummaryCount(parityPasses, 'parity passed', 'pass'),
|
|
786
|
+
formatSummaryCount(parityFailures, 'parity failed', 'fail'),
|
|
787
|
+
formatSummaryCount(parityUncovered, 'parity uncovered', parityCoverageFailed ? 'fail' : 'warn'),
|
|
788
|
+
formatSummaryCount(regressionUncovered, 'regression uncovered', regressionCoverageFailed ? 'fail' : 'warn'),
|
|
589
789
|
formatSummaryCount(skipped, 'skipped', 'warn'),
|
|
590
790
|
formatSummaryCount(loadFailures, 'load failures', 'fail'),
|
|
591
791
|
]) +
|
|
592
|
-
pc.dim(' (report only — nothing gated)') +
|
|
593
792
|
note);
|
|
594
|
-
|
|
793
|
+
// Name the entries behind an "N uncovered" count: a bare number tells a reader something is missing but
|
|
794
|
+
// not what to go look at, and the entries differ in why.
|
|
795
|
+
for (const [label, uncovered, failedTier] of [
|
|
796
|
+
['parity', parityUncovered, parityCoverageFailed],
|
|
797
|
+
['regression', regressionUncovered, regressionCoverageFailed],
|
|
798
|
+
]) {
|
|
799
|
+
if (uncovered === 0)
|
|
800
|
+
continue;
|
|
801
|
+
const names = [
|
|
802
|
+
...new Set(checks
|
|
803
|
+
.filter((check) => check.kind === (label === 'parity' ? 'parity' : 'baseline') && check.status === 'skipped')
|
|
804
|
+
.map((check) => check.entry)),
|
|
805
|
+
];
|
|
806
|
+
if (names.length > 0) {
|
|
807
|
+
const line = ` ${label} uncovered: ${names.join(', ')}`;
|
|
808
|
+
if (!options.quiet)
|
|
809
|
+
console.log(failedTier ? pc.red(line) : pc.yellow(line));
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
if (regressionCoverageFailed) {
|
|
813
|
+
console.error(pc.red(`\nRegression compared NOTHING across ${regressionUncovered} entr${regressionUncovered === 1 ? 'y' : 'ies'} — this tier is unconfigured, not clean. Re-capture with --update-fingerprints on a leg where the tier is valid, or remove the stale baselines so the gap is honest.`));
|
|
814
|
+
}
|
|
815
|
+
if (parityCoverageFailed) {
|
|
816
|
+
console.error(pc.red(`\nParity compared NOTHING across all ${parityUncovered} entr${parityUncovered === 1 ? 'y' : 'ies'} — this leg is unconfigured, not clean.`));
|
|
817
|
+
for (const reason of new Set(checks.filter((check) => check.kind === 'parity' && check.status === 'skipped').map((check) => check.message))) {
|
|
818
|
+
console.error(pc.red(` · ${reason}`));
|
|
819
|
+
}
|
|
595
820
|
}
|
|
596
|
-
const failed = regressionFailures > 0 || parityFailures > 0 || loadFailures > 0;
|
|
597
|
-
console.log('\n' +
|
|
598
|
-
formatSummaryLine(failed, [
|
|
599
|
-
formatSummaryCount(regressionPasses, 'regression passed', 'pass'),
|
|
600
|
-
formatSummaryCount(regressionFailures, 'regression failed', 'fail'),
|
|
601
|
-
formatSummaryCount(parityPasses, 'parity passed', 'pass'),
|
|
602
|
-
formatSummaryCount(parityFailures, 'parity failed', 'fail'),
|
|
603
|
-
formatSummaryCount(skipped, 'skipped', 'warn'),
|
|
604
|
-
formatSummaryCount(loadFailures, 'load failures', 'fail'),
|
|
605
|
-
]) +
|
|
606
|
-
note);
|
|
607
821
|
return createResult(failed);
|
|
608
822
|
function createResult(shouldFail) {
|
|
609
823
|
const reportPath = input.reportPath === false
|
|
@@ -614,7 +828,9 @@ export async function runCaptureValidation(input) {
|
|
|
614
828
|
loadFailures,
|
|
615
829
|
parityFailures,
|
|
616
830
|
parityPasses,
|
|
831
|
+
parityUncovered,
|
|
617
832
|
regressionFailures,
|
|
833
|
+
regressionUncovered,
|
|
618
834
|
regressionPasses,
|
|
619
835
|
shouldFail,
|
|
620
836
|
skipped,
|
|
@@ -628,4 +844,30 @@ export async function runCaptureValidation(input) {
|
|
|
628
844
|
return result;
|
|
629
845
|
}
|
|
630
846
|
}
|
|
847
|
+
function classifyCaptureBaselineFreshness(recordedSourceHash, currentSourceHash) {
|
|
848
|
+
if (recordedSourceHash === null) {
|
|
849
|
+
return {
|
|
850
|
+
message: 'scene-source freshness unavailable (baseline has no recorded source hash)',
|
|
851
|
+
status: 'unavailable',
|
|
852
|
+
};
|
|
853
|
+
}
|
|
854
|
+
if (currentSourceHash === null) {
|
|
855
|
+
return {
|
|
856
|
+
message: 'scene-source freshness unavailable (current scene source could not be read)',
|
|
857
|
+
status: 'unavailable',
|
|
858
|
+
};
|
|
859
|
+
}
|
|
860
|
+
if (recordedSourceHash !== currentSourceHash) {
|
|
861
|
+
return {
|
|
862
|
+
message: 'scene source changed since baseline — recapture owed by the scene owner',
|
|
863
|
+
status: 'changed',
|
|
864
|
+
};
|
|
865
|
+
}
|
|
866
|
+
return {
|
|
867
|
+
message: 'scene source unchanged since baseline — environment drift; never rebaseline',
|
|
868
|
+
status: 'unchanged',
|
|
869
|
+
};
|
|
870
|
+
}
|
|
871
|
+
// Hex characters per fingerprint cell (one RGB triplet).
|
|
872
|
+
const CAPTURE_FINGERPRINT_CELL_CHARS = 6;
|
|
631
873
|
//# sourceMappingURL=captureValidation.js.map
|