@flighthq/tool-capture 0.3.0-next.906.07cea63 → 0.3.0

package/src/captureValidation.test.ts

@@ -1,9 +1,155 @@
+import { createHash } from 'node:crypto';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 
 import { describe, expect, it, vi } from 'vitest';
 
-import { runCaptureValidation } from './captureValidation';
+import { setBaselineField } from './baselineStore';
+import { setCaptureTimeoutMs } from './captureTimeout';
+import {
+  explainCaptureParityUncovered,
+  isCaptureRegressionCoverageFailure,
+  isUniformCaptureFingerprint,
+  explainCaptureVerificationStall,
+  isCaptureParityCoverageFailure,
+  runCaptureValidation,
+} from './captureValidation';
+
+const COVERED: Readonly<Parameters<typeof isCaptureParityCoverageFailure>[0]> = {
+  gateParity: true,
+  interrupted: false,
+  parityComparisons: 0,
+  parityUncovered: 1,
+  rendererFilterCount: 0,
+};
+
+describe('explainCaptureParityUncovered', () => {
+  it('names the remedy for an ineligible backend, which differs from the one-backend case', () => {
+    // The two states look identical in a "0 comparisons" summary but need opposite fixes, which is why
+    // the reason is data rather than a bare skip.
+    expect(explainCaptureParityUncovered(0, false)).toContain('parity group');
+    expect(explainCaptureParityUncovered(1, false)).toContain('nothing to compare it against');
+  });
+
+  it('points at group membership rather than baselines when groups are declared', () => {
+    expect(explainCaptureParityUncovered(0, true)).toContain('parity group');
+    expect(explainCaptureParityUncovered(0, true)).not.toContain('fingerprint baseline');
+  });
+
+  it('blames the skip list when several backends were eligible and still produced no pair', () => {
+    expect(explainCaptureParityUncovered(3, false)).toContain('parity skip');
+  });
+});
+
+describe('explainCaptureVerificationStall', () => {
+  it('distinguishes a verifier that never registered from one that started and stalled', () => {
+    // Opposite remedies: the first is a page/module failure, the second a readback that never finished.
+    // The bare "verifier did not run" it replaces covered both.
+    expect(explainCaptureVerificationStall(null, 15_000)).toContain('never registered');
+    expect(explainCaptureVerificationStall({ state: 'running' }, 15_000)).toContain('stalled');
+  });
+
+  it('reports what it waited AGAINST the budget, the number that decides if cost is the cause', () => {
+    const reason = explainCaptureVerificationStall({ state: 'running' }, 15_000);
+    expect(reason).toContain('15000ms of 15000ms');
+    // A short wait is a different story from one that burned the whole budget, and the reason shows it.
+    expect(explainCaptureVerificationStall({ state: 'running' }, 900)).toContain('900ms of 15000ms');
+  });
+
+  // The reason and the wait must never disagree about what the budget was — a message that names a
+  // budget the wait did not use is worse than no message, since the whole point of it is to be trusted
+  // about how long the wait actually had. Both read the same seam, so a raised budget moves both.
+  it('reports the configured budget, not the compiled-in default', () => {
+    setCaptureTimeoutMs(45_000);
+    try {
+      expect(explainCaptureVerificationStall({ state: 'running' }, 20_000)).toContain('20000ms of 45000ms');
+    } finally {
+      setCaptureTimeoutMs(null);
+    }
+  });
+
+  it('names the empty-readback case, which looks like success until the fingerprint is read', () => {
+    expect(explainCaptureVerificationStall({ fingerprint: null, state: 'passed' }, 4_000)).toContain('no fingerprint');
+  });
+
+  it('flags a stateless verifier object as a protocol mismatch rather than a stall', () => {
+    expect(explainCaptureVerificationStall({}, 4_000)).toContain('protocol');
+  });
+});
+
+describe('isCaptureParityCoverageFailure', () => {
+  it('FAILS a gated run that compared nothing while entries wanted a comparison', () => {
+    // The defect this gate exists for: 107 entries skipped, leg green.
+    expect(isCaptureParityCoverageFailure({ ...COVERED, parityUncovered: 107 })).toBe(true);
+  });
+
+  it('passes as soon as a single comparison actually ran', () => {
+    expect(isCaptureParityCoverageFailure({ ...COVERED, parityComparisons: 1 })).toBe(false);
+  });
+
+  it('does not fire when parity is not being gated', () => {
+    expect(isCaptureParityCoverageFailure({ ...COVERED, gateParity: false })).toBe(false);
+  });
+
+  it('exempts an interrupted run, whose remaining entries never ran', () => {
+    expect(isCaptureParityCoverageFailure({ ...COVERED, interrupted: true })).toBe(false);
+  });
+
+  it('exempts a run narrowed to ONE renderer, which cannot compare by construction', () => {
+    expect(isCaptureParityCoverageFailure({ ...COVERED, rendererFilterCount: 1 })).toBe(false);
+    // Two named renderers can still disagree, so that narrowing stays gated.
+    expect(isCaptureParityCoverageFailure({ ...COVERED, rendererFilterCount: 2 })).toBe(true);
+  });
+
+  it('does not fire when nothing wanted a comparison in the first place', () => {
+    expect(isCaptureParityCoverageFailure({ ...COVERED, parityUncovered: 0 })).toBe(false);
+  });
+});
+
+describe('isCaptureRegressionCoverageFailure', () => {
+  const UNCOVERED = {
+    gateRegression: true,
+    interrupted: false,
+    regressionComparisons: 0,
+    regressionUncovered: 100,
+  };
+
+  it('FAILS a gated regression leg that compared nothing, the inert-tier defect', () => {
+    // 0 passed / 0 failed / 100 skipped used to read as a clean pass.
+    expect(isCaptureRegressionCoverageFailure(UNCOVERED)).toBe(true);
+  });
+
+  it('passes as soon as one comparison actually ran', () => {
+    expect(isCaptureRegressionCoverageFailure({ ...UNCOVERED, regressionComparisons: 1 })).toBe(false);
+  });
+
+  it('does not fire when regression is not being gated, or when the run was interrupted', () => {
+    expect(isCaptureRegressionCoverageFailure({ ...UNCOVERED, gateRegression: false })).toBe(false);
+    expect(isCaptureRegressionCoverageFailure({ ...UNCOVERED, interrupted: true })).toBe(false);
+  });
+
+  it('does not fire when nothing wanted a comparison', () => {
+    expect(isCaptureRegressionCoverageFailure({ ...UNCOVERED, regressionUncovered: 0 })).toBe(false);
+  });
+});
+
+describe('isUniformCaptureFingerprint', () => {
+  it('rejects a fingerprint whose cells are all identical, the blank frame a stability check cannot catch', () => {
+    // The real shape that was blessed once: every cell the same colour.
+    expect(isUniformCaptureFingerprint('16:' + 'eeddcc'.repeat(256))).toBe(true);
+  });
+
+  it('accepts a frame that varies anywhere, including in only one cell', () => {
+    expect(isUniformCaptureFingerprint('16:' + 'eeddcc'.repeat(255) + '112233')).toBe(false);
+    expect(isUniformCaptureFingerprint('16:112233' + 'eeddcc'.repeat(255))).toBe(false);
+  });
+
+  it('treats a single-cell or empty payload as uniform, since it can distinguish nothing', () => {
+    expect(isUniformCaptureFingerprint('1:aabbcc')).toBe(true);
+    expect(isUniformCaptureFingerprint('16:')).toBe(true);
+  });
+});
 
 describe('runCaptureValidation', () => {
   it('is a callable fingerprint-validation orchestrator', () => {
@@ -19,6 +165,7 @@ describe('runCaptureValidation', () => {
       server: { url: 'http://unused.invalid', kill },
       root: join(tmpdir(), 'tool-capture-reuse-fixture'),
       report: true,
+      quiet: true,
       fingerprints: { sample: { canvas: '1:000000' } },
       browserSession: {
         browser: { close: vi.fn() } as never,
@@ -31,4 +178,77 @@ describe('runCaptureValidation', () => {
     expect(result.skipped).toBe(1);
     expect(kill).toHaveBeenCalledOnce();
   });
+
+  it('classifies a failed regression with a changed scene source as recapture debt', async () => {
+    const fixture = createRegressionFixture('changed scene', sha256('captured scene'));
+    try {
+      const result = await validateRegressionFixture(fixture.root, fixture.kill);
+      const failure = result.checks.find((check) => check.kind === 'regression');
+
+      expect(result.shouldFail).toBe(true);
+      expect(failure).toMatchObject({
+        currentSourceHash: sha256('changed scene'),
+        recordedSourceHash: sha256('captured scene'),
+        sourceHashStatus: 'changed',
+      });
+      expect(failure?.message).toContain('recapture owed by the scene owner');
+    } finally {
+      rmSync(fixture.root, { recursive: true, force: true });
+    }
+  });
+
+  it('classifies a failed regression with unchanged scene source as environment drift', async () => {
+    const source = 'unchanged scene';
+    const fixture = createRegressionFixture(source, sha256(source));
+    try {
+      const result = await validateRegressionFixture(fixture.root, fixture.kill);
+      const failure = result.checks.find((check) => check.kind === 'regression');
+
+      expect(result.shouldFail).toBe(true);
+      expect(failure).toMatchObject({
+        currentSourceHash: sha256(source),
+        recordedSourceHash: sha256(source),
+        sourceHashStatus: 'unchanged',
+      });
+      expect(failure?.message).toContain('environment drift; never rebaseline');
+    } finally {
+      rmSync(fixture.root, { recursive: true, force: true });
+    }
+  });
 });
+
+function createRegressionFixture(
+  currentSource: string,
+  recordedSourceHash: string,
+): { root: string; kill: () => void } {
+  const root = mkdtempSync(join(tmpdir(), 'capture-regression-freshness-'));
+  const scenes = join(root, 'functional', 'scenes');
+  mkdirSync(scenes, { recursive: true });
+  writeFileSync(join(scenes, 'sample.canvas.ts'), currentSource);
+  setBaselineField(root, 'functional', 'sample', 'canvas', 'fingerprint', '1:000000');
+  setBaselineField(root, 'functional', 'sample', 'canvas', 'sourceHash', recordedSourceHash);
+  return { root, kill: vi.fn() };
+}
+
+function sha256(source: string): string {
+  return createHash('sha256').update(source).digest('hex');
+}
+
+async function validateRegressionFixture(root: string, kill: () => void) {
+  return runCaptureValidation({
+    subject: 'functional',
+    entries: [{ name: 'sample', renderers: ['canvas'] }],
+    server: { url: 'http://unused.invalid', kill },
+    root,
+    gateParity: false,
+    // These assert how a FAILURE is classified, so the run legitimately produces a failing summary.
+    // Printing it would put a real-looking "✗ FAILED" block in the CI log of a file whose tests all
+    // pass, which is the kind of noise that teaches a reader to scroll past a genuine one.
+    quiet: true,
+    fingerprints: { sample: { canvas: '1:ffffff' } },
+    browserSession: {
+      browser: { close: vi.fn() } as never,
+      context: { newPage: vi.fn() } as never,
+    },
+  });
+}

package/src/captureWorkerCount.test.ts

@@ -0,0 +1,28 @@
+import { resolveCaptureWorkerCount } from './captureWorkerCount';
+
+describe('resolveCaptureWorkerCount', () => {
+  it('derives the default from available parallelism with host headroom', () => {
+    expect(resolveCaptureWorkerCount(undefined, undefined, 4)).toBe(3);
+    expect(resolveCaptureWorkerCount(undefined, undefined, 1)).toBe(1);
+  });
+
+  it('keeps the automatic default at the four-worker expedient ceiling', () => {
+    expect(resolveCaptureWorkerCount(undefined, undefined, 16)).toBe(4);
+  });
+
+  it('lets the environment pin a worker count above the automatic ceiling', () => {
+    expect(resolveCaptureWorkerCount(undefined, '8', 16)).toBe(8);
+  });
+
+  it('lets the command-line flag override the environment', () => {
+    expect(resolveCaptureWorkerCount('2', '8', 16)).toBe(2);
+  });
+
+  it('falls back to adaptive sizing for an invalid environment override', () => {
+    expect(resolveCaptureWorkerCount(undefined, 'invalid', 8)).toBe(4);
+  });
+
+  it('preserves the existing minimum of one for an explicit zero', () => {
+    expect(resolveCaptureWorkerCount('0', undefined, 8)).toBe(1);
+  });
+});

package/src/captureWorkflow.test.ts

@@ -8,11 +8,17 @@ describe('runCaptureBatch', () => {
   });
 
   it('rejects an empty batch before launching browser work', async () => {
-    await expect(runCaptureBatch({ subjects: [] })).rejects.toThrow(/No capture batch subjects/);
+    await expect(
+      runCaptureBatch({
+        quiet: true,
+        subjects: [],
+      }),
+    ).rejects.toThrow(/No capture batch subjects/);
   });
 
   it('continues after a subject setup error and aggregates the verdict', async () => {
     const result = await runCaptureBatch({
+      quiet: true,
       subjectWorkerCount: 2,
       subjects: [
         {
@@ -46,6 +52,7 @@ describe('runCaptureWorkflow', () => {
   it('owns the subject server once across the workflow', async () => {
     let kills = 0;
     const result = await runCaptureWorkflow({
+      quiet: true,
       subject: 'fixture',
       entries: [],
       server: { url: 'http://localhost:1', kill: () => kills++ },

package/src/functionalVerify.test.ts

@@ -1,7 +1,7 @@
 // @vitest-environment jsdom
 
 import type { DomRenderState } from '@flighthq/types/contract';
-import { afterEach, describe, expect, it } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
 import type { FunctionalTarget } from './functionalVerify';
 import {
@@ -18,19 +18,52 @@ interface VerificationWindowLike {
   __ftVerification?: unknown;
   __ftRenderImage?: unknown;
   __ftBenchmarkTarget?: { run(): void | Promise<void> };
+  __ftCaptureTimeoutMs?: number;
+  __ftRealRequestAnimationFrame?: (callback: FrameRequestCallback) => number;
+}
+
+// Stands in for the browser having stopped delivering frames: callbacks are accepted and never invoked.
+// The verifier reads the harness-stashed rAF, so replacing that one covers the path it actually takes.
+function stubNeverFiringAnimationFrames(): void {
+  (window as unknown as VerificationWindowLike).__ftRealRequestAnimationFrame = () => 0;
+}
+
+// Drives the frame wait to its deadline under fake timers and asserts the bound it actually used. The
+// elapsed time is the assertion: advancing just short of the expected deadline must not settle it.
+async function expectFrameWaitTimeout(expectedMs: number): Promise<void> {
+  vi.useFakeTimers();
+  try {
+    const run = runRenderVerification({}, 'webgl');
+    const rejected = expect(run).rejects.toThrow(
+      new RegExp(`stalled in stage awaitingFrame: no animation frame within ${expectedMs}ms`),
+    );
+    let settled = false;
+    void run.catch(() => (settled = true));
+    await vi.advanceTimersByTimeAsync(expectedMs - 1);
+    expect(settled).toBe(false);
+    await vi.advanceTimersByTimeAsync(1);
+    await rejected;
+  } finally {
+    vi.useRealTimers();
+  }
 }
 
 function verification(): Record<string, unknown> {
   return (window as unknown as VerificationWindowLike).__ftVerification as Record<string, unknown>;
 }
 
-afterEach(() => {
+function resetVerificationWindow(): void {
   const w = window as unknown as VerificationWindowLike;
   w.__ftTarget = undefined;
   w.__ftVerification = undefined;
   w.__ftRenderImage = undefined;
   w.__ftBenchmarkTarget = undefined;
-});
+  w.__ftRealRequestAnimationFrame = undefined;
+  w.__ftCaptureTimeoutMs = undefined;
+}
+
+beforeEach(resetVerificationWindow);
+afterEach(resetVerificationWindow);
 
 // A minimal DOM target — the only backend snapshot/verification handles without a real GPU context.
 function domTarget(element: HTMLElement): FunctionalTarget {
@@ -102,6 +135,43 @@ describe('runRenderVerification', () => {
     await runRenderVerification({}, 'dom');
     expect(verification()).toMatchObject({ render: 'dom', state: 'passed', error: null });
   });
+
+  it('does not wait for an animation frame on webgpu', async () => {
+    // webgpu pixels are already in the retained capture buffer when the verifier runs, so it must not
+    // depend on the browser scheduling a frame for a canvas it never presents. With rAF stubbed out
+    // entirely, this reaches the readback and fails there — if it still waited, it would never return.
+    stubNeverFiringAnimationFrames();
+
+    await expect(runRenderVerification({}, 'webgpu')).rejects.toThrow(/no readable render bitmap/);
+    expect(verification()).toMatchObject({ state: 'failed', stage: 'readingBack' });
+  });
+
+  it('fails by name when the animation frame never arrives, naming the stage it sat in', async () => {
+    stubNeverFiringAnimationFrames();
+
+    // No injected budget: the wait falls back to the runner's default ceiling and takes its share of it.
+    await expectFrameWaitTimeout(4_000);
+
+    // The failure is terminal and carries its location — the runner's outer wait would otherwise have
+    // reported the bare `state: pending` this replaces.
+    expect(verification()).toMatchObject({ state: 'failed', stage: 'awaitingFrame' });
+  });
+
+  it('scales its wait with the budget the runner injected', async () => {
+    // The regression this prevents: the runner raised its budget for a contended machine and the page's
+    // own bounds stayed pinned to the old ceiling, turning slow-but-working captures into hard failures.
+    stubNeverFiringAnimationFrames();
+    (window as unknown as VerificationWindowLike).__ftCaptureTimeoutMs = 45_000;
+
+    await expectFrameWaitTimeout(12_000);
+  });
+
+  it('ignores an injected budget that is not a usable number', async () => {
+    stubNeverFiringAnimationFrames();
+    (window as unknown as VerificationWindowLike).__ftCaptureTimeoutMs = 0;
+
+    await expectFrameWaitTimeout(4_000);
+  });
 });
 
 describe('snapshotFunctionalRender', () => {