@flighthq/tool-capture 0.3.0 → 0.3.1-next.1041.35b950c

package/src/captureServer.test.ts

@@ -5,13 +5,36 @@ vi.mock('node:child_process', async (importOriginal) => ({
   spawnSync: childProcessMocks.spawnSync,
 }));
 
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { join } from 'node:path';
+import { dirname, join } from 'node:path';
 
 import { describe, expect, it } from 'vitest';
 
-import { resolveCaptureDirectoryServer, resolveServer, resolveStaticServer } from './captureServer';
+import {
+  explainCaptureDistStaleness,
+  resolveCaptureDirectoryServer,
+  resolveServer,
+  resolveStaticServer,
+} from './captureServer';
+
+describe('explainCaptureDistStaleness', () => {
+  it('reports when the build predates the source it was built from', () => {
+    expect(explainCaptureDistStaleness(1_000, 2_000)).toContain('measures the PREVIOUS code');
+  });
+
+  it('stays silent when the build is at least as new as the source', () => {
+    expect(explainCaptureDistStaleness(2_000, 1_000)).toBeNull();
+    expect(explainCaptureDistStaleness(1_000, 1_000)).toBeNull();
+  });
+
+  it('stays silent when a timestamp could not be read', () => {
+    // A comparison that could not be made must not print a reassurance OR a warning: the caller
+    // learns nothing either way, and a warning nobody can act on trains people to ignore it.
+    expect(explainCaptureDistStaleness(null, 2_000)).toBeNull();
+    expect(explainCaptureDistStaleness(1_000, null)).toBeNull();
+  });
+});
 
 describe('resolveCaptureDirectoryServer', () => {
   it('serves an already-built directory and shuts down', async () => {
@@ -36,6 +59,83 @@ describe('resolveServer', () => {
 });
 
 describe('resolveStaticServer', () => {
+  it('refuses a stale functional dist that omits currently discovered routes', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'capture-static-stale-'));
+    const dist = join(root, 'tools', 'functional', 'dist');
+    const scenes = join(root, 'functional', 'scenes');
+    mkdirSync(dist, { recursive: true });
+    mkdirSync(scenes, { recursive: true });
+    writeFileSync(join(dist, 'index.html'), '<h1>old build</h1>');
+    writeFileSync(join(scenes, 'new-scene.webgl.ts'), 'export {};');
+    writeFileSync(join(scenes, 'new-scene.webgpu.ts'), 'export {};');
+
+    try {
+      await expect(resolveStaticServer({ tool: 'functional', root })).rejects.toThrow(
+        /Functional dist is missing 2 of 2 discovered routes.*npm run build:functional.*tests\/new-scene\/webgl\/, tests\/new-scene\/webgpu\//,
+      );
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  it.each([
+    ['an existing scene', 'functional/scenes/existing-scene.webgl.ts'],
+    ['its Vite config', 'tools/functional/vite.config.ts'],
+    ['the shared harness', 'tools/harness/render.ts'],
+  ])('refuses a functional dist older than %s', async (_description, newerSource) => {
+    const root = mkdtempSync(join(tmpdir(), 'capture-static-old-'));
+    const dist = join(root, 'tools', 'functional', 'dist');
+    const route = join(dist, 'tests', 'existing-scene', 'webgl');
+    const scene = join(root, 'functional', 'scenes', 'existing-scene.webgl.ts');
+    mkdirSync(route, { recursive: true });
+    mkdirSync(join(root, 'functional', 'scenes'), { recursive: true });
+    writeFileSync(join(dist, 'index.html'), '<h1>old build</h1>');
+    writeFileSync(join(route, 'index.html'), '<h1>old route</h1>');
+    writeFileSync(scene, 'export {};');
+    mkdirSync(dirname(join(root, newerSource)), { recursive: true });
+    if (join(root, newerSource) !== scene) writeFileSync(join(root, newerSource), 'export {};');
+    utimesSync(scene, 1, 1);
+    utimesSync(join(dist, 'index.html'), 2, 2);
+    utimesSync(join(route, 'index.html'), 2, 2);
+    utimesSync(join(root, newerSource), 3, 3);
+
+    try {
+      await expect(resolveStaticServer({ tool: 'functional', root })).rejects.toThrow(
+        /static build is older.*npm run build:functional/,
+      );
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  it('serves a complete functional dist newer than its build inputs', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'capture-static-fresh-'));
+    const dist = join(root, 'tools', 'functional', 'dist');
+    const route = join(dist, 'tests', 'existing-scene', 'webgl');
+    const scene = join(root, 'functional', 'scenes', 'existing-scene.webgl.ts');
+    const config = join(root, 'tools', 'functional', 'vite.config.ts');
+    const harness = join(root, 'tools', 'harness', 'render.ts');
+    mkdirSync(route, { recursive: true });
+    mkdirSync(join(root, 'functional', 'scenes'), { recursive: true });
+    mkdirSync(join(root, 'tools', 'harness'), { recursive: true });
+    writeFileSync(join(dist, 'index.html'), '<h1>fresh build</h1>');
+    writeFileSync(join(route, 'index.html'), '<h1>fresh route</h1>');
+    writeFileSync(scene, 'export {};');
+    writeFileSync(config, 'export default {};');
+    writeFileSync(harness, 'export {};');
+    for (const source of [scene, config, harness]) utimesSync(source, 1, 1);
+    for (const output of [join(dist, 'index.html'), join(route, 'index.html')]) utimesSync(output, 2, 2);
+
+    let server;
+    try {
+      server = await resolveStaticServer({ tool: 'functional', root });
+      expect(await (await fetch(`${server.url}/tests/existing-scene/webgl/`)).text()).toContain('fresh route');
+    } finally {
+      server?.kill();
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
   it('uses npm rather than treating an inherited npx CLI as the npm CLI', async () => {
     const root = mkdtempSync(join(tmpdir(), 'capture-static-'));
     const dist = join(root, 'examples', 'runners', 'web', 'dist');

package/src/captureValidation.test.ts

@@ -6,9 +6,14 @@ import { join } from 'node:path';
 import { describe, expect, it, vi } from 'vitest';
 
 import { setBaselineField } from './baselineStore';
+import {
+  readCaptureBaselineCoverageManifest,
+  writeCaptureBaselineCoverageManifest,
+} from './captureBaselineCoverageManifest';
 import { setCaptureTimeoutMs } from './captureTimeout';
 import {
   explainCaptureParityUncovered,
+  formatCaptureParityRanking,
   isCaptureRegressionCoverageFailure,
   isUniformCaptureFingerprint,
   explainCaptureVerificationStall,
@@ -40,6 +45,23 @@ describe('explainCaptureParityUncovered', () => {
   it('blames the skip list when several backends were eligible and still produced no pair', () => {
     expect(explainCaptureParityUncovered(3, false)).toContain('parity skip');
   });
+
+  it('names the missing reference instead of blaming the skip list, which is a different remedy', () => {
+    // Same eligible count as the skip case above, opposite fix: shortening the skip list does nothing
+    // while the reference itself is absent, and the reference can be absent without any skip involved.
+    const reason = explainCaptureParityUncovered(3, true, ['webgl-vs-canvas → canvas']);
+    expect(reason).toContain('webgl-vs-canvas → canvas');
+    expect(reason).toContain('NO pairs');
+    expect(reason).not.toContain('parity skip');
+  });
+
+  it('still blames ineligibility when NOTHING is eligible, since that is the root cause', () => {
+    // The reference is trivially absent when no renderer is eligible at all — reporting the reference
+    // there would name a symptom and hide the cause.
+    expect(explainCaptureParityUncovered(0, true, ['webgl-vs-canvas → canvas'])).toContain(
+      'no renderer in any parity group is eligible',
+    );
+  });
 });
 
 describe('explainCaptureVerificationStall', () => {
@@ -78,6 +100,38 @@ describe('explainCaptureVerificationStall', () => {
   });
 });
 
+describe('formatCaptureParityRanking', () => {
+  it('ranks measured distances widest first and states the median beside them', () => {
+    const text = formatCaptureParityRanking([
+      { distance: 0.02, entry: 'shape-fill', kind: 'parity', renderers: ['webgl', 'webgpu'] },
+      { distance: 11.11, entry: 'effect-glitch', kind: 'parity', renderers: ['webgl', 'webgpu'] },
+      { distance: 0.5, entry: 'effect-dither', kind: 'parity', renderers: ['webgl', 'webgpu'] },
+    ]);
+
+    // The verdict for all three is "pass"; only the ranking distinguishes them.
+    expect(text).toContain('3 compared');
+    expect(text!.indexOf('effect-glitch')).toBeLessThan(text!.indexOf('effect-dither'));
+    expect(text!.indexOf('effect-dither')).toBeLessThan(text!.indexOf('shape-fill'));
+  });
+
+  it('returns null when nothing was compared', () => {
+    // An empty ranking would read as agreement; no ranking says the comparison did not happen.
+    expect(formatCaptureParityRanking([])).toBeNull();
+    expect(formatCaptureParityRanking([{ entry: 'shape-fill', kind: 'baseline' }])).toBeNull();
+  });
+
+  it('names how many rows it withheld', () => {
+    const many = Array.from({ length: 14 }, (_, index) => ({
+      distance: index,
+      entry: `scene-${index}`,
+      kind: 'parity',
+      renderers: ['webgl', 'webgpu'],
+    }));
+
+    expect(formatCaptureParityRanking(many)).toContain('4 more not shown');
+  });
+});
+
 describe('isCaptureParityCoverageFailure', () => {
   it('FAILS a gated run that compared nothing while entries wanted a comparison', () => {
     // The defect this gate exists for: 107 entries skipped, leg green.
@@ -151,7 +205,170 @@ describe('isUniformCaptureFingerprint', () => {
   });
 });
 
+function createRegressionFixture(
+  currentSource: string,
+  recordedSourceHash: string,
+): { root: string; kill: () => void } {
+  const root = mkdtempSync(join(tmpdir(), 'capture-regression-freshness-'));
+  const scenes = join(root, 'functional', 'scenes');
+  mkdirSync(scenes, { recursive: true });
+  writeFileSync(join(scenes, 'sample.canvas.ts'), currentSource);
+  setBaselineField(root, 'functional', 'sample', 'canvas', 'fingerprint', '2:000000000000000000ffffff');
+  setBaselineField(root, 'functional', 'sample', 'canvas', 'sourceHash', recordedSourceHash);
+  return { root, kill: vi.fn() };
+}
+
+function sha256(source: string): string {
+  return createHash('sha256').update(source).digest('hex');
+}
+
+async function validateRegressionFixture(root: string, kill: () => void) {
+  return runCaptureValidation({
+    subject: 'functional',
+    entries: [{ name: 'sample', renderers: ['canvas'] }],
+    server: { url: 'http://unused.invalid', kill },
+    root,
+    gateParity: false,
+    // These assert how a FAILURE is classified, so the run legitimately produces a failing summary.
+    // Printing it would put a real-looking "✗ FAILED" block in the CI log of a file whose tests all
+    // pass, which is the kind of noise that teaches a reader to scroll past a genuine one.
+    quiet: true,
+    fingerprints: { sample: { canvas: '2:ffffffffffffffffff000000' } },
+    browserSession: {
+      browser: { close: vi.fn() } as never,
+      context: { newPage: vi.fn() } as never,
+    },
+  });
+}
+
+async function validateParityFixture(input: Readonly<Record<string, unknown>>) {
+  return runCaptureValidation({
+    subject: 'functional',
+    entries: [{ name: 'sample', renderers: ['canvas', 'webgl'] }],
+    server: { url: 'http://unused.invalid', kill: vi.fn() },
+    root: join(tmpdir(), 'tool-capture-parity-reference-fixture'),
+    gateParity: true,
+    quiet: true,
+    // Supplied so neither renderer reloads a page; both are parity-eligible and IDENTICAL, so any pair
+    // that gets built passes. A failing pair would confound "no comparison" with "a failed comparison".
+    fingerprints: { sample: { canvas: '1:000000', webgl: '1:000000' } },
+    browserSession: {
+      browser: { close: vi.fn() } as never,
+      context: { newPage: vi.fn() } as never,
+    },
+    ...input,
+  } as never);
+}
+
 describe('runCaptureValidation', () => {
+  // The acceptance path must not become the hole. Writing the manifest from an early return reported
+  // exit 0 over a leg with real regression failures — the same "reports green" defect the manifest
+  // exists to close, one level up. Coverage is accepted; the run's own verdict still stands.
+  it('accepts new coverage WITHOUT masking the regression failures of the same run', async () => {
+    const { root, kill } = createRegressionFixture('export const scene = 1;\n', sha256('export const scene = 1;\n'));
+    mkdirSync(join(root, 'scripts'), { recursive: true });
+    const result = await runCaptureValidation({
+      subject: 'functional',
+      entries: [{ name: 'sample', renderers: ['canvas'] }],
+      server: { url: 'http://unused.invalid', kill },
+      root,
+      gateParity: false,
+      quiet: true,
+      updateCoverage: true,
+      fingerprints: { sample: { canvas: '2:ffffffffffffffffff000000' } },
+      browserSession: {
+        browser: { close: vi.fn() } as never,
+        context: { newPage: vi.fn() } as never,
+      },
+    });
+    expect(result.regressionFailures).toBe(1);
+    expect(result.shouldFail).toBe(true);
+    // A target with a FAILING comparison still has baseline evidence, so it is still covered.
+    expect(readCaptureBaselineCoverageManifest(root).subjects.functional).toEqual({
+      'sample/canvas': ['fingerprint'],
+    });
+  });
+
+  // The blanket refusal this replaces was too coarse: one flaky target blocked every acceptance. The
+  // precise rule is that a run retires only what it POSITIVELY DETERMINED to be uncovered, so a target
+  // that never loaded keeps its pin — it cannot be retired by its own flakiness, and it blocks nothing.
+  it('keeps the pin of a target it could not load, instead of retiring it', async () => {
+    const { root, kill } = createRegressionFixture('export const scene = 1;\n', sha256('export const scene = 1;\n'));
+    mkdirSync(join(root, 'scripts'), { recursive: true });
+    writeCaptureBaselineCoverageManifest(root, 'functional', { 'sample/canvas': ['fingerprint'] });
+    const result = await runCaptureValidation({
+      subject: 'functional',
+      entries: [{ name: 'sample', renderers: ['canvas'] }],
+      server: { url: 'http://unused.invalid', kill },
+      root,
+      gateParity: false,
+      quiet: true,
+      updateCoverage: true,
+      fingerprints: {},
+      browserSession: {
+        browser: { close: vi.fn() } as never,
+        context: { newPage: vi.fn().mockRejectedValue(new Error('page unavailable')) } as never,
+      },
+    });
+    expect(result.loadFailures).toBeGreaterThan(0);
+    expect(result.shouldFail).toBe(true);
+    expect(readCaptureBaselineCoverageManifest(root).subjects.functional).toEqual({
+      'sample/canvas': ['fingerprint'],
+    });
+  });
+
+  it('yields NO PAIRS when a declared parity reference is skipped, instead of falling back to all-pairs', async () => {
+    // The defect: `present.includes(reference)` is false both when a group declares no reference and
+    // when a skip removed the one it declared, so the second case inherited the first case's all-pairs
+    // fallback. The run then compared a DIFFERENT set of backends under the group's name and reported
+    // success. Two eligible renderers here, so a fallback would produce exactly one pair — the
+    // assertion is that it produces none.
+    const result = await validateParityFixture({
+      parityGroups: { 'ref-group': { targets: ['canvas', 'webgl'], reference: 'canvas' } },
+      paritySkip: { sample: ['canvas'] },
+    });
+
+    expect(result.parityPasses + result.parityFailures).toBe(0);
+    expect(result.parityUncovered).toBe(1);
+    const uncovered = result.checks.find((check) => check.kind === 'parity');
+    expect(uncovered?.status).toBe('skipped');
+    expect(uncovered?.message).toContain('ref-group → canvas');
+  });
+
+  it("still compares all pairs for a group that declares NO reference, which is that branch's real job", async () => {
+    // The negative control for the fix: the all-pairs branch must survive for the case it was written
+    // for. Same two renderers, same skip absent — one pair, and the scene is covered.
+    const result = await validateParityFixture({
+      parityGroups: { 'open-group': { targets: ['canvas', 'webgl'] } },
+      paritySkip: {},
+    });
+
+    expect(result.parityPasses + result.parityFailures).toBe(1);
+    expect(result.parityUncovered).toBe(0);
+  });
+
+  it('KEEPS all-pairs when the declared reference is simply not a column in the scene', async () => {
+    // NOT the same as a skip, and the difference is 85 real comparisons. The built-in group declares
+    // reference 'canvas' once for EVERY scene, and 83 functional scenes have no canvas column at all —
+    // 3D material, mesh, light and shadow scenes are webgl/webgpu only. Treating "never there" as
+    // "removed" drops the entire 3D suite's cross-backend coverage to a WARN: measured 253 → 168
+    // comparisons on the functional suite before this case was split back out.
+    const result = await validateParityFixture({
+      parityGroups: { visual: { targets: ['dom', 'canvas', 'webgl', 'webgpu'], reference: 'canvas' } },
+      paritySkip: {},
+      entries: [{ name: 'sample', renderers: ['webgl', 'webgpu'] }],
+      fingerprints: { sample: { webgl: '1:000000', webgpu: '1:000000' } },
+    });
+
+    expect(result.parityPasses + result.parityFailures).toBe(1);
+    expect(result.parityUncovered).toBe(0);
+    // The coverage is KEPT and the claim is made TRUE. Keeping the pairs while still labelling them
+    // 'visual:webgl·webgpu' is the false-claim state: the group asserts a canvas reference it never
+    // used. Asserting the label is what stops coverage and honesty from drifting apart.
+    const passed = result.checks.find((check) => check.kind === 'parity' && check.status === 'passed');
+    expect(passed?.message).toContain('all-pairs, no canvas column');
+  });
+
   it('is a callable fingerprint-validation orchestrator', () => {
     expect(typeof runCaptureValidation).toBe('function');
   });
@@ -216,39 +433,3 @@ describe('runCaptureValidation', () => {
     }
   });
 });
-
-function createRegressionFixture(
-  currentSource: string,
-  recordedSourceHash: string,
-): { root: string; kill: () => void } {
-  const root = mkdtempSync(join(tmpdir(), 'capture-regression-freshness-'));
-  const scenes = join(root, 'functional', 'scenes');
-  mkdirSync(scenes, { recursive: true });
-  writeFileSync(join(scenes, 'sample.canvas.ts'), currentSource);
-  setBaselineField(root, 'functional', 'sample', 'canvas', 'fingerprint', '1:000000');
-  setBaselineField(root, 'functional', 'sample', 'canvas', 'sourceHash', recordedSourceHash);
-  return { root, kill: vi.fn() };
-}
-
-function sha256(source: string): string {
-  return createHash('sha256').update(source).digest('hex');
-}
-
-async function validateRegressionFixture(root: string, kill: () => void) {
-  return runCaptureValidation({
-    subject: 'functional',
-    entries: [{ name: 'sample', renderers: ['canvas'] }],
-    server: { url: 'http://unused.invalid', kill },
-    root,
-    gateParity: false,
-    // These assert how a FAILURE is classified, so the run legitimately produces a failing summary.
-    // Printing it would put a real-looking "✗ FAILED" block in the CI log of a file whose tests all
-    // pass, which is the kind of noise that teaches a reader to scroll past a genuine one.
-    quiet: true,
-    fingerprints: { sample: { canvas: '1:ffffff' } },
-    browserSession: {
-      browser: { close: vi.fn() } as never,
-      context: { newPage: vi.fn() } as never,
-    },
-  });
-}

package/src/functionalScene3Ds.test.ts

@@ -28,6 +28,12 @@ describe('discoverFunctionalScene3Ds', () => {
     expect(discoverFunctionalScene3Ds(dir)).toEqual([{ name: 'foo', renderers: ['dom', 'canvas', 'webgl', 'webgpu'] }]);
   });
 
+  it('keeps every default backend when one has a specific-file override', () => {
+    touch('foo.ts');
+    touch('foo.dom.ts');
+    expect(discoverFunctionalScene3Ds(dir)).toEqual([{ name: 'foo', renderers: ['dom', 'canvas', 'webgl', 'webgpu'] }]);
+  });
+
   it('collects backend-specific files into one entry in default-backend order', () => {
     touch('bar.webgl.ts');
     touch('bar.canvas.ts');
@@ -44,6 +50,7 @@ describe('discoverFunctionalScene3Ds', () => {
 
 describe('functionalScene3DFile', () => {
   it('prefers the backend-specific file when it exists', () => {
+    touch('bar.ts');
     touch('bar.webgl.ts');
     expect(functionalScene3DFile(dir, 'bar', 'webgl')).toBe(join(dir, 'bar.webgl.ts'));
   });

package/src/functionalVerify.test.ts

@@ -17,6 +17,7 @@ interface VerificationWindowLike {
   __ftTarget?: unknown;
   __ftVerification?: unknown;
   __ftRenderImage?: unknown;
+  __ftProvideDomRenderPixels?: (readback: { data: Uint8ClampedArray; height: number; width: number } | null) => void;
   __ftBenchmarkTarget?: { run(): void | Promise<void> };
   __ftCaptureTimeoutMs?: number;
   __ftRealRequestAnimationFrame?: (callback: FrameRequestCallback) => number;
@@ -57,6 +58,7 @@ function resetVerificationWindow(): void {
   w.__ftTarget = undefined;
   w.__ftVerification = undefined;
   w.__ftRenderImage = undefined;
+  w.__ftProvideDomRenderPixels = undefined;
   w.__ftBenchmarkTarget = undefined;
   w.__ftRealRequestAnimationFrame = undefined;
   w.__ftCaptureTimeoutMs = undefined;
@@ -77,6 +79,12 @@ function domTarget(element: HTMLElement): FunctionalTarget {
   };
 }
 
+function provideDomRenderPixels(data = new Uint8ClampedArray([0, 0, 0, 255, 255, 255, 255, 255])): void {
+  const provide = (window as unknown as VerificationWindowLike).__ftProvideDomRenderPixels;
+  expect(provide).toBeTypeOf('function');
+  provide?.({ data, height: 1, width: 2 });
+}
+
 describe('publishFunctionalRenderSync', () => {
   it('returns false when no target is registered', () => {
     expect(publishFunctionalRenderSync('webgl')).toBe(false);
@@ -128,12 +136,34 @@ describe('runRenderVerification', () => {
     expect(verification()).toMatchObject({ state: 'failed' });
   });
 
-  it('passes a DOM render that emitted content and records the verification', async () => {
+  it('runs DOM pixels through coverage, the scene oracle, and fingerprinting', async () => {
+    const host = document.createElement('div');
+    host.appendChild(document.createElement('span'));
+    registerFunctionalTarget(domTarget(host));
+    const assertRender = vi.fn();
+    const run = runRenderVerification({ assertRender }, 'dom');
+    provideDomRenderPixels();
+    await run;
+
+    expect(assertRender).toHaveBeenCalledOnce();
+    expect(verification()).toMatchObject({
+      render: 'dom',
+      state: 'passed',
+      coverage: 0.5,
+      fingerprint: expect.stringMatching(/^16:/),
+      error: null,
+    });
+  });
+
+  it('rejects nonempty DOM structure whose supplied pixels are blank', async () => {
     const host = document.createElement('div');
     host.appendChild(document.createElement('span'));
     registerFunctionalTarget(domTarget(host));
-    await runRenderVerification({}, 'dom');
-    expect(verification()).toMatchObject({ render: 'dom', state: 'passed', error: null });
+    const run = runRenderVerification({}, 'dom');
+    provideDomRenderPixels(new Uint8ClampedArray([0, 0, 0, 255, 0, 0, 0, 255]));
+
+    await expect(run).rejects.toThrow(/coverage/);
+    expect(verification()).toMatchObject({ state: 'failed', stage: 'measuring' });
   });
 
   it('does not wait for an animation frame on webgpu', async () => {