@monoes/monobrowse 1.0.12 → 1.0.14

package/src/__tests__/report-evidence.test.ts

@@ -0,0 +1,276 @@
+/**
+ * RIG-11 — recorded evidence around a failure.
+ *
+ * Only the pure half is tested here (selection and timeline assembly); the
+ * recorder itself is exercised with a stub CDP client, so nothing launches
+ * Chrome. The bounds are the point: a report must never grow without limit,
+ * however long the run or however noisy the page.
+ */
+import { describe, expect, it, vi } from 'vitest';
+import type { CdpClient } from '../browser/cdp.js';
+import {
+  type BuildEvidenceInput,
+  buildEvidence,
+  findFocusOffset,
+  type RawFrame,
+  selectFrames,
+  startEvidenceRecorder,
+} from '../report/evidence.js';
+
+const START = 1_000_000;
+
+function frames(count: number, spacingMs = 100, bytes = 1000): RawFrame[] {
+  return Array.from({ length: count }, (_, i) => ({
+    offsetMs: i * spacingMs,
+    data: 'x'.repeat(bytes),
+    bytes,
+  }));
+}
+
+function input(partial: Partial<BuildEvidenceInput> = {}): BuildEvidenceInput {
+  return {
+    reason: 'budget-failure',
+    startedAtMs: START,
+    frames: frames(20),
+    console: [],
+    pageErrors: [],
+    requests: [],
+    failures: [],
+    ...partial,
+  };
+}
+
+describe('findFocusOffset', () => {
+  it('picks the earliest hard error as the moment to explain', () => {
+    const offset = findFocusOffset(
+      input({
+        console: [
+          { type: 'error', text: 'late', timestamp: START + 1800 },
+          { type: 'error', text: 'early', timestamp: START + 600 },
+        ],
+        pageErrors: [{ text: 'uncaught', timestamp: START + 900 }],
+      }),
+    );
+    expect(offset).toBe(600);
+  });
+
+  it('ignores console warnings — a warning is not the failure', () => {
+    expect(
+      findFocusOffset(input({ console: [{ type: 'warn', text: 'meh', timestamp: START + 100 }] })),
+    ).toBeNull();
+  });
+
+  it('uses a failed request when it carries a clock reading', () => {
+    expect(
+      findFocusOffset(
+        input({
+          requests: [
+            { url: 'https://a.test/x', method: 'GET', failed: true, startedAtMs: START + 450 },
+          ],
+        }),
+      ),
+    ).toBe(450);
+  });
+
+  it('skips a failed request with no clock reading rather than guessing', () => {
+    expect(
+      findFocusOffset(
+        input({ requests: [{ url: 'https://a.test/x', method: 'GET', failed: true }] }),
+      ),
+    ).toBeNull();
+  });
+
+  it('has no focus when the breach is a load-wide metric', () => {
+    expect(
+      findFocusOffset(
+        input({ failures: [{ budget: 'lcpMs', expected: '<= 2500ms', actual: '4100ms' }] }),
+      ),
+    ).toBeNull();
+  });
+});
+
+describe('selectFrames', () => {
+  it('keeps the frames nearest the failure', () => {
+    const picked = selectFrames(frames(40), 2000, { maxFrames: 5 });
+    expect(picked.frames.map((f) => f.offsetMs)).toEqual([1800, 1900, 2000, 2100, 2200]);
+    expect(picked.dropped).toBe(35);
+  });
+
+  it('samples the whole run when there is no single moment to centre on', () => {
+    const picked = selectFrames(frames(21), null, { maxFrames: 5 });
+    expect(picked.frames.map((f) => f.offsetMs)).toEqual([0, 500, 1000, 1500, 2000]);
+  });
+
+  it('returns the frames in time order even though it ranks them by distance', () => {
+    const picked = selectFrames(frames(40), 2000, { maxFrames: 7 });
+    const offsets = picked.frames.map((f) => f.offsetMs);
+    expect([...offsets].sort((a, b) => a - b)).toEqual(offsets);
+  });
+
+  it('falls back to the nearest frames when none fall inside the window', () => {
+    const picked = selectFrames(frames(10), 60_000, { maxFrames: 3, windowMs: 500 });
+    expect(picked.frames.map((f) => f.offsetMs)).toEqual([700, 800, 900]);
+  });
+
+  it('enforces the byte budget by evicting the least relevant frame first', () => {
+    const picked = selectFrames(frames(40, 100, 1000), 2000, { maxFrames: 10, maxBytes: 3000 });
+    expect(picked.frames.length).toBe(3);
+    expect(picked.frames.map((f) => f.offsetMs)).toEqual([1900, 2000, 2100]);
+  });
+
+  it('handles an empty recording without throwing', () => {
+    expect(selectFrames([], 100)).toEqual({ frames: [], dropped: 0 });
+  });
+});
+
+describe('buildEvidence', () => {
+  it('attaches frames around the failure and lines the events up against them', () => {
+    const evidence = buildEvidence(
+      input({
+        frames: frames(40),
+        console: [{ type: 'error', text: 'Cannot read x of undefined', timestamp: START + 2000 }],
+        requests: [
+          {
+            url: 'https://api.test/cart',
+            method: 'GET',
+            status: 503,
+            failed: true,
+            startedAtMs: START + 2100,
+          },
+        ],
+        failures: [{ budget: 'maxConsoleErrors', expected: '<= 0', actual: '1' }],
+        select: { maxFrames: 5 },
+      }),
+    );
+
+    expect(evidence.focusOffsetMs).toBe(2000);
+    expect(evidence.frames.length).toBe(5);
+    expect(evidence.frames[0].dataUrl.startsWith('data:image/jpeg;base64,')).toBe(true);
+
+    const kinds = evidence.timeline.map((e) => e.kind);
+    expect(kinds).toContain('console');
+    expect(kinds).toContain('request');
+    expect(kinds).toContain('marker');
+    // Everything is in time order, which is what makes it readable.
+    const offsets = evidence.timeline.map((e) => e.offsetMs);
+    expect([...offsets].sort((a, b) => a - b)).toEqual(offsets);
+  });
+
+  it('indexes each frame row back to the attached frame it shows', () => {
+    const evidence = buildEvidence(input({ frames: frames(6), select: { maxFrames: 3 } }));
+    const frameRows = evidence.timeline.filter((e) => e.kind === 'frame');
+    expect(frameRows.map((e) => e.frameIndex)).toEqual([0, 1, 2]);
+    for (const row of frameRows) {
+      expect(evidence.frames[row.frameIndex as number].offsetMs).toBe(row.offsetMs);
+    }
+  });
+
+  it('bounds the total attached bytes', () => {
+    const evidence = buildEvidence(
+      input({
+        frames: frames(200, 50, 100_000),
+        console: [{ type: 'error', text: 'boom', timestamp: START + 5000 }],
+        select: { maxFrames: 12, maxBytes: 500_000 },
+      }),
+    );
+    expect(evidence.totalBytes).toBeLessThanOrEqual(500_000);
+    expect(evidence.droppedFrames).toBeGreaterThan(0);
+    expect(evidence.notes.join(' ')).toMatch(/not attached/);
+  });
+
+  it('trims a noisy timeline worst-first so the real error survives', () => {
+    const warnings = Array.from({ length: 200 }, (_, i) => ({
+      type: 'warn',
+      text: `deprecation ${i}`,
+      timestamp: START + i,
+    }));
+    const evidence = buildEvidence(
+      input({
+        frames: frames(4),
+        console: [...warnings, { type: 'error', text: 'the real one', timestamp: START + 9999 }],
+        maxEvents: 10,
+        select: { maxFrames: 4 },
+      }),
+    );
+    const details = evidence.timeline.map((e) => e.detail ?? '');
+    expect(details).toContain('the real one');
+    expect(evidence.timeline.filter((e) => e.kind !== 'frame').length).toBe(10);
+    expect(evidence.notes.join(' ')).toMatch(/Timeline trimmed/);
+  });
+
+  it('explains itself when the breach has no single moment', () => {
+    const evidence = buildEvidence(
+      input({ failures: [{ budget: 'lcpMs', expected: '<= 2500ms', actual: '4100ms' }] }),
+    );
+    expect(evidence.focusOffsetMs).toBeNull();
+    expect(evidence.notes.join(' ')).toMatch(/load-wide metric/);
+  });
+
+  it('says so plainly when nothing was recorded', () => {
+    const evidence = buildEvidence(input({ frames: [] }));
+    expect(evidence.frames).toEqual([]);
+    expect(evidence.notes.join(' ')).toMatch(/No screencast frames/);
+  });
+
+  it('carries the dropped-frame count from the recorder into the total', () => {
+    const evidence = buildEvidence(
+      input({ frames: frames(4), bufferDropped: 30, select: { maxFrames: 4 } }),
+    );
+    expect(evidence.droppedFrames).toBe(30);
+  });
+});
+
+describe('startEvidenceRecorder', () => {
+  function stubClient() {
+    const handlers = new Map<string, (params: unknown, sid: string) => void>();
+    const send = vi.fn(async () => ({}));
+    const client = {
+      send,
+      on: (event: string, handler: (params: unknown, sid: string) => void) => {
+        handlers.set(event, handler);
+        return () => handlers.delete(event);
+      },
+    } as unknown as CdpClient;
+    return { client, handlers, send };
+  }
+
+  it('starts a screencast and collects timestamped frames', async () => {
+    const { client, handlers, send } = stubClient();
+    const recorder = await startEvidenceRecorder(client, 'S1', Date.now());
+    expect(send).toHaveBeenCalledWith('Page.startScreencast', expect.anything(), 'S1');
+
+    handlers.get('Page.screencastFrame')?.({ data: 'abc', sessionId: 1 }, 'S1');
+    expect(recorder.frames.length).toBe(1);
+    expect(recorder.frames[0].bytes).toBe(3);
+    // Failing to ack stalls the screencast, so it must happen every frame.
+    expect(send).toHaveBeenCalledWith('Page.screencastFrameAck', { sessionId: 1 }, 'S1');
+  });
+
+  it('ignores frames from another session', async () => {
+    const { client, handlers } = stubClient();
+    const recorder = await startEvidenceRecorder(client, 'S1', Date.now());
+    handlers.get('Page.screencastFrame')?.({ data: 'abc', sessionId: 1 }, 'S2');
+    expect(recorder.frames.length).toBe(0);
+  });
+
+  it('decimates rather than dropping the head, so early frames survive a long run', async () => {
+    const { client, handlers } = stubClient();
+    const recorder = await startEvidenceRecorder(client, 'S1', Date.now(), { maxFrames: 8 });
+    const frame = handlers.get('Page.screencastFrame');
+    for (let i = 0; i < 20; i++) frame?.({ data: `f${i}`, sessionId: i }, 'S1');
+
+    expect(recorder.frames.length).toBeLessThanOrEqual(8);
+    expect(recorder.droppedFrames).toBeGreaterThan(0);
+    // The very first frame is still there — a ring buffer would have lost it.
+    expect(recorder.frames[0].data).toBe('f0');
+  });
+
+  it('unsubscribes and stops the screencast on stop', async () => {
+    const { client, handlers, send } = stubClient();
+    const recorder = await startEvidenceRecorder(client, 'S1', Date.now());
+    await recorder.stop();
+    expect(send).toHaveBeenCalledWith('Page.stopScreencast', {}, 'S1');
+    handlers.get('Page.screencastFrame')?.({ data: 'abc', sessionId: 1 }, 'S1');
+    expect(recorder.frames.length).toBe(0);
+  });
+});

package/src/__tests__/report-flake.test.ts

@@ -0,0 +1,214 @@
+/**
+ * RIG-14 — flake detection across repeated runs.
+ *
+ * The load-bearing assertion in here is that a check which passed some runs
+ * and failed others never produces a `pass` verdict. That is the whole story:
+ * the failure mode is a red check being re-run until it is green.
+ */
+import { describe, expect, it } from 'vitest';
+import { buildReport } from '../report/analyze.js';
+import { DEFAULT_BUDGET, parseBudget } from '../report/budget.js';
+import {
+  analyzeFlake,
+  ruledOutFailureRate,
+  signalSignature,
+  summarizeFlake,
+} from '../report/flake.js';
+import type { Budget, CaptureData, Report } from '../report/types.js';
+
+function capture(partial: Partial<CaptureData> = {}): CaptureData {
+  return {
+    url: 'https://shop.test/checkout',
+    finalUrl: 'https://shop.test/checkout',
+    title: 'Checkout',
+    capturedAt: '2026-09-21T10:00:00.000Z',
+    durationMs: 1000,
+    console: [],
+    pageErrors: [],
+    requests: [],
+    vitals: { lcp: 2000, cls: 0.01, inp: 50 },
+    a11y: [],
+    screenshots: [],
+    notes: [],
+    ...partial,
+  };
+}
+
+const run = (partial: Partial<CaptureData> = {}, budget: Budget = DEFAULT_BUDGET): Report =>
+  buildReport(capture(partial), budget);
+
+const err = (text: string) => ({ type: 'error', text, timestamp: 0 });
+
+describe('analyzeFlake verdict', () => {
+  it('passes when every check held across every run', () => {
+    const flake = analyzeFlake([run(), run(), run(), run(), run()]);
+    expect(flake.verdict).toBe('pass');
+    expect(flake.runs).toBe(5);
+    expect(flake.checks.every((c) => c.stable)).toBe(true);
+    expect(flake.headline).toContain('PASS');
+  });
+
+  it('refuses to call a check that passed sometimes a pass', () => {
+    // Three clean runs, two with a console error — exactly the case that gets
+    // shipped as green after a re-run.
+    const runs = [
+      run(),
+      run({ console: [err('boom')] }),
+      run(),
+      run({ console: [err('boom')] }),
+      run(),
+    ];
+    const flake = analyzeFlake(runs);
+    expect(flake.verdict).toBe('flaky');
+    const check = flake.checks.find((c) => c.key === 'maxConsoleErrors');
+    expect(check).toMatchObject({ runs: 5, passed: 3, failed: 2, stable: false });
+    expect(check?.flakeRate).toBeCloseTo(0.4, 5);
+    expect(flake.headline).toContain('FLAKY');
+    expect(flake.headline).toContain('failed 2 of 5');
+  });
+
+  it('fails outright when a check failed in every run', () => {
+    const runs = [run({ console: [err('boom')] }), run({ console: [err('boom')] })];
+    const flake = analyzeFlake(runs);
+    expect(flake.verdict).toBe('fail');
+    expect(flake.headline).toBe('FAIL — 1 check(s) failed in all 2 runs.');
+  });
+
+  it('prefers flaky over fail when one check is unstable and another always fails', () => {
+    const budget = parseBudget({ lcp: 1000 });
+    const runs = [
+      run({ console: [err('boom')] }, budget),
+      run({}, budget), // lcp still breaches; the console error does not recur
+    ];
+    const flake = analyzeFlake(runs);
+    expect(flake.checks.find((c) => c.key === 'lcpMs')?.failed).toBe(2);
+    expect(flake.verdict).toBe('flaky');
+  });
+
+  it('records the measured value for every run, not just the failing ones', () => {
+    const runs = [run({ vitals: { lcp: 1900 } }), run({ vitals: { lcp: 3400 } })];
+    const lcp = analyzeFlake(runs).checks.find((c) => c.key === 'lcpMs');
+    expect(lcp?.actuals).toEqual(['1900ms', '3400ms']);
+  });
+
+  it('ignores budgets that are not enforced', () => {
+    const budget = parseBudget({ ttfb: null, fcp: null });
+    const keys = analyzeFlake([run({}, budget)]).checks.map((c) => c.key);
+    expect(keys).not.toContain('ttfbMs');
+    expect(keys).not.toContain('fcpMs');
+  });
+
+  it('throws rather than inventing a verdict from no runs', () => {
+    expect(() => analyzeFlake([])).toThrow(/at least one run/);
+  });
+});
+
+describe('analyzeFlake signals', () => {
+  it('reports how many runs an intermittent console error appeared in', () => {
+    const runs = [run(), run({ console: [err('Failed to fetch /cart')] }), run(), run(), run()];
+    const signal = analyzeFlake(runs).signals.find((s) => s.kind === 'console');
+    expect(signal).toMatchObject({ runs: 1, total: 5, signature: 'Failed to fetch /cart' });
+  });
+
+  it('does not call an error that happened every run a flake', () => {
+    const runs = [run({ console: [err('always')] }), run({ console: [err('always')] })];
+    expect(analyzeFlake(runs).signals).toEqual([]);
+  });
+
+  it('groups the same error carrying a different request id', () => {
+    const runs = [
+      run({ console: [err('Request 1043321 failed')] }),
+      run({ console: [err('Request 9910412 failed')] }),
+      run(),
+    ];
+    const signals = analyzeFlake(runs).signals;
+    expect(signals.length).toBe(1);
+    expect(signals[0]).toMatchObject({ runs: 2, total: 3 });
+  });
+
+  it('reports an intermittently failing request', () => {
+    const failing = {
+      requests: [
+        { url: 'https://api.test/cart', method: 'GET', status: 503, failed: true },
+        { url: 'https://api.test/ok', method: 'GET', status: 200, failed: false },
+      ],
+    };
+    const signals = analyzeFlake([run(), run(failing), run()]).signals;
+    expect(signals).toEqual([
+      { kind: 'request', signature: '503 https://api.test/cart', runs: 1, total: 3 },
+    ]);
+  });
+});
+
+describe('signalSignature', () => {
+  it('normalises ids, hex and long numbers so one error is one signal', () => {
+    expect(signalSignature('Request 1043321 failed')).toBe('Request <n> failed');
+    expect(signalSignature('trace 3f2a1b4c-0000-4aaa-8bbb-1234567890ab lost')).toBe(
+      'trace <uuid> lost',
+    );
+    expect(signalSignature('at 0xDEADBEEF')).toBe('at <hex>');
+  });
+
+  it('leaves short numbers alone — a 404 is part of the message', () => {
+    expect(signalSignature('GET returned 404')).toBe('GET returned 404');
+  });
+});
+
+describe('analyzeFlake metrics', () => {
+  it('reports the range a metric varied across, which is the readable form', () => {
+    const runs = [1900, 3400, 2100, 2000, 2050].map((lcp) => run({ vitals: { lcp } }));
+    const lcp = analyzeFlake(runs).metrics.find((m) => m.key === 'lcp');
+    expect(lcp).toMatchObject({ min: 1900, max: 3400, measured: 5, total: 5, unstable: true });
+    expect(lcp?.spread).toBe(1500);
+  });
+
+  it('does not flag a metric that barely moved', () => {
+    const runs = [2000, 2050, 2010].map((lcp) => run({ vitals: { lcp } }));
+    expect(analyzeFlake(runs).metrics.find((m) => m.key === 'lcp')?.unstable).toBe(false);
+  });
+
+  it('counts how many runs actually measured the metric', () => {
+    const runs = [
+      run({ vitals: { lcp: 2000 } }),
+      run({ vitals: {} }),
+      run({ vitals: { lcp: 2100 } }),
+    ];
+    expect(analyzeFlake(runs).metrics.find((m) => m.key === 'lcp')?.measured).toBe(2);
+  });
+});
+
+describe('confidence', () => {
+  it('states what a clean window actually rules out rather than implying certainty', () => {
+    const flake = analyzeFlake([run(), run(), run(), run(), run()]);
+    expect(flake.confidence).toBe('high');
+    expect(flake.confidenceNote).toContain('95% confidence');
+    expect(flake.confidenceNote).toContain('45%');
+  });
+
+  it('is low after a single run, and says why', () => {
+    const flake = analyzeFlake([run()]);
+    expect(flake.confidence).toBe('low');
+    expect(flake.confidenceNote).toContain('1 run');
+  });
+
+  it('is high for an observed flake, because both outcomes were seen', () => {
+    const flake = analyzeFlake([run(), run({ console: [err('boom')] })]);
+    expect(flake.confidence).toBe('high');
+    expect(flake.confidenceNote).toContain('observed, not inferred');
+  });
+
+  it('shrinks the ruled-out rate as runs are added', () => {
+    expect(ruledOutFailureRate(1)).toBeCloseTo(0.95, 5);
+    expect(ruledOutFailureRate(5)).toBeCloseTo(0.4507, 3);
+    expect(ruledOutFailureRate(20)).toBeCloseTo(0.1391, 3);
+    expect(ruledOutFailureRate(5)).toBeGreaterThan(ruledOutFailureRate(20));
+  });
+});
+
+describe('summarizeFlake', () => {
+  it('is one line naming the verdict and the confidence', () => {
+    const line = summarizeFlake(analyzeFlake([run(), run({ console: [err('boom')] })]));
+    expect(line).toMatch(/^FLAKY — /);
+    expect(line).toContain('Confidence: high.');
+  });
+});