@broberg/lens-daemon 0.0.0-stage → 0.1.5

package/src/validate-vision.ts

@@ -0,0 +1,97 @@
+// Vision-model validation — runs each candidate against TWO cases:
+//   1. spatial-overlap  — the ✕ over the version badge (geometry's home turf;
+//      geometry nails it free — here we see if vision can even see it).
+//   2. fuzzy-aesthetic   — inverted hierarchy + low-contrast-on-gradient +
+//      clashing illegible button (NO overlap; geometry is blind here — THIS is
+//      vision's real job). A gradient bg means the geometry contrast check skips.
+//
+// Pick cheapest-that-DUER for vision's actual domain. Add Pixtral to CANDIDATES
+// when it lands in the SDK. EU/Mistral only — no paid-Anthropic route (house rule).
+//
+//   MISTRAL_API_KEY auto-loaded from .env:
+//     bun run apps/agent/src/lens/validate-vision.ts
+
+import { withPage } from './browser.ts';
+
+const FIXTURES = [
+  {
+    name: 'spatial-overlap',
+    html: `<!doctype html><html><head><meta charset="utf-8"></head><body style="font-family:system-ui;margin:0;background:#fff">
+      <div style="position:relative;width:420px;height:160px;margin:48px;background:#f6f7f9;border:1px solid #ccc;border-radius:12px;padding:20px">
+        <h2 style="margin:0 0 8px">Release notes</h2>
+        <span style="position:absolute;left:24px;top:74px;width:160px;height:30px;line-height:30px;background:#e7ecff;border-radius:8px;padding:0 10px;color:#1a2b4a">shipped v0.2.0</span>
+        <button style="position:absolute;left:78px;top:68px;width:120px;height:42px;background:#fff;border:1px solid #ddd;font-size:18px">✕ close</button>
+      </div></body></html>`,
+    looksFor: /overlap|on top|cover|hidden|behind|close button|truncat|cut off|clip|awkward/i,
+  },
+  {
+    name: 'fuzzy-aesthetic',
+    html: `<!doctype html><html><head><meta charset="utf-8"></head><body style="margin:0;background:#fff">
+      <div style="width:460px;margin:40px;background:linear-gradient(135deg,#7b2ff7,#f107a3);padding:28px;border-radius:14px;font-family:system-ui">
+        <p style="font-size:11px;color:#d8b8ff;margin:0 0 4px">UPGRADE</p>
+        <h1 style="font-size:13px;color:#e3c0ff;margin:0 0 18px;font-weight:400">Pro plan</h1>
+        <p style="font-size:23px;color:#c79fe6;margin:0 0 6px;line-height:1.15">Everything in Free, plus unlimited projects, priority support, and advanced analytics for your whole team.</p>
+        <button style="background:#e8ff00;color:#fbff66;border:none;padding:12px 20px;border-radius:8px;font-size:16px;margin-top:14px">Get Pro</button>
+      </div></body></html>`,
+    looksFor: /hierarch|contrast|legib|readab|illegible|cramp|spacing|clash|color|heading|small|tiny|button/i,
+  },
+];
+
+const SYSTEM =
+  'You are a UI composition critic. You are shown ONE screenshot of a single app screen. ' +
+  'Report ONLY genuine VISUAL composition problems a careful designer would flag: overlapping ' +
+  'or colliding elements, a control sitting on top of text, text that is clipped/truncated/' +
+  'unreadable, content cut off or off-screen, low contrast, broken hierarchy, clashing/illegible ' +
+  'colours, anything that plainly looks broken. If the screen looks coherent, return an empty list. Be terse.';
+const PROMPT =
+  'Return ONLY a JSON object: {"findings":[{"severity":"high|medium|low","region":"<where>","issue":"<what>"}]}. ' +
+  'Empty findings array if the composition holds together.';
+
+const CANDIDATES = [
+  // ai-sdk: Pixtral is retired; magistral-medium (reasoning + vision) is the
+  // strongest GDPR-safe bet for a critic that must judge aesthetics — try first.
+  { key: 'magistral-medium ($2/$5, EU, reasoning)', provider: 'mistral', model: 'magistral-medium-latest', envKey: 'MISTRAL_API_KEY' },
+  { key: 'mistral-large ($0.5/$1.5, EU)', provider: 'mistral', model: 'mistral-large-2512', envKey: 'MISTRAL_API_KEY' },
+];
+
+const sdk = await import('@broberg/ai-sdk');
+const providers: Record<string, unknown> = {};
+if (process.env.MISTRAL_API_KEY) providers.mistral = sdk.mistralAdapter({ apiKey: process.env.MISTRAL_API_KEY });
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const ai = sdk.createAI({ providers: providers as any }) as any;
+
+for (const fx of FIXTURES) {
+  const url = 'data:text/html;charset=utf-8,' + encodeURIComponent(fx.html);
+  const shot = await withPage({ viewport: { width: 560, height: 320 } }, async (page) => {
+    await page.goto(url, { waitUntil: 'domcontentloaded' });
+    await page.waitForTimeout(200);
+    return Buffer.from(await page.screenshot());
+  });
+  console.log(`\n\n######## CASE: ${fx.name} ########`);
+  for (const c of CANDIDATES) {
+    if (!process.env[c.envKey]) {
+      console.log(`\n=== ${c.key} — SKIPPED (no ${c.envKey}) ===`);
+      continue;
+    }
+    console.log(`\n=== ${c.key} ===`);
+    try {
+      const res = await ai.vision({
+        image: shot,
+        mimeType: 'image/png',
+        prompt: SYSTEM + '\n\n' + PROMPT,
+        override: { provider: c.provider, model: c.model, transport: 'http' },
+        purpose: 'vision-validation',
+      });
+      const text = typeof res.text === 'string' ? res.text : JSON.stringify(res.text ?? '');
+      const parsed = sdk.parseJsonLoose(text) as { findings?: unknown };
+      const findings = Array.isArray(parsed?.findings) ? parsed.findings : [];
+      const onTarget = fx.looksFor.test(JSON.stringify(findings));
+      console.log('cost   :', res.usage?.costUsd != null ? `$${res.usage.costUsd.toFixed(5)}` : '—');
+      console.log('on-target finding?', onTarget ? 'YES ✓' : 'NO ✗', `· ${findings.length} finding(s)`);
+      console.log(JSON.stringify(findings, null, 2));
+    } catch (err) {
+      console.log('ERROR:', err instanceof Error ? err.message : String(err));
+    }
+  }
+}
+process.exit(0);

package/src/verify.ts

@@ -0,0 +1,411 @@
+// F074.2 — Lens verify primitives.
+//
+// Verification = diff AND assertion, not just a raw screenshot. One page
+// visit yields: a screenshot, an optional pixel-diff against a baseline, an
+// optional DOM measurement (assert.js via page.evaluate), a structural
+// domHash, and console/network capture (reported, never auto-fatal).
+//
+// Status logic (prioritised — see F074 plan):
+//   1. assert set and assertResult.pass === false        → fail
+//   2. else baselineKey set + baseline exists → pixelmatch:
+//        diffRatio < threshold → pass, else                → fail
+//   3. else baselineKey set, no baseline                   → no-baseline
+//   4. else (plain capture) page loaded w/o uncaught error → pass
+//
+// Console/network errors are reported but never downgrade status on their
+// own — "assertion failed" is separated from "page logged a warning". The
+// caller (cc) decides what to do with the console/network signal.
+
+import { createHash } from 'node:crypto';
+import { PNG } from 'pngjs';
+import pixelmatch from 'pixelmatch';
+import type { Page } from 'playwright';
+import { withPage } from './browser.ts';
+import { navigateAndAct, resolveExtensionUrl, resolvePrepareContext, takeShot, type CaptureMode, type CaptureOptions } from './capture.ts';
+import { storagePayload } from './seed-storage.ts';
+import { preflight, type PreflightResult } from './preflight.ts';
+import { runCritic, topFindingMessage, type CriticReport } from './critic.ts';
+import { checkVideo, type VideoCheckOptions, type VideoResult } from './video.ts';
+// F074.39 — ONE assert evaluator, shared with the flow surface (see runAssert).
+import { noVerdictMessage } from '@broberg/lens-engine';
+import { evalAssertBody } from './flow.ts';
+import { observeBuild } from './build-marker.ts';
+
+export type LensStatus = 'pass' | 'fail' | 'no-baseline' | 'error';
+
+export interface VerifyOptions extends CaptureOptions {
+  /** A JS statement-body evaluated in the page. Should `return` either a
+   *  boolean or `{ pass: boolean, detail?: any }`. Runs in the browser
+   *  context (sandboxed to the page), e.g.:
+   *    "const el = document.querySelector('[data-testid=cta]');
+   *     return { pass: el.getBoundingClientRect().width > 100, detail: el.offsetWidth }" */
+  assert?: string;
+  /** PNG bytes of the approved baseline for this surface, if one exists. */
+  baseline?: Buffer | null;
+  /** A key naming the baseline slot. When set without `baseline`, status is
+   *  `no-baseline` (the shot becomes promotable). */
+  baselineKey?: string;
+  /** Pixel-diff pass threshold as a fraction of total pixels. Default 0.01. */
+  diffThreshold?: number;
+  /** F074.57 — BCP-47 locale + IANA timezone for the browser context. Unset
+   *  passes nothing to Playwright (existing baselines must not move). */
+  locale?: string;
+  timezone?: string;
+  /** F074.59 — prefers-reduced-motion for this capture. Default 'reduce', which is
+   *  what every capture did before this option existed, so no baseline can move. */
+  reducedMotion?: 'reduce' | 'no-preference';
+  /** Collect console messages (default true). */
+  collectConsole?: boolean;
+  /** Collect failed/non-2xx network responses (default true). */
+  collectNetwork?: boolean;
+  /** Skip the pre-flight stage (used when the caller already proved reachability). */
+  skipPreflight?: boolean;
+  /** Video-liveness: drive a <video> and assert it actually plays (currentTime
+   *  advances + frames move + no MediaError). A not-playing verdict fails the
+   *  verify. Presence of this object turns the check on. */
+  video?: VideoCheckOptions;
+}
+
+export interface DiffResult {
+  diffRatio: number;
+  diffPixels: number;
+  totalPixels: number;
+  /** PNG bytes highlighting changed pixels, or null on a dimension mismatch. */
+  diffPng: Buffer | null;
+  /** True when baseline + current differ in size (an automatic fail). */
+  dimensionMismatch: boolean;
+}
+
+export interface ConsoleEntry {
+  type: string;
+  text: string;
+}
+export interface NetworkEntry {
+  url: string;
+  status: number;
+  ok: boolean;
+  failure?: string;
+}
+
+export interface VerifyResult {
+  status: LensStatus;
+  reason: string;
+  mode: CaptureMode;
+  selector: string | null;
+  url: string;
+  finalUrl: string | null;
+  title: string | null;
+  screenshot: Buffer | null;
+  width: number;
+  height: number;
+  elementBox: { x: number; y: number; width: number; height: number } | null;
+  diff: DiffResult | null;
+  assertResult: { pass: boolean; detail: unknown } | null;
+  domHash: string | null;
+  console: ConsoleEntry[];
+  network: NetworkEntry[];
+  pageErrors: string[];
+  preflight: PreflightResult | null;
+  /** F074.69 — the extension this run loaded and the id Chromium generated for
+   *  it, so a caller can cite the exact surface photographed. Null otherwise. */
+  extension?: { id: string; path: string } | null;
+  durationMs: number;
+  /** F126.4 — composition-critic findings, when a critic was requested. A
+   *  high-severity finding folds the status to `fail` (it blocks Review→Done). */
+  critic: CriticReport | null;
+  /** Video-liveness result, when a `video` check was requested. A not-playing
+   *  verdict folds the status to `fail`. */
+  video: VideoResult | null;
+  /** F074.30 — the build sha this capture was looking at, read at capture time.
+   *  NULL when the target exposes no marker: "not measured", never "same build
+   *  as the other run". */
+  build: string | null;
+}
+
+/** sha256 over the target's normalized outerHTML — a cheap structural
+ *  fingerprint that changes when the DOM changes but is stable across
+ *  re-renders of identical markup. */
+async function computeDomHash(page: Page, selector: string | null): Promise<string> {
+  const html = await page.evaluate((sel) => {
+    const node = sel ? document.querySelector(sel) : document.documentElement;
+    return node ? (node as HTMLElement).outerHTML : '';
+  }, selector);
+  const normalized = html.replace(/\s+/g, ' ').trim();
+  return createHash('sha256').update(normalized, 'utf-8').digest('hex');
+}
+
+/**
+ * Evaluate the assert body in the page; normalize boolean → {pass, detail}.
+ *
+ * F074.39 — this runs the SAME `evalAssertBody` the flow assert runs, because
+ * an assert contract that differs per surface is one nobody can hold in their
+ * head. It had diverged in a way that produced a false RED: verify used only
+ * the block wrapper, so a bare-expression assert (`document.title.length > 0` —
+ * exactly what the flow surface accepts, and what our own docs show) returned
+ * `undefined` here and failed a page that was fine. One evaluator, one contract:
+ * expression-or-block, `await` allowed, `{ pass, detail }` honoured.
+ *
+ * A broken PROBE stays distinguishable from a failing app: syntax + throw are
+ * re-thrown, so the run reports `error` with the reason rather than `fail` —
+ * "your predicate is false" and "your predicate never ran" are different
+ * sentences, and this surface has always been honest about that.
+ */
+export async function runAssert(page: Page, assertBody: string): Promise<{ pass: boolean; detail: unknown }> {
+  const out = await page.evaluate(evalAssertBody, assertBody);
+  if (out.kind === 'syntax') {
+    throw new Error(`assert body is not valid JavaScript (${out.message})`);
+  }
+  if (out.kind === 'threw') {
+    // No polling on this surface (verify is a single measurement), so a throw
+    // is terminal — and it is the INSTRUMENT that broke, never a verdict.
+    throw new Error(`assert body threw: ${out.message}`);
+  }
+  if (out.kind === 'no-verdict') {
+    // Same author error as on the flow path, and it throws here for the same
+    // reason a syntax error does: the instrument was handed something that
+    // cannot express a verdict, so reporting pass OR fail would be inventing
+    // one. Wording from the engine (0.6.1) so the two surfaces cannot drift.
+    throw new Error(noVerdictMessage(out.keys));
+  }
+  return { pass: out.value, detail: out.detail ?? null };
+}
+
+/** pixelmatch the current shot against the baseline. Dimension mismatch is an
+ *  automatic fail (the surface changed size). */
+function diffAgainstBaseline(current: Buffer, baseline: Buffer): DiffResult {
+  const a = PNG.sync.read(baseline);
+  const b = PNG.sync.read(current);
+  if (a.width !== b.width || a.height !== b.height) {
+    return {
+      diffRatio: 1,
+      diffPixels: a.width * a.height,
+      totalPixels: a.width * a.height,
+      diffPng: null,
+      dimensionMismatch: true,
+    };
+  }
+  const { width, height } = a;
+  const diff = new PNG({ width, height });
+  const diffPixels = pixelmatch(a.data, b.data, diff.data, width, height, {
+    threshold: 0.1, // per-pixel colour sensitivity (not the pass/fail ratio)
+    // F094.7 — detect + ignore anti-aliased pixels (the default, made explicit):
+    // text edges differ sub-pixel between captures and must not count as a diff.
+    includeAA: false,
+  });
+  const totalPixels = width * height;
+  return {
+    diffRatio: totalPixels === 0 ? 0 : diffPixels / totalPixels,
+    diffPixels,
+    totalPixels,
+    diffPng: PNG.sync.write(diff),
+    dimensionMismatch: false,
+  };
+}
+
+/** F074.92 — A SELECTOR WITHOUT A MODE USED TO PHOTOGRAPH THE WHOLE PAGE.
+ *
+ *  `mode` defaulted to 'viewport' unconditionally, and takeShot() reads the
+ *  selector ONLY in 'element' mode — so a caller who passed `selector` and no
+ *  `mode` got a full-page shot. The response gave no sign of it: the selector
+ *  came back echoed, `surface_key` was set to it, and `dom_hash` was computed
+ *  from that very element. Four fields agreed it was an element shot; only the
+ *  picture disagreed. Reported by broberg-id, who measured two shots of the same
+ *  size and concluded, correctly, that the element had been ignored.
+ *
+ *  Nobody passes a selector meaning "and ignore it", so the mode is inferred.
+ *  That is also what our OWN manifest meant: its `landing-hero` surface carries
+ *  a selector and no mode, and has been storing a full-page baseline under an
+ *  element's name ever since it was written. Expect that one baseline to diff
+ *  on its next run — the new shot is right and the stored one never was.
+ *
+ *  An explicit mode ALWAYS wins, so a surface that deliberately pairs a selector
+ *  with viewport (to key a baseline by name) is untouched. And the inference is
+ *  visible rather than magic: `mode` is echoed in the result. */
+export function resolveCaptureMode(opts: { mode?: CaptureMode; selector?: string | null }): CaptureMode {
+  return opts.mode ?? (opts.selector ? 'element' : 'viewport');
+}
+
+export async function verify(opts: VerifyOptions): Promise<VerifyResult> {
+  const startedAt = Date.now();
+  const mode: CaptureMode = resolveCaptureMode(opts);
+  const timeoutMs = opts.timeoutMs ?? 30_000;
+  const threshold = opts.diffThreshold ?? 0.01;
+  const collectConsole = opts.collectConsole ?? true;
+  const collectNetwork = opts.collectNetwork ?? true;
+
+  // F074.30 — stamped HERE, at capture, not read back later: asking after the
+  // fact returns today's build for an OLD recording. Best-effort: null when the
+  // target has no marker, and never able to fail a capture that would pass.
+  // F074.69 — an extension:// url is not reachable over the network and has no
+  // build marker. Probing it would fail for a reason that has nothing to do with
+  // the surface, and the failure would read as "the extension is broken".
+  const isExtension = !!opts.extension || opts.url.startsWith('extension://');
+  const build = isExtension ? null : await observeBuild(opts.url);
+
+  const empty = {
+    mode,
+    selector: opts.selector ?? null,
+    url: opts.url,
+    build,
+    finalUrl: null,
+    title: null,
+    screenshot: null,
+    width: 0,
+    height: 0,
+    elementBox: null,
+    diff: null,
+    assertResult: null,
+    domHash: null,
+    console: [] as ConsoleEntry[],
+    network: [] as NetworkEntry[],
+    pageErrors: [] as string[],
+    critic: null as CriticReport | null,
+    video: null as VideoResult | null,
+  };
+
+  // 1. Pre-flight (unless the caller already proved reachability).
+  let pre: PreflightResult | null = null;
+  if (!opts.skipPreflight && !isExtension) {
+    pre = await preflight(opts.url, Math.min(timeoutMs, 5_000));
+    if (!pre.ok) {
+      return { ...empty, status: 'error', reason: pre.reason ?? 'pre-flight failed', preflight: pre, durationMs: Date.now() - startedAt };
+    }
+  }
+
+  try {
+    const result = await withPage(
+      {
+        engine: opts.engine,
+        locale: opts.locale,
+        timezone: opts.timezone,
+        reducedMotion: opts.reducedMotion,
+        viewport: opts.viewport,
+        deviceScaleFactor: opts.deviceScaleFactor,
+        prepareContext: resolvePrepareContext(opts),
+        // F074.97 — seeded state reaches lens_verify too: the tool that GATES a
+        // card is the one that most needs to photograph a saved-state bug.
+        seedStorage: storagePayload(opts.storage, opts.url),
+        extension: opts.extension,
+      },
+      async (page, _context, extension) => {
+        const pageErrors: string[] = [];
+        const consoleEntries: ConsoleEntry[] = [];
+        const network: NetworkEntry[] = [];
+        page.on('pageerror', (err) => pageErrors.push(err.message));
+        if (collectConsole) {
+          page.on('console', (msg) => consoleEntries.push({ type: msg.type(), text: msg.text() }));
+        }
+        if (collectNetwork) {
+          page.on('response', (res) => {
+            if (!res.ok()) network.push({ url: res.url(), status: res.status(), ok: false });
+          });
+          page.on('requestfailed', (req) => {
+            network.push({ url: req.url(), status: 0, ok: false, failure: req.failure()?.errorText ?? 'failed' });
+          });
+        }
+
+        // F074.69 — substitute the generated extension id only now that the
+        // browser has reported it. A caller cannot know it in advance, so
+        // `extension://popup.html` is the shorthand they write instead.
+        await navigateAndAct(page, { ...opts, url: resolveExtensionUrl(opts.url, extension?.id) });
+        const title = await page.title();
+        const finalUrl = page.url();
+        const shot = await takeShot(page, mode, opts.selector ?? null, timeoutMs);
+        const domHash = await computeDomHash(page, opts.selector ?? null);
+        const assertResult = opts.assert ? await runAssert(page, opts.assert) : null;
+        const critic =
+          opts.critic && opts.critic !== 'off'
+            ? await runCritic(page, shot.screenshot, opts.critic, opts.criticVisionModel)
+            : null;
+        const video = opts.video ? await checkVideo(page, opts.video) : null;
+
+        return { title, finalUrl, shot, domHash, assertResult, pageErrors, console: consoleEntries, network, critic, video, extension: extension ?? null };
+      },
+    );
+
+    // 3. Diff against baseline (pure, post-session).
+    let diff: DiffResult | null = null;
+    if (opts.baselineKey && opts.baseline) {
+      diff = diffAgainstBaseline(result.shot.screenshot, opts.baseline);
+    }
+
+    // 4. Prioritised status logic.
+    let status: LensStatus;
+    let reason: string;
+    if (opts.assert && result.assertResult && !result.assertResult.pass) {
+      status = 'fail';
+      reason = `assertion failed: ${JSON.stringify(result.assertResult.detail)}`;
+    } else if (opts.baselineKey && opts.baseline && diff) {
+      if (diff.dimensionMismatch) {
+        status = 'fail';
+        reason = 'baseline dimension mismatch — surface changed size';
+      } else if (diff.diffRatio < threshold) {
+        status = 'pass';
+        reason = `diff ${(diff.diffRatio * 100).toFixed(3)}% < ${(threshold * 100).toFixed(2)}% threshold`;
+      } else {
+        status = 'fail';
+        reason = `diff ${(diff.diffRatio * 100).toFixed(3)}% ≥ ${(threshold * 100).toFixed(2)}% threshold`;
+      }
+    } else if (opts.baselineKey && !opts.baseline) {
+      status = 'no-baseline';
+      reason = `no baseline for "${opts.baselineKey}" — shot is promotable`;
+    } else {
+      const ok = result.pageErrors.length === 0;
+      status = ok ? 'pass' : 'fail';
+      reason = ok ? 'page loaded with no uncaught error' : `uncaught page error(s): ${result.pageErrors.join('; ')}`;
+    }
+
+    // F126.4 — a high-severity composition finding folds an otherwise-passing
+    // (or promotable) surface to `fail`: the screen is visibly broken
+    // (something overlapping / covered / clipped / off-screen) even though its
+    // pixels/assert passed. This is what makes the critic BLOCK Review→Done. An
+    // existing assert/baseline/error fail keeps its (more specific) reason.
+    if (result.critic && result.critic.high > 0 && status !== 'fail') {
+      status = 'fail';
+      const top = topFindingMessage(result.critic);
+      reason = `composition: ${result.critic.high} high-severity issue(s)${top ? ` — ${top}` : ''}`;
+    }
+
+    // Video-liveness: if a video check was requested and the video isn't
+    // playing, that's a definitive fail (it's the point of the check). Keep an
+    // earlier assert/baseline fail's more-specific reason.
+    if (result.video && !result.video.playing && status !== 'fail') {
+      status = 'fail';
+      reason = `video: ${result.video.reason}`;
+    }
+
+    return {
+      status,
+      reason,
+      mode,
+      selector: opts.selector ?? null,
+      url: opts.url,
+      build,
+      finalUrl: result.finalUrl,
+      title: result.title,
+      screenshot: result.shot.screenshot,
+      width: result.shot.width,
+      height: result.shot.height,
+      elementBox: result.shot.elementBox,
+      diff,
+      assertResult: result.assertResult,
+      domHash: result.domHash,
+      console: result.console,
+      network: result.network,
+      pageErrors: result.pageErrors,
+      preflight: pre,
+      durationMs: Date.now() - startedAt,
+      critic: result.critic,
+      video: result.video,
+      extension: result.extension,
+    };
+  } catch (err) {
+    return {
+      ...empty,
+      status: 'error',
+      reason: `capture/verify threw: ${err instanceof Error ? err.message : String(err)}`,
+      preflight: pre,
+      durationMs: Date.now() - startedAt,
+    };
+  }
+}