@broberg/lens-daemon 0.0.0-stage → 0.1.0

package/src/video.ts

@@ -0,0 +1,211 @@
+// Lens video-liveness — verify a <video> actually PLAYS, not just exists.
+//
+// expectVisible/expectText/pixel-diff all pass on a frozen or decode-failed
+// video — the element is in the DOM and a poster frame renders. "Works" for a
+// video is a BEHAVIOUR: time advances, frames move, no MediaError. This module
+// drives the element and checks exactly that, with two complementary signals:
+//
+//   1. Element state (deterministic): play(), then currentTime advanced,
+//      readyState >= HAVE_CURRENT_DATA, !error, videoWidth/Height > 0.
+//   2. Frame motion (visual): screenshot the element, let it play, screenshot
+//      again, pixel-diff. Motion = genuinely rendering video; identical frames =
+//      frozen (the inverse of a normal baseline diff — here a diff is PASS).
+//
+// Audio cannot be verified headlessly (no output capture); muted/volume/
+// audioTracks are reported as context, never the verdict. It also can't judge
+// whether it's the RIGHT video — that's vision/human. Plugs into lens_verify as
+// a `video` option, folding a not-playing verdict into status='fail'.
+
+import type { Page } from 'playwright';
+import { PNG } from 'pngjs';
+import pixelmatch from 'pixelmatch';
+
+export interface VideoCheckOptions {
+  /** [data-testid] / CSS selector of the <video> (or a container holding one).
+   *  Omitted → the first <video> on the page. */
+  selector?: string;
+  /** How long to let it play before re-sampling (ms). Default 1200. */
+  playMs?: number;
+  /** Require visible frame motion between the two screenshots (default true).
+   *  Motion is the strong signal a poster-frozen video can't fake. */
+  requireMotion?: boolean;
+}
+
+export interface VideoResult {
+  /** The verdict: the video is genuinely playing (or played to completion). */
+  playing: boolean;
+  reason: string;
+  found: boolean;
+  currentTimeStart: number | null;
+  currentTimeEnd: number | null;
+  advancedSec: number | null;
+  readyState: number | null; // 0..4 (HAVE_ENOUGH_DATA = 4)
+  paused: boolean | null;
+  ended: boolean | null;
+  duration: number | null;
+  videoWidth: number | null;
+  videoHeight: number | null;
+  errorCode: number | null; // MediaError.code, or null
+  muted: boolean | null;
+  volume: number | null;
+  audioTracks: number | null;
+  /** Fraction of pixels that changed between the two frames while playing. */
+  frameMotionRatio: number | null;
+}
+
+interface VideoSample {
+  ct: number;
+  rs: number;
+  paused: boolean;
+  ended: boolean;
+  err: number | null;
+  dur: number | null;
+  vw: number;
+  vh: number;
+  muted: boolean;
+  vol: number;
+  at: number | null;
+}
+
+const empty = (reason: string, found = false): VideoResult => ({
+  playing: false,
+  reason,
+  found,
+  currentTimeStart: null,
+  currentTimeEnd: null,
+  advancedSec: null,
+  readyState: null,
+  paused: null,
+  ended: null,
+  duration: null,
+  videoWidth: null,
+  videoHeight: null,
+  errorCode: null,
+  muted: null,
+  volume: null,
+  audioTracks: null,
+  frameMotionRatio: null,
+});
+
+/** Drive the <video> at `selector` and report whether it actually plays. */
+export async function checkVideo(page: Page, opts: VideoCheckOptions = {}): Promise<VideoResult> {
+  const sel = opts.selector ?? null;
+  const playMs = opts.playMs ?? 1200;
+  const requireMotion = opts.requireMotion ?? true;
+
+  // Play + sample t0 (all in-page; resolve the <video> at/under the selector).
+  const start = (await page.evaluate((s) => {
+    const root = s ? document.querySelector(s) : document.querySelector('video');
+    const v = (root && root.tagName === 'VIDEO' ? root : root?.querySelector('video')) as HTMLVideoElement | null;
+    if (!v) return null;
+    try {
+      const p = v.play();
+      if (p && typeof p.catch === 'function') p.catch(() => {});
+    } catch {
+      /* autoplay policy / already playing */
+    }
+    return {
+      ct: v.currentTime,
+      rs: v.readyState,
+      paused: v.paused,
+      ended: v.ended,
+      err: v.error ? v.error.code : null,
+      dur: Number.isFinite(v.duration) ? v.duration : null,
+      vw: v.videoWidth,
+      vh: v.videoHeight,
+      muted: v.muted,
+      vol: v.volume,
+      at: (v as HTMLVideoElement & { audioTracks?: { length: number } }).audioTracks?.length ?? null,
+    } as VideoSample;
+  }, sel)) as VideoSample | null;
+
+  if (!start) return empty(`no <video> found for ${sel ?? 'video'}`);
+
+  // Frame 1 (screenshot the selector's element — container or the video).
+  const loc = sel ? page.locator(sel).first() : page.locator('video').first();
+  const frame1 = await loc.screenshot().catch(() => null);
+  await page.waitForTimeout(playMs);
+  const frame2 = await loc.screenshot().catch(() => null);
+
+  // Sample t1.
+  const end = (await page.evaluate((s) => {
+    const root = s ? document.querySelector(s) : document.querySelector('video');
+    const v = (root && root.tagName === 'VIDEO' ? root : root?.querySelector('video')) as HTMLVideoElement | null;
+    if (!v) return null;
+    return {
+      ct: v.currentTime,
+      rs: v.readyState,
+      paused: v.paused,
+      ended: v.ended,
+      err: v.error ? v.error.code : null,
+      dur: Number.isFinite(v.duration) ? v.duration : null,
+      vw: v.videoWidth,
+      vh: v.videoHeight,
+      muted: v.muted,
+      vol: v.volume,
+      at: (v as HTMLVideoElement & { audioTracks?: { length: number } }).audioTracks?.length ?? null,
+    } as VideoSample;
+  }, sel)) as VideoSample | null;
+
+  const e = end ?? start;
+  const advancedSec = +(e.ct - start.ct).toFixed(3);
+  const frameMotionRatio = frame1 && frame2 ? motionRatio(frame1, frame2) : null;
+
+  // ── verdict ──
+  let playing = false;
+  let reason: string;
+  if (e.err !== null) {
+    reason = `MediaError code ${e.err} — source not playable / decode failed`;
+  } else if (e.vw === 0 || e.vh === 0) {
+    reason = 'video has 0×0 dimensions — nothing decoded';
+  } else if (e.rs < 2) {
+    reason = `readyState ${e.rs} (< HAVE_CURRENT_DATA) — stalled / no data`;
+  } else {
+    const advanced = advancedSec > 0.1;
+    const playedToEnd = e.ended && (start.ct > 0 || advancedSec > 0 || frameMotionRatio === null);
+    const moved = frameMotionRatio === null ? true : frameMotionRatio > 0.002;
+    if (!advanced && !playedToEnd) {
+      reason = `currentTime did not advance (stuck at ${e.ct.toFixed(2)}s) — not playing`;
+    } else if (requireMotion && !moved) {
+      reason = `currentTime advanced ${advancedSec}s but frames are identical — frozen (poster / decode stall)`;
+    } else {
+      playing = true;
+      reason = e.ended
+        ? `played to end (${frameMotionRatio === null ? 'no frame diff' : `${(frameMotionRatio * 100).toFixed(1)}% frame motion`})`
+        : `playing — currentTime +${advancedSec}s, readyState ${e.rs}${frameMotionRatio === null ? '' : `, ${(frameMotionRatio * 100).toFixed(1)}% frame motion`}`;
+    }
+  }
+
+  return {
+    playing,
+    reason,
+    found: true,
+    currentTimeStart: start.ct,
+    currentTimeEnd: e.ct,
+    advancedSec,
+    readyState: e.rs,
+    paused: e.paused,
+    ended: e.ended,
+    duration: e.dur,
+    videoWidth: e.vw,
+    videoHeight: e.vh,
+    errorCode: e.err,
+    muted: e.muted,
+    volume: e.vol,
+    audioTracks: e.at,
+    frameMotionRatio,
+  };
+}
+
+/** Fraction of pixels that differ between two PNG frames. Dimension mismatch
+ *  (the element resized between shots) counts as full motion. */
+function motionRatio(a: Buffer, b: Buffer): number {
+  const pa = PNG.sync.read(a);
+  const pb = PNG.sync.read(b);
+  if (pa.width !== pb.width || pa.height !== pb.height) return 1;
+  const { width, height } = pa;
+  const total = width * height;
+  if (total === 0) return 0;
+  const changed = pixelmatch(pa.data, pb.data, null, width, height, { threshold: 0.1, includeAA: false });
+  return changed / total;
+}