@broberg/lens-daemon 0.0.0-stage → 0.1.0

package/src/schema.ts

@@ -0,0 +1,300 @@
+// lens F001.5 — the Lens tables and their row types, owned by the Lens package.
+//
+// Moved verbatim from cardmem apps/agent/src/state.ts (ed10516f): the lens_* DDL,
+// the additive lens migrations, the F276.1 upload queue and the F094.3 baseline
+// backfill. cardmem's getDb() calls ensureLensSchema(db) once the daemon runs on
+// this package; until that cutover cardmem keeps its own copy, and the cutover
+// deletes it (no second copy survives F001.5).
+
+import type { Database } from 'bun:sqlite';
+import { randomUUID } from 'node:crypto';
+import { ensureUploadQueue } from './artifact-uploads.ts';
+
+/** Create/upgrade every Lens table. Idempotent; call inside the daemon's schema transaction. */
+export function ensureLensSchema(db: Database): void {
+  db.exec(`
+    -- F074.3 — Lens runs + baselines. Metadata lives here (daemon-local
+    -- SQLite); screenshot/diff BYTES live in the storage adapter (local dir
+    -- or Tigris). Hard-scoped to project_slug so a project only ever sees
+    -- its own runs. The SPA dashboard (F074.8) reads these via the daemon's
+    -- HTTP, the same pattern as fleet-status / audit.
+    CREATE TABLE IF NOT EXISTS lens_runs (
+      id TEXT PRIMARY KEY,
+      project_slug TEXT NOT NULL,
+      surface_key TEXT,
+      url TEXT NOT NULL,
+      mode TEXT NOT NULL,
+      selector TEXT,
+      status TEXT NOT NULL,
+      reason TEXT,
+      diff_ratio REAL,
+      artifact_key TEXT,
+      diff_artifact_key TEXT,
+      dom_hash TEXT,
+      assert_pass INTEGER,
+      assert_detail_json TEXT,
+      console_json TEXT,
+      network_json TEXT,
+      page_errors_json TEXT,
+      width INTEGER,
+      height INTEGER,
+      duration_ms INTEGER,
+      critic_json TEXT,
+      created_at INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_lens_runs_project ON lens_runs(project_slug, created_at DESC);
+
+    CREATE TABLE IF NOT EXISTS lens_baselines (
+      id TEXT PRIMARY KEY,
+      project_slug TEXT NOT NULL,
+      baseline_key TEXT NOT NULL,
+      artifact_key TEXT NOT NULL,
+      width INTEGER,
+      height INTEGER,
+      approved_from_run_id TEXT,
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL,
+      UNIQUE(project_slug, baseline_key)
+    );
+
+    -- F094.3 — immutable baseline version history. Every approve APPENDS a row
+    -- here (never overwrites); lens_baselines.current_version_id points at the
+    -- active one, so a revert is a re-point with no data loss. The artifact_key
+    -- is content-addressed per version (baselines/<key>/v<n>.png).
+    CREATE TABLE IF NOT EXISTS lens_baseline_versions (
+      id TEXT PRIMARY KEY,
+      project_slug TEXT NOT NULL,
+      baseline_key TEXT NOT NULL,
+      version_no INTEGER NOT NULL,
+      artifact_key TEXT NOT NULL,
+      width INTEGER,
+      height INTEGER,
+      approved_from_run_id TEXT,
+      commit_sha TEXT,
+      note TEXT,
+      approved_at INTEGER NOT NULL,
+      UNIQUE(project_slug, baseline_key, version_no)
+    );
+    CREATE INDEX IF NOT EXISTS idx_lens_baseline_versions
+      ON lens_baseline_versions(project_slug, baseline_key, version_no DESC);
+
+    -- F082 — E2E flow runs (manuscript executions). One row per flow run; the
+    -- per-step report (status + screenshot artifact key per step) lives in
+    -- steps_json. Same project-scoping + artifact-storage pattern as lens_runs.
+    CREATE TABLE IF NOT EXISTS lens_flow_runs (
+      id TEXT PRIMARY KEY,
+      project_slug TEXT NOT NULL,
+      flow_name TEXT NOT NULL,
+      status TEXT NOT NULL,
+      failed_at INTEGER,
+      reason TEXT,
+      steps_json TEXT NOT NULL,
+      step_count INTEGER NOT NULL,
+      duration_ms INTEGER,
+      created_at INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_lens_flow_runs_project ON lens_flow_runs(project_slug, created_at DESC);
+  `);
+  // F094.3 — additive: pointer from a baseline to its active version row.
+  try {
+    db.exec('ALTER TABLE lens_baselines ADD COLUMN current_version_id TEXT');
+  } catch {
+    // column already exists
+  }
+  // F126.4 — additive: composition-critic findings (JSON) per run, so
+  // lens_list_runs / the report can show them historically.
+  try {
+    db.exec('ALTER TABLE lens_runs ADD COLUMN critic_json TEXT');
+  } catch {
+    // column already exists
+  }
+  // F255.5 — additive: artifact key of a flow run's .webm recording (null unless
+  // the run asked for record:true).
+  try {
+    db.exec('ALTER TABLE lens_flow_runs ADD COLUMN video_key TEXT');
+  } catch {
+    // column already exists
+  }
+  // F074.30 — additive: the build sha the run MEASURED, stamped at capture time.
+  // NULL means "not measured" (the target exposes no marker) and must never be
+  // read as "same build as the other run" — a comparison spanning a null has to
+  // refuse rather than reassure. Without this, a run-vs-run diff can compare a
+  // build with itself and answer "no regression".
+  for (const table of ['lens_runs', 'lens_flow_runs']) {
+    try {
+      db.exec(`ALTER TABLE ${table} ADD COLUMN build TEXT`);
+    } catch {
+      // column already exists
+    }
+  }
+  // F074.36 — additive: WHERE the run ended, and WHICH credential it used (or why
+  // it had none). Without these, a past verdict cannot be re-checked for the one
+  // question that invalidates it: "was that run even logged in, and was it
+  // looking at the page I think it was?"
+  // F074.99 — additive: the run-level FINDINGS a reader can act on (a password
+  // that came back in the URL, a page that was never listening). Both were
+  // produced in flow.ts and written nowhere, which is this file's own recurring
+  // lesson: a field produced in one place and not persisted in the other reaches
+  // nobody. fd-sundhed measured exactly that from a consumer's seat.
+  for (const col of ['final_url TEXT', 'auth_json TEXT', 'reduced_motion TEXT', 'findings_json TEXT']) {
+    try {
+      db.exec(`ALTER TABLE lens_flow_runs ADD COLUMN ${col}`);
+    } catch {
+      // column already exists
+    }
+  }
+  // F074.59 — reduced_motion is deliberately NULLABLE with no default. A run
+  // recorded before the column existed genuinely does not know which motion state
+  // it used, and back-filling it with 'reduce' would turn "not measured" into a
+  // measurement — the exact collapse this card exists to prevent one layer up.
+
+  // F272.2 — additive: WHICH STORAGE BACKEND held this run's screenshots, stamped
+  // at capture time and never re-derived from current config.
+  //
+  // The re-derived version is the trap. Read the daemon's CURRENT backend and
+  // apply it to every row, and the day LENS_TIGRIS_BUCKET is finally set, every
+  // historic run whose PNGs are still on this Mac starts reporting `tigris`. The
+  // bytes have not moved; the config changed; the citation goes green. That is
+  // the false-green direction, so the value is written once, at the moment the
+  // bytes are actually put somewhere.
+  //
+  // NULL is the same third fact as reduced_motion above: a run older than this
+  // column does not know where its bytes went, and `unknown` must never be
+  // rounded up to `durable`. An evidence citation over a null is exactly as
+  // unproven as one over 'local' — it just fails for a different reason.
+  try {
+    db.exec('ALTER TABLE lens_flow_runs ADD COLUMN artifact_backend TEXT');
+  } catch {
+    // column already exists
+  }
+
+  // F276.1 — the upload queue that makes every artifact reachable from a phone.
+  //
+  // A QUEUE RATHER THAN AN INLINE UPLOAD, for three reasons that all bite:
+  //   · a capture must not get slower, or nobody will run one;
+  //   · a failed upload must not fail the RUN — the local copy is still valid
+  //     evidence, and losing a run because R2 hiccuped is a worse outcome than
+  //     a late upload;
+  //   · the backfill of 6588 historic files (F276.4, Christian: "ALT") is the
+  //     same operation at a different scale, so it gets the same machinery
+  //     instead of a second one that drifts.
+  //
+  // `artifact_key` is the primary key, so enqueueing the same key twice is a
+  // no-op and the backfill can be re-run without duplicating uploads. That is
+  // an ACCEPTANCE CRITERION on F276.4, expressed in the schema rather than in a
+  // guard someone can forget to call.
+  ensureUploadQueue(db);
+
+  // F094.3 — backfill: every pre-existing baseline becomes version 1, pointing at
+  // its current (stable-key) artifact, so the visual history starts from what's
+  // live. Runs once (gated on current_version_id IS NULL).
+  const legacy = db
+    .query('SELECT * FROM lens_baselines WHERE current_version_id IS NULL')
+    .all() as LensBaselineRow[];
+  for (const b of legacy) {
+    const vid = randomUUID();
+    db.prepare(
+      `INSERT INTO lens_baseline_versions
+        (id, project_slug, baseline_key, version_no, artifact_key, width, height,
+         approved_from_run_id, commit_sha, note, approved_at)
+       VALUES (?,?,?,1,?,?,?,?,NULL,'backfilled v1',?)`,
+    ).run(vid, b.project_slug, b.baseline_key, b.artifact_key, b.width, b.height, b.approved_from_run_id, b.updated_at);
+    db.prepare('UPDATE lens_baselines SET current_version_id = ? WHERE id = ?').run(vid, b.id);
+  }
+}
+
+export interface LensRunRow {
+  id: string;
+  project_slug: string;
+  surface_key: string | null;
+  url: string;
+  mode: string;
+  selector: string | null;
+  status: string;
+  reason: string | null;
+  diff_ratio: number | null;
+  artifact_key: string | null;
+  diff_artifact_key: string | null;
+  dom_hash: string | null;
+  assert_pass: number | null;
+  assert_detail_json: string | null;
+  console_json: string | null;
+  network_json: string | null;
+  page_errors_json: string | null;
+  width: number | null;
+  height: number | null;
+  duration_ms: number | null;
+  critic_json: string | null;
+  created_at: number;
+  /** F074.30 — the build sha this run measured, stamped at capture time. NULL =
+   *  not measured (no marker on the target); never read it as "same build". */
+  build?: string | null;
+}
+
+export interface LensBaselineRow {
+  id: string;
+  project_slug: string;
+  baseline_key: string;
+  artifact_key: string;
+  width: number | null;
+  height: number | null;
+  approved_from_run_id: string | null;
+  created_at: number;
+  updated_at: number;
+  current_version_id: string | null; // F094.3 — active version pointer
+}
+
+// F094.3 — one row per approved baseline version (append-only history).
+export interface LensBaselineVersionRow {
+  id: string;
+  project_slug: string;
+  baseline_key: string;
+  version_no: number;
+  artifact_key: string;
+  width: number | null;
+  height: number | null;
+  approved_from_run_id: string | null;
+  commit_sha: string | null;
+  note: string | null;
+  approved_at: number;
+}
+
+export interface LensFlowRunRow {
+  id: string;
+  project_slug: string;
+  flow_name: string;
+  status: string;
+  failed_at: number | null;
+  reason: string | null;
+  steps_json: string;
+  step_count: number;
+  duration_ms: number | null;
+  created_at: number;
+  /** F255.5 — artifact key of the run's .webm recording (null unless record:true). */
+  video_key?: string | null;
+  /** F074.30 — the build sha this run measured, stamped at capture time. NULL =
+   *  not measured (no marker on the target); never read it as "same build". */
+  build?: string | null;
+  /** F074.36 — the URL the run ended on. Answers "which page was that verdict
+   *  about?" for a run nobody is watching any more. */
+  final_url?: string | null;
+  /** F074.36 — JSON {source, adapter, note}: the credential the run used, or the
+   *  reason it had none. So a past GREEN run can be asked whether it was even
+   *  logged in. */
+  auth_json?: string | null;
+  /** F074.59 — 'reduce' | 'no-preference': the motion state the run executed under.
+   *  NULL means NOT RECORDED (a run older than the column), which is a third fact,
+   *  not a synonym for 'reduce'. A reader comparing two runs must refuse on a null
+   *  rather than assume they saw the same browser. */
+  reduced_motion?: string | null;
+  /** F272.2 — 'local' | 'tigris': where this run's step screenshots were PUT, stamped
+   *  at capture time. NULL means NOT RECORDED (a run older than the column) and is a
+   *  third fact: `unknown`, never rounded up to durable. Read from the row, never
+   *  re-derived from the daemon's current config — a config flip must not retroactively
+   *  bless bytes that never moved. */
+  artifact_backend?: string | null;
+  /** F074.99 — JSON {password_in_url?, not_listening?}: the run-level findings.
+   *  `'{}'` means the run LOOKED and found nothing; NULL means NOT RECORDED (a run
+   *  older than the column). A reader must not read the second as the first. */
+  findings_json?: string | null;
+}

package/src/scrape.ts

@@ -0,0 +1,116 @@
+// F125.2 — Lens scrape adapter.
+//
+// Extract an arbitrary article URL → { title, markdown }. This is the Research
+// Adapter worker's "get the body" step. It runs INSIDE Lens (the daemon owns
+// the warm Playwright browser) — never a raw one-off Playwright script (F112).
+//
+// Strategy = exactly the trail web-clipper recipe, server-side: Playwright
+// renders the page (handles JS + auth), Readability runs IN the page on a DOM
+// clone (injected as a global, no jsdom needed under Bun), and Turndown turns
+// the article HTML into markdown Node-side. The Medium adapter authes via a
+// stored storageState (Christian's Medium session) reusing the F074 auth layer.
+
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import TurndownService from 'turndown';
+import { withPage } from './browser.ts';
+import { authPrepare } from './auth/index.ts';
+
+const require = createRequire(import.meta.url);
+// Readability.js is a standalone class file (top-level `function Readability`),
+// so injecting it as a <script> defines window.Readability in the page. Read it
+// once at module load — node_modules is present (the daemon runs from source).
+const READABILITY_SRC = readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf8');
+
+export interface ScrapeResult {
+  url: string;
+  title: string;
+  markdown: string;
+  byline: string | null;
+  length: number;
+  excerpt: string | null;
+}
+
+export interface ScrapeOptions {
+  /** F074.12 storageState path on the daemon host — the Medium session for
+   *  paywalled posts. The path is not a secret; the gitignored file it points
+   *  at is. Generic (non-authed) scrapes omit it. */
+  storageStatePath?: string;
+  /** Override the User-Agent — must match the cf_clearance cookie's UA on
+   *  Cloudflare-fronted sites (Medium). */
+  userAgent?: string;
+  timeoutMs?: number;
+}
+
+export async function scrapeArticle(url: string, opts: ScrapeOptions = {}): Promise<ScrapeResult> {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new Error(`invalid url: ${url}`);
+  }
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    throw new Error(`unsupported protocol: ${parsed.protocol} (only http/https)`);
+  }
+
+  const timeout = opts.timeoutMs ?? 30_000;
+  const article = await withPage(
+    {
+      // Tall viewport so lazy-loaded article bodies render before extraction.
+      viewport: { width: 1280, height: 2200 },
+      // CSP-strict sites (Medium) block addScriptTag; bypass so Readability injects.
+      bypassCSP: true,
+      // Look like a real browser past anti-bot walls (Cloudflare on Medium).
+      stealth: true,
+      ...(opts.userAgent ? { userAgent: opts.userAgent } : {}),
+      prepareContext: opts.storageStatePath
+        ? authPrepare({ adapter: 'storageState', statePath: opts.storageStatePath })
+        : undefined,
+    },
+    async (page) => {
+      await page.goto(url, { waitUntil: 'domcontentloaded', timeout });
+      // Let late content settle, but don't fail if the page never goes idle
+      // (Medium/analytics keep long-lived connections open).
+      await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});
+      await page.addScriptTag({ content: READABILITY_SRC });
+      return page.evaluate(() => {
+        // Readability was injected as a page global by addScriptTag.
+        const R = (globalThis as unknown as { Readability?: new (doc: Document, opts?: unknown) => { parse(): { title: string; content: string; byline?: string; length?: number; excerpt?: string } | null } }).Readability;
+        if (!R) return null;
+        const clone = document.cloneNode(true) as Document;
+        const result = new R(clone, { keepClasses: false, charThreshold: 0 }).parse();
+        if (!result) return null;
+        return {
+          title: result.title,
+          content: result.content,
+          byline: result.byline ?? null,
+          length: result.length ?? 0,
+          excerpt: result.excerpt ?? null,
+        };
+      });
+    },
+  );
+
+  if (!article || !article.content) {
+    throw new Error(
+      'extraction failed — Readability found no article content (page may require auth, be JS-gated, or not be an article)',
+    );
+  }
+
+  const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced', bulletListMarker: '-' });
+  turndown.addRule('strip-noise', {
+    filter: ['script', 'style', 'noscript', 'iframe', 'nav', 'footer', 'header'],
+    replacement: () => '',
+  });
+  const markdown = turndown.turndown(article.content).trim();
+  if (!markdown) throw new Error('extraction produced empty markdown');
+
+  return {
+    url,
+    title: article.title || parsed.hostname,
+    markdown,
+    byline: article.byline,
+    length: article.length,
+    excerpt: article.excerpt,
+  };
+}

package/src/seed-storage.ts

@@ -0,0 +1,149 @@
+// F074.97 — seed localStorage / sessionStorage BEFORE the first navigation.
+//
+// A whole class of bug is otherwise unphotographable by our own instrument:
+// anything whose input is state the browser held BEFORE the page loaded. Lens
+// could drive, fill, click and measure — but every run started from a clean
+// profile, so a restored tab strip, a remembered filter, an unsent draft, a
+// dismissed banner, or F178.4's seven-day prune (whose entire input is a
+// persisted timestamp) could only be verified by a unit fixture, while the
+// report read as if a browser had confirmed it.
+//
+// THE ORDERING IS THE WHOLE FEATURE. Writing the value AFTER navigation is the
+// trap this card exists to avoid: the app has already booted on an empty store,
+// so the run photographs the DEFAULT state while reporting that it seeded the
+// stale one — green, wrong, and indistinguishable from success. addInitScript
+// runs at document-start in every document the context loads, which is before
+// any app script.
+//
+// ORIGIN-GUARDED, AND THE CALLER DOES NOT NAME THE ORIGIN. An init script runs
+// in EVERY document, including a cross-origin iframe and a redirect to
+// checkout.stripe.com. The origin is derived from the run's own target URL, so
+// a seed cannot be aimed at somebody else's site even by mistake.
+
+import type { BrowserContext } from 'playwright';
+
+export interface StorageSeed {
+  /** localStorage keys to write for the run's own origin. */
+  local?: Record<string, unknown>;
+  /** sessionStorage keys to write for the run's own origin. */
+  session?: Record<string, unknown>;
+}
+
+/** The shape addInitScript receives — everything already stringified, so the
+ *  page-side function is trivial and has nothing left to decide. */
+export interface StoragePayload {
+  origin: string;
+  local: Array<[string, string]>;
+  session: Array<[string, string]>;
+}
+
+/** localStorage stores STRINGS. A non-string value is JSON-stringified rather
+ *  than left to `String()`, because the realistic payload (a tab list, a filter,
+ *  a saved draft) IS json and `String({})` yields "[object Object]" — a value the
+ *  app would fail to parse, from a seed that reported success. */
+function toEntries(rec: Record<string, unknown> | undefined): Array<[string, string]> {
+  if (!rec) return [];
+  return Object.entries(rec).map(([k, v]) => [k, typeof v === 'string' ? v : JSON.stringify(v)]);
+}
+
+/** Tolerate an MCP harness that stringified the object param (the same coercion
+ *  coerceViewport exists for). Anything unparseable is refused with a reason —
+ *  a seed silently dropped would put us back in the false-green this card is
+ *  about, one layer further in. */
+export function coerceStorageSeed(raw: unknown): StorageSeed | undefined {
+  if (raw == null) return undefined;
+  let value: unknown = raw;
+  if (typeof value === 'string') {
+    const trimmed = value.trim();
+    if (!trimmed) return undefined;
+    try {
+      value = JSON.parse(trimmed) as unknown;
+    } catch {
+      throw new Error(
+        'storage must be an object like {local:{key:"value"}} (or its JSON text) — ' +
+          `could not parse: ${trimmed.slice(0, 80)}`,
+      );
+    }
+  }
+  if (typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error('storage must be an object like {local:{…}, session:{…}}');
+  }
+  const o = value as Record<string, unknown>;
+  const bad = Object.keys(o).filter((k) => k !== 'local' && k !== 'session');
+  if (bad.length) {
+    throw new Error(
+      `storage accepts only "local" and "session" — unknown key(s): ${bad.join(', ')}. ` +
+        'Silently ignoring one would report a seeded run that never seeded anything.',
+    );
+  }
+  const seed: StorageSeed = {};
+  if (o.local != null) {
+    if (typeof o.local !== 'object' || Array.isArray(o.local)) throw new Error('storage.local must be an object of key → value');
+    seed.local = o.local as Record<string, unknown>;
+  }
+  if (o.session != null) {
+    if (typeof o.session !== 'object' || Array.isArray(o.session)) throw new Error('storage.session must be an object of key → value');
+    seed.session = o.session as Record<string, unknown>;
+  }
+  return seed;
+}
+
+/** Build the payload for a seed against a run's target URL, or null when there
+ *  is nothing to write. Null is what keeps the NEGATIVE CONTROL honest: no
+ *  payload → no init script → a context byte-identical to every run before this
+ *  option existed, so no baseline in the fleet can move. */
+export function storagePayload(seed: StorageSeed | undefined, targetUrl: string): StoragePayload | null {
+  if (!seed) return null;
+  const local = toEntries(seed.local);
+  const session = toEntries(seed.session);
+  if (!local.length && !session.length) return null;
+  let origin: string;
+  try {
+    origin = new URL(targetUrl).origin;
+  } catch {
+    throw new Error(
+      `storage seeding needs an absolute target url to derive its origin from — got "${targetUrl}". ` +
+        'The origin is derived, never passed, so a seed cannot land on a third-party site.',
+    );
+  }
+  if (origin === 'null') {
+    throw new Error(`storage seeding cannot target an opaque origin (${targetUrl}) — it has no storage to write.`);
+  }
+  return { origin, local, session };
+}
+
+/** The page-side function, exported so the ordering can be asserted without a
+ *  browser. It runs at document-start in every document; the origin guard is
+ *  what stops it writing into a cross-origin iframe or a payment redirect.
+ *  Storage access THROWS in a sandboxed/partitioned document, so each write is
+ *  guarded — a seed that cannot land must not take the whole run with it. */
+export function seedStorageInPage(payload: StoragePayload): void {
+  try {
+    if (location.origin !== payload.origin) return;
+  } catch {
+    return;
+  }
+  for (const [k, v] of payload.local) {
+    try {
+      localStorage.setItem(k, v);
+    } catch {
+      /* storage unavailable in this document — nothing to seed */
+    }
+  }
+  for (const [k, v] of payload.session) {
+    try {
+      sessionStorage.setItem(k, v);
+    } catch {
+      /* ditto */
+    }
+  }
+}
+
+/** Register the seed on a context. Called by withPage AFTER prepareContext, so a
+ *  per-run key outranks the same key inside an auth storageState blob: init
+ *  scripts run in registration order, and a per-call seed is the more specific
+ *  instruction. */
+export async function applyStorageSeed(context: BrowserContext, payload: StoragePayload | null): Promise<void> {
+  if (!payload) return;
+  await context.addInitScript(seedStorageInPage, payload);
+}