@broberg/lens-daemon 0.1.15 → 0.1.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@broberg/lens-daemon",
-  "version": "0.1.15",
+  "version": "0.1.18",
   "description": "The Lens module of the cardmem daemon (flows, critic, coverage, capture sessions, the /lens/* routes and the cardmem-lens MCP) as a package. The daemon mounts createLensRouter(deps) + createLensMcpRouter(deps). Bun runtime.",
   "type": "module",
   "license": "MIT",
@@ -15,8 +15,8 @@
     "sharp": "^0.33.5",
     "turndown": "^7.2.4",
     "zod": "^3.23.8",
-    "@broberg/lens-svg": "0.1.0",
-    "@broberg/lens-engine": "0.13.1"
+    "@broberg/lens-engine": "0.13.1",
+    "@broberg/lens-svg": "0.1.0"
   },
   "devDependencies": {
     "@happy-dom/global-registrator": "^20.10.6",

package/src/extension-flow.ts

@@ -88,16 +88,77 @@ export const SIMULATED_ACCESS_NOTE =
  *  Read back and compared, so a seed that did not land fails the run here instead
  *  of showing up later as a popup that is mysteriously logged out. */
 export async function seedExtensionStorage(context: BrowserContext, values: Record<string, unknown>): Promise<void> {
-  const target = context.serviceWorkers()[0] ?? context.backgroundPages()[0];
-  if (!target) throw new Error('storage_local: the extension has no service worker or background page to write chrome.storage.local from');
   const keys = Object.keys(values);
-  const back = await target.evaluate(async (v: Record<string, unknown>) => {
-    const c = (globalThis as unknown as { chrome: { storage?: { local: { set(x: object): Promise<void>; get(k: string[]): Promise<Record<string, unknown>> } } } }).chrome;
-    if (!c.storage) return null;
-    await c.storage.local.set(v);
-    return c.storage.local.get(Object.keys(v));
-  }, values);
+  // An MV3 service worker can be stopped and restarted by Chrome at any moment, and an
+  // evaluate on the stopped one never settles — observed as a 90 s hang under a loaded
+  // test suite, never in isolation. So every attempt has a deadline, and each retry asks
+  // the context for the worker that is live NOW instead of reusing the old handle.
+  let back: Record<string, unknown> | null | undefined;
+  let lastError = 'no attempt made';
+  for (let attempt = 1; attempt <= SEED_ATTEMPTS && back === undefined; attempt++) {
+    const target =
+      context.serviceWorkers()[0] ??
+      context.backgroundPages()[0] ??
+      (await context.waitForEvent('serviceworker', { timeout: SEED_DEADLINE_MS }).catch(() => undefined));
+    if (!target) {
+      lastError = 'no service worker or background page to write chrome.storage.local from';
+      continue;
+    }
+    try {
+      back = await deadline(
+        target.evaluate(async (v: Record<string, unknown>) => {
+          const c = (globalThis as unknown as { chrome: { storage?: { local: { set(x: object): Promise<void>; get(k: string[]): Promise<Record<string, unknown>> } } } }).chrome;
+          if (!c.storage) return null;
+          await c.storage.local.set(v);
+          return c.storage.local.get(Object.keys(v));
+        }, values),
+        SEED_DEADLINE_MS,
+      );
+    } catch (err) {
+      lastError = err instanceof Error ? err.message : String(err);
+    }
+  }
+  if (back === undefined) throw new Error(`storage_local: could not write chrome.storage.local after ${SEED_ATTEMPTS} attempts — ${lastError}`);
   if (!back) throw new Error('storage_local: the extension has no chrome.storage — its manifest does not request the "storage" permission, so there is nowhere to seed');
-  const missing = keys.filter((k) => JSON.stringify(back[k]) !== JSON.stringify(values[k]));
+  const missing = keys.filter((k) => !sameJson(back![k], values[k]));
   if (missing.length) throw new Error(`storage_local: chrome.storage.local did not keep ${missing.join(', ')} — read back differs from what was written`);
 }
+
+const SEED_ATTEMPTS = 3;
+const SEED_DEADLINE_MS = 5_000;
+
+/** Reject after `ms` instead of waiting forever on a call that will never settle. */
+export function deadline<T>(p: Promise<T>, ms: number): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    const t = setTimeout(() => reject(new Error(`no answer within ${ms} ms`)), ms);
+    t.unref?.();
+    p.then(
+      (v) => {
+        clearTimeout(t);
+        resolve(v);
+      },
+      (e) => {
+        clearTimeout(t);
+        reject(e);
+      },
+    );
+  });
+}
+
+/** F009.127 — structural equality for JSON values, blind to object key ORDER.
+ *  chrome.storage hands an object back with its keys reordered, so comparing
+ *  JSON text refused a login object that had landed perfectly. Array order and
+ *  every value still count. Exported for test. */
+export function sameJson(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (typeof a !== typeof b || a === null || b === null || typeof a !== 'object') return false;
+  if (Array.isArray(a) !== Array.isArray(b)) return false;
+  if (Array.isArray(a)) {
+    const bb = b as unknown[];
+    return a.length === bb.length && a.every((v, i) => sameJson(v, bb[i]));
+  }
+  const ao = a as Record<string, unknown>;
+  const bo = b as Record<string, unknown>;
+  const ak = Object.keys(ao);
+  return ak.length === Object.keys(bo).length && ak.every((k) => Object.hasOwn(bo, k) && sameJson(ao[k], bo[k]));
+}

package/src/flow-store.ts

@@ -9,7 +9,7 @@ import { randomUUID } from 'node:crypto';
 import type { LensFlowRunRow } from './schema.ts';
 import { artifactKeys, type LensStorage } from './storage.ts';
 import { enqueueArtifacts } from './artifact-uploads.ts';
-import type { FlowResult, FailedResponse } from './flow.ts';
+import type { FlowDownload, FlowResult, FailedResponse } from './flow.ts';
 
 /** A per-step entry as persisted (screenshot bytes already offloaded to an
  *  artifact key — null when the step took no shot). */
@@ -110,6 +110,18 @@ export async function recordFlowRun(
     await storage.put(videoKey, result.video);
   }
 
+  // F009.128 — downloaded files go where the stills go; the row keeps only the key.
+  const storedDownloads: Array<Omit<FlowDownload, 'bytes'> & { key?: string }> = [];
+  for (const [n, d] of (result.downloads ?? []).entries()) {
+    const { bytes, ...meta } = d;
+    let key: string | undefined;
+    if (bytes) {
+      key = artifactKeys.flowDownload(projectSlug, id, n, d.name);
+      await storage.put(key, bytes);
+    }
+    storedDownloads.push({ ...meta, ...(key ? { key } : {}) });
+  }
+
   const row: LensFlowRunRow = {
     id,
     project_slug: projectSlug,
@@ -157,6 +169,8 @@ export async function recordFlowRun(
       ...(result.extension ? { extension: result.extension } : {}),
       ...(result.password_in_url ? { password_in_url: result.password_in_url } : {}),
       ...(result.not_listening ? { not_listening: result.not_listening } : {}),
+      // F009.128 — always written when the run reported it, empty included.
+      ...(result.downloads ? { downloads: storedDownloads } : {}),
     }),
   };
 

package/src/flow.ts

@@ -24,7 +24,7 @@ import {
   type Target,
 } from '@broberg/lens-engine';
 import { withPage, type LensEngine } from './browser.ts';
-import { defaultPopupOf, prepareExtension, seedExtensionStorage, SIMULATED_ACCESS_NOTE } from './extension-flow.ts';
+import { deadline, defaultPopupOf, prepareExtension, seedExtensionStorage, SIMULATED_ACCESS_NOTE } from './extension-flow.ts';
 import { authPrepare, type AuthConfig, type AuthEnv } from './auth/index.ts';
 import { takeShot, settleForShot } from './capture.ts';
 import { runCritic, topFindingMessage, type CriticMode, type CriticReport } from './critic.ts';
@@ -359,7 +359,36 @@ export class InstrumentError extends Error {
   }
 }
 
+/** F009.128 — one file the run's browser downloaded. `bytes` is dropped once the
+ *  store has written it; a reader gets `url` instead. */
+export interface FlowDownload {
+  name: string;
+  size: number;
+  /** Which page triggered it: the page under test, the extension popup, or another. */
+  page: 'page' | 'popup' | 'other';
+  /** Index of the step that was running when it started; null before the first. */
+  step: number | null;
+  bytes?: Buffer;
+  error?: string;
+  /** Not kept because a limit was hit (MAX_DOWNLOADS / MAX_DOWNLOAD_BYTES). */
+  skipped?: 'cap';
+}
+export const MAX_DOWNLOADS = 10;
+export const MAX_DOWNLOAD_BYTES = 25 * 1024 * 1024;
+const DOWNLOAD_SETTLE_MS = 5_000;
+const DOWNLOAD_FINISH_MS = 30_000;
+
+/** F009.128 — wait for the run's downloads, bounded. Whatever has not finished by
+ *  then is marked as such, never dropped and never waited on forever. Exported for test. */
+export async function finishDownloads(pending: Promise<void>[], downloads: FlowDownload[], ms: number): Promise<void> {
+  await deadline(Promise.allSettled(pending), ms).catch(() => {
+    for (const d of downloads) if (!d.bytes && !d.error && !d.skipped) d.error = `did not finish within ${ms} ms`;
+  });
+}
+
 export interface FlowResult {
+  /** F009.128 — files the run downloaded. Always present on a finished run. */
+  downloads?: FlowDownload[];
   /** F009.125 — the extension this run loaded: its id (read at load time), the
    *  build path the caller named, and whether host access was SIMULATED on a copy. */
   extension?: { id: string; path: string; host_access: 'as-built' | 'simulated'; note?: string } | null;
@@ -511,6 +540,8 @@ function requestRegex(u: string): RegExp {
 export interface SeenRequest {
   method: string;
   url: string;
+  /** Playwright's resource type; 'document' is a navigation (and what a link download is). */
+  type?: string;
   /** null while in flight, or when it failed before a response. */
   status: number | null;
   failed?: string;
@@ -1562,6 +1593,14 @@ export async function clickAndActivate(
     if (t === 'xhr' || t === 'fetch' || t === 'document') requests += 1;
   };
   page.on('request', onRequest);
+  // F009.128 — a download IS an effect. A button whose only job is a download
+  // (a blob handed to <a download>) touches no DOM and makes no request, so it
+  // read as a silent no-op and got pressed twice: two files for one click.
+  let downloaded = false;
+  const onDownload = () => {
+    downloaded = true;
+  };
+  page.on('download', onDownload);
   const urlBefore = page.url();
   await page
     .evaluate(() => {
@@ -1606,7 +1645,7 @@ export async function clickAndActivate(
       const mutations = await page
         .evaluate(() => (window as unknown as { __lensMut?: number }).__lensMut ?? 0)
         .catch(() => 1); // page navigated away → that IS an effect
-      if (mutations > 0 || requests > 0 || page.url() !== urlBefore) return false;
+      if (mutations > 0 || requests > 0 || downloaded || page.url() !== urlBefore) return false;
       const focusMoved = await page
         .evaluate(
           ([before, clicked]) => {
@@ -1629,6 +1668,7 @@ export async function clickAndActivate(
     return true;
   } finally {
     page.off('request', onRequest);
+    page.off('download', onDownload);
     await focusBefore.dispose().catch(() => {});
     await target?.dispose().catch(() => {});
     await page
@@ -2741,6 +2781,7 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
       // F213.11 — read BEFORE the step runs, so the number is "how long after the
       // page loaded did this start", never "how long did it take".
       const stepStartedAt = Date.now();
+      currentStep = i;
       const msAfterNav = stepStartedAt - lastNavAt;
       // F213.13 — a goto's reading is measured from the PREVIOUS navigation, so it
       // is not the same quantity as its siblings and must not sit in their column.
@@ -2928,12 +2969,58 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
     extCtx.popup = popup;
     return null;
   };
+  // F009.128 — files the run downloads. A download is awaited before the context
+  // closes; otherwise the bytes die with the browser and the report says nothing.
+  const downloads: FlowDownload[] = [];
+  const pendingDownloads: Promise<void>[] = [];
+  let currentStep: number | null = null;
+  const watchDownloads = (context: BrowserContext): void => {
+    const hooked = new WeakSet<Page>();
+    const hook = (p: Page) => {
+      if (hooked.has(p)) return;
+      hooked.add(p);
+      p.on('download', (d) => {
+        const entry: FlowDownload = {
+          name: d.suggestedFilename(),
+          size: 0,
+          page: p === extCtx?.popup ? 'popup' : p === extCtx?.main || p === videoPage ? 'page' : 'other',
+          step: currentStep,
+        };
+        downloads.push(entry);
+        if (downloads.length > MAX_DOWNLOADS) {
+          entry.skipped = 'cap';
+          pendingDownloads.push(d.cancel().catch(() => {}));
+          return;
+        }
+        pendingDownloads.push(
+          (async () => {
+            const failure = await d.failure();
+            // `??=`: once the run gave up waiting, the context closing cancels the
+            // download — and 'canceled' would hide that it simply never finished.
+            if (failure) {
+              entry.error ??= failure;
+              return;
+            }
+            const path = await d.path();
+            const bytes = await readFile(path);
+            entry.size = bytes.length;
+            if (bytes.length > MAX_DOWNLOAD_BYTES) entry.skipped = 'cap';
+            else entry.bytes = bytes;
+          })().catch((err) => {
+            entry.error ??= err instanceof Error ? err.message : String(err);
+          }),
+        );
+      });
+    };
+    context.pages().forEach(hook);
+    context.on('page', hook);
+  };
   // F009.126 — every request the run's main browser makes, from the first step on.
   const seenRequests: SeenRequest[] = [];
   const watchRequests = (context: BrowserContext): void => {
     const byReq = new Map<object, SeenRequest>();
     context.on('request', (req) => {
-      const r: SeenRequest = { method: req.method(), url: req.url(), status: null };
+      const r: SeenRequest = { method: req.method(), url: req.url(), status: null, type: req.resourceType() };
       byReq.set(req, r);
       seenRequests.push(r);
     });
@@ -3017,6 +3104,7 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
         videoPage = page;
         collectFailures(page);
         watchRequests(context);
+        watchDownloads(context);
         if (ext && flow.extension && prepared) {
           extCtx = { context, id: ext.id, main: page, popup: null };
           if (flow.extension.storage_local) await seedExtensionStorage(context, flow.extension.storage_local);
@@ -3118,6 +3206,22 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
         } catch {
           // page already closed — final_url stays null (not measured)
         }
+        // F009.128 — before the context (and the files) go away. A link download is a
+        // navigation whose 'download' event fires only once its headers arrive, which can
+        // be AFTER the last step returned; so wait for navigations still in flight (bounded)
+        // and only then for the downloads they turned into. Runs with nothing in flight
+        // pay nothing.
+        // A navigation Chromium turns into a download is ABORTED as a request
+        // (net::ERR_ABORTED) and only then raises 'download' — so count those too.
+        const settleUntil = Date.now() + DOWNLOAD_SETTLE_MS;
+        const navInFlight = () => seenRequests.some((r) => r.type === 'document' && r.status === null && !r.failed);
+        const becameDownload = () => seenRequests.filter((r) => r.type === 'document' && (r.failed ?? '').includes('ERR_ABORTED')).length;
+        while (Date.now() < settleUntil && (navInFlight() || downloads.length < becameDownload())) {
+          await new Promise((res) => setTimeout(res, 100));
+        }
+        // Bounded: a download that never finishes must not hang the run. Whatever is
+        // still unfinished is reported as such, never dropped.
+        await finishDownloads(pendingDownloads, downloads, DOWNLOAD_FINISH_MS);
       },
     );
   } catch (err) {
@@ -3125,7 +3229,7 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
     // Keep the recording even on a crash — watching HOW it broke is the whole point.
     const video = await readVideo();
     await prepared?.cleanup();
-    return { ...(extensionInfo ? { extension: extensionInfo } : {}), name: flow.name, status: 'error', steps: results.sort((a, b) => a.index - b.index), failed_at: firstFailedAt, reason: leadWithRedirect(reason, firstRedirect), duration_ms: Date.now() - startedAt, video, build: observedBuild, final_url: finalUrl, redirected: firstRedirect, password_in_url: passwordInUrl(finalUrl, filledPasswords), not_listening: notListening[0] ?? null, reduced_motion: opts.reducedMotion ?? 'reduce' };
+    return { downloads, ...(extensionInfo ? { extension: extensionInfo } : {}), name: flow.name, status: 'error', steps: results.sort((a, b) => a.index - b.index), failed_at: firstFailedAt, reason: leadWithRedirect(reason, firstRedirect), duration_ms: Date.now() - startedAt, video, build: observedBuild, final_url: finalUrl, redirected: firstRedirect, password_in_url: passwordInUrl(finalUrl, filledPasswords), not_listening: notListening[0] ?? null, reduced_motion: opts.reducedMotion ?? 'reduce' };
   }
 
   const video = await readVideo();
@@ -3179,6 +3283,7 @@ export async function runFlow(flow: Flow, env: AuthEnv, opts: RunFlowOptions = {
     // context can never disagree about it.
     reduced_motion: opts.reducedMotion ?? 'reduce',
     ...(extensionInfo ? { extension: extensionInfo } : {}),
+    downloads,
   };
 }
 

package/src/service.ts

@@ -666,11 +666,16 @@ function parseAuthJson(raw: string | null | undefined): { source: string; adapte
 /** NULL = the row predates the column, so we have no answer; `{}` = the run
  *  looked and found nothing. Malformed JSON is reported as unknown rather than
  *  as "clean": a parse failure must never read as an all-clear. */
-function parseFindingsJson(raw: string | null | undefined): Record<string, unknown> | null {
+function parseFindingsJson(raw: string | null | undefined, port?: number): Record<string, unknown> | null {
   if (raw == null) return null;
   try {
     const v = JSON.parse(raw);
-    return v && typeof v === 'object' ? (v as Record<string, unknown>) : null;
+    if (!v || typeof v !== 'object') return null;
+    // F009.128 — a stored download carries its storage key; a reader gets a url.
+    if (port !== undefined && Array.isArray(v.downloads)) {
+      v.downloads = v.downloads.map(({ key, ...d }: { key?: string }) => (key ? { ...d, url: artifactUrl(port, key) } : d));
+    }
+    return v as Record<string, unknown>;
   } catch {
     return null;
   }
@@ -748,7 +753,7 @@ export function flowRunView(row: LensFlowRunRow, port: number) {
     // and lens_list_flow_runs for the string and found neither — "the instrument
     // measures right and nobody is told", which is the same shape as the probe
     // that sat unwired to `fill` for a month.
-    findings: parseFindingsJson(row.findings_json),
+    findings: parseFindingsJson(row.findings_json, port),
     steps,
     // F255.5 — present only when the run was recorded. Same artifact mechanism as
     // the step screenshots, so a consumer fetches it exactly the same way.
@@ -869,7 +874,7 @@ export function toCloudFlowResult(result: EngineFlowResult, row: LensFlowRunRow,
     // (a CI runner with no daemon) is the consumer LEAST able to go and look
     // somewhere else, so leaving them off this shape would be the gap at its
     // worst.
-    findings: parseFindingsJson(row.findings_json),
+    findings: parseFindingsJson(row.findings_json, port),
     steps: result.steps.map((s) => ({
       index: s.index,
       action: s.action,

package/src/storage.ts

@@ -118,6 +118,9 @@ export const artifactKeys = {
     `${project}/flows/${flowRunId}/step-${stepIndex}.png`,
   // F255.5 — the whole driven run as a video, alongside its step stills.
   flowVideo: (project: string, flowRunId: string) => `${project}/flows/${flowRunId}/run.webm`,
+  // F009.128 — a file the run downloaded. Numbered so two files with one name both survive.
+  flowDownload: (project: string, flowRunId: string, n: number, name: string) =>
+    `${project}/flows/${flowRunId}/download-${n}-${name.replace(/[^\w.-]+/g, '_').slice(0, 120)}`,
   // F074.63 — a vector export (PDF). Its own prefix, not runs/: an export is not
   // a capture, carries no baseline and must never be diffed against one.
   exportDoc: (project: string, exportId: string, ext: string) =>