@ultimat3/scraping 22.15.0 → 24.0.0

package/README.md

@@ -6,10 +6,8 @@ no ninth primitive, and no ordinal here to go stale when the next factory lands.
 
 ```ts
 import { t } from '@ultimat3/schema';
-import type { StorageDriver } from '@ultimat3/storage';
 import { scrape, storageSessionStore } from '@ultimat3/scraping';
-
-declare const disk: StorageDriver;
+import { disk } from '@ultimat3/storage';
 
 const orderPage = t.object({
   rows: t.array(t.object({ id: t.string, total: t.number })),
@@ -29,7 +27,8 @@ export const dailyOrders = scrape({
   secrets: ['SHOP_PASSWORD'],
   expect: { minRows: 1, maxDrop: 0.5 },
   auth: {
-    store: storageSessionStore(disk),
+    // A thunk: the app's disks exist once boot ran `defineStorage()`, after this module loaded.
+    store: storageSessionStore(() => disk('sessions')),
     login: async ({ page, secrets, prompt }) => {
       await page.goto('https://shop.example.com/login');
       await page.fill('#user', 'ops@example.com');
@@ -85,6 +84,168 @@ Both legs replay from **one** fixture directory (`fixtureBrowser(dir)`), so a hy
 login, session handoff, HTTP bulk fetch — is tested end to end. Both legs apply the same robots
 gate, offline included.
 
+## A session a service is built on
+
+One connection, one run at a time, on its own exit, in a browser rented for that run — with a
+person or an OTP source answering the site mid-run and a `usage` block on the result. Every piece
+is a field on `scrape()` or on its driver; there is no second factory.
+
+```ts
+import { t } from '@ultimat3/schema';
+import type {
+  CdpLauncherLike,
+  CdpResolver,
+  PromptRequest,
+  ScrapeDriver,
+  ScrapeUsage,
+} from '@ultimat3/scraping';
+import {
+  answerPrompt,
+  eventPrompt,
+  fixtureBrowser,
+  remoteBrowser,
+  scrape,
+  storageSessionStore,
+} from '@ultimat3/scraping';
+import { disk } from '@ultimat3/storage';
+
+declare const puppeteer: CdpLauncherLike;
+/** Set where recordings stand in for the browser: `bun test`, `x dev`, the app's own gate. */
+declare const fixtures: string | undefined;
+/** The app's own provider client, and its own "a prompt is pending" row. Neither ships. */
+declare const provider: {
+  create(request: {
+    tag: string | undefined;
+    proxy: string | undefined;
+    signal: AbortSignal | undefined;
+  }): Promise<{ id: string; connectUrl: string; priceCents: number }>;
+  end(id: string): Promise<void>;
+};
+declare function promptPending(request: PromptRequest): Promise<void>;
+/** The app's own rows: the exit is a `.sealed()` column on the connection, read by id. */
+declare function exitOf(connectionId: string): Promise<string | undefined>;
+declare function recordRun(run: {
+  runId: string;
+  connectionId: string;
+  outcome: string;
+  code?: string | undefined;
+  usage?: ScrapeUsage | undefined;
+}): Promise<void>;
+
+// A browser rented per session. `release` runs exactly once, whatever the run did.
+const rent: CdpResolver = async ({ runId, egress, signal }) => {
+  const rented = await provider.create({ tag: runId, proxy: egress, signal });
+  return {
+    cdpUrl: rented.connectUrl,
+    release: () => provider.end(rented.id),
+    cost: { minor: rented.priceCents, currency: 'USD' },
+  };
+};
+
+const browser: ScrapeDriver =
+  fixtures === undefined
+    ? remoteBrowser({ launcher: puppeteer, cdpUrl: rent })
+    : fixtureBrowser(fixtures);
+
+const waitForAnswer = eventPrompt({ timeout: 300_000 });
+
+export const syncAccounts = scrape({
+  name: 'accounts.sync',
+  // IDS only. A proxy URL carries its account, and the input is the queue row.
+  input: t.object({ connectionId: t.uuid, orgId: t.uuid }),
+  extract: t.object({ id: t.string }),
+  idempotencyKey: ({ connectionId }) => `accounts:${connectionId}`,
+  tenant: ({ orgId }) => orgId,
+  allowHosts: ['bank.example'],
+  driver: browser,
+  // The exit THIS run leaves through, looked up IN THE WORKER under the job's tenant. Both legs
+  // and the robots read dial it.
+  egress: ({ connectionId }) => exitOf(connectionId),
+  // One session per connection: a second run of the same one is `failed`, X_JOB_KEY_BUSY.
+  concurrency: { key: ({ connectionId }) => connectionId, limit: 1, whenBusy: 'fail' },
+  auth: {
+    // Declaring a store IS what persists the session — sealed under the app's master key.
+    store: storageSessionStore(() => disk('sessions')),
+    key: ({ connectionId }) => connectionId,
+    login: async ({ page, prompt }) => {
+      await page.goto('https://bank.example/login');
+      await page.fill('#otp', await prompt('sms code'));
+    },
+  },
+  // Say a prompt is pending, then wait for its answer on the job event bus — browser open.
+  prompt: async (request) => {
+    await promptPending(request);
+    return waitForAnswer(request);
+  },
+  async run({ page, http }) {
+    await page.goto('https://bank.example/accounts');
+    const body = await (await http.request('https://bank.example/api/accounts')).parse(
+      t.object({ rows: t.array(t.object({ id: t.string })) }),
+    );
+    return body.rows;
+  },
+  // How the run ENDED, told once: the report with its usage, or the code it failed with.
+  async onSettled(settled) {
+    if (settled.input === undefined) return;
+    const run = { runId: settled.runId, connectionId: settled.input.connectionId };
+    await recordRun(
+      settled.outcome === 'completed'
+        ? { ...run, outcome: settled.outcome, usage: settled.result.usage }
+        : { ...run, outcome: settled.outcome, code: settled.code, usage: settled.usage },
+    );
+  },
+});
+
+/** The body of the app's own policy-checked action: the event name is not an authorization. */
+export const answer = (input: { runId: string; index: number; answer: string }) =>
+  answerPrompt(input);
+```
+
+| Field | Rule |
+|---|---|
+| `egress` | `(input, ctx) => string \| undefined`, sync or async, called in the WORKER under the job's tenant — so the input carries an id and a row carries the exit. An exit whose password is also in the input rode the queue payload and is refused before the browser opens: `X_SCRAPE_EGRESS_IN_PAYLOAD`, terminal. Reaches the driver as `SessionInit.proxy` and wins over the driver's `proxy`. A URL with credentials never reaches the process arguments: a launched browser answers the proxy through `page.authenticate()`, and a launcher without it is `X_SCRAPE_EGRESS_UNSUPPORTED` |
+| `remoteBrowser({ cdpUrl })` | a fixed URL, or a `CdpResolver`: called once per `open()` with `{ scrape, runId, egress, signal }`; `release()` runs once — in `close()` after the browser is quit, or at once when the attach fails. A `release()` that throws is logged `scrape.browser.release_failed` (`code`, `driver`, `origin` as scheme and host) through `SessionInit.logger`, which every driver now requires. A fixed URL handed a different exit is `X_SCRAPE_EGRESS_UNSUPPORTED` |
+| `concurrency` | the job's own field, passed through: a number, or `{ key, limit, whenBusy }` |
+| `auth.store` | `storageSessionStore(() => disk('sessions'), { prefix?, keySource? })` — a THUNK, read per call: a scrape is declared when its module loads and the app's disks exist after boot. Seals with core's `seal()`, purpose `SESSION_SEAL_PURPOSE`. Anything in the bucket that is not its own sealed record for that key is burned; no master key is `X_SEAL_KEY_MISSING` before the browser opens. `artifacts.storage` is the same thunk |
+| `onSettled` | the job's own hook (`@ultimat3/jobs`), with what a scrape adds: `completed` hands over the `ScrapeReport`; `dead-lettered` / `dropped` / `refused` carry `code` and the last attempt's `usage` (`undefined` for `refused` — the body never ran). After the row is settled, under the job's tenant, **at most once** across a crash, and a hook that throws changes nothing |
+| `run({ progress, finalAttempt })` | the job's own, passed through |
+| `prompt` | `eventPrompt({ timeout, pollMs?, bus?, keySource?, env? })` polls the event bus in process — never `step.waitForEvent`, which would close the browser. An in-memory bus (`stored: false`) is refused outside development and test (`X_DRIVER_UNAVAILABLE`): the answer comes from another process. Event `promptEventName(runId, index)`; answered by `answerPrompt({ runId, index, answer })`, sealed for that one name. Timeout is `X_SCRAPE_PROMPT_UNANSWERED` |
+| the bus | the **stored** one (`createPgEventBus`) wherever the worker and the answering process differ. Every `x` boot installs it |
+
+`PromptRequest` carries the run's `input` and `runId` — what ties a prompt to the app's own row
+for the run — and `index`, `clock`, `signal` and `keepAlive()` beside `label`, `scrape` and `url`:
+what any handler that waits needs: wait on the run's clock, stop on the run's
+signal, and call `keepAlive()` between polls so the wedge watchdog and a rented browser both keep
+seeing activity. `auth.login` and `auth.validate` are handed `runId` beside `input`.
+
+`ScrapeReport.usage` is `{ browserMs, navigations, httpRequests, bytesIn, promptsAnswered,
+browserCost? }`. `bytesIn` is the HTTP leg's response bodies; `browserCost` is the resolver's
+`cost`, a `Money` value parsed at the boundary, so a float is refused. The counts ride the
+`scrape.ok` and `scrape.failed` log lines too — a failed run has no report and was billed anyway,
+which is why `onSettled` hands a failed run its last attempt's `usage`. No quota reads them.
+
+### The clock, and a test's
+
+A run waits on the PROCESS's clock — `scrapeClock()`, the system's — unless its definition pins
+one with `clock:`. A test replaces it the way it replaces the queue driver:
+
+```ts
+import { noWaitClock, resetScrapeClock, setScrapeClock } from '@ultimat3/scraping';
+
+setScrapeClock(noWaitClock); // in a beforeEach
+resetScrapeClock(); // in the afterEach
+```
+
+| Clock | Sleeping | For |
+|---|---|---|
+| `systemScrapeClock` | waits | production |
+| `noWaitClock` | one turn of the event loop, whatever was asked; `now()` and deadlines are real | a test whose run waits on something that really happens in the same process — an answer the test publishes |
+| `testClock()` | advances virtual time and yields a microtask | a test of a timeout: thirty seconds pass in none |
+
+A third-party driver takes part by reading three `SessionInit` fields: dial `proxy` or throw
+`egressUnsupported()`, pass `usage` to `pageOverTarget` and its HTTP transport, and answer
+`browserCost` on the session it returns.
+
 ## Reading elements, frames, and the network condition
 
 | Read | Answers |
@@ -152,8 +313,9 @@ bound threw away, so a count taken from one is never quietly a floor.
 All three are **redacted by value** on the way out, the same pass `page.html()` makes: a login
 endpoint fetched with the password in its query string, or a site that logs the credential it
 rejected, would otherwise put the value in `page.network()`/`page.console()` verbatim and from
-there into the stored failure artifact. The HTTP leg's `X_SCRAPE_HTTP_FAILED` cause is redacted
-too, at its throw site. Values shorter than `MIN_REDACTABLE_LENGTH` are not — a 3-character PIN is
+there into the stored failure artifact — percent-encoded, form-encoded or HTML-escaped as much as
+raw, since a password in a query string is never the raw string. The HTTP leg's
+`X_SCRAPE_HTTP_FAILED` cause is redacted too, at its throw site. Values shorter than `MIN_REDACTABLE_LENGTH` are not — a 3-character PIN is
 a substring of ordinary prose — and pixels never can be, which is why a typed secret TAINTS the
 page and `screenshot()`/`pdf()` are refused outright.
 
@@ -199,14 +361,17 @@ on a machine with no browser — the case CI is.
 | `page.ts` / `page-over-target.ts` | the driver-blind vocabulary, implemented once |
 | `target.ts` / `driver.ts` | the two seams: what a driver answers, and what a session is |
 | `driver-cdp.ts` / `cdp-*.ts` | the real browser, over a structural CDP port |
-| `driver-fake.ts` / `driver-fixture.ts` / `html-*.ts` | the offline drivers, on Bun's `HTMLRewriter` |
+| `driver-fake.ts` / `driver-recorded.ts` / `html-*.ts` | the offline drivers, on Bun's `HTMLRewriter` |
 | `http.ts` / `http-recorded.ts` | the second transport, live and replayed |
-| `auth.ts` / `session-state.ts` | acquire → persist → reuse → validate → burn. The session key encodes each part (`<sanitised>.<digest>`), so two account names that differ only outside `[a-zA-Z0-9._-]` are two sessions |
-| `secrets.ts` / `browser-record.ts` | what may leave this package, and how a browser's own string map is read |
+| `auth.ts` / `session-state.ts` | acquire → persist → reuse → validate → burn. The session key encodes each part (`<sanitised>.<digest>`), so two account names that differ only outside `[a-zA-Z0-9._-]` are two sessions. A stored session is sealed. `burnSession(plan, seen)` takes `recordVersion(record)` of the record the run used, so a record another run saved since is not burned |
+| `event-prompt.ts` | `eventPrompt()` and `answerPrompt()`: a prompt answered from another process, over the job event bus, with the browser open |
+| `cdp-resolver.ts` | a browser rented per session: acquired once, released once |
+| `usage.ts` | what one run used, counted where the work happens |
+| `secrets.ts` / `url-secrets.ts` / `browser-record.ts` | what may leave this package — declared secrets, values learned mid-run (`conceal`), the credential-bearing parts of a URL — and how a browser's own string map is read |
 | `expect.ts` | the silent-green alarm |
 | `watchdog.ts` | the wedge and zombie discipline |
 | `capture-clip.ts` | the one framing rule — crop, whole page, or a refusal — checked before any driver sees it |
-| `errors.ts` / `error-throws.ts` | this package's `X_*` codes and their retry classification |
+| `errors.ts` / `error-throws.ts` / `error-throws-session.ts` | this package's `X_*` codes and their retry classification |
 
 ## Extending it — there is no plugin API, and none is needed
 
@@ -247,6 +412,7 @@ export const myBrowser = (options: MyOptions): ScrapeDriver => ({
         secrets: init.secrets,
         robots: init.robots,
         signal: init.signal,
+        usage: init.usage,
       }),
       http: myHttp(target, init),
       close: () => target.close(),
@@ -260,10 +426,8 @@ export const myBrowser = (options: MyOptions): ScrapeDriver => ({
 ```ts
 // apps/web/shared/base/bank-scrape.ts — the app's convention, written once
 import type { JobHandle } from '@ultimat3/jobs';
-import type { StorageDriver } from '@ultimat3/storage';
 import { scrape, type ScrapeDefinition } from '@ultimat3/scraping';
-
-declare const disk: StorageDriver;
+import { disk } from '@ultimat3/storage';
 
 export const bankScrape = <I, R>(over: ScrapeDefinition<I, R>): JobHandle<I> =>
   scrape<I, R>({
@@ -271,7 +435,7 @@ export const bankScrape = <I, R>(over: ScrapeDefinition<I, R>): JobHandle<I> =>
     rate: 0.5,
     block: ['image', 'media', 'font'],
     expect: { minRows: 1, maxDrop: 0.5 },
-    artifacts: { storage: disk },
+    artifacts: { storage: () => disk('artifacts') },
     ...over,
   });
 ```
@@ -298,6 +462,15 @@ Every code carries a cause, a runnable `fix:` and a retry classification — see
 `X_SCRAPE_YIELD_COLLAPSED`, `X_SCRAPE_AUTH_FAILED`, `X_SCRAPE_BLOCKED` and `X_SCRAPE_PAGE_CRASHED`
 are the four worth knowing by heart.
 
+`X_SCRAPE_EGRESS_IN_PAYLOAD` is the one that is about the DECLARATION rather than the site: the
+exit's password was readable out of the run's own input, so it is in `x_jobs` in the clear.
+Enqueue the id of the row that holds the exit, resolve it in `egress`, and rotate the credential.
+
+`X_SCRAPE_CDP_ATTACH_FAILED` names the endpoint by **scheme and host only**, for a fixed `cdpUrl` as
+for a resolved one, and cuts every credential-bearing part of the URL out of the launcher's own
+message: a provider's connect URL is its access token, and this cause is written to the dead-letter
+row.
+
 ### Error classes
 
 Every error class `src/index.ts` exports, for `instanceof` inside one process. Across a wire or

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/scraping",
-  "version": "22.15.0",
+  "version": "24.0.0",
   "description": "Browser automation as a job: scrape() returns a JobHandle",
   "license": "MIT",
   "type": "module",
@@ -19,6 +19,7 @@
   "files": [
     "src",
     "!src/**/*.test.ts",
+    "!src/**/*-fixture.ts",
     "README.md",
     "LICENSE"
   ],
@@ -30,9 +31,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "22.15.0",
-    "@ultimat3/jobs": "22.15.0",
-    "@ultimat3/schema": "22.15.0",
-    "@ultimat3/storage": "22.15.0"
+    "@ultimat3/core": "24.0.0",
+    "@ultimat3/jobs": "24.0.0",
+    "@ultimat3/schema": "24.0.0",
+    "@ultimat3/storage": "24.0.0"
   }
 }

package/src/artifacts.ts

@@ -21,7 +21,8 @@ export interface ArtifactWriter {
 }
 
 export interface ArtifactWriterInit {
-  readonly storage: StorageDriver | undefined;
+  /** A thunk, read per save: the app's disk exists only after boot (`storageSessionStore`). */
+  readonly storage: (() => StorageDriver) | undefined;
   readonly scrape: string;
   readonly runId: string;
   /** Optional prefix, so an app can keep scrape artifacts out of its user-upload namespace. */
@@ -69,7 +70,7 @@ export function createArtifactWriter(init: ArtifactWriterInit): ArtifactWriter {
         contentType: contentType ?? contentTypeFor(name),
       };
       if (init.storage !== undefined) {
-        await init.storage.put(ref.key, bytes, { contentType: ref.contentType });
+        await init.storage().put(ref.key, bytes, { contentType: ref.contentType });
       }
       saved.push(ref);
       return ref;

package/src/auth.ts

@@ -23,23 +23,49 @@ import type { ScrapeClock } from './clock';
 import { authFailed, promptUnanswered, sessionExpired } from './error-throws';
 import type { ScrapePage } from './page';
 import type { ScrapeSecrets } from './secrets';
-import { type ScrapeSessionStore, type SessionState, sessionDigest } from './session-state';
+import {
+  recordVersion,
+  type ScrapeSessionStore,
+  type SessionState,
+  sessionDigest,
+} from './session-state';
 
-export interface PromptRequest {
+export interface PromptRequest<I = unknown> {
+  /**
+   * The run's own input — what ties this prompt to the app's identity for the run (a connection
+   * id, an org). A handler typed on the definition reads it typed; a shared one reads `unknown`.
+   */
+  readonly input: I;
   /** What the site is asking for: `'sms code'`, `'authenticator code'`, `'security question 3'`. */
   readonly label: string;
   readonly scrape: string;
   readonly url: string;
+  /** The job run asking. With `index` it names this one prompt: `promptEventName(runId, index)`. */
+  readonly runId: string;
+  /** 1 for the first prompt of this attempt, 2 for the next. */
+  readonly index: number;
+  /** The run's clock. A handler that waits, waits on this — never on a timer of its own. */
+  readonly clock: ScrapeClock;
+  /** The run's cancellation. A handler that waits past it holds a browser nobody is using. */
+  readonly signal: AbortSignal | undefined;
+  /**
+   * One cheap round trip to the browser. A handler that waits calls it between polls: the site is
+   * waiting INSIDE this session, so the wedge watchdog must keep seeing activity and a rented
+   * browser must not idle out — and a browser that died is found now, not after the answer.
+   */
+  keepAlive(): Promise<void>;
 }
 
 /**
  * Where an out-of-band code comes from. A declared callback field on the definition — typed,
  * discoverable, one per concern — and never a global "register a plugin" call.
  */
-export type PromptHandler = (request: PromptRequest) => Promise<string> | string;
+export type PromptHandler<I = unknown> = (request: PromptRequest<I>) => Promise<string> | string;
 
 export interface AuthContext<I> {
   readonly input: I;
+  /** The job run logging in — the id the run's events, steps and prompts are keyed by. */
+  readonly runId: string;
   readonly page: ScrapePage;
   /**
    * The declared secrets, boxed. A login body needs the credential it is about to type, and
@@ -78,15 +104,50 @@ export interface ScrapeAuth<I> {
   readonly maxAge?: number | undefined;
 }
 
-export function createPrompt(
-  scrape: string,
-  handler: PromptHandler | undefined,
-  page: ScrapePage,
-): (label: string) => Promise<string> {
+export interface PromptInit<I = unknown> {
+  readonly scrape: string;
+  readonly handler: PromptHandler<I> | undefined;
+  readonly input: I;
+  readonly page: ScrapePage;
+  readonly runId: string;
+  readonly clock: ScrapeClock;
+  readonly signal?: AbortSignal | undefined;
+  /** The run's secret set. Every answer is concealed in it before the body sees the answer. */
+  readonly secrets?: ScrapeSecrets | undefined;
+  /** Called once per answered prompt — `ScrapeReport.usage.promptsAnswered`. */
+  readonly onAnswered?: (() => void) | undefined;
+}
+
+/**
+ * Matches no element on any page, which is the point: the read costs one round trip and carries
+ * nothing back. It goes through the page vocabulary, so it reports activity like any other verb.
+ */
+const KEEPALIVE_SELECTOR = 'ultimate-keepalive';
+
+export function createPrompt<I>(init: PromptInit<I>): (label: string) => Promise<string> {
+  let asked = 0;
   return async (label: string): Promise<string> => {
+    const { scrape, handler, page } = init;
     if (handler === undefined) throw promptUnanswered(scrape, label);
-    const answer = await handler({ label, scrape, url: page.url() });
+    asked += 1;
+    const answer = await handler({
+      input: init.input,
+      label,
+      scrape,
+      url: page.url(),
+      runId: init.runId,
+      index: asked,
+      clock: init.clock,
+      signal: init.signal,
+      keepAlive: async (): Promise<void> => {
+        await page.count(KEEPALIVE_SELECTOR);
+      },
+    });
     if (typeof answer !== 'string' || answer === '') throw promptUnanswered(scrape, label);
+    // Before the body can type it: an OTP echoed into the page it was typed on is in `page.html()`,
+    // and the failure artifact is written from exactly that.
+    init.secrets?.conceal(answer);
+    init.onAnswered?.();
     return answer;
   };
 }
@@ -111,10 +172,30 @@ export interface AuthPlanInput<I> {
 export async function restorableSession<I>(
   plan: AuthPlanInput<I>,
 ): Promise<SessionState | undefined> {
+  return (await observeSession(plan)).restorable;
+}
+
+/** What `observeSession` answers: the session to restore, and the version of what was found. */
+export interface ObservedSession {
+  readonly restorable: SessionState | undefined;
+  /**
+   * `recordVersion` of the record found — restorable or NOT. A `reuse: false` run, or one whose
+   * record is past `maxAge`, still saw that record: a refused login must tombstone over it, and a
+   * run that compared against "nothing" let the rejected credential through again.
+   */
+  readonly seen: string | undefined;
+}
+
+/** `restorableSession`, plus the version of the record found, for the run's later writes. */
+export async function observeSession<I>(plan: AuthPlanInput<I>): Promise<ObservedSession> {
   const store = plan.auth?.store;
-  if (store === undefined) return undefined;
+  if (store === undefined) return { restorable: undefined, seen: undefined };
   const found = await store.load(plan.key);
-  if (found === undefined) return undefined;
+  if (found === undefined) return { restorable: undefined, seen: undefined };
+  return { restorable: restorableFrom(plan, found), seen: recordVersion(found) };
+}
+
+function restorableFrom<I>(plan: AuthPlanInput<I>, found: SessionState): SessionState | undefined {
   // The tombstone is read BEFORE `reuse` is honoured. `reuse: false` says "do not restore this
   // session"; it does not say "present the rejected credential again", and reading it first meant
   // a `reuse: false` scrape walked a refused password back to the login form on every requeue.
@@ -138,13 +219,42 @@ export async function restorableSession<I>(
   return found;
 }
 
-/** Written down so the next attempt cannot reach the login form with the same credentials. */
-export async function markRefused<I>(plan: AuthPlanInput<I>): Promise<void> {
+/**
+ * True when the stored record is still the one this run last saw — `seen` is the `recordVersion`
+ * it found or persisted, `undefined` when it found none. The store has no compare-and-set (nor has
+ * `StorageDriver` a conditional write to build one from), so a
+ * burn or a tombstone is a blind write: two runs on one key, and the one that failed destroyed the
+ * session the other had just persisted. Compared before the write, which narrows the race to the
+ * gap between this read and that write rather than the whole run. A record moved on is left alone.
+ */
+async function stillSeen<I>(
+  plan: AuthPlanInput<I>,
+  store: ScrapeSessionStore,
+  seen: string | undefined,
+  step: 'session.burn' | 'session.refuse',
+): Promise<boolean> {
+  const current = await store.load(plan.key);
+  if (current === undefined || recordVersion(current) === seen) return true;
+  plan.logger.warn('scrape.session.superseded', { step });
+  return false;
+}
+
+/**
+ * Written down so the next attempt cannot reach the login form with the same credentials — unless
+ * another run saved a session after this one read the record: that login SUCCEEDED, so this
+ * refusal is older news than the record it would overwrite.
+ */
+export async function markRefused<I>(
+  plan: AuthPlanInput<I>,
+  seen: string | undefined,
+): Promise<void> {
   const store = plan.auth?.store;
   if (store === undefined) return;
+  if (!(await stillSeen(plan, store, seen, 'session.refuse'))) return;
   await store.save({
     key: plan.key,
     savedAt: plan.clock.now().toISOString(),
+    version: crypto.randomUUID(),
     refusedAt: plan.clock.now().toISOString(),
     cookies: [],
     headers: {},
@@ -154,15 +264,25 @@ export async function markRefused<I>(plan: AuthPlanInput<I>): Promise<void> {
   });
 }
 
-/** New identity, from scratch. One call, because a flagged profile is unusable and must go. */
-export async function burnSession<I>(plan: AuthPlanInput<I>): Promise<void> {
-  if (plan.auth?.store === undefined) return;
-  await plan.auth.store.burn(plan.key);
+/**
+ * New identity, from scratch. One call, because a flagged profile is unusable and must go — the
+ * profile THIS run used: `seen` is that record's `recordVersion` (`observeSession`'s, or the one
+ * `persistSession` answered), and a record another run saved since is not burned.
+ */
+export async function burnSession<I>(
+  plan: AuthPlanInput<I>,
+  seen: string | undefined,
+): Promise<void> {
+  const store = plan.auth?.store;
+  if (store === undefined) return;
+  if (!(await stillSeen(plan, store, seen, 'session.burn'))) return;
+  await store.burn(plan.key);
   plan.logger.warn('scrape.session.burned', { burned: true });
 }
 
 export interface EnsureAuthInput<I> extends AuthPlanInput<I> {
   readonly input: I;
+  readonly runId: string;
   readonly page: ScrapePage;
   readonly secrets: ScrapeSecrets;
   readonly restored: SessionState | undefined;
@@ -182,6 +302,7 @@ export async function ensureAuthenticated<I>(args: EnsureAuthInput<I>): Promise<
   if (auth === undefined) return false;
   const context: AuthContext<I> = {
     input: args.input,
+    runId: args.runId,
     page: args.page,
     secrets: args.secrets,
     prompt: args.prompt,
@@ -193,18 +314,31 @@ export async function ensureAuthenticated<I>(args: EnsureAuthInput<I>): Promise<
       return false;
     }
     args.logger.info('scrape.session.expired', { reused: false });
-    await burnSession(args);
+    await burnSession(args, recordVersion(args.restored));
   }
   if (auth.login === undefined) throw sessionExpired(args.scrape, args.key);
   await auth.login(context);
   return true;
 }
 
-/** After a login: keep what it produced. A 2FA code somebody typed is worth exactly this. */
-export async function persistSession<I>(plan: AuthPlanInput<I>, page: ScrapePage): Promise<void> {
+/**
+ * After a login: keep what it produced. A 2FA code somebody typed is worth exactly this. Answers
+ * the `version` it wrote — the record this run now owns, for a later burn to compare against.
+ */
+export async function persistSession<I>(
+  plan: AuthPlanInput<I>,
+  page: ScrapePage,
+): Promise<string | undefined> {
   const store = plan.auth?.store;
-  if (store === undefined) return;
+  if (store === undefined) return undefined;
   const snapshot = await page.session();
-  await store.save({ ...snapshot, key: plan.key, savedAt: plan.clock.now().toISOString() });
+  const version = crypto.randomUUID();
+  await store.save({
+    ...snapshot,
+    key: plan.key,
+    savedAt: plan.clock.now().toISOString(),
+    version,
+  });
   plan.logger.info('scrape.session.saved', sessionDigest(snapshot));
+  return version;
 }

package/src/cdp-arm.ts

@@ -4,7 +4,7 @@
 // three verbs `press`/`focus`/`accessibility` did not fit under it. The extraction is VERBATIM —
 // the request, console, `pageerror` and `error` handlers are the same code, one file over.
 
-import type { CdpPageLike, CdpRequestLike } from './cdp-port';
+import type { CdpBrowserLike, CdpPageLike, CdpRequestLike } from './cdp-port';
 import type { ScrapeClock } from './clock';
 import type { InterceptRules } from './intercept';
 import { interceptVerdict, refusalEntry } from './intercept';
@@ -174,3 +174,67 @@ export async function arm(init: CdpArmInit, sinks: CdpSinks): Promise<void> {
     crashed.value = readStringFrom(payload, 'message') ?? 'renderer crashed';
   });
 }
+
+/** A `Fetch.requestPaused` payload, read structurally — the fields the verdict needs, or nothing. */
+interface PausedRequest {
+  readonly requestId: string;
+  readonly resourceType: string;
+  readonly url: string;
+  readonly method: string;
+}
+
+const asPaused = (payload: unknown): PausedRequest | undefined => {
+  if (typeof payload !== 'object' || payload === null) return undefined;
+  const event = payload as Record<string, unknown>;
+  const request = event['request'];
+  if (typeof request !== 'object' || request === null) return undefined;
+  const { url, method } = request as Record<string, unknown>;
+  const { requestId, resourceType } = event;
+  if (typeof requestId !== 'string' || typeof url !== 'string') return undefined;
+  return {
+    requestId,
+    url,
+    resourceType: typeof resourceType === 'string' ? resourceType : 'Other',
+    method: typeof method === 'string' ? method : 'GET',
+  };
+};
+
+/**
+ * `allowHosts` and `block` on EVERY target the browser opens. Page-level interception (`arm`)
+ * covers the one page this package created; a popup — `window.open`, a `target=_blank` link — is
+ * a new target it never sees, and on Chrome 150 a scraped page's `window.open` to an off-list host
+ * reached that host. Browser-level `Fetch` pauses those requests too, and the same
+ * `interceptVerdict` decides them. The page's own requests are decided at the page first, so an
+ * allowed one is continued here as well and a refused one never arrives: one ring entry each.
+ *
+ * Enabled BEFORE the first navigation; a browser that refuses `Fetch.enable` rejects here, and
+ * `driver-cdp.ts`'s `opened()` closes it — a session whose rule would hold on one tab is refused.
+ */
+export async function armBrowser(
+  init: {
+    readonly browser: CdpBrowserLike;
+    readonly rules: InterceptRules;
+    readonly clock: ScrapeClock;
+  },
+  network: NetworkRing,
+): Promise<void> {
+  const session = await init.browser.target().createCDPSession();
+  session.on('Fetch.requestPaused', (payload) => {
+    const paused = asPaused(payload);
+    if (paused === undefined) return;
+    const type = asResourceType(paused.resourceType.toLowerCase());
+    const verdict = interceptVerdict(paused.url, type, init.rules);
+    const requestId = { requestId: paused.requestId };
+    // Caught, never floated, for `arm`'s reason: the target may be gone by the time this answers.
+    if (verdict === 'allow') {
+      session.send('Fetch.continueRequest', requestId).catch(() => undefined);
+      return;
+    }
+    const at = init.clock.now().getTime();
+    network.push(refusalEntry(paused.url, type, verdict, at, paused.method));
+    session
+      .send('Fetch.failRequest', { ...requestId, errorReason: 'BlockedByClient' })
+      .catch(() => undefined);
+  });
+  await session.send('Fetch.enable', { patterns: [{ urlPattern: '*' }] });
+}