@ultimat3/scraping 2.0.0

package/src/scrape-run.ts

@@ -0,0 +1,226 @@
+// One attempt, assembled: driver, secrets, robots gate, pacing, session lifecycle, the body, the
+// extract schema, the yield alarm, the artifacts, the teardown.
+//
+// RESTARTABILITY, stated once because it is the property everything here is arranged around: a
+// killed attempt resumes from the caller's own `step.run` checkpoints, and what a step may persist
+// is a CURSOR — a page number, an id, an offset. Never a page, never a live handle, never a
+// session. The session lives in the session store and is re-probed on every attempt, which is what
+// makes a resumed run skip the login when the session is still good and re-login when it is not.
+// A step record saying "logged in" would be a checkpoint asserting something about a session that
+// may have expired an hour ago.
+
+import type { JobRunArgs } from '@ultimat3/jobs';
+import { parse } from '@ultimat3/schema';
+import { createArtifactWriter } from './artifacts';
+import type { AuthPlanInput } from './auth';
+import {
+  burnSession,
+  createPrompt,
+  ensureAuthenticated,
+  markRefused,
+  persistSession,
+  restorableSession,
+} from './auth';
+import { systemScrapeClock } from './clock';
+import type { ScrapeSession } from './driver';
+import { scrapeDriver } from './driver';
+import { driverUnknown, outputInvalid } from './error-throws';
+import { scrapeLogger, withStepEvent } from './events';
+import { guardYield } from './expect';
+import { burnsSession, errorCode, neverRetried } from './failures';
+import { createPacer, DEFAULT_NAVIGATION_RATE } from './rate';
+import { runRecovery } from './recover';
+import { createRobotsGate } from './robots';
+import type { ScrapeDefinition, ScrapeReport } from './scrape';
+import { createSecretBag } from './secrets';
+import { sessionKeyFor } from './session-state';
+
+/** `ctx.actor` is a structural read: this package never imports the auth types (tier 2). */
+const orgOf = (ctx: unknown): string | undefined => {
+  const actor = (ctx as { actor?: { orgId?: unknown } }).actor;
+  return typeof actor?.orgId === 'string' ? actor.orgId : undefined;
+};
+
+const toMillis = (value: string | number | undefined, fallback: number): number => {
+  if (value === undefined) return fallback;
+  if (typeof value === 'number') return value;
+  const match = /^(\d+(?:\.\d+)?)(ms|s|m|h)?$/.exec(value.trim());
+  if (match === null) return fallback;
+  const scale = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000 }[match[2] ?? 'ms'] ?? 1;
+  return Number(match[1]) * scale;
+};
+
+export const DEFAULT_PAGE_TIMEOUT_MS = 30_000;
+
+export async function runScrape<I, Row>(
+  definition: ScrapeDefinition<I, Row>,
+  args: JobRunArgs<I>,
+): Promise<ScrapeReport<Row>> {
+  const clock = definition.clock ?? systemScrapeClock;
+  const driver = definition.driver ?? scrapeDriver();
+  // The scrape's name goes in the SCRAPE slot: there is no driver here to name, which is the
+  // whole failure.
+  if (driver === undefined) throw driverUnknown(undefined, [], definition.name);
+  const logger = scrapeLogger(args.ctx.logger, {
+    scrape: definition.name,
+    runId: args.runId,
+    attempt: args.attempt,
+    driver: driver.name,
+  });
+  const secrets = createSecretBag(definition.secrets ?? []);
+  const rules = { allowHosts: definition.allowHosts, block: definition.block };
+  const pace = createPacer(definition.rate ?? DEFAULT_NAVIGATION_RATE, clock);
+  const artifact = createArtifactWriter({
+    storage: definition.artifacts?.storage,
+    scrape: definition.name,
+    runId: args.runId,
+    prefix: definition.artifacts?.prefix,
+  });
+  const plan: AuthPlanInput<I> = {
+    scrape: definition.name,
+    auth: definition.auth,
+    // `auth.key` DISCRIMINATES inside the tenant's key space; it never replaces it. Letting it
+    // replace the key gave two tenants declaring the same account name one authenticated session,
+    // and skipped the sanitising `sessionKeyFor` does to a value that is also a storage path.
+    key: sessionKeyFor({
+      scrape: definition.name,
+      tenant: orgOf(args.ctx),
+      discriminator: definition.auth?.key?.(args.input),
+    }),
+    clock,
+    logger,
+  };
+
+  // Read BEFORE the browser opens: a refused credential must not reach a login form again, and
+  // opening a session first would already have spent an identity on a run that cannot succeed.
+  const restored = await restorableSession(plan);
+  const session = await driver.open({
+    name: definition.name,
+    rules,
+    clock,
+    timeoutMs: toMillis(definition.pageTimeout, DEFAULT_PAGE_TIMEOUT_MS),
+    secrets,
+    robots: createRobotsGate({ policy: definition.robots ?? 'obey' }),
+    signal: args.ctx.signal,
+    restore: restored,
+    pace: (signal) => pace(signal),
+    watchdog: definition.watchdog,
+  });
+
+  try {
+    if (definition.auth !== undefined) {
+      const loggedIn = await withStepEvent(
+        { name: 'auth', logger, clock, attempt: args.attempt },
+        () =>
+          ensureAuthenticated({
+            ...plan,
+            input: args.input,
+            page: session.page,
+            secrets,
+            restored,
+            prompt: createPrompt(definition.name, definition.prompt, session.page),
+          }),
+      );
+      // Persisted after a LOGIN only, never after a reuse: rewriting the record on every run
+      // refreshes `savedAt` without refreshing the session, so `maxAge` would never expire it.
+      if (loggedIn) await persistSession(plan, session.page);
+    }
+
+    const raw = await bodyWithRecovery(definition, args, session, logger, artifact, secrets);
+    const rows = raw.map((row): Row => {
+      try {
+        return parse(definition.extract, row);
+      } catch (thrown) {
+        throw outputInvalid(definition.name, errorCode(thrown) ?? 'the row did not parse');
+      }
+    });
+    await guardYield({
+      scrape: definition.name,
+      rows: rows.length,
+      expect: definition.expect,
+      history: definition.history,
+    });
+    const refused = session.page.network().filter((entry) => entry.refused !== undefined).length;
+    // The ring is bounded, so `refused` is a FLOOR and this is what says so. Reporting the count
+    // alone made a run that blocked 5,000 images print 200 and discarded the one number
+    // (`Ring.dropped`) that exists to say "you are not seeing it all".
+    const networkDropped = session.page.networkDropped();
+    logger.info('scrape.ok', { rows: rows.length, refused });
+    return {
+      scrape: definition.name,
+      rows,
+      artifacts: artifact.saved.map((ref) => ref.key),
+      refused,
+      networkDropped,
+    };
+  } catch (thrown) {
+    logger.error('scrape.failed', { code: errorCode(thrown) });
+    if (errorCode(thrown) === 'X_SCRAPE_AUTH_FAILED') await markRefused(plan);
+    else if (burnsSession(thrown)) await burnSession(plan);
+    if (definition.artifacts?.onFailure !== false) await saveFailureArtifact(session, artifact);
+    throw thrown;
+  } finally {
+    // Always, and it never throws: `close()` ends the local connection AND the remote browser.
+    await session.close();
+  }
+}
+
+/**
+ * The body, with at most ONE recovery pass. `recover` never runs for a failure that must not be
+ * retried — a rejected credential is the case, and asking a model to "fix" a wrong password is
+ * how an account gets locked.
+ */
+async function bodyWithRecovery<I, Row>(
+  definition: ScrapeDefinition<I, Row>,
+  args: JobRunArgs<I>,
+  session: ScrapeSession,
+  logger: ReturnType<typeof scrapeLogger>,
+  artifact: ReturnType<typeof createArtifactWriter>,
+  secrets: ReturnType<typeof createSecretBag>,
+): Promise<readonly unknown[]> {
+  const body = (): Promise<readonly unknown[]> =>
+    definition.run({
+      input: args.input,
+      page: session.page,
+      http: session.http,
+      step: args.step,
+      ctx: args.ctx,
+      secrets,
+      artifact,
+      attempt: args.attempt,
+      runId: args.runId,
+    });
+  try {
+    return await withStepEvent(
+      { name: 'body', logger, clock: definition.clock ?? systemScrapeClock, attempt: args.attempt },
+      body,
+    );
+  } catch (thrown) {
+    if (definition.recover === undefined || neverRetried(thrown)) throw thrown;
+    const recovered = await runRecovery(definition.recover, {
+      scrape: definition.name,
+      page: session.page,
+      failure: thrown,
+      attempt: args.attempt,
+    });
+    if (!recovered) throw thrown;
+    logger.warn('scrape.recovered', { code: errorCode(thrown) });
+    return await body();
+  }
+}
+
+/**
+ * The page HTML, on failure — redacted by value, password fields blanked. It is the only forensic
+ * left once the browser is gone, and by the time somebody looks the page has changed.
+ */
+async function saveFailureArtifact(
+  session: ScrapeSession,
+  artifact: ReturnType<typeof createArtifactWriter>,
+): Promise<void> {
+  try {
+    await artifact.save('page.html', await session.page.html());
+  } catch {
+    // A failed artifact must never replace the failure that caused it. The run's own error is the
+    // one the reader needs; this is a best effort on the way out.
+  }
+}

package/src/scrape.ts

@@ -0,0 +1,151 @@
+// `scrape()` — a browser run, declared as a `job` and NOT as a ninth primitive. The rule's fourth
+// instance after `llm()` (an action factory) and `backfill()` (a job factory).
+//
+// It is a job by every field of the definition, not by analogy: a scrape has an input schema, a
+// tenant, a retry policy, a timeout, a concurrency cap, a queue, and — decisively — a REQUIRED
+// idempotency key and `step.run` checkpoints. Logging into a bank twice because a worker was
+// killed between the login and its checkpoint is not a hypothetical; three wrong attempts locks
+// the account. So this file is a FACTORY over `job()`, and a scrape inherits `.enqueue()`, the
+// worker's cancellation, the dead-letter path, `x jobs show` and its manifest row for free.
+
+import type { Ctx } from '@ultimat3/core';
+import { assert } from '@ultimat3/core';
+import type { JobHandle, JobTenant, RetryPolicy, StepApi } from '@ultimat3/jobs';
+import { DEFAULT_RETRY, job } from '@ultimat3/jobs';
+import type { StandardSchemaV1 } from '@ultimat3/schema';
+import type { StorageDriver } from '@ultimat3/storage';
+import type { ArtifactWriter } from './artifacts';
+import type { PromptHandler, ScrapeAuth } from './auth';
+import type { ScrapeClock } from './clock';
+import type { ScrapeDriver } from './driver';
+import type { YieldExpectation, YieldHistory } from './expect';
+import type { HostRule } from './hosts';
+import type { ScrapeHttp } from './http';
+import type { ScrapePage } from './page';
+import type { Recovery } from './recover';
+import type { ResourceType } from './rings';
+import type { RobotsPolicy } from './robots';
+import { runScrape } from './scrape-run';
+import type { ScrapeSecrets } from './secrets';
+
+export interface ScrapeRunArgs<I> {
+  readonly input: I;
+  /** Driver-blind. The same body runs on a browser, on a recording and on a string of HTML. */
+  readonly page: ScrapePage;
+  /**
+   * The same session, over HTTP. Drive the browser through login and navigation, then pull the
+   * bulk off the site's own JSON endpoints: the cookies, headers, proxy, host allow list, rate
+   * limit and cancellation are the page's, so the authenticated browser session simply continues.
+   */
+  readonly http: ScrapeHttp;
+  /** The job's own step api: one `step.run` per page, so a kill resumes where it stopped. */
+  readonly step: StepApi;
+  readonly ctx: Ctx;
+  /** Declared names, resolved in the worker. A value never enters the queue row. */
+  readonly secrets: ScrapeSecrets;
+  readonly artifact: ArtifactWriter;
+  readonly attempt: number;
+  readonly runId: string;
+}
+
+export interface ScrapeArtifacts {
+  readonly storage?: StorageDriver | undefined;
+  /** Save the page's HTML when the run fails. On by default — it is the only forensic left. */
+  readonly onFailure?: boolean | undefined;
+  readonly prefix?: string | undefined;
+}
+
+export interface ScrapeDefinition<I, Row> {
+  /** REQUIRED, exactly as a backfill's is: the name is a durable queue key, never an export name. */
+  readonly name: string;
+  readonly input: StandardSchemaV1<unknown, I>;
+  /**
+   * Every row, parsed. A scrape's output is somebody else's HTML, so "it came back" and "it came
+   * back in the shape this app stores" are different questions and this is the second one.
+   * A row the schema rejects is `X_SCRAPE_OUTPUT_INVALID`, never a silently stored partial.
+   */
+  readonly extract: StandardSchemaV1<unknown, Row>;
+  /** REQUIRED by the type, like every job's. */
+  readonly idempotencyKey: (input: I) => string;
+  /** REQUIRED by the type, like every job's. */
+  readonly tenant: JobTenant<I>;
+  /**
+   * REQUIRED, and the field this package is most opinionated about. A headless browser with no
+   * host list is the widest SSRF surface an app can own. `['*']` is the explicit escape hatch —
+   * spelled out, visible in review — and there is no way to omit the decision.
+   */
+  readonly allowHosts: readonly HostRule[];
+  readonly block?: readonly ResourceType[] | undefined;
+  /** Navigations per second. Defaults to `DEFAULT_NAVIGATION_RATE`; there is no unpaced mode. */
+  readonly rate?: number | undefined;
+  readonly robots?: RobotsPolicy | undefined;
+  /** The silent-green alarm. See `expect.ts` — this is the most valuable field here. */
+  readonly expect?: YieldExpectation | undefined;
+  readonly history?: YieldHistory | undefined;
+  readonly artifacts?: ScrapeArtifacts | undefined;
+  /** NAMES, never values. */
+  readonly secrets?: readonly string[] | undefined;
+  readonly recover?: Recovery | undefined;
+  /**
+   * Session lifecycle: acquire, persist, reuse, validate, burn. Authenticated scraping is the
+   * primary case, so this is declared rather than hand-rolled per app. See `auth.ts`.
+   */
+  readonly auth?: ScrapeAuth<I> | undefined;
+  /** Where an out-of-band code comes from, for a site that asks for one after the password. */
+  readonly prompt?: PromptHandler | undefined;
+  readonly driver?: ScrapeDriver | undefined;
+  readonly retry?: RetryPolicy | undefined;
+  /** Per attempt, whole-run. `'5m'` or ms. */
+  readonly timeout?: string | number | undefined;
+  /** Per browser operation. `'30s'` or ms. */
+  readonly pageTimeout?: string | number | undefined;
+  /** Kill the browser after this much silence from it. See `watchdog.ts`. */
+  readonly watchdog?: { readonly idleMs?: number; readonly graceMs?: number } | undefined;
+  readonly concurrency?: number | undefined;
+  readonly queue?: string | undefined;
+  readonly clock?: ScrapeClock | undefined;
+  run(args: ScrapeRunArgs<I>): Promise<readonly unknown[]>;
+}
+
+/** What one completed scrape reports — bounded, so `x jobs show` can print it. */
+export interface ScrapeReport<Row> {
+  readonly scrape: string;
+  readonly rows: readonly Row[];
+  readonly artifacts: readonly string[];
+  /**
+   * Requests interception refused, by reason. A zero-row run usually explains itself here — and it
+   * is a FLOOR, not a total, whenever `networkDropped` is non-zero.
+   */
+  readonly refused: number;
+  /**
+   * Entries the bounded network ring dropped to stay bounded (`rings.ts`). Non-zero is the honest
+   * "you are not seeing it all": `refused` was counted from what survived the bound.
+   */
+  readonly networkDropped: number;
+}
+
+export function scrape<I, Row>(definition: ScrapeDefinition<I, Row>): JobHandle<I> {
+  // Refused where it is written, in the voice `backfill()` uses: a scrape with an empty host list
+  // can never navigate anywhere, and finding that out is otherwise a dead-lettered job.
+  assert(
+    definition.allowHosts.length > 0,
+    `scrape "${definition.name}" declares allowHosts: [] — nothing can be navigated to`,
+    `list the hosts on scrape("${definition.name}") — allowHosts: ['example.com'] — or state the decision with allowHosts: ['*']`,
+  );
+  assert(
+    definition.rate === undefined || (Number.isFinite(definition.rate) && definition.rate > 0),
+    `scrape "${definition.name}" declares rate: ${String(definition.rate)} — a rate is navigations per second, greater than zero`,
+    `set rate: 1 on scrape("${definition.name}"), or leave it out — to go faster raise the number, there is no unpaced mode`,
+  );
+  return job<I>({
+    name: definition.name,
+    input: definition.input,
+    idempotencyKey: definition.idempotencyKey,
+    tenant: definition.tenant,
+    retry: definition.retry ?? DEFAULT_RETRY,
+    ...(definition.queue === undefined ? {} : { queue: definition.queue }),
+    ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
+    ...(definition.concurrency === undefined ? {} : { concurrency: definition.concurrency }),
+    run: (args) => runScrape(definition, args),
+  });
+}

package/src/secrets.ts

@@ -0,0 +1,91 @@
+// Secrets in a browser run, and the one place this package decides what may leave it.
+//
+// A scrape logs in. That means a password crosses this package on its way into a form field, and
+// there are exactly three ways it gets out: an event stream, a log line, and — the one nobody
+// remembers — a SCREENSHOT of the filled form. The first two are solved by carrying the value in
+// core's `Secret` box, which renders `[redacted]` through `String`, `JSON.stringify` and the
+// logger. The third is not, because pixels cannot be redacted after the fact.
+
+import type { Secret } from '@ultimat3/core';
+import { revealSecret, secret, UltimateError } from '@ultimat3/core';
+
+export interface ScrapeSecrets {
+  readonly names: readonly string[];
+  /** The boxed value. Reading it out is `revealSecret()`, one greppable call site. */
+  get(name: string): Secret;
+}
+
+/**
+ * Names only ever reach a job payload — the values are resolved in the worker, per attempt, from
+ * the environment. A queue row that carried the value would put a bank password in a durable
+ * table, in the outbox, and in every `x jobs show` an operator runs.
+ */
+export type SecretResolver = (name: string) => string | undefined;
+
+const fromEnvironment: SecretResolver = (name) => Bun.env[name];
+
+export function createSecretBag(
+  names: readonly string[],
+  resolve: SecretResolver = fromEnvironment,
+): ScrapeSecrets {
+  const boxed = new Map<string, Secret>();
+  for (const name of names) {
+    const value = resolve(name);
+    if (value === undefined || value === '') {
+      throw new UltimateError({
+        code: 'X_ENV_MISSING',
+        cause: `scrape secret "${name}" is declared and the environment has no value for it`,
+        fix: `add ${name}= to .env.local, or drop "${name}" from secrets: on the scrape() definition`,
+        meta: { name },
+      });
+    }
+    boxed.set(name, secret(value, name));
+  }
+  return {
+    names: [...names],
+    get(name: string): Secret {
+      const found = boxed.get(name);
+      if (found === undefined) {
+        throw new UltimateError({
+          code: 'X_ENV_MISSING',
+          cause: `scrape secret "${name}" was read and never declared`,
+          fix: `add "${name}" to secrets: on the scrape() definition — a run resolves only what it declares`,
+          meta: { name },
+        });
+      }
+      return found;
+    },
+  };
+}
+
+export const SECRET_PLACEHOLDER = '[redacted]';
+
+/**
+ * Redaction BY VALUE, over any text this package is about to persist: page HTML, a console line,
+ * a request URL, an error cause. Name-based redaction only catches a secret travelling under a
+ * name somebody remembered to list, and a password pasted into a query string travels under none.
+ *
+ * Longest first, so a secret that contains another one does not leave its tail behind.
+ */
+export function redactSecrets(text: string, secrets: ScrapeSecrets | undefined): string {
+  if (secrets === undefined || secrets.names.length === 0) return text;
+  const values = secrets.names
+    .map((name) => revealSecret(secrets.get(name)))
+    .filter((value) => value.length >= 4)
+    .sort((a, b) => b.length - a.length);
+  let out = text;
+  for (const value of values) out = out.split(value).join(SECRET_PLACEHOLDER);
+  return out;
+}
+
+/** `<input type="password" value="hunter2">` -> `value=""`, whatever the value happened to be. */
+export function blankPasswordFields(html: string): string {
+  return html.replaceAll(
+    /(<input\b[^>]*\btype\s*=\s*['"]?password['"]?[^>]*?)\bvalue\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]*)/gi,
+    '$1value=""',
+  );
+}
+
+/** Both passes, in the order that matters: value redaction first, then the structural blank. */
+export const safeHtml = (html: string, secrets: ScrapeSecrets | undefined): string =>
+  blankPasswordFields(redactSecrets(html, secrets));

package/src/session-state.ts

@@ -0,0 +1,181 @@
+// The session: what an authenticated browser carries, as a value that can be persisted, restored,
+// validated and BURNED.
+//
+// Reuse is both the fast path and the safe path — logging in on every run is slow and is itself
+// the signal anti-bot systems look for. Burning is the counter-intuitive half and it is
+// evidence-backed: a persisted profile that has been flagged stays flagged, so a retry that
+// reloads it re-trips the same block, forever. "New identity, from scratch" has to be one call.
+//
+// Session material is CREDENTIAL material. It is tenant-scoped, it never reaches a log line, an
+// event field, an artifact or a screenshot, and it is summarised for logs by `sessionDigest()` —
+// counts and an origin, never a value.
+
+import type { StorageDriver } from '@ultimat3/storage';
+import { assertSafeKey } from '@ultimat3/storage';
+import type { ScrapeCookie } from './target';
+
+export interface SessionSnapshot {
+  readonly cookies: readonly ScrapeCookie[];
+  /**
+   * Headers the HTTP leg must send to stay the same client — user-agent, language, site tokens.
+   *
+   * Only an OFFLINE driver can fill it: CDP exposes no read for "the headers this site now
+   * expects", so the puppeteer driver answers `{}` and `driver-parity.test.ts` pins the
+   * divergence. A token the HTTP leg must carry belongs on the request (`http.request(url, {
+   * headers })`), not here — a fixture that proves otherwise proves it only offline.
+   */
+  readonly headers: Readonly<Record<string, string>>;
+  /** `localStorage`, flattened. Many sites keep the bearer token here and not in a cookie. */
+  readonly storage: Readonly<Record<string, string>>;
+  readonly userAgent: string;
+  /** The origin this session belongs to. A session is never replayed against another one. */
+  readonly origin: string;
+}
+
+export interface SessionState extends SessionSnapshot {
+  readonly key: string;
+  /** ISO 8601. What a `maxAge` on reuse is measured against. */
+  readonly savedAt: string;
+  /**
+   * ISO 8601, set when this site REFUSED these credentials — and the reason the record survives
+   * the failure instead of being deleted with it.
+   *
+   * A site that locks an account after three wrong attempts makes a retrying framework the thing
+   * that destroys the user's account. Two mechanisms stop that, at two different distances, and
+   * this is the near one:
+   *
+   * | Mechanism | Stops | Cost still paid |
+   * |---|---|---|
+   * | `X_SCRAPE_AUTH_FAILED` registered `terminal` (`errors.ts`) | the NEXT attempt — `executeJob` reads the thrown code's classification and dead-letters on the attempt that failed | this attempt ran in full: process, CDP attach, navigation, one wrong password at the site |
+   * | this field | THIS attempt, before `driver.open()` — `restorableSession()` reads it first | nothing; the run refuses without a browser and without a request |
+   *
+   * So the queue-level classification is what makes the failure terminal, and this is what makes
+   * the failure CHEAP: a replay after a worker restart, a manual `x jobs retry`, or any second
+   * enqueue of the same input never reaches a login form at all. Clearing it is deliberate —
+   * `burn()` — because the thing that fixed it was a human changing the credential.
+   */
+  readonly refusedAt?: string | undefined;
+}
+
+export const EMPTY_SESSION: SessionSnapshot = Object.freeze({
+  cookies: [],
+  headers: {},
+  storage: {},
+  userAgent: '',
+  origin: '',
+});
+
+/**
+ * Per tenant, per scrape, per site. The tenant is FIRST because the key is also a storage path,
+ * and a prefix that starts with the tenant is one an object-store policy can scope.
+ */
+export function sessionKeyFor(input: {
+  readonly scrape: string;
+  readonly tenant: string | undefined;
+  readonly discriminator?: string | undefined;
+}): string {
+  const parts = [input.tenant ?? 'no-tenant', input.scrape, input.discriminator ?? 'default'];
+  return parts.map((part) => part.replaceAll(/[^a-zA-Z0-9._-]+/g, '-')).join('/');
+}
+
+/** What a log line may say about a session: shape, never content. */
+export const sessionDigest = (
+  state: SessionSnapshot | undefined,
+): Readonly<Record<string, unknown>> =>
+  state === undefined
+    ? { session: 'none' }
+    : {
+        session: 'present',
+        origin: state.origin,
+        cookies: state.cookies.length,
+        storageKeys: Object.keys(state.storage).length,
+      };
+
+export interface ScrapeSessionStore {
+  load(key: string): Promise<SessionState | undefined>;
+  save(state: SessionState): Promise<void>;
+  /** Delete it. Called on a block, and by an author who knows the identity is spent. */
+  burn(key: string): Promise<void>;
+}
+
+export function memorySessionStore(
+  seed: Readonly<Record<string, SessionState>> = {},
+): ScrapeSessionStore {
+  const states = new Map<string, SessionState>(Object.entries(seed));
+  return {
+    load: (key) => Promise.resolve(states.get(key)),
+    save: (state) => {
+      states.set(state.key, state);
+      return Promise.resolve();
+    },
+    burn: (key) => {
+      states.delete(key);
+      return Promise.resolve();
+    },
+  };
+}
+
+export const DEFAULT_SESSION_PREFIX = 'scrape-session';
+
+/**
+ * Sessions on the app's own disk, through `@ultimat3/storage` — the seam that already knows about
+ * tenant-scoped keys and refuses one that escapes its prefix. This package owns no upload path.
+ *
+ * NOT encrypted at rest: the bytes are as sensitive as the bucket they sit in, so the bucket must
+ * be private. Sealing them under core's secrets envelope is the obvious next step and is
+ * deliberately not guessed at here — it needs a key the app declares, and a wrong guess about
+ * where that key lives is worse than the honest note.
+ */
+export function storageSessionStore(
+  storage: StorageDriver,
+  options: { readonly prefix?: string | undefined } = {},
+): ScrapeSessionStore {
+  const keyFor = (key: string): string => {
+    const path = `${options.prefix ?? DEFAULT_SESSION_PREFIX}/${key}.json`;
+    assertSafeKey(path);
+    return path;
+  };
+  return {
+    async load(key: string): Promise<SessionState | undefined> {
+      try {
+        const read = await storage.get(keyFor(key));
+        return parseSessionState(JSON.parse(new TextDecoder().decode(read.bytes)) as unknown, key);
+      } catch {
+        // A session that cannot be read is a session that does not exist. Refusing the run over a
+        // missing cache would make reuse — the fast path — the fragile path.
+        return undefined;
+      }
+    },
+    async save(state: SessionState): Promise<void> {
+      await storage.put(keyFor(state.key), new TextEncoder().encode(JSON.stringify(state)), {
+        contentType: 'application/json',
+      });
+    },
+    async burn(key: string): Promise<void> {
+      await storage.delete(keyFor(key));
+    },
+  };
+}
+
+const isCookie = (value: unknown): value is ScrapeCookie =>
+  typeof value === 'object' &&
+  value !== null &&
+  typeof (value as { name?: unknown }).name === 'string' &&
+  typeof (value as { value?: unknown }).value === 'string';
+
+/** Stored JSON is `unknown`. Read structurally, and answer `undefined` rather than half a session. */
+export function parseSessionState(raw: unknown, key: string): SessionState | undefined {
+  if (typeof raw !== 'object' || raw === null) return undefined;
+  const value = raw as Partial<SessionState>;
+  if (!Array.isArray(value.cookies) || typeof value.savedAt !== 'string') return undefined;
+  return {
+    key,
+    savedAt: value.savedAt,
+    ...(typeof value.refusedAt === 'string' ? { refusedAt: value.refusedAt } : {}),
+    cookies: value.cookies.filter((cookie): cookie is ScrapeCookie => isCookie(cookie)),
+    headers: value.headers ?? {},
+    storage: value.storage ?? {},
+    userAgent: value.userAgent ?? '',
+    origin: value.origin ?? '',
+  };
+}