@ultimat3/scraping 2.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 developerz.ai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,194 @@
+# @ultimat3/scraping
+
+Browser automation as a **job**. `scrape()` returns a `JobHandle` — the rule's fourth instance
+after `llm()` (an action factory) and `backfill()` (a job factory). There is no ninth primitive.
+
+```ts
+import { t } from '@ultimat3/schema';
+import type { StorageDriver } from '@ultimat3/storage';
+import { scrape, storageSessionStore } from '@ultimat3/scraping';
+
+declare const disk: StorageDriver;
+
+const orderPage = t.object({
+  rows: t.array(t.object({ id: t.string, total: t.number })),
+  hasMore: t.boolean,
+});
+
+export const dailyOrders = scrape({
+  name: 'orders.daily',
+  input: t.object({ orgId: t.uuid, day: t.string }),
+  extract: t.object({ id: t.string, total: t.number }),
+  idempotencyKey: ({ orgId, day }) => `orders:${orgId}:${day}`,
+  tenant: ({ orgId }) => orgId,
+  allowHosts: ['shop.example.com', '*.api.example.com'],
+  block: ['image', 'media', 'font'],
+  rate: 1,
+  robots: 'obey',
+  secrets: ['SHOP_PASSWORD'],
+  expect: { minRows: 1, maxDrop: 0.5 },
+  auth: {
+    store: storageSessionStore(disk),
+    login: async ({ page, secrets, prompt }) => {
+      await page.goto('https://shop.example.com/login');
+      await page.fill('#user', 'ops@example.com');
+      await page.fill('#pass', secrets.get('SHOP_PASSWORD'));
+      await page.click('#submit');
+      await page.fill('#otp', await prompt('sms code'));
+    },
+    validate: async ({ page }) => (await page.count('#logout')) > 0,
+  },
+  async run({ page, http, step }) {
+    await page.goto('https://shop.example.com/orders');
+    const rows: { id: string; total: number }[] = [];
+    for (let pageNumber = 1; ; pageNumber += 1) {
+      // One `step.run` per page: a killed worker resumes at the page it stopped on. What a step
+      // persists is a CURSOR — never a page, never a live handle, never a session.
+      const batch = await step.run(`page:${pageNumber}`, async () =>
+        (await http.request(`https://api.example.com/orders?page=${pageNumber}`)).parse(orderPage),
+      );
+      rows.push(...batch.rows);
+      if (!batch.hasMore) break;
+    }
+    return rows;
+  },
+});
+```
+
+## Why a job
+
+| A scrape has | So it is |
+|---|---|
+| an input schema, a tenant, a retry policy, a timeout, a queue, a concurrency cap | a `job` |
+| a **required** idempotency key | a `job` — re-logging into a bank after a worker kill is the exact bug; three wrong attempts locks the account |
+| `step.run` checkpoints, because recovery must resume at the broken page | a `job` |
+
+It therefore inherits `.enqueue()`, the worker's cancellation, the dead-letter path, `x jobs show`
+and its manifest row without a line of code here.
+
+## The two transports, one session
+
+Drive the **browser** through login, 2FA and navigation; then reverse-engineer the site's own JSON
+endpoints and pull the bulk over **HTTP**. `http` is session-bound, never a bare `fetch`: the
+browser's cookies, headers and proxy, the same `allowHosts`, the same robots gate, the same rate
+limit, the same cancellation. `page.session()` exposes the handoff so an author can see what
+carried over.
+
+The jar is scoped per request, RFC 6265: a cookie stored for `bank.test` is sent to `bank.test`
+and to nothing else — not `evilbank.test`, not `sub.bank.test` — and only a domain-scoped
+`.bank.test` reaches subdomains. `SessionSnapshot.headers` is the one field the real driver cannot
+fill (CDP exposes no read for it, and `driver-parity.test.ts` pins that): a token the HTTP leg must
+carry goes on the request, `http.request(url, { headers })`.
+
+Both legs replay from **one** fixture directory (`fixtureBrowser(dir)`), so a hybrid run — browser
+login, session handoff, HTTP bulk fetch — is tested end to end. Both legs apply the same robots
+gate, offline included.
+
+## What it owns
+
+| Module | Owns |
+|---|---|
+| `scrape.ts` / `scrape-run.ts` | the factory over `job()`, and one attempt's assembly |
+| `page.ts` / `page-over-target.ts` | the driver-blind vocabulary, implemented once |
+| `target.ts` / `driver.ts` | the two seams: what a driver answers, and what a session is |
+| `driver-cdp.ts` / `cdp-*.ts` | the real browser, over a structural CDP port |
+| `driver-fake.ts` / `driver-fixture.ts` / `html-*.ts` | the offline drivers, on Bun's `HTMLRewriter` |
+| `http.ts` / `http-recorded.ts` | the second transport, live and replayed |
+| `auth.ts` / `session-state.ts` | acquire → persist → reuse → validate → burn |
+| `expect.ts` | the silent-green alarm |
+| `watchdog.ts` | the wedge and zombie discipline |
+| `errors.ts` / `error-throws.ts` | this package's `X_*` codes and their retry classification |
+
+## Extending it — there is no plugin API, and none is needed
+
+Two mechanisms, both of which the framework already ships:
+
+**1. The driver seam.** `ScrapeDriver` is the extension point, exactly as `jobs`' driver is.
+Everything a third-party driver needs is a named export of `src/index.ts`:
+
+```ts
+import {
+  pageOverTarget,
+  type ScrapeDriver,
+  type ScrapeHttp,
+  type ScrapeSession,
+  type ScrapeTarget,
+  type SessionInit,
+} from '@ultimat3/scraping';
+
+interface MyOptions {
+  readonly endpoint: string;
+}
+
+// Implement `ScrapeTarget` — twelve methods, all driver-blind — and the vocabulary above it,
+// actionability and frame re-resolution included, comes from `pageOverTarget` unchanged.
+declare function myTarget(options: MyOptions, init: SessionInit): Promise<ScrapeTarget>;
+declare function myHttp(target: ScrapeTarget, init: SessionInit): ScrapeHttp;
+
+export const myBrowser = (options: MyOptions): ScrapeDriver => ({
+  name: 'my-browser',
+  async open(init: SessionInit): Promise<ScrapeSession> {
+    const target = await myTarget(options, init);
+    return {
+      driver: 'my-browser',
+      page: pageOverTarget(target, {
+        clock: init.clock,
+        allowHosts: init.rules.allowHosts,
+        defaultTimeoutMs: init.timeoutMs,
+        secrets: init.secrets,
+        robots: init.robots,
+        signal: init.signal,
+      }),
+      http: myHttp(target, init),
+      close: () => target.close(),
+    };
+  },
+});
+```
+
+**2. Primitives are functions returning values.** An app's house style is a wrapper, not a fork:
+
+```ts
+// apps/web/shared/base/bank-scrape.ts — the app's convention, written once
+import type { JobHandle } from '@ultimat3/jobs';
+import type { StorageDriver } from '@ultimat3/storage';
+import { scrape, type ScrapeDefinition } from '@ultimat3/scraping';
+
+declare const disk: StorageDriver;
+
+export const bankScrape = <I, R>(over: ScrapeDefinition<I, R>): JobHandle<I> =>
+  scrape<I, R>({
+    robots: 'obey',
+    rate: 0.5,
+    block: ['image', 'media', 'font'],
+    expect: { minRows: 1, maxDrop: 0.5 },
+    artifacts: { storage: disk },
+    ...over,
+  });
+```
+
+Nothing downstream can tell the difference: the value is still a `JobHandle`, so the registry, the
+manifest and `x jobs show` work on it unchanged.
+
+Where a genuine hook is needed it is a **declared callback field** on the definition — `recover`,
+`auth.login`, `auth.validate`, `prompt` — typed, discoverable, one per concern. Never a global
+register-a-plugin call ([`docs/idea/19-mechanism-not-convention.md`](../../docs/idea/19-mechanism-not-convention.md)).
+
+## What does NOT ship, and why
+
+| Not shipped | Reason |
+|---|---|
+| stealth payloads | a framework-shipped payload is a shared, fingerprintable signature handed to every user. Two independent teams reached the same conclusion — one moved stealth into a Chromium fork, another *removed* an injected script because the injection was itself detectable. The hook ships; the payload never does |
+| captcha solving | same argument, plus it is not a mechanism — it is a business decision about somebody else's site |
+| credential stuffing, lockout-defeating retry | `X_SCRAPE_AUTH_FAILED` is **terminal**, and the refusal is written into the session record so the next attempt fails before reaching a login form. A site that locks an account after three wrong attempts makes a retrying framework the thing that destroys the user's account |
+| a `puppeteer-core` dependency | the launcher is passed in (`localBrowser({ launcher: puppeteer })`), the port is structural, and no puppeteer type can reach the vocabulary |
+
+## Errors
+
+Every code carries a cause, a runnable `fix:` and a retry classification — see `src/errors.ts`.
+`X_SCRAPE_YIELD_COLLAPSED`, `X_SCRAPE_AUTH_FAILED`, `X_SCRAPE_BLOCKED` and `X_SCRAPE_PAGE_CRASHED`
+are the four worth knowing by heart.
+
+## Boundary
+
+Tier 5. May import tiers 0-4 only — enforced by `bun run scripts/boundaries.ts`.

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@ultimat3/scraping",
+  "version": "2.0.0",
+  "description": "Browser automation as a job: scrape() returns a JobHandle",
+  "license": "MIT",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/developerz-ai/ultimate.git",
+    "directory": "packages/scraping"
+  },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "bun": ">=1.3.0"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit -p tsconfig.json",
+    "test": "bun test"
+  },
+  "dependencies": {
+    "@ultimat3/core": "2.0.0",
+    "@ultimat3/jobs": "2.0.0",
+    "@ultimat3/schema": "2.0.0",
+    "@ultimat3/storage": "2.0.0"
+  }
+}

package/src/actionability.ts

@@ -0,0 +1,106 @@
+// What "the element is ready" means, in one place. `page.click` in a raw driver waits for the
+// SELECTOR and nothing else, so every app that has ever used one re-learns the same lesson and
+// writes the same `waitForTimeout(800)` — or the same hand-rolled
+// `waitForSelector('#aceptar:not([disabled])')`, which is this rule, spelled once, by hand, at
+// one call site out of forty.
+
+import type { ScrapeClock } from './clock';
+import { deadline } from './clock';
+import { notActionable, selectorMissing } from './error-throws';
+import type { ElementSnapshot } from './target';
+
+/** What a caller needs to be true before it acts. Each level implies the ones above it. */
+export type ActionabilityState = 'attached' | 'visible' | 'enabled' | 'actionable';
+
+export const DEFAULT_POLL_MS = 50;
+
+const sameBox = (a: ElementSnapshot, b: ElementSnapshot): boolean =>
+  a.box === undefined || b.box === undefined
+    ? true
+    : a.box.x === b.box.x &&
+      a.box.y === b.box.y &&
+      a.box.width === b.box.width &&
+      a.box.height === b.box.height;
+
+/**
+ * Stability, as an observation rather than as a timer: two consecutive polls that agree on
+ * everything this package can see. A layout-carrying driver compares boxes and so catches an
+ * element still sliding in from a CSS transition; a DOM-only driver compares text, value and
+ * attributes and so catches one still being re-rendered. Neither sleeps 800ms and hopes.
+ */
+export const isStable = (
+  current: ElementSnapshot,
+  previous: ElementSnapshot | undefined,
+): boolean =>
+  previous !== undefined &&
+  sameBox(current, previous) &&
+  current.text === previous.text &&
+  current.value === previous.value &&
+  current.enabled === previous.enabled &&
+  current.visible === previous.visible;
+
+/**
+ * Why this snapshot is not yet ready for `state`, or `undefined` when it is.
+ *
+ * `hitTarget === undefined` means the driver has no layout engine, so nothing here can decide
+ * whether something covers the element — and it is NOT read as "covered". The divergence is
+ * deliberate, and `driver-parity.test.ts` pins it in one place: a fake that answered `false` would
+ * make every click in every offline test fail, and one that fabricated `true` on a real browser
+ * would hide the cookie banner that eats the click.
+ */
+export function actionabilityProblem(
+  current: ElementSnapshot,
+  previous: ElementSnapshot | undefined,
+  state: ActionabilityState,
+): string | undefined {
+  if (state === 'attached') return undefined;
+  if (!current.visible) return 'not visible';
+  if (current.box !== undefined && (current.box.width === 0 || current.box.height === 0))
+    return 'has a zero-sized box';
+  if (state === 'visible') return undefined;
+  if (!current.enabled) return 'disabled';
+  if (state === 'enabled') return undefined;
+  if (current.hitTarget === false) return 'covered by another element at its centre';
+  if (!isStable(current, previous)) return 'still moving';
+  return undefined;
+}
+
+export interface ActionabilityWait {
+  readonly selector: string;
+  readonly url: string;
+  readonly state: ActionabilityState;
+  readonly timeoutMs: number;
+  readonly clock: ScrapeClock;
+  readonly pollMs?: number | undefined;
+  readonly signal?: AbortSignal | undefined;
+  /** Re-read from the live page on EVERY poll. Never a handle captured before the loop. */
+  snapshot(): Promise<ElementSnapshot | undefined>;
+}
+
+/**
+ * Poll until the element reaches `state`, or refuse with the reason it never did. Two codes and
+ * they are genuinely different questions: `X_SCRAPE_SELECTOR_MISSING` says the page does not have
+ * this element (the markup changed), `X_SCRAPE_NOT_ACTIONABLE` says it does and something is in
+ * the way (a modal, a spinner, a disabled submit). Collapsing them into one timeout is what makes
+ * a scraper failure take an afternoon.
+ */
+export async function awaitActionable(wait: ActionabilityWait): Promise<ElementSnapshot> {
+  const budget = deadline(wait.clock, wait.timeoutMs);
+  const pollMs = wait.pollMs ?? DEFAULT_POLL_MS;
+  let previous: ElementSnapshot | undefined;
+  let lastProblem: string | undefined;
+  let everSeen = false;
+  for (;;) {
+    const current = await wait.snapshot();
+    if (current !== undefined) {
+      everSeen = true;
+      lastProblem = actionabilityProblem(current, previous, wait.state);
+      if (lastProblem === undefined) return current;
+      previous = current;
+    }
+    if (budget.expired()) break;
+    await wait.clock.sleep(Math.min(pollMs, budget.remainingMs()), wait.signal);
+  }
+  if (!everSeen) throw selectorMissing(wait.selector, wait.url, wait.timeoutMs);
+  throw notActionable(wait.selector, lastProblem ?? 'never became actionable', wait.timeoutMs);
+}

package/src/artifacts.ts

@@ -0,0 +1,69 @@
+// Run artifacts: the page HTML, the screenshot, the downloaded file. A scrape that failed and
+// kept nothing is a scrape somebody has to reproduce by hand, and the page it failed on no longer
+// exists by the time they look.
+//
+// Written through `@ultimat3/storage`'s driver, so the same call lands on a local disk in
+// development and on S3 in production, and this package owns no upload path of its own.
+
+import type { StorageDriver } from '@ultimat3/storage';
+
+export interface ArtifactRef {
+  readonly key: string;
+  readonly bytes: number;
+  readonly contentType: string;
+}
+
+export interface ArtifactWriter {
+  /** Bytes under this run's own prefix. Answers the key, which is what a report links to. */
+  save(name: string, body: Uint8Array | string, contentType?: string): Promise<ArtifactRef>;
+  /** Everything written by this run, in order. Bounded by how many the body asked for. */
+  readonly saved: readonly ArtifactRef[];
+}
+
+export interface ArtifactWriterInit {
+  readonly storage: StorageDriver | undefined;
+  readonly scrape: string;
+  readonly runId: string;
+  /** Optional prefix, so an app can keep scrape artifacts out of its user-upload namespace. */
+  readonly prefix?: string | undefined;
+}
+
+export const DEFAULT_ARTIFACT_PREFIX = 'scrape';
+
+const CONTENT_TYPES: Readonly<Record<string, string>> = {
+  html: 'text/html; charset=utf-8',
+  png: 'image/png',
+  pdf: 'application/pdf',
+  json: 'application/json',
+  csv: 'text/csv',
+  txt: 'text/plain; charset=utf-8',
+};
+
+export const contentTypeFor = (name: string): string =>
+  CONTENT_TYPES[name.split('.').pop()?.toLowerCase() ?? ''] ?? 'application/octet-stream';
+
+/**
+ * A writer with no storage driver is a NO-OP that still answers a key — never a throw. An app
+ * that has not configured storage should still be able to run a scrape; losing the artifact is a
+ * cost, and refusing the run over it is a bigger one.
+ */
+export function createArtifactWriter(init: ArtifactWriterInit): ArtifactWriter {
+  const saved: ArtifactRef[] = [];
+  const prefix = `${init.prefix ?? DEFAULT_ARTIFACT_PREFIX}/${init.scrape}/${init.runId}`;
+  return {
+    saved,
+    async save(name, body, contentType): Promise<ArtifactRef> {
+      const bytes = typeof body === 'string' ? new TextEncoder().encode(body) : body;
+      const ref: ArtifactRef = {
+        key: `${prefix}/${name}`,
+        bytes: bytes.byteLength,
+        contentType: contentType ?? contentTypeFor(name),
+      };
+      if (init.storage !== undefined) {
+        await init.storage.put(ref.key, bytes, { contentType: ref.contentType });
+      }
+      saved.push(ref);
+      return ref;
+    },
+  };
+}

package/src/auth.ts

@@ -0,0 +1,200 @@
+// Session lifecycle: acquire -> persist -> reuse -> validate -> burn. Authenticated scraping is
+// the primary case, not an edge case, so this is a declared part of the API rather than something
+// every app hand-rolls.
+//
+// The order below is the whole design, and each step is there because skipping it costs something
+// real:
+//
+// | Step | Why it is not optional |
+// |---|---|
+// | reuse | logging in every run is slow, and the login itself is the anti-bot signal |
+// | validate | an expired session must be caught by a cheap probe, not by a failure 40 steps later |
+// | login | only when the probe says the session is gone — the author drives the browser |
+// | persist | a 2FA code the user typed is worth keeping; that is the whole economic argument |
+// | burn | a flagged identity stays flagged, so a retry on it re-trips the same block forever |
+//
+// What is NOT here, deliberately: credential stuffing, captcha solving, and any retry of a
+// rejected credential. A rejected credential is `X_SCRAPE_AUTH_FAILED`, terminal, and the refusal
+// is written into the session record so the NEXT attempt fails before reaching the login form.
+
+import type { Logger } from '@ultimat3/core';
+import type { ScrapeClock } from './clock';
+import { authFailed, promptUnanswered, sessionExpired } from './error-throws';
+import type { ScrapePage } from './page';
+import type { ScrapeSecrets } from './secrets';
+import { type ScrapeSessionStore, type SessionState, sessionDigest } from './session-state';
+
+export interface PromptRequest {
+  /** What the site is asking for: `'sms code'`, `'authenticator code'`, `'security question 3'`. */
+  readonly label: string;
+  readonly scrape: string;
+  readonly url: string;
+}
+
+/**
+ * Where an out-of-band code comes from. A declared callback field on the definition — typed,
+ * discoverable, one per concern — and never a global "register a plugin" call.
+ */
+export type PromptHandler = (request: PromptRequest) => Promise<string> | string;
+
+export interface AuthContext<I> {
+  readonly input: I;
+  readonly page: ScrapePage;
+  /**
+   * The declared secrets, boxed. A login body needs the credential it is about to type, and
+   * without this field there was no way to reach one — the README's own first example could not
+   * have compiled, which is how the omission was found.
+   */
+  readonly secrets: ScrapeSecrets;
+  /** The out-of-band code seam. Throws `X_SCRAPE_PROMPT_UNANSWERED` if nothing was declared. */
+  prompt(label: string): Promise<string>;
+}
+
+export interface ScrapeAuth<I> {
+  /**
+   * Drives the browser through the login. Runs ONLY when there is no valid session — never on
+   * every run, and never as a retry of a refused credential.
+   */
+  login(context: AuthContext<I>): Promise<void>;
+  /**
+   * The cheap probe: is the restored session still good? Site knowledge, so the author declares
+   * it and the framework decides what to do with the answer. Omitted means a restored session is
+   * trusted until something else fails.
+   */
+  validate?(context: AuthContext<I>): Promise<boolean>;
+  /** Where sessions live. Omitted means no reuse at all — every run logs in. */
+  readonly store?: ScrapeSessionStore | undefined;
+  /**
+   * The DISCRIMINATOR inside this tenant's key space — a second account, say. It is one segment of
+   * `sessionKeyFor({ scrape, tenant, discriminator })` and never the whole key: a value that
+   * replaced the key would put two tenants declaring the same account name on one authenticated
+   * session. Sanitised like every other segment, so it cannot escape the key space either.
+   */
+  key?(input: I): string;
+  /** Reuse a stored session. `false` forces a fresh login every run. Defaults to `true`. */
+  readonly reuse?: boolean | undefined;
+  /** Milliseconds. A stored session older than this is not restored. */
+  readonly maxAge?: number | undefined;
+}
+
+export function createPrompt(
+  scrape: string,
+  handler: PromptHandler | undefined,
+  page: ScrapePage,
+): (label: string) => Promise<string> {
+  return async (label: string): Promise<string> => {
+    if (handler === undefined) throw promptUnanswered(scrape, label);
+    const answer = await handler({ label, scrape, url: page.url() });
+    if (typeof answer !== 'string' || answer === '') throw promptUnanswered(scrape, label);
+    return answer;
+  };
+}
+
+export interface AuthPlanInput<I> {
+  readonly scrape: string;
+  readonly auth: ScrapeAuth<I> | undefined;
+  readonly key: string;
+  readonly clock: ScrapeClock;
+  readonly logger: Logger;
+}
+
+/**
+ * The stored session this run may restore, or `undefined`.
+ *
+ * THROWS when the record carries a refusal: those credentials were rejected, and reaching the
+ * login form again with them is how an account gets locked. It runs BEFORE `driver.open()`, which
+ * is what it buys: `X_SCRAPE_AUTH_FAILED` is registered `terminal` so the queue will not retry,
+ * and this makes a replay, a manual requeue or a second enqueue refuse without spending a browser,
+ * a CDP attach or a request on an answer that cannot change.
+ */
+export async function restorableSession<I>(
+  plan: AuthPlanInput<I>,
+): Promise<SessionState | undefined> {
+  const store = plan.auth?.store;
+  if (store === undefined) return undefined;
+  const found = await store.load(plan.key);
+  if (found === undefined) return undefined;
+  // The tombstone is read BEFORE `reuse` is honoured. `reuse: false` says "do not restore this
+  // session"; it does not say "present the rejected credential again", and reading it first meant
+  // a `reuse: false` scrape walked a refused password back to the login form on every requeue.
+  if (found.refusedAt !== undefined) throw authFailed(plan.scrape, `refused at ${found.refusedAt}`);
+  if (plan.auth?.reuse === false) return undefined;
+  const maxAge = plan.auth?.maxAge;
+  if (maxAge !== undefined) {
+    const age = plan.clock.now().getTime() - new Date(found.savedAt).getTime();
+    if (age > maxAge) return undefined;
+  }
+  plan.logger.debug('scrape.session.restored', sessionDigest(found));
+  return found;
+}
+
+/** Written down so the next attempt cannot reach the login form with the same credentials. */
+export async function markRefused<I>(plan: AuthPlanInput<I>): Promise<void> {
+  const store = plan.auth?.store;
+  if (store === undefined) return;
+  await store.save({
+    key: plan.key,
+    savedAt: plan.clock.now().toISOString(),
+    refusedAt: plan.clock.now().toISOString(),
+    cookies: [],
+    headers: {},
+    storage: {},
+    userAgent: '',
+    origin: '',
+  });
+}
+
+/** New identity, from scratch. One call, because a flagged profile is unusable and must go. */
+export async function burnSession<I>(plan: AuthPlanInput<I>): Promise<void> {
+  if (plan.auth?.store === undefined) return;
+  await plan.auth.store.burn(plan.key);
+  plan.logger.warn('scrape.session.burned', { burned: true });
+}
+
+export interface EnsureAuthInput<I> extends AuthPlanInput<I> {
+  readonly input: I;
+  readonly page: ScrapePage;
+  readonly secrets: ScrapeSecrets;
+  readonly restored: SessionState | undefined;
+  readonly prompt: (label: string) => Promise<string>;
+}
+
+/**
+ * Log in if — and only if — this run has no session it can prove is still good.
+ *
+ * A resumed attempt takes the same path and reaches the same conclusion: the session lives in the
+ * store, not in a step record, so restarting the job re-reads it and re-probes it rather than
+ * replaying a checkpoint that says "logged in" about a session that has since expired. What a
+ * step may persist is a cursor; a session is neither a cursor nor a page.
+ */
+export async function ensureAuthenticated<I>(args: EnsureAuthInput<I>): Promise<boolean> {
+  const auth = args.auth;
+  if (auth === undefined) return false;
+  const context: AuthContext<I> = {
+    input: args.input,
+    page: args.page,
+    secrets: args.secrets,
+    prompt: args.prompt,
+  };
+  if (args.restored !== undefined) {
+    if (auth.validate === undefined) return false;
+    if (await auth.validate(context)) {
+      args.logger.info('scrape.session.reused', { reused: true });
+      return false;
+    }
+    args.logger.info('scrape.session.expired', { reused: false });
+    await burnSession(args);
+  }
+  if (auth.login === undefined) throw sessionExpired(args.scrape, args.key);
+  await auth.login(context);
+  return true;
+}
+
+/** After a login: keep what it produced. A 2FA code somebody typed is worth exactly this. */
+export async function persistSession<I>(plan: AuthPlanInput<I>, page: ScrapePage): Promise<void> {
+  const store = plan.auth?.store;
+  if (store === undefined) return;
+  const snapshot = await page.session();
+  await store.save({ ...snapshot, key: plan.key, savedAt: plan.clock.now().toISOString() });
+  plan.logger.info('scrape.session.saved', sessionDigest(snapshot));
+}