@ultimat3/scraping 2.0.0

package/src/error-throws.ts

@@ -0,0 +1,258 @@
+// One constructor per failure mode, so a throw site is one call and every `cause`/`fix` pair for
+// a code is written once. Nothing here interpolates an `unknown` into a cause: an exception from
+// a browser is rendered through core's `renderThrowable`, which is the rule `bun run error-render`
+// enforces.
+
+import { renderThrowable, UltimateError } from '@ultimat3/core';
+import { ScrapeError } from './errors';
+
+/**
+ * Two shapes of the same failure, one code. `name` is the driver a definition ASKED for; `scrape`
+ * is passed when there is no driver at all to name — and that is the reachable case, so the cause
+ * has to read as one. It said `no scrape driver named "orders.daily" is installed`, naming the
+ * scrape as a driver, which sends its reader hunting for a driver nobody ever declared.
+ */
+export const driverUnknown = (
+  name: string | undefined,
+  installed: readonly string[],
+  scrape?: string,
+): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_DRIVER_UNKNOWN',
+    cause:
+      scrape === undefined
+        ? `no scrape driver named "${name ?? 'none'}" is installed; installed: ${installed.join(', ') || 'none'}`
+        : `scrape "${scrape}" has no browser driver: nothing called setScrapeDriver() and the definition declares no driver; installed: ${installed.join(', ') || 'none'}`,
+    fix: 'call setScrapeDriver(localBrowser()) at boot, or pass driver: fakeBrowser() on the scrape() definition',
+    meta: { driver: name ?? 'none', ...(scrape === undefined ? {} : { scrape }) },
+  });
+
+export const cdpAttachFailed = (cdpUrl: string, thrown: unknown): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_CDP_ATTACH_FAILED',
+    cause: `the CDP endpoint ${cdpUrl} refused the attach: ${renderThrowable(thrown)}`,
+    fix: 'curl "$CDP_URL/json/version" to confirm the endpoint answers, then pass that webSocketDebuggerUrl to remoteBrowser({ cdpUrl })',
+    meta: { cdpUrl },
+  });
+
+export const browserUnreachable = (driver: string, thrown: unknown): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_BROWSER_UNREACHABLE',
+    cause: `the ${driver} browser stopped answering: ${renderThrowable(thrown)}`,
+    fix: 'raise watchdog: { idleMs } on the scrape() definition if the site is genuinely slow, otherwise re-run once the browser host is back',
+    meta: { driver },
+  });
+
+export const profileLocked = (profileDir: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_PROFILE_LOCKED',
+    cause: `another browser process holds the profile at ${profileDir}`,
+    fix: `rm -f ${profileDir}/SingletonLock once no browser is using it, or give this run its own localBrowser({ profileDir })`,
+    meta: { profileDir },
+  });
+
+export const hostBlocked = (url: string, allowed: readonly string[]): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_HOST_BLOCKED',
+    cause: `the page requested ${url}, and allowHosts lists ${allowed.join(', ') || 'nothing'}`,
+    fix: `add the host to allowHosts on the scrape() definition — allowHosts: [${allowed.map((host) => `'${host}'`).join(', ')}, '<the host above>']`,
+    meta: { url, allowed },
+  });
+
+export const selectorMissing = (selector: string, url: string, waitedMs: number): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_SELECTOR_MISSING',
+    cause: `"${selector}" never appeared on ${url} within ${String(waitedMs)}ms`,
+    fix: "open the run's page.html artifact and re-derive the selector, or raise the timeout on this waitFor({ timeout })",
+    meta: { selector, url, waitedMs },
+  });
+
+export const notActionable = (selector: string, reason: string, waitedMs: number): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_NOT_ACTIONABLE',
+    cause: `"${selector}" is present and ${reason} after ${String(waitedMs)}ms`,
+    fix: 'wait for the state that unblocks it — page.waitFor(selector, { state: "enabled" }) — or dismiss whatever overlays it before the click',
+    meta: { selector, reason, waitedMs },
+  });
+
+export const scrapeTimeout = (what: string, ms: number): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_TIMEOUT',
+    cause: `${what} exceeded its ${String(ms)}ms budget`,
+    fix: 'raise timeout: on the scrape() definition, or split the pass so each step.run stays inside one budget',
+    meta: { what, ms },
+  });
+
+export const wedged = (what: string, idleMs: number): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_WEDGED',
+    cause: `${what} produced no browser activity for ${String(idleMs)}ms, so the process was killed`,
+    fix: 'raise watchdog: { idleMs } on the scrape() definition if the site is genuinely this slow, otherwise re-run — the browser was killed and its session is gone',
+    meta: { what, idleMs },
+  });
+
+export const pageCrashed = (url: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_PAGE_CRASHED',
+    cause: `the renderer for ${url} died mid-run, so this attempt's page state is gone`,
+    fix: 'lower concurrency: on the scrape() definition or raise the container memory limit in docker-compose.prod.yml — a crashed renderer is out of memory far more often than it is a bug',
+    meta: { url },
+  });
+
+export const outputInvalid = (name: string, detail: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_OUTPUT_INVALID',
+    cause: `scrape "${name}" produced rows its extract schema rejects: ${detail}`,
+    fix: "align the extract schema with the page — the run's page.html artifact holds the markup the rows came from",
+    meta: { scrape: name },
+  });
+
+export const downloadTimeout = (ms: number, url: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_DOWNLOAD_TIMEOUT',
+    cause: `no download landed within ${String(ms)}ms of the trigger on ${url}`,
+    fix: 'raise the timeout on page.download({ timeout }), or confirm the trigger is the element that starts the download',
+    meta: { ms, url },
+  });
+
+/**
+ * The silent-green alarm. A scraper that succeeds and returns nothing stays green for weeks, and
+ * every field of this cause is there so the first read answers "was it always this low, or did it
+ * fall off a cliff today?" without opening a dashboard.
+ */
+export const yieldCollapsed = (input: {
+  readonly scrape: string;
+  readonly rows: number;
+  readonly reason: 'min-rows' | 'drop';
+  readonly minRows?: number | undefined;
+  readonly baseline?: number | undefined;
+  readonly maxDrop?: number | undefined;
+}): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_YIELD_COLLAPSED',
+    cause:
+      input.reason === 'min-rows'
+        ? `scrape "${input.scrape}" returned ${String(input.rows)} rows and declares expect.minRows ${String(input.minRows)}`
+        : `scrape "${input.scrape}" returned ${String(input.rows)} rows against a trailing median of ${String(input.baseline)}, past the ${String(Math.round((input.maxDrop ?? 0) * 100))}% drop expect.maxDrop allows`,
+    fix: "open the run's page.html artifact and compare it with the extract selectors — a collapse is the page changing far more often than the data changing",
+    meta: {
+      scrape: input.scrape,
+      rows: input.rows,
+      reason: input.reason,
+      minRows: input.minRows,
+      baseline: input.baseline,
+      maxDrop: input.maxDrop,
+    },
+  });
+
+export const robotsDisallowed = (url: string, agent: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_ROBOTS_DISALLOWED',
+    cause: `robots.txt disallows ${url} for user-agent "${agent}"`,
+    fix: "declare robots: { ignore: '<the written reason this run is permitted>' } on the scrape() definition, or scrape a path robots.txt allows",
+    meta: { url, agent },
+  });
+
+export const fixtureMissing = (url: string, dir: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_FIXTURE_MISSING',
+    cause: `the fixture driver has no recording of ${url} under ${dir}, and an offline driver never reaches the network`,
+    fix: `add ${dir}/<the recording file> with { "url": "${url}", "html": "…" }, or point fixtureBrowser() at the directory that already holds it`,
+    meta: { url, dir },
+  });
+
+export const fixtureStale = (url: string, ageMs: number, maxAgeMs: number): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_FIXTURE_STALE',
+    cause: `the recording of ${url} is ${String(Math.round(ageMs / 86_400_000))} days old and fixtureBrowser declares maxAge ${String(Math.round(maxAgeMs / 86_400_000))} days`,
+    fix: 're-record the fixture directory, or raise maxAge on fixtureBrowser({ maxAge }) with the reason an old recording still proves something',
+    meta: { url, ageMs, maxAgeMs },
+  });
+
+export const remoteRequired = (driver: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_REMOTE_REQUIRED',
+    cause: `the ${driver} driver attaches to a browser somebody else started and was given no cdpUrl`,
+    fix: 'pass remoteBrowser({ cdpUrl: env.SCRAPE_CDP_URL }), or use localBrowser({ launcher }) to start one in this container',
+    meta: { driver },
+  });
+
+export const recoverRefused = (name: string, reason: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_RECOVER_REFUSED',
+    cause: `the recover hook on scrape "${name}" declined this failure: ${reason}`,
+    fix: "widen recover() to handle this failure, or drop recover: and let the job's retry policy own it",
+    meta: { scrape: name, reason },
+  });
+
+export const secretExposed = (artifact: string, url: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_SECRET_EXPOSED',
+    cause: `a ${artifact} of ${url} was requested after a secret was typed into this page, and pixels cannot be redacted afterwards`,
+    fix: 'call artifact.html() instead — page HTML is redacted by value and password fields are blanked — or take the capture before the secret is typed',
+    meta: { artifact, url },
+  });
+
+/** A non-2xx from the HTTP leg. 4xx below 429 is terminal; everything else may be tried again. */
+export const httpFailed = (url: string, status: number, body: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_HTTP_FAILED',
+    cause: `${url} answered ${String(status)}: ${body}`,
+    fix: 'confirm the endpoint the browser leg calls is the one this request names — page.network() lists every URL the page actually fetched',
+    meta: { url, status },
+    retry: status >= 400 && status < 500 && status !== 429 ? 'terminal' : 'retryable',
+  });
+
+/**
+ * TERMINAL, and the retry table cannot be talked out of it. A site that locks an account after
+ * three wrong attempts turns a retrying framework into the thing that destroys the user's
+ * account, so this failure ends the run — no backoff, no recovery hook, no second attempt.
+ */
+export const authFailed = (scrape: string, detail: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_AUTH_FAILED',
+    cause: `scrape "${scrape}" was refused by the site's login: ${detail}`,
+    fix: 'correct the credential in .env.local for the name listed in secrets: on the scrape() definition — this run will NOT be retried, deliberately: a second wrong attempt is how an account gets locked',
+    meta: { scrape },
+  });
+
+export const sessionExpired = (scrape: string, key: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_SESSION_EXPIRED',
+    cause: `the stored session ${key} for scrape "${scrape}" failed its validate() probe and the definition declares no auth.login`,
+    fix: `add auth: { login } to scrape("${scrape}") so an expired session can be replaced, or drop auth.validate and let the body handle the logged-out page`,
+    meta: { scrape, key },
+  });
+
+export const promptUnanswered = (scrape: string, label: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_PROMPT_UNANSWERED',
+    cause: `scrape "${scrape}" asked for "${label}" and no prompt handler was declared`,
+    fix: `add prompt: async ({ label }) => await otpFor(label) to scrape("${scrape}") — a code that arrives out of band needs somewhere to come from`,
+    meta: { scrape, label },
+  });
+
+/**
+ * The identity is spent. Retryable — and the run BURNS the session before the retry, because a
+ * flagged profile stays flagged and reloading it re-trips the same block every time.
+ */
+export const blocked = (scrape: string, url: string, detail: string): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_BLOCKED',
+    cause: `scrape "${scrape}" was refused by ${url}: ${detail}`,
+    fix: 'the session is burned and the next attempt starts a new identity — lower rate: on the scrape() definition if this repeats',
+    meta: { scrape, url },
+  });
+
+/**
+ * The honest stub, in the shape `packages/jobs/src/driver-redis.ts` uses: correct types so an app
+ * can be written against the seam, and one labelled throw so nobody discovers the gap from a
+ * silently-skipped recovery.
+ */
+export const scrapeNotImplemented = (feature: string, fix: string): UltimateError =>
+  new UltimateError({
+    code: 'X_NOT_IMPLEMENTED',
+    cause: `${feature} is declared and not implemented in @ultimat3/scraping`,
+    fix,
+    meta: { feature },
+  });

package/src/errors.ts

@@ -0,0 +1,180 @@
+// The X_* codes owned by @ultimat3/scraping, and — the half that carries the weight — their
+// retry classification. A scraper's whole operational question is "was that the site being slow
+// or the site being different?", so every code here is registered as `retryable` or `terminal`
+// once, in this file, rather than re-decided by whichever `catch` saw it.
+
+import type { ErrorRetry } from '@ultimat3/core';
+import {
+  errorDocsUrl,
+  registerErrorCodes,
+  registerErrorRetry,
+  UltimateError,
+} from '@ultimat3/core';
+
+/** Codes this package declares and owns. */
+export const SCRAPE_OWNED_ERROR_CODES = [
+  'X_SCRAPE_DRIVER_UNKNOWN',
+  'X_SCRAPE_CDP_ATTACH_FAILED',
+  'X_SCRAPE_BROWSER_UNREACHABLE',
+  'X_SCRAPE_PROFILE_LOCKED',
+  'X_SCRAPE_HOST_BLOCKED',
+  'X_SCRAPE_SELECTOR_MISSING',
+  'X_SCRAPE_NOT_ACTIONABLE',
+  'X_SCRAPE_TIMEOUT',
+  'X_SCRAPE_WEDGED',
+  'X_SCRAPE_PAGE_CRASHED',
+  'X_SCRAPE_OUTPUT_INVALID',
+  'X_SCRAPE_YIELD_COLLAPSED',
+  'X_SCRAPE_DOWNLOAD_TIMEOUT',
+  'X_SCRAPE_ROBOTS_DISALLOWED',
+  'X_SCRAPE_FIXTURE_MISSING',
+  'X_SCRAPE_FIXTURE_STALE',
+  'X_SCRAPE_REMOTE_REQUIRED',
+  'X_SCRAPE_RECOVER_REFUSED',
+  'X_SCRAPE_SECRET_EXPOSED',
+  'X_SCRAPE_HTTP_FAILED',
+  'X_SCRAPE_AUTH_FAILED',
+  'X_SCRAPE_SESSION_EXPIRED',
+  'X_SCRAPE_PROMPT_UNANSWERED',
+  'X_SCRAPE_BLOCKED',
+] as const;
+
+/**
+ * `X_NOT_IMPLEMENTED` is `@ultimat3/core`'s and `recover.ts` throws it for the agent seam, the
+ * way `packages/jobs/src/driver-redis.ts` does for its stub. `X_ENV_MISSING` is core's too: a
+ * declared secret with no value in the environment is a missing environment variable, not a
+ * scraping concept needing its own code. No title is kept here for either — one code, one owner,
+ * or the two copies drift.
+ */
+export const SCRAPE_BORROWED_ERROR_CODES = ['X_NOT_IMPLEMENTED', 'X_ENV_MISSING'] as const;
+
+export const SCRAPE_ERROR_CODES = [
+  ...SCRAPE_OWNED_ERROR_CODES,
+  ...SCRAPE_BORROWED_ERROR_CODES,
+] as const;
+
+export type ScrapeOwnedErrorCode = (typeof SCRAPE_OWNED_ERROR_CODES)[number];
+export type ScrapeErrorCode = (typeof SCRAPE_ERROR_CODES)[number];
+
+export const SCRAPE_ERROR_TITLES: Readonly<Record<ScrapeOwnedErrorCode, string>> = {
+  X_SCRAPE_DRIVER_UNKNOWN: 'no browser driver is installed for this run',
+  X_SCRAPE_CDP_ATTACH_FAILED: 'the CDP endpoint refused the attach',
+  X_SCRAPE_BROWSER_UNREACHABLE: 'the browser went away mid-run',
+  X_SCRAPE_PROFILE_LOCKED: 'another process holds this browser profile',
+  X_SCRAPE_HOST_BLOCKED: 'the page asked for a host allowHosts does not list',
+  X_SCRAPE_SELECTOR_MISSING: 'the selector never appeared inside its window',
+  X_SCRAPE_NOT_ACTIONABLE: 'the element is present and cannot be acted on',
+  X_SCRAPE_TIMEOUT: 'the step exceeded its wall-clock budget',
+  X_SCRAPE_WEDGED: 'the browser stopped answering and was killed',
+  X_SCRAPE_PAGE_CRASHED: 'the renderer process died',
+  X_SCRAPE_OUTPUT_INVALID: 'the extracted rows do not match the extract schema',
+  X_SCRAPE_YIELD_COLLAPSED: 'the run succeeded and returned far too little',
+  X_SCRAPE_DOWNLOAD_TIMEOUT: 'the download never landed',
+  X_SCRAPE_ROBOTS_DISALLOWED: 'robots.txt disallows this path',
+  X_SCRAPE_FIXTURE_MISSING: 'the fixture driver has no recording for this request',
+  X_SCRAPE_FIXTURE_STALE: 'the recording is older than the fixture max age',
+  X_SCRAPE_REMOTE_REQUIRED: 'this driver needs a cdpUrl and was given none',
+  X_SCRAPE_RECOVER_REFUSED: 'the recovery hook declined to recover this failure',
+  X_SCRAPE_SECRET_EXPOSED: 'an artifact would have carried a secret this run typed',
+  X_SCRAPE_HTTP_FAILED: 'the site answered the HTTP leg with a non-2xx status',
+  X_SCRAPE_AUTH_FAILED: 'the credentials were rejected',
+  X_SCRAPE_SESSION_EXPIRED: 'the restored session is no longer valid and nothing can renew it',
+  X_SCRAPE_PROMPT_UNANSWERED: 'a login step asked for a code and nothing answered',
+  X_SCRAPE_BLOCKED: 'the site refused this client — the identity is spent',
+};
+
+// One unconditional call, so a second package claiming one of these codes throws
+// X_ERROR_CODE_DUPLICATE instead of losing silently to whichever module imported first.
+registerErrorCodes(
+  Object.fromEntries(Object.entries(SCRAPE_ERROR_TITLES).map(([code, title]) => [code, { title }])),
+);
+
+/**
+ * The retry table, which is this package's most load-bearing declaration: a worker reads it to
+ * decide whether attempt 2 is worth running, and both wrong answers cost real money.
+ *
+ * The rule used, stated once: **retryable means the same code, run again, has a real chance of a
+ * different answer.** Transport, budget and liveness faults qualify. Everything that says "the
+ * page is not the page this scraper was written against" does not — attempt 5 hammers a site with
+ * a request that cannot succeed, and on an authenticated target that is how three wrong attempts
+ * lock an account. `X_SCRAPE_YIELD_COLLAPSED` is deliberately terminal for the same reason: a run
+ * that returned nothing is a human's problem, not a queue's.
+ */
+export const SCRAPE_ERROR_RETRY = {
+  X_SCRAPE_CDP_ATTACH_FAILED: 'retryable',
+  X_SCRAPE_BROWSER_UNREACHABLE: 'retryable',
+  X_SCRAPE_TIMEOUT: 'retryable',
+  X_SCRAPE_WEDGED: 'retryable',
+  X_SCRAPE_DOWNLOAD_TIMEOUT: 'retryable',
+  // Retryable AND it burns the session first (`scrape-run.ts`). Retrying a block on the SAME
+  // persisted identity re-trips it every time: the flagged cookies are the thing being refused,
+  // so the retry has to arrive as somebody else or it is arithmetic, not a retry.
+  X_SCRAPE_BLOCKED: 'retryable',
+  // Non-2xx is transient far more often than not (429, 502, a deploy). A 4xx that is genuinely
+  // permanent is thrown with a per-instance `terminal` override, which `UltimateError` supports —
+  // one code, and the throw site decides, because the same status is both at different sites.
+  X_SCRAPE_HTTP_FAILED: 'retryable',
+  // Everything below is terminal, and each one is listed rather than left to the default so that
+  // deleting a line is a visible decision.
+  X_SCRAPE_DRIVER_UNKNOWN: 'terminal',
+  X_SCRAPE_PROFILE_LOCKED: 'terminal',
+  X_SCRAPE_HOST_BLOCKED: 'terminal',
+  X_SCRAPE_SELECTOR_MISSING: 'terminal',
+  X_SCRAPE_NOT_ACTIONABLE: 'terminal',
+  // A renderer that died takes its tab's state with it. Every retry so far in this package is a
+  // retry of an attempt that could still be somewhere; this one cannot, and a re-run of a
+  // half-submitted form is the incident, not the recovery.
+  X_SCRAPE_PAGE_CRASHED: 'terminal',
+  X_SCRAPE_OUTPUT_INVALID: 'terminal',
+  X_SCRAPE_YIELD_COLLAPSED: 'terminal',
+  X_SCRAPE_ROBOTS_DISALLOWED: 'terminal',
+  X_SCRAPE_FIXTURE_MISSING: 'terminal',
+  X_SCRAPE_FIXTURE_STALE: 'terminal',
+  X_SCRAPE_REMOTE_REQUIRED: 'terminal',
+  X_SCRAPE_RECOVER_REFUSED: 'terminal',
+  X_SCRAPE_SECRET_EXPOSED: 'terminal',
+  // THE hard rule of this package. A site that locks an account after three wrong attempts turns
+  // a retrying framework into the thing that destroys the user's account — so a rejected
+  // credential is terminal, always, and no retry policy, recovery hook or backoff can reach it.
+  X_SCRAPE_AUTH_FAILED: 'terminal',
+  X_SCRAPE_SESSION_EXPIRED: 'terminal',
+  X_SCRAPE_PROMPT_UNANSWERED: 'terminal',
+} as const satisfies Readonly<Record<ScrapeOwnedErrorCode, 'retryable' | 'terminal'>>;
+
+registerErrorRetry(SCRAPE_ERROR_RETRY);
+
+export interface ScrapeErrorInit {
+  readonly code: ScrapeErrorCode;
+  readonly cause: string;
+  readonly fix: string;
+  readonly meta?: Readonly<Record<string, unknown>> | undefined;
+  /**
+   * Per-instance override of the table above, for the one case where a code is genuinely both:
+   * a 429 and a 404 share `X_SCRAPE_HTTP_FAILED`, and only the throw site knows which it saw.
+   */
+  readonly retry?: ErrorRetry | undefined;
+}
+
+export class ScrapeError extends UltimateError {
+  override readonly name = 'ScrapeError';
+
+  constructor(init: ScrapeErrorInit) {
+    super({
+      code: init.code,
+      cause: init.cause,
+      fix: init.fix,
+      docs: errorDocsUrl(init.code),
+      meta: init.meta,
+      ...(init.retry === undefined ? {} : { retry: init.retry }),
+    });
+  }
+}
+
+export function isScrapeError(value: unknown): value is ScrapeError {
+  return value instanceof ScrapeError;
+}
+
+/** True when the failure is one a second attempt could survive — the table above, read back. */
+export function isRetryableScrapeError(value: unknown): boolean {
+  return isScrapeError(value) && value.retry !== 'terminal';
+}

package/src/events.ts

@@ -0,0 +1,74 @@
+// The run's event stream — and it goes through `@ultimat3/core`'s `Logger`, which `Ctx` already
+// carries. This package ships CALL SITES and a FIELD VOCABULARY; it ships no sink, no transport
+// and no second logger. Where the lines go is the app's existing logger configuration (axiom 1:
+// one way to do each thing, and logging already has one).
+//
+// The event stream is the observability model: a debug report, an operator asking "where did it
+// stop", and any future recovery pass all read the same lines. So every step emits start, then
+// exactly one of success or failure, with its duration.
+//
+// What may be in a field is a CLOSED type. That is the mechanism that keeps a session cookie out
+// of a log line: `ScrapeEventFields` has no key to put one in, and core's logger redacts the
+// remaining spellings by key and every `Secret` by value.
+
+import type { Logger } from '@ultimat3/core';
+import type { ScrapeClock } from './clock';
+import { errorCode } from './failures';
+
+export interface ScrapeEventFields {
+  readonly scrape?: string | undefined;
+  readonly runId?: string | undefined;
+  readonly attempt?: number | undefined;
+  readonly step?: string | undefined;
+  readonly durationMs?: number | undefined;
+  readonly rows?: number | undefined;
+  readonly driver?: string | undefined;
+  readonly url?: string | undefined;
+  readonly code?: string | undefined;
+  /** Shape of a session, never its content — `sessionDigest()` builds these. */
+  readonly session?: string | undefined;
+  readonly cookies?: number | undefined;
+  readonly storageKeys?: number | undefined;
+  readonly origin?: string | undefined;
+  readonly refused?: number | undefined;
+  readonly reused?: boolean | undefined;
+  readonly burned?: boolean | undefined;
+}
+
+/** One child logger per run, so every line downstream carries the run's identity unasked. */
+export const scrapeLogger = (logger: Logger, fields: ScrapeEventFields): Logger =>
+  logger.child({ ...fields, component: 'scrape' });
+
+export interface StepEvent {
+  readonly name: string;
+  readonly logger: Logger;
+  readonly clock: ScrapeClock;
+  readonly attempt?: number | undefined;
+}
+
+/**
+ * Start, then exactly one of success or failure, with a duration. The failure line carries the
+ * error's CODE and never its message: a code is stable, greppable and safe, and a message is
+ * whatever a site put in its HTML.
+ */
+export async function withStepEvent<T>(event: StepEvent, run: () => Promise<T>): Promise<T> {
+  const startedAt = event.clock.monotonic();
+  event.logger.debug('scrape.step.start', { step: event.name, attempt: event.attempt });
+  try {
+    const result = await run();
+    event.logger.info('scrape.step.ok', {
+      step: event.name,
+      attempt: event.attempt,
+      durationMs: Math.round(event.clock.monotonic() - startedAt),
+    });
+    return result;
+  } catch (thrown) {
+    event.logger.warn('scrape.step.failed', {
+      step: event.name,
+      attempt: event.attempt,
+      durationMs: Math.round(event.clock.monotonic() - startedAt),
+      code: errorCode(thrown),
+    });
+    throw thrown;
+  }
+}

package/src/expect.ts

@@ -0,0 +1,133 @@
+// The silent-green alarm. The worst failure a scraper has is not a crash — it is a run that
+// succeeds, returns nothing, writes nothing, alerts nobody, and stays green until somebody asks
+// where the data went. `expect` turns that into a red run: a yield under `minRows`, or under
+// `maxDrop` of what this scrape normally returns, throws.
+
+import { yieldCollapsed } from './error-throws';
+import type { ScrapeError } from './errors';
+
+export interface YieldExpectation {
+  /**
+   * The floor, in rows. A scrape whose real answer is legitimately sometimes zero declares
+   * `minRows: 0` — explicitly, so the reader can tell "zero is fine here" from "nobody thought
+   * about it". Omitted means only `maxDrop` applies.
+   */
+  readonly minRows?: number | undefined;
+  /**
+   * How far below the trailing median a run may fall, as a fraction: `0.5` allows half. The
+   * comparison is against the MEDIAN and not the mean because one 12,000-row backfill run in the
+   * history would drag a mean high enough to fire the alarm on every ordinary day afterwards.
+   */
+  readonly maxDrop?: number | undefined;
+  /** How many past runs form the baseline. */
+  readonly window?: number | undefined;
+}
+
+export const DEFAULT_YIELD_WINDOW = 7;
+
+/**
+ * Below this many samples there is no baseline, so `maxDrop` cannot fire. A median of one run
+ * would make the SECOND run of a brand-new scraper alarm on any variation at all, and an alarm
+ * that fires on day two of every new scraper is an alarm somebody turns off.
+ */
+export const MIN_BASELINE_RUNS = 3;
+
+/** What a run's yield is measured against. Persisted by the app, never by this package. */
+export interface YieldHistory {
+  /** Most recent first or last — order is irrelevant to a median, and this says so. */
+  recent(scrape: string, limit: number): Promise<readonly number[]>;
+  record(scrape: string, rows: number): Promise<void>;
+}
+
+export function median(values: readonly number[]): number | undefined {
+  if (values.length === 0) return undefined;
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  const low = sorted[mid - 1];
+  const high = sorted[mid];
+  if (high === undefined) return undefined;
+  return sorted.length % 2 === 1 ? high : ((low ?? high) + high) / 2;
+}
+
+export interface YieldCheck {
+  readonly scrape: string;
+  readonly rows: number;
+  readonly expect: YieldExpectation;
+  readonly history: readonly number[];
+}
+
+/**
+ * The whole rule, pure: the error this yield earns, or `undefined`. Pure so a test can hand it
+ * fifty histories without a queue, a browser or a clock.
+ */
+export function yieldProblem(check: YieldCheck): ScrapeError | undefined {
+  const { minRows, maxDrop } = check.expect;
+  if (minRows !== undefined && check.rows < minRows) {
+    return yieldCollapsed({ scrape: check.scrape, rows: check.rows, reason: 'min-rows', minRows });
+  }
+  if (maxDrop === undefined || check.history.length < MIN_BASELINE_RUNS) return undefined;
+  const baseline = median(check.history);
+  if (baseline === undefined || baseline <= 0) return undefined;
+  if (check.rows >= baseline * (1 - maxDrop)) return undefined;
+  return yieldCollapsed({
+    scrape: check.scrape,
+    rows: check.rows,
+    reason: 'drop',
+    baseline,
+    maxDrop,
+  });
+}
+
+export interface YieldGuardInput {
+  readonly scrape: string;
+  readonly rows: number;
+  readonly expect: YieldExpectation | undefined;
+  readonly history: YieldHistory | undefined;
+}
+
+/**
+ * Check, then record — and record ONLY a run that passed.
+ *
+ * That order is the mechanism, not an implementation detail. Recording a collapsed run would let
+ * the baseline follow the collapse down: three broken runs at 2 rows and the median IS 2, so the
+ * fourth broken run is within `maxDrop` of it and the alarm has silenced itself exactly when it
+ * was working. A scraper that quietly re-baselines onto its own failure is the bug this file
+ * exists to prevent, one level up.
+ */
+export async function guardYield(input: YieldGuardInput): Promise<void> {
+  // No `expect` is no baseline either, deliberately: with no floor and no drop rule there is
+  // nothing deciding whether a run was good, so recording it would let a stretch of silent
+  // zero-row runs become the median an `expect` added later is measured against. The cost is that
+  // `maxDrop` needs `MIN_BASELINE_RUNS` runs after it is declared before it can fire — a delay,
+  // not a hole. `expect.test.ts` pins both halves.
+  if (input.expect === undefined) return;
+  const window = input.expect.window ?? DEFAULT_YIELD_WINDOW;
+  const history =
+    input.history === undefined ? [] : await input.history.recent(input.scrape, window);
+  const problem = yieldProblem({
+    scrape: input.scrape,
+    rows: input.rows,
+    expect: input.expect,
+    history,
+  });
+  if (problem !== undefined) throw problem;
+  await input.history?.record(input.scrape, input.rows);
+}
+
+/** In-memory history: what `fakeBrowser()` runs against, and what a test asserts on. */
+export function memoryYieldHistory(
+  seed: Readonly<Record<string, readonly number[]>> = {},
+): YieldHistory {
+  const runs = new Map<string, number[]>(
+    Object.entries(seed).map(([name, values]) => [name, [...values]]),
+  );
+  return {
+    recent: (scrape, limit) => Promise.resolve((runs.get(scrape) ?? []).slice(-limit)),
+    record: (scrape, rows) => {
+      const existing = runs.get(scrape) ?? [];
+      existing.push(rows);
+      runs.set(scrape, existing);
+      return Promise.resolve();
+    },
+  };
+}