@ultimat3/scraping 22.15.0 → 23.0.0

package/src/driver-cdp.ts

@@ -10,14 +10,22 @@
 // The library is passed IN (`launcher`), never imported — see `cdp-port.ts` for why that seam is
 // what keeps a puppeteer type out of the vocabulary and this package free of a dependency.
 
+import type { Logger } from '@ultimat3/core';
+import type { MoneyValue } from '@ultimat3/schema';
 import type { CdpBrowserLike, CdpLauncherLike } from './cdp-port';
+import type { CdpResolver } from './cdp-resolver';
+import { acquireCdp } from './cdp-resolver';
 import { CDP_DRIVER, cdpTarget } from './cdp-target';
 import type { ScrapeDriver, ScrapeSession, SessionInit } from './driver';
 import { browserUnreachable, cdpAttachFailed, remoteRequired } from './error-throws';
+import { egressUnsupported } from './error-throws-session';
 import { isScrapeError } from './errors';
+import type { ScrapeEventFields } from './events';
+import { errorCode } from './failures';
 import { httpOverFetch } from './http';
 import { pageOverTarget } from './page-over-target';
 import type { ScrapeTarget } from './target';
+import { endpointLabel, hasCredentials, splitCredentials, urlSecretValues } from './url-secrets';
 import { createWedgeGuard } from './watchdog';
 
 export { CDP_DRIVER } from './cdp-target';
@@ -25,6 +33,10 @@ export { CDP_DRIVER } from './cdp-target';
 export interface BrowserOptions {
   /** `puppeteer` itself, or anything with the same two methods. */
   readonly launcher: CdpLauncherLike;
+  /**
+   * The exit every session of this driver dials when the run names none. A scrape's own
+   * `egress(input)` — `SessionInit.proxy` — wins over it, per session.
+   */
   readonly proxy?: string | undefined;
   /** Extra launch/connect arguments, passed through untouched. */
   readonly options?: Record<string, unknown> | undefined;
@@ -43,8 +55,23 @@ export interface LocalBrowserOptions extends BrowserOptions {
 }
 
 export interface RemoteBrowserOptions extends BrowserOptions {
-  /** The `webSocketDebuggerUrl` from `/json/version`, or a provider's connect URL. */
-  readonly cdpUrl: string;
+  /**
+   * The `webSocketDebuggerUrl` from `/json/version` or a provider's connect URL — or a
+   * `CdpResolver`, for a browser rented PER SESSION: called once per `open()`, told the run's
+   * exit, and its `release()` runs exactly once when the session ends. A fixed URL is one browser
+   * already bound to one exit, so a run asking for another is `X_SCRAPE_EGRESS_UNSUPPORTED`.
+   */
+  readonly cdpUrl: string | CdpResolver;
+}
+
+/** What one session was opened WITH, decided before the browser exists. */
+interface SessionWiring {
+  /** The exit both legs dial: the run's, else the driver's. */
+  readonly proxy: string | undefined;
+  /** True when THIS driver must answer the proxy's auth challenge — a launched browser only. */
+  readonly authenticate: boolean;
+  readonly release?: (() => Promise<void>) | undefined;
+  readonly cost?: MoneyValue | undefined;
 }
 
 /**
@@ -53,9 +80,27 @@ export interface RemoteBrowserOptions extends BrowserOptions {
  * outside `cdpTarget`'s own `guard()`, so a bare library `Error` would otherwise reach the job's
  * retry classifier with no code at all.
  */
-async function opened(browser: CdpBrowserLike, init: SessionInit): Promise<ScrapeTarget> {
+async function opened(
+  browser: CdpBrowserLike,
+  init: SessionInit,
+  wiring: SessionWiring,
+): Promise<ScrapeTarget> {
   try {
     const page = await browser.newPage();
+    if (wiring.authenticate && wiring.proxy !== undefined) {
+      // Before the first navigation, and through its owner: the challenge arrives with the first
+      // request, and `--proxy-server` was deliberately handed the exit WITHOUT its userinfo.
+      if (typeof page.authenticate !== 'function') {
+        throw egressUnsupported({
+          driver: CDP_DRIVER,
+          egress: wiring.proxy,
+          reason:
+            'the exit carries credentials and this launcher has no page.authenticate() to answer the proxy with',
+        });
+      }
+      const { username, password } = splitCredentials(wiring.proxy);
+      await page.authenticate({ username, password });
+    }
     const target = await cdpTarget({ page, browser, rules: init.rules, clock: init.clock });
     if (init.restore !== undefined) await target.restore(init.restore);
     return target;
@@ -84,13 +129,14 @@ async function sessionOver(
   browser: CdpBrowserLike,
   init: SessionInit,
   options: BrowserOptions,
+  wiring: SessionWiring,
 ): Promise<ScrapeSession> {
   // Acquire, then roll back on ANY throw — the shape `releaseBoot` uses in `packages/cli/src/
   // serve.ts`. Between the launch and the `WedgeGuard` below, nothing else holds this browser:
   // `runScrape`'s `finally { session.close() }` never runs for a session `open()` did not return,
   // so a tab limit, a refused interception or a restore that threw left a real Chrome process —
   // or a remote session somebody is billing for — running per attempt, unattributed.
-  const target = await opened(browser, init);
+  const target = await opened(browser, init, wiring);
   const guard = createWedgeGuard({
     clock: init.clock,
     what: `scrape "${init.name}"`,
@@ -116,10 +162,11 @@ async function sessionOver(
   return {
     driver: CDP_DRIVER,
     // The exit the browser was launched or attached with, handed back so the run's robots read
-    // presents the SAME client identity to the origin the page loads from. `options.proxy` and
-    // not `init.proxy`: this is what the launch args and the HTTP leg below actually carry, and a
-    // reported exit that nothing dialled would be worse than none.
-    ...(options.proxy === undefined ? {} : { proxy: options.proxy }),
+    // presents the SAME client identity to the origin the page loads from. `wiring.proxy` is what
+    // the launch args and the HTTP leg below actually carry — the run's exit when it named one,
+    // the driver's otherwise — and a reported exit that nothing dialled would be worse than none.
+    ...(wiring.proxy === undefined ? {} : { proxy: wiring.proxy }),
+    ...(wiring.cost === undefined ? {} : { browserCost: wiring.cost }),
     page: pageOverTarget(target, {
       clock: init.clock,
       allowHosts: init.rules.allowHosts,
@@ -129,6 +176,7 @@ async function sessionOver(
       signal,
       onActivity,
       pace: init.pace,
+      usage: init.usage,
     }),
     http: httpOverFetch({
       rules: init.rules,
@@ -142,16 +190,66 @@ async function sessionOver(
       pace: init.pace,
       signal,
       onActivity,
-      proxy: options.proxy,
+      proxy: wiring.proxy,
+      usage: init.usage,
       // The run's secrets, so the ONE thing this leg quotes from the site — the first 200 bytes
       // of a non-2xx body, in `X_SCRAPE_HTTP_FAILED`'s cause — cannot carry the password the
       // login endpoint echoed back. The bag was in scope here and unread.
       secrets: init.secrets,
     }),
-    close: () => guard.shutdown(),
+    // Both halves, in order, and neither can throw: the browser is quit under its ceiling, THEN
+    // the rental is handed back — a provider told "done" while the socket is still open bills the
+    // tail to nobody. `release` is latched, so a second `close()` does nothing.
+    close: async (): Promise<void> => {
+      await guard.shutdown();
+      await wiring.release?.();
+    },
   };
 }
 
+/** The line a failed hand-back of a rented browser writes. Stable: alerts are keyed on it. */
+const RELEASE_FAILED_EVENT = 'scrape.browser.release_failed';
+
+/**
+ * The resolver with its `release` WATCHED. `acquireCdp` swallows a rejected hand-back on purpose
+ * — it runs on the way out of a run whose own outcome is the one the reader needs — and swallowed
+ * silently, a browser its provider is still billing for left no line anywhere. The error's CODE
+ * and the endpoint's scheme and host, nothing else: a provider's failure text quotes its connect
+ * URL, and that URL is its access token. Rethrown, so `acquireCdp`'s contract is untouched.
+ */
+const watched =
+  (resolver: CdpResolver, logger: Logger): CdpResolver =>
+  async (request) => {
+    const resolution = await resolver(request);
+    if (resolution.release === undefined) return resolution;
+    return {
+      cdpUrl: resolution.cdpUrl,
+      cost: resolution.cost,
+      release: async (): Promise<void> => {
+        try {
+          await resolution.release?.();
+        } catch (thrown) {
+          const fields: ScrapeEventFields = {
+            code: errorCode(thrown),
+            driver: CDP_DRIVER,
+            origin: endpointLabel(resolution.cdpUrl),
+          };
+          logger.warn(RELEASE_FAILED_EVENT, { ...fields });
+          throw thrown;
+        }
+      },
+    };
+  };
+
+/** The run's exit over the driver's, and its credential-bearing parts into the redaction set. */
+function exitFor(init: SessionInit, options: BrowserOptions): string | undefined {
+  const exit = init.proxy ?? options.proxy;
+  if (exit !== undefined && hasCredentials(exit)) {
+    for (const value of urlSecretValues(exit)) init.secrets?.conceal(value);
+  }
+  return exit;
+}
+
 /**
  * A browser in this container. `executablePath` is required by every puppeteer-core build — it
  * ships no browser — so it is passed through rather than guessed at.
@@ -164,6 +262,11 @@ export function localBrowser(options: LocalBrowserOptions): ScrapeDriver {
       if (launch === undefined) {
         throw remoteRequired('local browser: the launcher has no launch()');
       }
+      const proxy = exitFor(init, options);
+      const authenticate = proxy !== undefined && hasCredentials(proxy);
+      // The userinfo never reaches the argument list: Chrome ignores it there, and an argument is
+      // readable by every process on the box. `opened()` answers the challenge instead.
+      const proxyArg = proxy !== undefined && authenticate ? splitCredentials(proxy).bare : proxy;
       let browser: CdpBrowserLike;
       try {
         browser = await launch.call(options.launcher, {
@@ -172,13 +275,13 @@ export function localBrowser(options: LocalBrowserOptions): ScrapeDriver {
             ? {}
             : { executablePath: options.executablePath }),
           ...(options.profileDir === undefined ? {} : { userDataDir: options.profileDir }),
-          ...(options.proxy === undefined ? {} : { args: [`--proxy-server=${options.proxy}`] }),
+          ...(proxyArg === undefined ? {} : { args: [`--proxy-server=${proxyArg}`] }),
           ...options.options,
         });
       } catch (thrown) {
         throw browserUnreachable(CDP_DRIVER, thrown);
       }
-      return sessionOver(browser, init, options);
+      return sessionOver(browser, init, options, { proxy, authenticate });
     },
   };
 }
@@ -192,16 +295,59 @@ export function remoteBrowser(options: RemoteBrowserOptions): ScrapeDriver {
       if (connect === undefined || options.cdpUrl === '') {
         throw remoteRequired(CDP_DRIVER);
       }
-      let browser: CdpBrowserLike;
+      // Refused BEFORE anything is rented or attached. A fixed URL is a browser that already
+      // exists on whatever exit it was started with; attaching and dialling the HTTP leg through
+      // the run's exit would be one session presenting two client identities.
+      if (
+        typeof options.cdpUrl === 'string' &&
+        init.proxy !== undefined &&
+        init.proxy !== options.proxy
+      ) {
+        throw egressUnsupported({
+          driver: CDP_DRIVER,
+          egress: init.proxy,
+          reason: `remoteBrowser({ cdpUrl }) was given a fixed URL, and the browser behind it is already bound to ${
+            options.proxy === undefined ? 'a direct connection' : endpointLabel(options.proxy)
+          }`,
+        });
+      }
+      const proxy = exitFor(init, options);
+      const source =
+        typeof options.cdpUrl === 'string' ? options.cdpUrl : watched(options.cdpUrl, init.logger);
+      const acquired = await acquireCdp(source, {
+        scrape: init.name,
+        runId: init.runId,
+        egress: proxy,
+        signal: init.signal,
+      });
+      // From here the rental exists, and `runScrape`'s `finally` cannot hand back a session
+      // `open()` never returned — so every throw below releases it first. `sessionOver` rolls the
+      // BROWSER back on its own; this is the half only the resolver knows how to end.
       try {
-        browser = await connect.call(options.launcher, {
-          browserWSEndpoint: options.cdpUrl,
-          ...options.options,
+        if (acquired.cdpUrl === '') throw remoteRequired(CDP_DRIVER);
+        // A provider's connect URL is its access token. Into the run's secret set before the
+        // attach, so not even the failure artifact of a run that died here can carry it.
+        for (const value of urlSecretValues(acquired.cdpUrl)) init.secrets?.conceal(value);
+        let browser: CdpBrowserLike;
+        try {
+          browser = await connect.call(options.launcher, {
+            browserWSEndpoint: acquired.cdpUrl,
+            ...options.options,
+          });
+        } catch (thrown) {
+          throw cdpAttachFailed(acquired.cdpUrl, thrown);
+        }
+        return await sessionOver(browser, init, options, {
+          proxy,
+          // The provider launched this browser; answering its proxy is the provider's.
+          authenticate: false,
+          release: acquired.release,
+          cost: acquired.cost,
         });
       } catch (thrown) {
-        throw cdpAttachFailed(options.cdpUrl, thrown);
+        await acquired.release();
+        throw thrown;
       }
-      return sessionOver(browser, init, options);
     },
   };
 }

package/src/driver.ts

@@ -5,6 +5,8 @@
 // Two methods. `open` hands back a session; `close` ends it. Everything else a scraper does goes
 // through `ScrapePage`, which is driver-blind by construction.
 
+import type { Logger } from '@ultimat3/core';
+import type { MoneyValue } from '@ultimat3/schema';
 import type { ScrapeClock } from './clock';
 import type { ScrapeHttp } from './http';
 import type { InterceptRules } from './intercept';
@@ -12,12 +14,20 @@ import type { ScrapePage } from './page';
 import type { RobotsGate } from './robots';
 import type { ScrapeSecrets } from './secrets';
 import type { SessionSnapshot } from './session-state';
+import type { UsageMeter } from './usage';
 
 export interface SessionInit {
   /** The scrape's name — every error cause raised inside the session carries it. */
   readonly name: string;
   readonly rules: InterceptRules;
   readonly clock: ScrapeClock;
+  /**
+   * The run's logger — `scrapeLogger()`'s child, so every line carries the scrape and the run.
+   * Required, because `close()` never throws: a teardown that failed (a rented browser its
+   * provider would not take back) has no other way to be seen, and a session with nowhere to say
+   * so is how that bill went unattributed. Fields stay inside `ScrapeEventFields`.
+   */
+  readonly logger: Logger;
   /** Per-operation default, in ms. A `waitFor` with its own `timeout` overrides it. */
   readonly timeoutMs: number;
   readonly secrets?: ScrapeSecrets | undefined;
@@ -26,8 +36,17 @@ export interface SessionInit {
   readonly signal?: AbortSignal | undefined;
   /** A previously captured session, put back before the first navigation. */
   readonly restore?: SessionSnapshot | undefined;
-  /** BOTH transports dial through it. A different exit IP mid-session is a different client. */
+  /**
+   * The exit THIS run asked for — `egress(input)` on the definition. It wins over a driver's own
+   * `proxy` option, BOTH transports dial through it, and a driver that cannot is required to
+   * refuse with `X_SCRAPE_EGRESS_UNSUPPORTED` rather than dial another: a different exit IP
+   * mid-session is a different client. May carry credentials; never print it.
+   */
   readonly proxy?: string | undefined;
+  /** The job run this session belongs to. A `CdpResolver` tags the browser it rents with it. */
+  readonly runId?: string | undefined;
+  /** The run's usage meter. Pass it to `pageOverTarget` and the HTTP transport; see `usage.ts`. */
+  readonly usage?: UsageMeter | undefined;
   /** Awaited before every navigation AND every HTTP request — one budget across both legs. */
   readonly pace?: ((signal?: AbortSignal) => Promise<void>) | undefined;
   /** Called on every operation on either transport. The wedge watchdog measures the gaps. */
@@ -51,6 +70,11 @@ export interface ScrapeSession {
    * as allow-everything. A driver that omits it is asked for its rules directly, as before.
    */
   readonly proxy?: string | undefined;
+  /**
+   * What the browser behind this session cost, when the driver rented it and the provider said
+   * so at acquire time. `Money`, never a float. It reaches `ScrapeReport.usage.browserCost`.
+   */
+  readonly browserCost?: MoneyValue | undefined;
   readonly page: ScrapePage;
   /**
    * The second transport, bound to the SAME session as the page: the browser's cookies, headers

package/src/error-throws-session.ts

@@ -0,0 +1,74 @@
+// The failure modes of a SESSION a service is built on: an exit the driver cannot dial, and a
+// prompt answered over the event bus. Split from `error-throws.ts`, which stands at its size
+// ceiling; the codes themselves stay declared in `errors.ts` — one registry, one place.
+
+import { ScrapeError } from './errors';
+import { endpointLabel } from './url-secrets';
+
+/**
+ * A driver handed an exit it cannot dial. Scheme and host of both exits and nothing else: an exit
+ * URL carries the proxy account in its userinfo, and this cause is written to the dead-letter row.
+ */
+export const egressUnsupported = (input: {
+  readonly driver: string;
+  readonly egress: string;
+  readonly reason: string;
+}): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_EGRESS_UNSUPPORTED',
+    cause: `the ${input.driver} driver cannot give this session the exit ${endpointLabel(input.egress)}: ${input.reason}`,
+    fix: 'pass remoteBrowser({ cdpUrl: resolver }) so each run rents a browser on its own exit, or drop egress: from the scrape() definition and set the exit once on the driver — remoteBrowser({ proxy })',
+    meta: { driver: input.driver, egress: endpointLabel(input.egress) },
+  });
+
+/**
+ * The exit `egress` answered carries a credential, and that credential is in the job's own input —
+ * so it rode the queue payload and sits in `x_jobs` in the clear (`redactInput` hides a key by
+ * NAME for display; the column is untouched). Scheme and host only, like every exit this package
+ * prints: the cause is written to the dead-letter row, beside the payload it is about.
+ */
+export const egressInPayload = (input: {
+  readonly scrape: string;
+  readonly egress: string;
+}): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_EGRESS_IN_PAYLOAD',
+    cause: `scrape "${input.scrape}" was given the exit ${endpointLabel(input.egress)} with a credential that is also in the run's input, so the credential is stored in the queue row in the clear`,
+    fix: `enqueue the id of the row that holds the exit, and resolve it in the worker: egress: async ({ connectionId }) => (await repo.connectionById(connectionId))?.exit on scrape("${input.scrape}") — a .sealed() column keeps it encrypted at rest — then rotate the exposed credential`,
+    meta: { scrape: input.scrape, egress: endpointLabel(input.egress) },
+  });
+
+/**
+ * `eventPrompt()` waited its whole budget. The event name is in the cause because it is the one
+ * fact the answering side needs and cannot guess: the run id and the prompt's index within it.
+ */
+export const promptTimedOut = (input: {
+  readonly scrape: string;
+  readonly label: string;
+  readonly event: string;
+  readonly waitedMs: number;
+}): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_PROMPT_UNANSWERED',
+    cause: `scrape "${input.scrape}" asked for "${input.label}" and no answer was published to ${input.event} within ${String(input.waitedMs)}ms`,
+    fix: 'call answerPrompt({ runId, index, answer }) with the run id and index the event name in the cause ends with, from a process on the same STORED event bus — setEventBus(createPgEventBus({ executor })) — or raise eventPrompt({ timeout })',
+    meta: {
+      scrape: input.scrape,
+      label: input.label,
+      event: input.event,
+      waitedMs: input.waitedMs,
+    },
+  });
+
+/** Something was published to the prompt's event that `answerPrompt()` did not write. */
+export const promptAnswerInvalid = (input: {
+  readonly scrape: string;
+  readonly label: string;
+  readonly event: string;
+}): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_PROMPT_UNANSWERED',
+    cause: `scrape "${input.scrape}" asked for "${input.label}" and the event published to ${input.event} does not carry a sealed answer`,
+    fix: 'publish the answer with answerPrompt({ runId, index, answer }) and never publishEvent() by hand — the answer is sealed for this one prompt, so a plain payload is refused rather than typed into the site',
+    meta: { scrape: input.scrape, label: input.label, event: input.event },
+  });

package/src/error-throws.ts

@@ -12,6 +12,8 @@ import {
 } from '@ultimat3/core';
 import type { CaptureClip } from './capture-clip';
 import { ScrapeError } from './errors';
+import { SECRET_PLACEHOLDER } from './secrets';
+import { endpointLabel, urlSecretValues } from './url-secrets';
 
 /**
  * Two shapes of the same failure, one code. `name` is the driver a definition ASKED for; `scrape`
@@ -34,12 +36,43 @@ export const driverUnknown = (
     meta: { driver: name ?? 'none', ...(scrape === undefined ? {} : { scrape }) },
   });
 
+/**
+ * The launcher's own words with every credential-bearing part of `url` cut out. A library's
+ * connect error repeats the URL it was handed — `Unexpected server response: 401 for wss://…
+ * ?token=…` — so naming scheme and host in the cause is not enough on its own.
+ */
+const renderWithout = (thrown: unknown, url: string): string => {
+  let text = renderThrowable(thrown);
+  const values = [...urlSecretValues(url)].sort((a, b) => b.length - a.length);
+  for (const value of values) text = text.split(value).join(SECRET_PLACEHOLDER);
+  return text;
+};
+
+/**
+ * SCHEME AND HOST ONLY, for every CDP URL — a fixed one included. A provider's connect URL carries
+ * its access token in the query or the path, this error's cause is written to the dead-letter row
+ * and printed by `x jobs show`, and a `cdpUrl` read from the environment is that same URL. The
+ * host is what tells a reader which endpoint refused; nothing after it is theirs to print.
+ */
 export const cdpAttachFailed = (cdpUrl: string, thrown: unknown): ScrapeError =>
   new ScrapeError({
     code: 'X_SCRAPE_CDP_ATTACH_FAILED',
-    cause: `the CDP endpoint ${cdpUrl} refused the attach: ${renderThrowable(thrown)}`,
+    cause: `the CDP endpoint ${endpointLabel(cdpUrl)} refused the attach: ${renderWithout(thrown, cdpUrl)}`,
     fix: 'curl "$CDP_URL/json/version" to confirm the endpoint answers, then pass that webSocketDebuggerUrl to remoteBrowser({ cdpUrl })',
-    meta: { cdpUrl },
+    meta: { cdpUrl: endpointLabel(cdpUrl) },
+  });
+
+/**
+ * The resolver itself failed — the provider's API, not the browser it would have handed over. The
+ * same code and the same retryable classification: "no browser to attach to yet" is one fact,
+ * whichever call discovered it.
+ */
+export const cdpResolveFailed = (scrape: string, thrown: unknown): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_CDP_ATTACH_FAILED',
+    cause: `the CdpResolver on remoteBrowser({ cdpUrl }) could not rent a browser for scrape "${scrape}": ${renderThrowable(thrown)}`,
+    fix: "run the resolver's own request by hand against the provider to see its answer, then correct the credential or the endpoint the resolver passed to remoteBrowser({ cdpUrl }) uses",
+    meta: { scrape },
   });
 
 /**

package/src/errors.ts

@@ -39,6 +39,8 @@ export const SCRAPE_OWNED_ERROR_CODES = [
   'X_SCRAPE_PROMPT_UNANSWERED',
   'X_SCRAPE_BLOCKED',
   'X_SCRAPE_KEY_INVALID',
+  'X_SCRAPE_EGRESS_UNSUPPORTED',
+  'X_SCRAPE_EGRESS_IN_PAYLOAD',
 ] as const;
 
 /**
@@ -91,6 +93,8 @@ export const SCRAPE_ERROR_TITLES: Readonly<Record<ScrapeOwnedErrorCode, string>>
   X_SCRAPE_PROMPT_UNANSWERED: 'a login step asked for a code and nothing answered',
   X_SCRAPE_BLOCKED: 'the site refused this client — the identity is spent',
   X_SCRAPE_KEY_INVALID: 'the key chord names no key a browser can press',
+  X_SCRAPE_EGRESS_UNSUPPORTED: 'this driver cannot give the session the exit the run asked for',
+  X_SCRAPE_EGRESS_IN_PAYLOAD: 'the run exit carries a credential that is also in the job payload',
 };
 
 // One unconditional call, so a second package claiming one of these codes throws
@@ -192,6 +196,12 @@ export const SCRAPE_ERROR_RETRY = {
   // literal, attempt 2 passes the identical string, and a retry is a browser launch for no chance
   // of a different parse. Refused before any key goes down, so no modifier is left held.
   X_SCRAPE_KEY_INVALID: 'terminal',
+  // The exit is a function of the run's INPUT and the driver is a constant of the deploy: attempt
+  // 2 computes the same exit and meets the same driver. Dialling the driver's own exit instead
+  // would be the retry that "works", as a different client — which is the failure this code names.
+  X_SCRAPE_EGRESS_UNSUPPORTED: 'terminal',
+  // The payload is the row: every remaining attempt reads the same credential out of it.
+  X_SCRAPE_EGRESS_IN_PAYLOAD: 'terminal',
 } as const satisfies Readonly<Record<ScrapeOwnedErrorCode, 'retryable' | 'terminal'>>;
 
 registerErrorRetry(SCRAPE_ERROR_RETRY);

package/src/event-prompt.ts

@@ -0,0 +1,133 @@
+// A prompt answered from ANOTHER process while the browser stays open: `eventPrompt()` is a
+// `PromptHandler` that polls the stored job event bus, and `answerPrompt()` is the one way an
+// answer is published to it.
+//
+// It polls IN PROCESS and never uses `step.waitForEvent`: that suspends the run, the attempt ends,
+// `runScrape`'s `finally` closes the session — and the site is waiting for the code inside that
+// session. So the run keeps its claim (the worker's heartbeat is a timer, untouched by a poll),
+// keeps the browser alive between polls, and gives up on the run's own cancellation.
+//
+// The event name is DERIVED — `scrape-prompt:<runId>:<n>` — and it is not an authorization: the
+// app's answering action is policy-checked like any action and calls `answerPrompt()`.
+
+import type { SealKeySource } from '@ultimat3/core';
+import { finiteCount, isLocal, isSealed, openText, seal } from '@ultimat3/core';
+import type { EventBus, JobEvent } from '@ultimat3/jobs';
+import { DriverUnavailableError, eventBus } from '@ultimat3/jobs';
+import type { PromptHandler, PromptRequest } from './auth';
+import { deadline, throwIfAborted } from './clock';
+import { promptAnswerInvalid, promptTimedOut } from './error-throws-session';
+
+/** How often the bus is asked. One indexed read and one browser round trip per interval. */
+export const DEFAULT_PROMPT_POLL_MS = 1_000;
+
+/**
+ * How long a published answer stays matchable. Short on purpose: an answer is a one-time code,
+ * and the stored bus's own default keeps an event for a week.
+ */
+export const DEFAULT_PROMPT_ANSWER_TTL_MS = 600_000;
+
+/** The one spelling. `index` is `PromptRequest.index`: 1 for the first prompt of an attempt. */
+export const promptEventName = (runId: string, index: number): string =>
+  `scrape-prompt:${runId}:${String(index)}`;
+
+export interface EventPromptOptions {
+  /** Milliseconds one prompt waits for its answer before `X_SCRAPE_PROMPT_UNANSWERED`. */
+  readonly timeout: number;
+  /** Milliseconds between two looks at the bus. Defaults to `DEFAULT_PROMPT_POLL_MS`. */
+  readonly pollMs?: number | undefined;
+  /**
+   * The bus to read. Defaults to the ambient one, read when the prompt is ASKED — boot installs
+   * the stored bus after this module is evaluated. It has to be the stored one in any deployment
+   * with more than one process: the answer is published by a web process, never by this worker.
+   */
+  readonly bus?: EventBus | undefined;
+  /** Where the master key is read from, for opening the answer. `seal()`'s own two fields. */
+  readonly keySource?: SealKeySource | undefined;
+  /** Where the environment is read from. Defaults to the process's own. */
+  readonly env?: Readonly<Record<string, string | undefined>> | undefined;
+}
+
+/**
+ * `prompt: eventPrompt({ timeout: 300_000 })` on a `scrape()` definition.
+ *
+ * Only an answer published AFTER the prompt was asked is consumed. The run id is the same on
+ * every attempt, so a code left over from attempt 1 would otherwise be typed into the site by
+ * attempt 2 — and a stale code is a refused login. "After" is decided on ONE clock, the bus's
+ * (`bus.now()`); the timeout and the poll stay on the run's own.
+ */
+export function eventPrompt(options: EventPromptOptions): PromptHandler {
+  // Floors of 1, both: a zero timeout is a prompt that can never be answered and a zero interval
+  // is the poll loop spinning on the worker's only thread.
+  const timeoutMs = finiteCount('eventPrompt', 'timeout', options.timeout, 1);
+  const pollMs = finiteCount('eventPrompt', 'pollMs', options.pollMs ?? DEFAULT_PROMPT_POLL_MS, 1);
+  return async (request: PromptRequest): Promise<string> => {
+    const bus = options.bus ?? eventBus();
+    // Refused when the prompt is ASKED, before the wait: the answer is published by a web process
+    // and read by this worker, so an in-memory bus can only ever time out — minutes later, as
+    // `X_SCRAPE_PROMPT_UNANSWERED`, blaming the human who did answer. Development and test run
+    // every role in one process, where the in-memory bus is the right one. `=== false`, never
+    // `!bus.stored`: only a bus that SAYS it is not stored is refused.
+    if (bus.stored === false && !isLocal({ env: options.env, fallback: 'production' })) {
+      throw new DriverUnavailableError({
+        driver: 'event bus',
+        cause:
+          'eventPrompt() was asked on the in-memory event bus — the answer is published by another process and can never reach this one',
+        fix: 'call setEventBus(createPgEventBus({ executor })) at boot — x dev and the served roles do — or pass bus: createPgEventBus({ executor }) to eventPrompt()',
+      });
+    }
+    const event = promptEventName(request.runId, request.index);
+    // The BUS's clock, read once: an answer's `publishedAt` is stamped by it — the database, for
+    // the stored bus — and `request.clock` is this worker's. Comparing the two let a few seconds
+    // of skew between pods decide that an answer given after the ask had come before it.
+    const askedAt = await bus.now();
+    const budget = deadline(request.clock, timeoutMs, 'eventPrompt');
+    for (;;) {
+      if (request.signal !== undefined) throwIfAborted(request.signal);
+      const found = await bus.find(event, undefined, askedAt);
+      if (found !== undefined) {
+        if (!isSealed(found.payload)) {
+          throw promptAnswerInvalid({ scrape: request.scrape, label: request.label, event });
+        }
+        // The purpose IS the event name, so an answer sealed for another run or another prompt of
+        // this one fails its tag instead of being typed.
+        return await openText(found.payload, { purpose: event, ...options.keySource });
+      }
+      if (budget.expired()) {
+        throw promptTimedOut({
+          scrape: request.scrape,
+          label: request.label,
+          event,
+          waitedMs: timeoutMs,
+        });
+      }
+      await request.keepAlive();
+      await request.clock.sleep(Math.min(pollMs, budget.remainingMs()), request.signal);
+    }
+  };
+}
+
+export interface AnswerPromptInput {
+  readonly runId: string;
+  /** Which prompt of the run: `PromptRequest.index`. */
+  readonly index: number;
+  readonly answer: string;
+  /** Milliseconds the answer stays matchable. Defaults to `DEFAULT_PROMPT_ANSWER_TTL_MS`. */
+  readonly ttl?: number | undefined;
+  readonly bus?: EventBus | undefined;
+  readonly keySource?: SealKeySource | undefined;
+}
+
+/**
+ * Publish one prompt's answer. Called from the app's own action, after its policy passed.
+ *
+ * The answer is SEALED before it is published: the stored bus is a table, and an OTP or a
+ * security answer written there in the clear is a credential in a queue row.
+ */
+export async function answerPrompt(input: AnswerPromptInput): Promise<JobEvent> {
+  const event = promptEventName(input.runId, input.index);
+  const sealed = await seal(input.answer, { purpose: event, ...input.keySource });
+  return (input.bus ?? eventBus()).publish(event, sealed, {
+    ttl: finiteCount('answerPrompt', 'ttl', input.ttl ?? DEFAULT_PROMPT_ANSWER_TTL_MS, 1),
+  });
+}

package/src/events.ts

@@ -33,6 +33,12 @@ export interface ScrapeEventFields {
   readonly refused?: number | undefined;
   readonly reused?: boolean | undefined;
   readonly burned?: boolean | undefined;
+  /** `ScrapeUsage`'s counts — numbers only, so a failed run still says what it spent. */
+  readonly browserMs?: number | undefined;
+  readonly navigations?: number | undefined;
+  readonly httpRequests?: number | undefined;
+  readonly bytesIn?: number | undefined;
+  readonly promptsAnswered?: number | undefined;
 }
 
 /** One child logger per run, so every line downstream carries the run's identity unasked. */