@ultimat3/scraping 22.15.0 → 23.0.0

package/src/http-recorded.ts

@@ -15,6 +15,7 @@ import type { HttpRecording } from './recording';
 import type { NetworkRing } from './rings';
 import type { RobotsGate } from './robots';
 import type { ScrapeSecrets } from './secrets';
+import type { UsageMeter } from './usage';
 
 export type HttpRecordingLookup = (
   method: string,
@@ -36,6 +37,8 @@ export interface RecordedHttpInit {
   readonly maxAgeMs?: number | undefined;
   /** The SAME bag the live leg holds, so a recorded 4xx body redacts the way a real one does. */
   readonly secrets?: ScrapeSecrets | undefined;
+  /** The SAME meter the live leg counts into, so a fixture run reports the usage a real one would. */
+  readonly usage?: UsageMeter | undefined;
 }
 
 /** `GET https://api.example.com/v1/orders?page=2` -> `http-get-api-example-com-v1-orders-page-2`. */
@@ -70,6 +73,7 @@ export function recordedHttp(init: RecordedHttpInit): ScrapeHttp {
         resourceType: 'fetch',
         at: init.clock.now().getTime(),
       });
+      init.usage?.httpRequest(new TextEncoder().encode(found.body).length);
       return responseOver(
         url,
         found.status,

package/src/http.ts

@@ -25,6 +25,7 @@ import type { RobotsGate } from './robots';
 import type { ScrapeSecrets } from './secrets';
 import { redactSecrets } from './secrets';
 import type { SessionSnapshot } from './session-state';
+import type { UsageMeter } from './usage';
 
 /**
  * Just the call. `typeof fetch` also carries `preconnect`, which no test double and no app wrapper
@@ -110,6 +111,8 @@ export interface HttpTransportInit {
    * until 2026-08-24, so nothing on this leg COULD redact.
    */
   readonly secrets?: ScrapeSecrets | undefined;
+  /** The run's meter: one `httpRequest` per hop on the wire, with the body bytes read for it. */
+  readonly usage?: UsageMeter | undefined;
   readonly fetch?: ScrapeFetch | undefined;
 }
 
@@ -213,8 +216,11 @@ const readResponse = async (
   response: Response,
   maxBytes: number,
   secrets: ScrapeSecrets | undefined,
+  usage: UsageMeter | undefined,
 ): Promise<ScrapeResponse> => {
   const capped = await readWithinLimit(response.body, maxBytes);
+  // Counted on BOTH branches: the bytes a refused body cost were read before the refusal.
+  usage?.httpRequest('over' in capped ? capped.over : capped.bytes.length);
   if ('over' in capped) throw bodyTooLarge(url, capped.over, maxBytes);
   const body = new TextDecoder().decode(capped.bytes);
   return responseOver(
@@ -345,7 +351,9 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
             hop.body,
           );
           if (next === undefined)
-            return await readResponse(hop.url, response, maxBytes, init.secrets);
+            return await readResponse(hop.url, response, maxBytes, init.secrets, init.usage);
+          // A followed hop was a request on the wire too; its body is discarded unread.
+          init.usage?.httpRequest(0);
           // Discarded BEFORE the refusal, not after it: a throw over an unread stream holds that
           // hop's socket until the collector reaches it, and the refusal path is exactly the one a
           // hostile chain drives ten times per request.

package/src/index.ts

@@ -12,6 +12,7 @@ export {
 export type {
   AuthContext,
   PromptHandler,
+  PromptInit,
   PromptRequest,
   ScrapeAuth,
 } from './auth';
@@ -29,11 +30,21 @@ export type {
   CdpScreenshotOptions,
   CdpSessionLike,
 } from './cdp-port';
+export type { CdpResolution, CdpResolver, CdpResolverRequest } from './cdp-resolver';
 export { snapshotExpression } from './cdp-snapshot';
 export type { CdpTargetInit } from './cdp-target';
 export { CDP_DRIVER, cdpTarget } from './cdp-target';
 export type { Deadline, ScrapeClock, TestScrapeClock } from './clock';
-export { deadline, systemScrapeClock, testClock, throwIfAborted } from './clock';
+export {
+  deadline,
+  noWaitClock,
+  resetScrapeClock,
+  scrapeClock,
+  setScrapeClock,
+  systemScrapeClock,
+  testClock,
+  throwIfAborted,
+} from './clock';
 export type { ColorScheme } from './color-scheme';
 export { COLOR_SCHEMES, isColorScheme } from './color-scheme';
 export { cookieHeaderFor } from './cookie-scope';
@@ -43,14 +54,15 @@ export type { BrowserOptions, LocalBrowserOptions, RemoteBrowserOptions } from '
 export { localBrowser, remoteBrowser } from './driver-cdp';
 export type { FakeBrowserOptions, FakePageOptions, FakePages } from './driver-fake';
 export { FAKE_DRIVER, FAKE_PAGE_URL, fakeBrowser, fakePage, recordingsOf } from './driver-fake';
-export type { FixtureBrowserOptions } from './driver-fixture';
-export { FIXTURE_DRIVER, fixtureBrowser } from './driver-fixture';
+export type { FixtureBrowserOptions } from './driver-recorded';
+export { FIXTURE_DRIVER, fixtureBrowser } from './driver-recorded';
 export {
   authFailed,
   blocked,
   bodyTooLarge,
   browserUnreachable,
   cdpAttachFailed,
+  cdpResolveFailed,
   downloadTimeout,
   driverUnknown,
   fixtureMissing,
@@ -75,6 +87,12 @@ export {
   wedged,
   yieldCollapsed,
 } from './error-throws';
+export {
+  egressInPayload,
+  egressUnsupported,
+  promptAnswerInvalid,
+  promptTimedOut,
+} from './error-throws-session';
 export type { ScrapeErrorCode, ScrapeErrorInit, ScrapeOwnedErrorCode } from './errors';
 export {
   isRetryableScrapeError,
@@ -86,6 +104,14 @@ export {
   SCRAPE_OWNED_ERROR_CODES,
   ScrapeError,
 } from './errors';
+export type { AnswerPromptInput, EventPromptOptions } from './event-prompt';
+export {
+  answerPrompt,
+  DEFAULT_PROMPT_ANSWER_TTL_MS,
+  DEFAULT_PROMPT_POLL_MS,
+  eventPrompt,
+  promptEventName,
+} from './event-prompt';
 export type { ScrapeEventFields, StepEvent } from './events';
 export type { YieldCheck, YieldExpectation, YieldGuardInput, YieldHistory } from './expect';
 export {
@@ -164,6 +190,7 @@ export type {
   ScrapeDefinition,
   ScrapeReport,
   ScrapeRunArgs,
+  ScrapeSettled,
 } from './scrape';
 export { scrape } from './scrape';
 export { DEFAULT_PAGE_TIMEOUT_MS, runScrape } from './scrape-run';
@@ -177,10 +204,16 @@ export {
   safeNetwork,
   safePageErrors,
 } from './secrets';
-export type { ScrapeSessionStore, SessionSnapshot, SessionState } from './session-state';
+export type {
+  ScrapeSessionStore,
+  SessionSnapshot,
+  SessionState,
+  StorageSessionStoreOptions,
+} from './session-state';
 export {
   memorySessionStore,
   parseSessionState,
+  SESSION_SEAL_PURPOSE,
   sessionDigest,
   sessionKeyFor,
   storageSessionStore,
@@ -196,5 +229,8 @@ export type {
   ScrapeDownloadFile,
   ScrapeTarget,
 } from './target';
+export { endpointLabel, urlSecretValues } from './url-secrets';
+export type { RunUsageMeter, ScrapeUsage, UsageMeter } from './usage';
+export { createUsageMeter } from './usage';
 export type { WedgeGuard, WedgeGuardInit } from './watchdog';
 export { DEFAULT_GRACE_MS } from './watchdog';

package/src/offline-session.ts

@@ -42,6 +42,10 @@ export async function openOfflineSession(init: OfflineSessionInit): Promise<Scra
   if (init.session.restore !== undefined) await target.restore(init.session.restore);
   return {
     driver: init.driver,
+    // An offline session dials nothing — except the run's own `/robots.txt` read, which is a real
+    // request. Reporting the run's exit is what makes that one request leave through it, exactly
+    // as it would on the driver this recording stands in for.
+    ...(init.session.proxy === undefined ? {} : { proxy: init.session.proxy }),
     page: pageOverTarget(target, {
       clock: init.session.clock,
       allowHosts: init.session.rules.allowHosts,
@@ -51,6 +55,7 @@ export async function openOfflineSession(init: OfflineSessionInit): Promise<Scra
       signal: init.session.signal,
       onActivity: init.session.onActivity,
       pace: init.session.pace,
+      usage: init.session.usage,
     }),
     http: recordedHttp({
       lookup: init.http,
@@ -63,6 +68,7 @@ export async function openOfflineSession(init: OfflineSessionInit): Promise<Scra
       // The same bag, for the same reason: a redaction only the live leg performs is one no
       // fixture can prove.
       secrets: init.session.secrets,
+      usage: init.session.usage,
       maxAgeMs: init.maxAgeMs,
     }),
     close: () => target.close(),

package/src/page-over-target.ts

@@ -35,6 +35,7 @@ import type {
   ScrapeTarget,
 } from './target';
 import { ROOT_SELECTOR } from './target';
+import type { UsageMeter } from './usage';
 
 export interface PageContext {
   readonly clock: ScrapeClock;
@@ -51,6 +52,8 @@ export interface PageContext {
   readonly secrets?: ScrapeSecrets | undefined;
   readonly robots?: RobotsGate | undefined;
   readonly signal?: AbortSignal | undefined;
+  /** The run's meter. Counts a navigation where the wedge watchdog is told about one. */
+  readonly usage?: UsageMeter | undefined;
 }
 
 /** Mutable, shared by the page and every frame under it: a taint is a property of the SESSION. */
@@ -251,6 +254,7 @@ export function pageOverTarget(target: ScrapeTarget, ctx: PageContext): ScrapePa
       await guardNavigation(url, ctx);
       await ctx.pace?.(ctx.signal);
       ctx.onActivity?.();
+      ctx.usage?.navigation();
       await target.goto(url, {
         timeoutMs: options?.timeout ?? ctx.defaultTimeoutMs,
         signal: ctx.signal,

package/src/scrape-run.ts

@@ -22,10 +22,12 @@ import {
   persistSession,
   restorableSession,
 } from './auth';
-import { systemScrapeClock } from './clock';
+import { scrapeClock } from './clock';
 import type { ScrapeSession } from './driver';
 import { scrapeDriver } from './driver';
 import { driverUnknown, outputInvalid } from './error-throws';
+import { egressInPayload } from './error-throws-session';
+import type { ScrapeEventFields } from './events';
 import { scrapeLogger, withStepEvent } from './events';
 import { guardYield } from './expect';
 import { burnsSession, errorCode, neverRetried } from './failures';
@@ -33,8 +35,12 @@ import { createPacer, DEFAULT_NAVIGATION_RATE } from './rate';
 import { runRecovery } from './recover';
 import { createRobotsGate } from './robots';
 import type { ScrapeDefinition, ScrapeReport } from './scrape';
-import { createSecretBag } from './secrets';
+import { containsSecret } from './secret-scan';
+import { createSecretBag, MIN_REDACTABLE_LENGTH } from './secrets';
 import { sessionKeyFor } from './session-state';
+import { hasCredentials, splitCredentials, urlSecretValues } from './url-secrets';
+import type { ScrapeUsage } from './usage';
+import { createUsageMeter, rememberFailedUsage } from './usage';
 
 /** `ctx.actor` is a structural read: this package never imports the auth types (tier 2). */
 const orgOf = (ctx: unknown): string | undefined => {
@@ -63,11 +69,26 @@ const toMillis = (value: string | number | undefined, fallback: number, option:
 
 export const DEFAULT_PAGE_TIMEOUT_MS = 30_000;
 
+/**
+ * True when the exit's password is readable out of the run's own input — which is the queue row.
+ * Both spellings, because a password with a quote or a backslash is stored JSON-escaped; and
+ * nothing shorter than `MIN_REDACTABLE_LENGTH`, which is a substring of ordinary payload text.
+ */
+function credentialInPayload(egress: string, input: unknown): boolean {
+  const { password } = splitCredentials(egress);
+  if (password.length < MIN_REDACTABLE_LENGTH) return false;
+  const payload = JSON.stringify(input ?? null);
+  const escaped = JSON.stringify(password).slice(1, -1);
+  // Both scans always run: an `||` would skip the second exactly when the first found the secret.
+  const plain = containsSecret(payload, password);
+  return containsSecret(payload, escaped) || plain;
+}
+
 export async function runScrape<I, Row>(
   definition: ScrapeDefinition<I, Row>,
   args: JobRunArgs<I>,
 ): Promise<ScrapeReport<Row>> {
-  const clock = definition.clock ?? systemScrapeClock;
+  const clock = definition.clock ?? scrapeClock();
   const driver = definition.driver ?? scrapeDriver();
   // The scrape's name goes in the SCRAPE slot: there is no driver here to name, which is the
   // whole failure.
@@ -79,6 +100,21 @@ export async function runScrape<I, Row>(
     driver: driver.name,
   });
   const secrets = createSecretBag(definition.secrets ?? []);
+  // The run's exit, resolved HERE — in the worker, under the job's tenant — so what the queue row
+  // carries is the id of whatever holds it. An empty answer is "none", the same as absent: a
+  // driver handed `''` would dial `--proxy-server=` and call that a decision.
+  const declared = await definition.egress?.(args.input, args.ctx);
+  const egress = declared === undefined || declared === '' ? undefined : declared;
+  if (egress !== undefined && hasCredentials(egress)) {
+    // Concealed HERE and not only in the driver that dials it: an offline or third-party driver
+    // never parses the exit, and the page artifact is redacted from this one bag either way.
+    for (const value of urlSecretValues(egress)) secrets.conceal(value);
+    // Before the browser opens: the credential is in `x_jobs` already, and running would only add
+    // a session to an exposure somebody has to rotate.
+    if (credentialInPayload(egress, args.input)) {
+      throw egressInPayload({ scrape: definition.name, egress });
+    }
+  }
   const rules = { allowHosts: definition.allowHosts, block: definition.block };
   // Screened here as well as in `scrape()`, and the two are not one check written twice: that one
   // refuses the DECLARATION and never sees a definition assembled by hand, which `runScrape` is
@@ -113,13 +149,19 @@ export async function runScrape<I, Row>(
   // opening a session first would already have spent an identity on a run that cannot succeed.
   const restored = await restorableSession(plan);
   const pageTimeoutMs = toMillis(definition.pageTimeout, DEFAULT_PAGE_TIMEOUT_MS, 'pageTimeout');
-  // The exit the session dials, readable only AFTER `driver.open()` — the proxy is a driver
-  // option and the gate below is an argument to `open()`, so the gate asks for it per read
-  // instead of being handed a value that cannot exist yet. Every read happens during a
-  // navigation, which is after this is assigned.
+  // The exit the session dials, readable only AFTER `driver.open()` — a driver with no run exit
+  // falls back to its own option, and the gate below is an argument to `open()`, so the gate asks
+  // for it per read instead of being handed a value that cannot exist yet. Every read happens
+  // during a navigation, which is after this is assigned.
   let sessionProxy: string | undefined;
+  // Before `open()`: the time a rented browser takes to arrive is time it was held.
+  const usage = createUsageMeter(clock);
   const session = await driver.open({
     name: definition.name,
+    logger,
+    runId: args.runId,
+    ...(egress === undefined ? {} : { proxy: egress }),
+    usage,
     rules,
     clock,
     timeoutMs: pageTimeoutMs,
@@ -151,10 +193,21 @@ export async function runScrape<I, Row>(
           ensureAuthenticated({
             ...plan,
             input: args.input,
+            runId: args.runId,
             page: session.page,
             secrets,
             restored,
-            prompt: createPrompt(definition.name, definition.prompt, session.page),
+            prompt: createPrompt({
+              scrape: definition.name,
+              handler: definition.prompt,
+              input: args.input,
+              page: session.page,
+              runId: args.runId,
+              clock,
+              signal: args.ctx.signal,
+              secrets,
+              onAnswered: () => usage.promptAnswered(),
+            }),
           }),
       );
       // Persisted after a LOGIN only, never after a reuse: rewriting the record on every run
@@ -181,16 +234,23 @@ export async function runScrape<I, Row>(
     // alone made a run that blocked 5,000 images print 200 and discarded the one number
     // (`Ring.dropped`) that exists to say "you are not seeing it all".
     const networkDropped = session.page.networkDropped();
-    logger.info('scrape.ok', { rows: rows.length, refused });
+    const used = usage.snapshot(session.browserCost);
+    logger.info('scrape.ok', { rows: rows.length, refused, ...usageFields(used) });
     return {
       scrape: definition.name,
       rows,
       artifacts: artifact.saved.map((ref) => ref.key),
       refused,
       networkDropped,
+      usage: used,
     };
   } catch (thrown) {
-    logger.error('scrape.failed', { code: errorCode(thrown) });
+    // A failed run has no report, and it was billed all the same: the counts ride the one line
+    // every failure already writes.
+    const used = usage.snapshot(session.browserCost);
+    logger.error('scrape.failed', { code: errorCode(thrown), ...usageFields(used) });
+    // Kept for `onSettled`, and only when somebody declared one to read it.
+    if (definition.onSettled !== undefined) rememberFailedUsage(args.runId, used);
     if (errorCode(thrown) === 'X_SCRAPE_AUTH_FAILED') {
       await recordSessionOutcome('session.refuse', logger, () => markRefused(plan));
     } else if (burnsSession(thrown)) {
@@ -204,6 +264,15 @@ export async function runScrape<I, Row>(
   }
 }
 
+/** The counts as log fields. `browserCost` stays off the line: it is on the report, as `Money`. */
+const usageFields = (used: ScrapeUsage): ScrapeEventFields => ({
+  browserMs: used.browserMs,
+  navigations: used.navigations,
+  httpRequests: used.httpRequests,
+  bytesIn: used.bytesIn,
+  promptsAnswered: used.promptsAnswered,
+});
+
 /**
  * The tombstone or the burn, on the way out — best effort, and it may NEVER replace the failure
  * that caused it. `markRefused` reaches `store.save()` reaches `storage.put()`, so an S3 503 or an
@@ -250,11 +319,13 @@ async function bodyWithRecovery<I, Row>(
       secrets,
       artifact,
       attempt: args.attempt,
+      finalAttempt: args.finalAttempt,
+      progress: args.progress,
       runId: args.runId,
     });
   try {
     return await withStepEvent(
-      { name: 'body', logger, clock: definition.clock ?? systemScrapeClock, attempt: args.attempt },
+      { name: 'body', logger, clock: definition.clock ?? scrapeClock(), attempt: args.attempt },
       body,
     );
   } catch (thrown) {

package/src/scrape.ts

@@ -11,7 +11,16 @@
 
 import type { Ctx } from '@ultimat3/core';
 import { assert } from '@ultimat3/core';
-import type { JobHandle, JobTenant, RetryPolicy, StepApi } from '@ultimat3/jobs';
+import type {
+  JobCompleted,
+  JobConcurrency,
+  JobFailed,
+  JobHandle,
+  JobTenant,
+  ProgressFn,
+  RetryPolicy,
+  StepApi,
+} from '@ultimat3/jobs';
 import { DEFAULT_RETRY, job } from '@ultimat3/jobs';
 import type { StandardSchemaV1 } from '@ultimat3/schema';
 import type { StorageDriver } from '@ultimat3/storage';
@@ -29,6 +38,8 @@ import type { ResourceType } from './rings';
 import type { RobotsPolicy } from './robots';
 import { runScrape } from './scrape-run';
 import type { ScrapeSecrets } from './secrets';
+import type { ScrapeUsage } from './usage';
+import { takeFailedUsage } from './usage';
 
 export interface ScrapeRunArgs<I> {
   readonly input: I;
@@ -47,11 +58,16 @@ export interface ScrapeRunArgs<I> {
   readonly secrets: ScrapeSecrets;
   readonly artifact: ArtifactWriter;
   readonly attempt: number;
+  /** The job's own: true when a failure of this attempt is not retried for want of attempts. */
+  readonly finalAttempt: boolean;
+  /** The job's own `progress(done, total, note?)` — what `x jobs show` and a dashboard read. */
+  readonly progress: ProgressFn;
   readonly runId: string;
 }
 
 export interface ScrapeArtifacts {
-  readonly storage?: StorageDriver | undefined;
+  /** A thunk, read per write — `() => disk('artifacts')`: the app's disk exists only after boot. */
+  readonly storage?: (() => StorageDriver) | undefined;
   /** Save the page's HTML when the run fails. On by default — it is the only forensic left. */
   readonly onFailure?: boolean | undefined;
   readonly prefix?: string | undefined;
@@ -93,9 +109,26 @@ export interface ScrapeDefinition<I, Row> {
    * primary case, so this is declared rather than hand-rolled per app. See `auth.ts`.
    */
   readonly auth?: ScrapeAuth<I> | undefined;
-  /** Where an out-of-band code comes from, for a site that asks for one after the password. */
-  readonly prompt?: PromptHandler | undefined;
+  /**
+   * Where an out-of-band code comes from, for a site that asks for one after the password. The
+   * request carries the run's `input` and `runId`: what a handler ties the prompt to.
+   */
+  readonly prompt?: PromptHandler<I> | undefined;
   readonly driver?: ScrapeDriver | undefined;
+  /**
+   * The exit THIS run leaves through — a proxy URL, credentials included — RESOLVED IN THE WORKER,
+   * under the job's tenant: `async ({ connectionId }) => (await repo.connectionById(connectionId))
+   * ?.exit`. The input names the row; the row (a `.sealed()` column) holds the exit. An exit whose
+   * credential is ALSO in the input rode the queue payload and sits in `x_jobs` in the clear, and
+   * is refused before the browser opens (`X_SCRAPE_EGRESS_IN_PAYLOAD`).
+   *
+   * It reaches the driver as `SessionInit.proxy` and wins over the driver's own `proxy`; both legs
+   * and the robots read dial it. `undefined` leaves the driver's exit in force. A driver that
+   * cannot dial it refuses with `X_SCRAPE_EGRESS_UNSUPPORTED` rather than dialling another.
+   */
+  readonly egress?:
+    | ((input: I, ctx: Ctx) => string | undefined | Promise<string | undefined>)
+    | undefined;
   readonly retry?: RetryPolicy | undefined;
   /** Per attempt, whole-run. `'5m'` or ms. */
   readonly timeout?: string | number | undefined;
@@ -103,12 +136,33 @@ export interface ScrapeDefinition<I, Row> {
   readonly pageTimeout?: string | number | undefined;
   /** Kill the browser after this much silence from it. See `watchdog.ts`. */
   readonly watchdog?: { readonly idleMs?: number; readonly graceMs?: number } | undefined;
-  readonly concurrency?: number | undefined;
+  /** The job's own field, unchanged: a number, or `{ key, limit, whenBusy }` for a cap per key. */
+  readonly concurrency?: JobConcurrency<I> | undefined;
   readonly queue?: string | undefined;
+  /**
+   * Pins this definition's clock. Omit it: a run waits on the process's (`scrapeClock()`), which
+   * is the system clock and what a test replaces with `setScrapeClock()`.
+   */
   readonly clock?: ScrapeClock | undefined;
   run(args: ScrapeRunArgs<I>): Promise<readonly unknown[]>;
+  /**
+   * The job's own `onSettled`, with what a scrape adds: a `completed` run hands over its
+   * `ScrapeReport` — rows, artifacts and `usage` — and a run that ended any other way carries the
+   * `usage` of its last attempt (`undefined` for a `refused` one, whose body never ran). Same
+   * guarantees as the job's: after the row is settled, under the job's tenant, AT MOST ONCE
+   * across a crash, and a hook that throws changes nothing.
+   */
+  onSettled?(settled: ScrapeSettled<I, Row>): Promise<void>;
 }
 
+/** How one scrape run ended, as `ScrapeDefinition.onSettled` is told. */
+export type ScrapeSettled<I, Row> =
+  | JobCompleted<I, ScrapeReport<Row>>
+  | (JobFailed<I> & {
+      /** What the last attempt used. A failed run has no report and was billed all the same. */
+      readonly usage: ScrapeUsage | undefined;
+    });
+
 /** What one completed scrape reports — bounded, so `x jobs show` can print it. */
 export interface ScrapeReport<Row> {
   readonly scrape: string;
@@ -124,6 +178,8 @@ export interface ScrapeReport<Row> {
    * "you are not seeing it all": `refused` was counted from what survived the bound.
    */
   readonly networkDropped: number;
+  /** What this run used. A measurement: no quota is read from it and none is enforced. */
+  readonly usage: ScrapeUsage;
 }
 
 export function scrape<I, Row>(definition: ScrapeDefinition<I, Row>): JobHandle<I> {
@@ -145,7 +201,8 @@ export function scrape<I, Row>(definition: ScrapeDefinition<I, Row>): JobHandle<
   if (definition.expect?.maxDrop !== undefined && definition.history === undefined) {
     throw yieldHistoryMissing(definition.name);
   }
-  return job<I>({
+  const onSettled = definition.onSettled?.bind(definition);
+  return job<I, ScrapeReport<Row>>({
     name: definition.name,
     input: definition.input,
     idempotencyKey: definition.idempotencyKey,
@@ -155,5 +212,15 @@ export function scrape<I, Row>(definition: ScrapeDefinition<I, Row>): JobHandle<
     ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
     ...(definition.concurrency === undefined ? {} : { concurrency: definition.concurrency }),
     run: (args) => runScrape(definition, args),
+    ...(onSettled === undefined
+      ? {}
+      : {
+          onSettled: (settled) => {
+            // Read on EVERY ending, the completed one too: a run that failed an attempt and then
+            // completed must not leave that attempt's counts behind in this process.
+            const usage = takeFailedUsage(settled.runId);
+            return onSettled(settled.outcome === 'completed' ? settled : { ...settled, usage });
+          },
+        }),
   });
 }

package/src/secret-scan.ts

@@ -0,0 +1,17 @@
+// Whether a secret occurs in a piece of text, without a search that stops at the first differing
+// byte. `.includes(secret)` answers sooner the more of a window matches the secret's prefix — the
+// oracle `scripts/secret-compare.ts` refuses — so every window is compared whole, in constant time.
+
+import { timingSafeEqual } from '@ultimat3/core';
+
+/** True when `secret` occurs in `text`. Time depends on the two lengths, never on their bytes. */
+export function containsSecret(text: string, secret: string): boolean {
+  const width = secret.length;
+  if (width === 0) return false;
+  let found = false;
+  for (let start = 0; start + width <= text.length; start += 1) {
+    // `timingSafeEqual` first, so a window after a hit is still compared rather than skipped.
+    found = timingSafeEqual(text.slice(start, start + width), secret) || found;
+  }
+  return found;
+}

package/src/secrets.ts

@@ -14,6 +14,14 @@ export interface ScrapeSecrets {
   readonly names: readonly string[];
   /** The boxed value. Reading it out is `revealSecret()`, one greppable call site. */
   get(name: string): Secret;
+  /**
+   * Add a value this run learned MID-RUN to the redaction set: a prompt answer, the credentials
+   * of the exit it dials, the connect URL of the browser it rented, a bearer the body read off a
+   * page. It has no name and cannot be read back — it exists only so `page.html()`, the rings and
+   * the failure artifact never carry it. The same floor applies: shorter than
+   * `MIN_REDACTABLE_LENGTH` is not redacted.
+   */
+  conceal(value: string): void;
 }
 
 /**
@@ -30,6 +38,7 @@ export function createSecretBag(
   resolve: SecretResolver = fromEnvironment,
 ): ScrapeSecrets {
   const boxed = new Map<string, Secret>();
+  const learned = new Set<string>();
   for (const name of names) {
     const value = resolve(name);
     if (value === undefined || value === '') {
@@ -42,8 +51,11 @@ export function createSecretBag(
     }
     boxed.set(name, secret(value, name));
   }
-  return {
+  const bag: ScrapeSecrets = {
     names: [...names],
+    conceal(value: string): void {
+      if (value !== '') learned.add(value);
+    },
     get(name: string): Secret {
       const found = boxed.get(name);
       if (found === undefined) {
@@ -57,8 +69,17 @@ export function createSecretBag(
       return found;
     },
   };
+  concealed.set(bag, learned);
+  return bag;
 }
 
+/**
+ * The values `conceal()` was handed, by bag — module-private, so the only way out of a bag is
+ * still `revealSecret(secrets.get(name))` for a DECLARED name. A `ScrapeSecrets` built by hand has
+ * no entry here and redacts its declared names alone.
+ */
+const concealed = new WeakMap<ScrapeSecrets, ReadonlySet<string>>();
+
 export const SECRET_PLACEHOLDER = '[redacted]';
 
 /**
@@ -93,9 +114,13 @@ export const MIN_REDACTABLE_LENGTH = 4;
  * Longest first, so a secret that contains another one does not leave its tail behind.
  */
 export function redactSecrets(text: string, secrets: ScrapeSecrets | undefined): string {
-  if (secrets === undefined || secrets.names.length === 0) return text;
-  const values = secrets.names
-    .map((name) => revealSecret(secrets.get(name)))
+  if (secrets === undefined) return text;
+  const learned = concealed.get(secrets);
+  if (secrets.names.length === 0 && (learned === undefined || learned.size === 0)) return text;
+  const values = [
+    ...secrets.names.map((name) => revealSecret(secrets.get(name))),
+    ...(learned ?? []),
+  ]
     .filter((value) => value.length >= MIN_REDACTABLE_LENGTH)
     .sort((a, b) => b.length - a.length);
   let out = text;