@ultimat3/scraping 22.15.0 → 24.0.0

package/src/pinned-host.ts

@@ -0,0 +1,65 @@
+// The DNS half of `allowHosts`, for the two legs this package dials itself — the HTTP leg and the
+// robots read. Core's `hostDecision` is synchronous and judges NAMES: a wildcard never admits an
+// address literal inside the network, but `allowHosts: ['*']` still admits a hostname that
+// resolves inward. So a name admitted by a wildcard alone is resolved here, every address
+// classified by core's one classifier, and the connection PINNED to the approved address — the
+// shape `@ultimat3/jobs`' `webhook-target.ts` takes for a webhook.
+
+import type { HostRule } from '@ultimat3/core';
+import { classifyAddress, hostMatches } from '@ultimat3/core';
+import { hostResolvesInward } from './error-throws-session';
+
+/** Every address a hostname answers. Injected in tests: the network is sealed in this repo. */
+export type HostResolve = (hostname: string) => Promise<readonly string[]>;
+
+/** `Bun.dns.lookup` — the runtime's own resolver, every family, every address. */
+export const resolveHost: HostResolve = async (hostname) =>
+  (await Bun.dns.lookup(hostname)).map((entry) => entry.address);
+
+/** What the transport connects to: the URL to dial, and the name it proves when pinned. */
+export interface DialTarget {
+  readonly url: string;
+  /** The original `host[:port]`, sent as the `Host` header of a pinned request. */
+  readonly host?: string | undefined;
+  /** For `https:` — the certificate is verified against the NAME, not the address. */
+  readonly serverName?: string | undefined;
+}
+
+const isWildcard = (rule: HostRule): boolean => {
+  const cleaned = rule.trim();
+  return cleaned === '*' || cleaned.startsWith('*.');
+};
+
+/**
+ * The connection for `url`, which `interceptVerdict` has ALREADY allowed. Unchanged when an exact
+ * rule names the host — the app said that host, internal or not, in a line a reviewer can see —
+ * or when the host is an address literal, which core's floor already judged. Otherwise resolved:
+ * one address that is not public refuses the request (`X_SCRAPE_HOST_BLOCKED`), and the dial is
+ * pinned to the first approved address, so a name that answers public here and private to the
+ * socket (DNS rebinding) is never reached. A resolver failure propagates: nothing unscreened is
+ * dialled, and the same name would not have resolved for `fetch` either.
+ */
+export async function dialTarget(
+  url: string,
+  allowHosts: readonly HostRule[],
+  resolve: HostResolve,
+): Promise<DialTarget> {
+  const parsed = new URL(url);
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return { url };
+  const hostname = parsed.hostname.replace(/^\[|\]$/g, '');
+  if (classifyAddress(hostname) !== undefined) return { url };
+  if (allowHosts.some((rule) => !isWildcard(rule) && hostMatches(hostname, rule))) return { url };
+  const addresses = await resolve(hostname);
+  const inward = addresses.map((address) => classifyAddress(address) ?? 'unclassifiable');
+  const refused =
+    addresses.length === 0 ? 'unresolvable' : inward.find((kind) => kind !== 'public');
+  if (refused !== undefined) throw hostResolvesInward(url, hostname, refused);
+  const pinned = addresses[0] ?? hostname;
+  const target = new URL(parsed.href);
+  target.hostname = pinned.includes(':') ? `[${pinned}]` : pinned;
+  return {
+    url: target.href,
+    host: parsed.host,
+    ...(parsed.protocol === 'https:' ? { serverName: hostname } : {}),
+  };
+}

package/src/recover.ts

@@ -6,7 +6,8 @@
 // `X_NOT_IMPLEMENTED` today, in the shape `packages/jobs/src/driver-redis.ts` uses: correct types
 // so an app can be written against it, one labelled throw, no silent no-op.
 
-import { recoverRefused, scrapeNotImplemented } from './error-throws';
+import { NotImplementedError } from '@ultimat3/core';
+import { recoverRefused } from './error-throws';
 import type { ScrapePage } from './page';
 
 export interface RecoveryAttempt {
@@ -41,7 +42,7 @@ export async function runRecovery(recovery: Recovery, attempt: RecoveryAttempt):
     // The agent path lands with `@ultimat3/ai`'s `llm()` behind it: page HTML in, a selector out,
     // re-run once. Until it does, this throws rather than answering `false` — a recovery that
     // silently declines is indistinguishable from one that was never configured.
-    throw scrapeNotImplemented("recover: 'agent'", AGENT_FIX);
+    throw new NotImplementedError({ cause: "recover: 'agent' is not implemented", fix: AGENT_FIX });
   }
   const verdict: unknown = await recovery(attempt);
   if (typeof verdict !== 'boolean') {

package/src/robots-fetch.ts

@@ -9,7 +9,12 @@
 // `driver.open()`, and the proxy is a driver option the session only reports on the way back out.
 
 import { finiteCount, readWithinLimit } from '@ultimat3/core';
+import type { HostRule } from './hosts';
 import type { ScrapeFetch } from './http';
+import { redirectHop } from './http-redirect';
+import { interceptVerdict } from './intercept';
+import type { HostResolve } from './pinned-host';
+import { dialTarget, resolveHost } from './pinned-host';
 import type { RobotsFetch } from './robots';
 
 /**
@@ -23,6 +28,9 @@ export const DEFAULT_ROBOTS_TIMEOUT_MS = 10_000;
 /** Google's own documented ceiling for the file, and generous for a list of path prefixes. */
 export const DEFAULT_ROBOTS_MAX_BYTES = 500 * 1024;
 
+/** RFC 9309 §2.3.1.2: a crawler SHOULD follow at least five consecutive redirects for robots.txt. */
+export const MAX_ROBOTS_REDIRECTS = 5;
+
 export interface RobotsFetchInit {
   /** Per-read wall clock. Defaults to `DEFAULT_ROBOTS_TIMEOUT_MS`. */
   readonly timeoutMs?: number | undefined;
@@ -41,6 +49,15 @@ export interface RobotsFetchInit {
    */
   readonly proxy?: (() => string | undefined) | undefined;
   readonly maxBytes?: number | undefined;
+  /**
+   * The run's `allowHosts`, asked about EVERY hop — the first URL included — through
+   * `interceptVerdict`, the one host rule. A hop off the list is never requested and the read
+   * answers "no robots". With none given, only a redirect that stays on the starting HOSTNAME is
+   * followed (a scheme upgrade, a moved path), still capped; another host is never dialled.
+   */
+  readonly allowHosts?: readonly HostRule[] | undefined;
+  /** The resolver a wildcard-admitted name is checked and pinned with. Defaults to `Bun.dns`. */
+  readonly resolve?: HostResolve | undefined;
   /**
    * The platform `fetch`, injectable so the default path itself is testable. `ScrapeFetch` and not
    * `typeof fetch`: the latter also carries `preconnect`, so nothing a caller can write satisfies
@@ -49,13 +66,38 @@ export interface RobotsFetchInit {
   readonly fetch?: ScrapeFetch | undefined;
 }
 
+const onList = (url: string, allowHosts: readonly HostRule[]): boolean =>
+  interceptVerdict(url, 'fetch', { allowHosts }) === 'allow';
+
+/** Same hostname over http(s) — the port may change with the scheme. Unparseable is not same. */
+const sameHost = (from: string, to: string): boolean => {
+  try {
+    const next = new URL(to);
+    if (next.protocol !== 'http:' && next.protocol !== 'https:') return false;
+    return new URL(from).hostname === next.hostname;
+  } catch {
+    return false;
+  }
+};
+
+/** The final hop's body under the cap, or `undefined` for a non-2xx answer. */
+const answerOf = async (response: Response, limit: number): Promise<string | undefined> => {
+  if (!response.ok) return undefined;
+  // Counted as it arrives rather than `.text()`, which materialises the whole body first: a
+  // multi-gigabyte robots.txt is a heap the worker never gets back.
+  const capped = await readWithinLimit(response.body, limit);
+  return 'over' in capped ? undefined : new TextDecoder().decode(capped.bytes);
+};
+
 /**
  * Reads `robotsUrl`, or answers `undefined` — which the gate reads as "no restrictions", the
- * standard's own answer for a file it cannot obtain. A deadline that fires, a body past the cap
- * and a 404 are all the same answer on purpose: none of them is evidence of a rule.
+ * standard's own answer for a file it cannot obtain. A deadline that fires, a body past the cap,
+ * a 404 and a redirect off `allowHosts` are all the same answer on purpose: none of them is
+ * evidence of a rule.
  */
 export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
   const call: ScrapeFetch = init.fetch ?? fetch;
+  const resolve = init.resolve ?? resolveHost;
   // Both bounds are screened HERE, at construction, and both floors are 1 — because every way this
   // read can fail is the same answer, `undefined`, which the gate reads as "no restrictions". A
   // `NaN` deadline throws a bare `TypeError` out of `AbortSignal.timeout` (measured: "Value NaN is
@@ -83,16 +125,42 @@ export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
     // Resolved here, at the read, because the session that owns the exit did not exist when this
     // fetcher was built. An empty string is not an exit and is dropped with the absent one.
     const proxy = init.proxy?.();
+    const dialled = proxy === undefined || proxy === '' ? undefined : proxy;
     try {
-      const response = await call(robotsUrl, {
-        signal,
-        ...(proxy === undefined || proxy === '' ? {} : { proxy }),
-      });
-      if (!response.ok) return undefined;
-      // Counted as it arrives rather than `.text()`, which materialises the whole body first: a
-      // multi-gigabyte robots.txt is a heap the worker never gets back.
-      const capped = await readWithinLimit(response.body, limit);
-      return 'over' in capped ? undefined : new TextDecoder().decode(capped.bytes);
+      let url = robotsUrl;
+      for (let hops = 0; ; hops += 1) {
+        if (init.allowHosts !== undefined && !onList(url, init.allowHosts)) return undefined;
+        // Pinned exactly as the HTTP leg pins (`pinned-host.ts`), and for its reason skipped
+        // through a proxy. A name resolving inward throws, which this read answers as "no robots".
+        const dial =
+          init.allowHosts === undefined || dialled !== undefined
+            ? { url }
+            : await dialTarget(url, init.allowHosts, resolve);
+        // `manual`, never the platform default: `follow` dials wherever the site points before
+        // anything here can ask whether that host is on the list.
+        const response = await call(dial.url, {
+          signal,
+          redirect: 'manual',
+          ...(dialled === undefined ? {} : { proxy: dialled }),
+          ...(dial.host === undefined ? {} : { headers: { host: dial.host } }),
+          ...(dial.serverName === undefined ? {} : { tls: { serverName: dial.serverName } }),
+        });
+        const next = redirectHop(
+          response.status,
+          response.headers.get('location'),
+          url,
+          'GET',
+          undefined,
+        );
+        if (next === undefined) return await answerOf(response, limit);
+        await response.body?.cancel().catch(() => undefined);
+        if (hops >= MAX_ROBOTS_REDIRECTS) return undefined;
+        // With no list to ask, only a hop that stays on the host the read started on is
+        // followed — an `http:` → `https:` upgrade or a moved path. Another host is a host
+        // nobody screened, and is never dialled.
+        if (init.allowHosts === undefined && !sameHost(robotsUrl, next.url)) return undefined;
+        url = next.url;
+      }
     } catch {
       return undefined;
     }

package/src/scrape-run.ts

@@ -19,13 +19,15 @@ import {
   createPrompt,
   ensureAuthenticated,
   markRefused,
+  observeSession,
   persistSession,
-  restorableSession,
 } from './auth';
-import { systemScrapeClock } from './clock';
+import { scrapeClock } from './clock';
 import type { ScrapeSession } from './driver';
 import { scrapeDriver } from './driver';
 import { driverUnknown, outputInvalid } from './error-throws';
+import { egressInPayload } from './error-throws-session';
+import type { ScrapeEventFields } from './events';
 import { scrapeLogger, withStepEvent } from './events';
 import { guardYield } from './expect';
 import { burnsSession, errorCode, neverRetried } from './failures';
@@ -33,8 +35,12 @@ import { createPacer, DEFAULT_NAVIGATION_RATE } from './rate';
 import { runRecovery } from './recover';
 import { createRobotsGate } from './robots';
 import type { ScrapeDefinition, ScrapeReport } from './scrape';
-import { createSecretBag } from './secrets';
+import { containsSecret } from './secret-scan';
+import { createSecretBag, MIN_REDACTABLE_LENGTH } from './secrets';
 import { sessionKeyFor } from './session-state';
+import { hasCredentials, splitCredentials, urlSecretValues } from './url-secrets';
+import type { ScrapeUsage } from './usage';
+import { createUsageMeter, rememberFailedUsage } from './usage';
 
 /** `ctx.actor` is a structural read: this package never imports the auth types (tier 2). */
 const orgOf = (ctx: unknown): string | undefined => {
@@ -63,11 +69,26 @@ const toMillis = (value: string | number | undefined, fallback: number, option:
 
 export const DEFAULT_PAGE_TIMEOUT_MS = 30_000;
 
+/**
+ * True when the exit's password is readable out of the run's own input — which is the queue row.
+ * Both spellings, because a password with a quote or a backslash is stored JSON-escaped; and
+ * nothing shorter than `MIN_REDACTABLE_LENGTH`, which is a substring of ordinary payload text.
+ */
+function credentialInPayload(egress: string, input: unknown): boolean {
+  const { password } = splitCredentials(egress);
+  if (password.length < MIN_REDACTABLE_LENGTH) return false;
+  const payload = JSON.stringify(input ?? null);
+  const escaped = JSON.stringify(password).slice(1, -1);
+  // Both scans always run: an `||` would skip the second exactly when the first found the secret.
+  const plain = containsSecret(payload, password);
+  return containsSecret(payload, escaped) || plain;
+}
+
 export async function runScrape<I, Row>(
   definition: ScrapeDefinition<I, Row>,
   args: JobRunArgs<I>,
 ): Promise<ScrapeReport<Row>> {
-  const clock = definition.clock ?? systemScrapeClock;
+  const clock = definition.clock ?? scrapeClock();
   const driver = definition.driver ?? scrapeDriver();
   // The scrape's name goes in the SCRAPE slot: there is no driver here to name, which is the
   // whole failure.
@@ -79,6 +100,21 @@ export async function runScrape<I, Row>(
     driver: driver.name,
   });
   const secrets = createSecretBag(definition.secrets ?? []);
+  // The run's exit, resolved HERE — in the worker, under the job's tenant — so what the queue row
+  // carries is the id of whatever holds it. An empty answer is "none", the same as absent: a
+  // driver handed `''` would dial `--proxy-server=` and call that a decision.
+  const declared = await definition.egress?.(args.input, args.ctx);
+  const egress = declared === undefined || declared === '' ? undefined : declared;
+  if (egress !== undefined && hasCredentials(egress)) {
+    // Concealed HERE and not only in the driver that dials it: an offline or third-party driver
+    // never parses the exit, and the page artifact is redacted from this one bag either way.
+    for (const value of urlSecretValues(egress)) secrets.conceal(value);
+    // Before the browser opens: the credential is in `x_jobs` already, and running would only add
+    // a session to an exposure somebody has to rotate.
+    if (credentialInPayload(egress, args.input)) {
+      throw egressInPayload({ scrape: definition.name, egress });
+    }
+  }
   const rules = { allowHosts: definition.allowHosts, block: definition.block };
   // Screened here as well as in `scrape()`, and the two are not one check written twice: that one
   // refuses the DECLARATION and never sees a definition assembled by hand, which `runScrape` is
@@ -111,15 +147,22 @@ export async function runScrape<I, Row>(
 
   // Read BEFORE the browser opens: a refused credential must not reach a login form again, and
   // opening a session first would already have spent an identity on a run that cannot succeed.
-  const restored = await restorableSession(plan);
+  const observed = await observeSession(plan);
+  const restored = observed.restorable;
   const pageTimeoutMs = toMillis(definition.pageTimeout, DEFAULT_PAGE_TIMEOUT_MS, 'pageTimeout');
-  // The exit the session dials, readable only AFTER `driver.open()` — the proxy is a driver
-  // option and the gate below is an argument to `open()`, so the gate asks for it per read
-  // instead of being handed a value that cannot exist yet. Every read happens during a
-  // navigation, which is after this is assigned.
+  // The exit the session dials, readable only AFTER `driver.open()` — a driver with no run exit
+  // falls back to its own option, and the gate below is an argument to `open()`, so the gate asks
+  // for it per read instead of being handed a value that cannot exist yet. Every read happens
+  // during a navigation, which is after this is assigned.
   let sessionProxy: string | undefined;
+  // Before `open()`: the time a rented browser takes to arrive is time it was held.
+  const usage = createUsageMeter(clock);
   const session = await driver.open({
     name: definition.name,
+    logger,
+    runId: args.runId,
+    ...(egress === undefined ? {} : { proxy: egress }),
+    usage,
     rules,
     clock,
     timeoutMs: pageTimeoutMs,
@@ -132,6 +175,8 @@ export async function runScrape<I, Row>(
     // this gate reads as "no restrictions".
     robots: createRobotsGate({
       policy: definition.robots ?? 'obey',
+      // Every hop of the read screened by the run's own host rule, as the HTTP leg's are.
+      allowHosts: rules.allowHosts,
       timeoutMs: pageTimeoutMs,
       signal: args.ctx.signal,
       proxy: () => sessionProxy,
@@ -142,6 +187,11 @@ export async function runScrape<I, Row>(
     watchdog: definition.watchdog,
   });
   sessionProxy = session.proxy;
+  // The VERSION of the stored record this run is on — the one it found (restorable or not), or the
+  // one it persisted after its own login. A burn or a tombstone on the way out compares against
+  // it, so a record another run on the key saved meanwhile is not destroyed by this run's failure
+  // (`auth.ts`'s `stillSeen`).
+  let seen = observed.seen;
 
   try {
     if (definition.auth !== undefined) {
@@ -151,15 +201,34 @@ export async function runScrape<I, Row>(
           ensureAuthenticated({
             ...plan,
             input: args.input,
+            runId: args.runId,
             page: session.page,
             secrets,
             restored,
-            prompt: createPrompt(definition.name, definition.prompt, session.page),
+            prompt: createPrompt({
+              scrape: definition.name,
+              handler: definition.prompt,
+              input: args.input,
+              page: session.page,
+              runId: args.runId,
+              clock,
+              signal: args.ctx.signal,
+              secrets,
+              onAnswered: () => usage.promptAnswered(),
+            }),
           }),
       );
       // Persisted after a LOGIN only, never after a reuse: rewriting the record on every run
       // refreshes `savedAt` without refreshing the session, so `maxAge` would never expire it.
-      if (loggedIn) await persistSession(plan, session.page);
+      //
+      // And a failure to SAVE never fails the attempt: the login happened, the body can run, and
+      // failing here made the retry log in again — one more arrival at the login form, and on a
+      // store that stays down, one per attempt. Logged, like the burn and the tombstone below.
+      if (loggedIn) {
+        await recordSessionOutcome('session.save', logger, async () => {
+          seen = await persistSession(plan, session.page);
+        });
+      }
     }
 
     const raw = await bodyWithRecovery(definition, args, session, logger, artifact, secrets);
@@ -181,20 +250,27 @@ export async function runScrape<I, Row>(
     // alone made a run that blocked 5,000 images print 200 and discarded the one number
     // (`Ring.dropped`) that exists to say "you are not seeing it all".
     const networkDropped = session.page.networkDropped();
-    logger.info('scrape.ok', { rows: rows.length, refused });
+    const used = usage.snapshot(session.browserCost);
+    logger.info('scrape.ok', { rows: rows.length, refused, ...usageFields(used) });
     return {
       scrape: definition.name,
       rows,
       artifacts: artifact.saved.map((ref) => ref.key),
       refused,
       networkDropped,
+      usage: used,
     };
   } catch (thrown) {
-    logger.error('scrape.failed', { code: errorCode(thrown) });
+    // A failed run has no report, and it was billed all the same: the counts ride the one line
+    // every failure already writes.
+    const used = usage.snapshot(session.browserCost);
+    logger.error('scrape.failed', { code: errorCode(thrown), ...usageFields(used) });
+    // Kept for `onSettled`, and only when somebody declared one to read it.
+    if (definition.onSettled !== undefined) rememberFailedUsage(args.runId, used);
     if (errorCode(thrown) === 'X_SCRAPE_AUTH_FAILED') {
-      await recordSessionOutcome('session.refuse', logger, () => markRefused(plan));
+      await recordSessionOutcome('session.refuse', logger, () => markRefused(plan, seen));
     } else if (burnsSession(thrown)) {
-      await recordSessionOutcome('session.burn', logger, () => burnSession(plan));
+      await recordSessionOutcome('session.burn', logger, () => burnSession(plan, seen));
     }
     if (definition.artifacts?.onFailure !== false) await saveFailureArtifact(session, artifact);
     throw thrown;
@@ -204,9 +280,18 @@ export async function runScrape<I, Row>(
   }
 }
 
+/** The counts as log fields. `browserCost` stays off the line: it is on the report, as `Money`. */
+const usageFields = (used: ScrapeUsage): ScrapeEventFields => ({
+  browserMs: used.browserMs,
+  navigations: used.navigations,
+  httpRequests: used.httpRequests,
+  bytesIn: used.bytesIn,
+  promptsAnswered: used.promptsAnswered,
+});
+
 /**
- * The tombstone or the burn, on the way out — best effort, and it may NEVER replace the failure
- * that caused it. `markRefused` reaches `store.save()` reaches `storage.put()`, so an S3 503 or an
+ * The save after a login, and the tombstone or the burn on the way out — best effort, and none
+ * may ever replace the run's own outcome. `markRefused` reaches `store.save()` reaches `storage.put()`, so an S3 503 or an
  * `X_STORAGE_PATH_UNSAFE` from a tenant whose key sanitises to nothing used to propagate out of
  * the catch and REPLACE a terminal `X_SCRAPE_AUTH_FAILED` with a retryable one. Attempt 2 then
  * found no tombstone — the save is what failed — and walked the same rejected password back to
@@ -214,15 +299,16 @@ export async function runScrape<I, Row>(
  * reason: the run's own error is the one the reader needs.
  */
 async function recordSessionOutcome(
-  step: 'session.refuse' | 'session.burn',
+  step: 'session.save' | 'session.refuse' | 'session.burn',
   logger: ReturnType<typeof scrapeLogger>,
   write: () => Promise<void>,
 ): Promise<void> {
   try {
     await write();
   } catch (thrown) {
-    // Logged rather than swallowed silently: the tombstone is missing, so the NEXT attempt will
-    // re-probe rather than refuse cheaply, and that is a fact an operator has to be able to see.
+    // Logged rather than swallowed silently: a missing tombstone means the NEXT attempt re-probes
+    // rather than refusing cheaply, a missing save means it logs in again, and an operator has to
+    // be able to see either.
     logger.error('scrape.session.write_failed', { step, code: errorCode(thrown) });
   }
 }
@@ -250,11 +336,13 @@ async function bodyWithRecovery<I, Row>(
       secrets,
       artifact,
       attempt: args.attempt,
+      finalAttempt: args.finalAttempt,
+      progress: args.progress,
       runId: args.runId,
     });
   try {
     return await withStepEvent(
-      { name: 'body', logger, clock: definition.clock ?? systemScrapeClock, attempt: args.attempt },
+      { name: 'body', logger, clock: definition.clock ?? scrapeClock(), attempt: args.attempt },
       body,
     );
   } catch (thrown) {

package/src/scrape.ts

@@ -11,7 +11,16 @@
 
 import type { Ctx } from '@ultimat3/core';
 import { assert } from '@ultimat3/core';
-import type { JobHandle, JobTenant, RetryPolicy, StepApi } from '@ultimat3/jobs';
+import type {
+  JobCompleted,
+  JobConcurrency,
+  JobFailed,
+  JobHandle,
+  JobTenant,
+  ProgressFn,
+  RetryPolicy,
+  StepApi,
+} from '@ultimat3/jobs';
 import { DEFAULT_RETRY, job } from '@ultimat3/jobs';
 import type { StandardSchemaV1 } from '@ultimat3/schema';
 import type { StorageDriver } from '@ultimat3/storage';
@@ -21,6 +30,7 @@ import type { ScrapeClock } from './clock';
 import type { ScrapeDriver } from './driver';
 import { yieldHistoryMissing } from './error-throws';
 import type { YieldExpectation, YieldHistory } from './expect';
+import { maxDropFraction } from './expect';
 import type { HostRule } from './hosts';
 import type { ScrapeHttp } from './http';
 import type { ScrapePage } from './page';
@@ -29,6 +39,8 @@ import type { ResourceType } from './rings';
 import type { RobotsPolicy } from './robots';
 import { runScrape } from './scrape-run';
 import type { ScrapeSecrets } from './secrets';
+import type { ScrapeUsage } from './usage';
+import { takeFailedUsage } from './usage';
 
 export interface ScrapeRunArgs<I> {
   readonly input: I;
@@ -47,11 +59,16 @@ export interface ScrapeRunArgs<I> {
   readonly secrets: ScrapeSecrets;
   readonly artifact: ArtifactWriter;
   readonly attempt: number;
+  /** The job's own: true when a failure of this attempt is not retried for want of attempts. */
+  readonly finalAttempt: boolean;
+  /** The job's own `progress(done, total, note?)` — what `x jobs show` and a dashboard read. */
+  readonly progress: ProgressFn;
   readonly runId: string;
 }
 
 export interface ScrapeArtifacts {
-  readonly storage?: StorageDriver | undefined;
+  /** A thunk, read per write — `() => disk('artifacts')`: the app's disk exists only after boot. */
+  readonly storage?: (() => StorageDriver) | undefined;
   /** Save the page's HTML when the run fails. On by default — it is the only forensic left. */
   readonly onFailure?: boolean | undefined;
   readonly prefix?: string | undefined;
@@ -93,9 +110,26 @@ export interface ScrapeDefinition<I, Row> {
    * primary case, so this is declared rather than hand-rolled per app. See `auth.ts`.
    */
   readonly auth?: ScrapeAuth<I> | undefined;
-  /** Where an out-of-band code comes from, for a site that asks for one after the password. */
-  readonly prompt?: PromptHandler | undefined;
+  /**
+   * Where an out-of-band code comes from, for a site that asks for one after the password. The
+   * request carries the run's `input` and `runId`: what a handler ties the prompt to.
+   */
+  readonly prompt?: PromptHandler<I> | undefined;
   readonly driver?: ScrapeDriver | undefined;
+  /**
+   * The exit THIS run leaves through — a proxy URL, credentials included — RESOLVED IN THE WORKER,
+   * under the job's tenant: `async ({ connectionId }) => (await repo.connectionById(connectionId))
+   * ?.exit`. The input names the row; the row (a `.sealed()` column) holds the exit. An exit whose
+   * credential is ALSO in the input rode the queue payload and sits in `x_jobs` in the clear, and
+   * is refused before the browser opens (`X_SCRAPE_EGRESS_IN_PAYLOAD`).
+   *
+   * It reaches the driver as `SessionInit.proxy` and wins over the driver's own `proxy`; both legs
+   * and the robots read dial it. `undefined` leaves the driver's exit in force. A driver that
+   * cannot dial it refuses with `X_SCRAPE_EGRESS_UNSUPPORTED` rather than dialling another.
+   */
+  readonly egress?:
+    | ((input: I, ctx: Ctx) => string | undefined | Promise<string | undefined>)
+    | undefined;
   readonly retry?: RetryPolicy | undefined;
   /** Per attempt, whole-run. `'5m'` or ms. */
   readonly timeout?: string | number | undefined;
@@ -103,12 +137,33 @@ export interface ScrapeDefinition<I, Row> {
   readonly pageTimeout?: string | number | undefined;
   /** Kill the browser after this much silence from it. See `watchdog.ts`. */
   readonly watchdog?: { readonly idleMs?: number; readonly graceMs?: number } | undefined;
-  readonly concurrency?: number | undefined;
+  /** The job's own field, unchanged: a number, or `{ key, limit, whenBusy }` for a cap per key. */
+  readonly concurrency?: JobConcurrency<I> | undefined;
   readonly queue?: string | undefined;
+  /**
+   * Pins this definition's clock. Omit it: a run waits on the process's (`scrapeClock()`), which
+   * is the system clock and what a test replaces with `setScrapeClock()`.
+   */
   readonly clock?: ScrapeClock | undefined;
   run(args: ScrapeRunArgs<I>): Promise<readonly unknown[]>;
+  /**
+   * The job's own `onSettled`, with what a scrape adds: a `completed` run hands over its
+   * `ScrapeReport` — rows, artifacts and `usage` — and a run that ended any other way carries the
+   * `usage` of its last attempt (`undefined` for a `refused` one, whose body never ran). Same
+   * guarantees as the job's: after the row is settled, under the job's tenant, AT MOST ONCE
+   * across a crash, and a hook that throws changes nothing.
+   */
+  onSettled?(settled: ScrapeSettled<I, Row>): Promise<void>;
 }
 
+/** How one scrape run ended, as `ScrapeDefinition.onSettled` is told. */
+export type ScrapeSettled<I, Row> =
+  | JobCompleted<I, ScrapeReport<Row>>
+  | (JobFailed<I> & {
+      /** What the last attempt used. A failed run has no report and was billed all the same. */
+      readonly usage: ScrapeUsage | undefined;
+    });
+
 /** What one completed scrape reports — bounded, so `x jobs show` can print it. */
 export interface ScrapeReport<Row> {
   readonly scrape: string;
@@ -124,6 +179,8 @@ export interface ScrapeReport<Row> {
    * "you are not seeing it all": `refused` was counted from what survived the bound.
    */
   readonly networkDropped: number;
+  /** What this run used. A measurement: no quota is read from it and none is enforced. */
+  readonly usage: ScrapeUsage;
 }
 
 export function scrape<I, Row>(definition: ScrapeDefinition<I, Row>): JobHandle<I> {
@@ -142,10 +199,12 @@ export function scrape<I, Row>(definition: ScrapeDefinition<I, Row>): JobHandle<
   // Two halves that must be set together. `maxDrop` is a fraction of a trailing median and only
   // `history:` can supply one, so declaring it alone is an alarm that cannot fire — refused here,
   // where it is written, rather than discovered as a scrape that never once went red.
-  if (definition.expect?.maxDrop !== undefined && definition.history === undefined) {
-    throw yieldHistoryMissing(definition.name);
+  if (definition.expect?.maxDrop !== undefined) {
+    if (definition.history === undefined) throw yieldHistoryMissing(definition.name);
+    maxDropFraction(definition.name, definition.expect.maxDrop);
   }
-  return job<I>({
+  const onSettled = definition.onSettled?.bind(definition);
+  return job<I, ScrapeReport<Row>>({
     name: definition.name,
     input: definition.input,
     idempotencyKey: definition.idempotencyKey,
@@ -155,5 +214,15 @@ export function scrape<I, Row>(definition: ScrapeDefinition<I, Row>): JobHandle<
     ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
     ...(definition.concurrency === undefined ? {} : { concurrency: definition.concurrency }),
     run: (args) => runScrape(definition, args),
+    ...(onSettled === undefined
+      ? {}
+      : {
+          onSettled: (settled) => {
+            // Read on EVERY ending, the completed one too: a run that failed an attempt and then
+            // completed must not leave that attempt's counts behind in this process.
+            const usage = takeFailedUsage(settled.runId);
+            return onSettled(settled.outcome === 'completed' ? settled : { ...settled, usage });
+          },
+        }),
   });
 }

package/src/secret-scan.ts

@@ -0,0 +1,17 @@
+// Whether a secret occurs in a piece of text, without a search that stops at the first differing
+// byte. `.includes(secret)` answers sooner the more of a window matches the secret's prefix — the
+// oracle `scripts/secret-compare.ts` refuses — so every window is compared whole, in constant time.
+
+import { timingSafeEqual } from '@ultimat3/core';
+
+/** True when `secret` occurs in `text`. Time depends on the two lengths, never on their bytes. */
+export function containsSecret(text: string, secret: string): boolean {
+  const width = secret.length;
+  if (width === 0) return false;
+  let found = false;
+  for (let start = 0; start + width <= text.length; start += 1) {
+    // `timingSafeEqual` first, so a window after a hit is still compared rather than skipped.
+    found = timingSafeEqual(text.slice(start, start + width), secret) || found;
+  }
+  return found;
+}