@ultimat3/scraping 23.0.0 → 25.0.0

package/src/robots-fetch.ts

@@ -8,9 +8,14 @@
 // The exit is a RESOLVER, not a string: `scrape-run.ts` builds this gate as an argument to
 // `driver.open()`, and the proxy is a driver option the session only reports on the way back out.
 
-import { finiteCount, readWithinLimit } from '@ultimat3/core';
+import type { HostRule } from '@ultimat3/core';
+import { finiteCount, isUltimateError, readWithinLimit } from '@ultimat3/core';
 import type { ScrapeFetch } from './http';
-import type { RobotsFetch } from './robots';
+import { redirectHop } from './http-redirect';
+import { interceptVerdict } from './intercept';
+import type { HostResolve } from './pinned-host';
+import { dialTarget, resolveHost } from './pinned-host';
+import type { RobotsAnswer, RobotsFetch } from './robots';
 
 /**
  * A deadline is applied ALWAYS, proxy or no proxy, session or no session: the failure it prevents
@@ -23,6 +28,9 @@ export const DEFAULT_ROBOTS_TIMEOUT_MS = 10_000;
 /** Google's own documented ceiling for the file, and generous for a list of path prefixes. */
 export const DEFAULT_ROBOTS_MAX_BYTES = 500 * 1024;
 
+/** RFC 9309 §2.3.1.2: a crawler SHOULD follow at least five consecutive redirects for robots.txt. */
+export const MAX_ROBOTS_REDIRECTS = 5;
+
 export interface RobotsFetchInit {
   /** Per-read wall clock. Defaults to `DEFAULT_ROBOTS_TIMEOUT_MS`. */
   readonly timeoutMs?: number | undefined;
@@ -41,6 +49,15 @@ export interface RobotsFetchInit {
    */
   readonly proxy?: (() => string | undefined) | undefined;
   readonly maxBytes?: number | undefined;
+  /**
+   * The run's `allowHosts`, asked about EVERY hop — the first URL included — through
+   * `interceptVerdict`, the one host rule. A hop off the list is never requested and the read
+   * answers "no robots". With none given, only a redirect that stays on the starting HOSTNAME is
+   * followed (a scheme upgrade, a moved path), still capped; another host is never dialled.
+   */
+  readonly allowHosts?: readonly HostRule[] | undefined;
+  /** The resolver a wildcard-admitted name is checked and pinned with. Defaults to `Bun.dns`. */
+  readonly resolve?: HostResolve | undefined;
   /**
    * The platform `fetch`, injectable so the default path itself is testable. `ScrapeFetch` and not
    * `typeof fetch`: the latter also carries `preconnect`, so nothing a caller can write satisfies
@@ -49,20 +66,56 @@ export interface RobotsFetchInit {
   readonly fetch?: ScrapeFetch | undefined;
 }
 
+const onList = (url: string, allowHosts: readonly HostRule[]): boolean =>
+  interceptVerdict(url, 'fetch', { allowHosts }) === 'allow';
+
+/** Same hostname over http(s) — the port may change with the scheme. Unparseable is not same. */
+const sameHost = (from: string, to: string): boolean => {
+  try {
+    const next = new URL(to);
+    if (next.protocol !== 'http:' && next.protocol !== 'https:') return false;
+    return new URL(from).hostname === next.hostname;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * RFC 9309 §2.3.1.4 names server errors; a 429 is the same condition spelled as a client status —
+ * the origin is shedding load, not reporting a missing file — and every major crawler reads it so.
+ */
+const isUnreachableStatus = (status: number): boolean => status >= 500 || status === 429;
+
+/** The final hop's body under the cap, unreachable for a 5xx/429, `undefined` for another non-2xx. */
+const answerOf = async (response: Response, limit: number): Promise<RobotsAnswer> => {
+  if (isUnreachableStatus(response.status)) {
+    await response.body?.cancel().catch(() => undefined);
+    return { unreachable: `status ${String(response.status)}` };
+  }
+  if (!response.ok) return undefined;
+  // Counted as it arrives rather than `.text()`, which materialises the whole body first: a
+  // multi-gigabyte robots.txt is a heap the worker never gets back.
+  const capped = await readWithinLimit(response.body, limit);
+  return 'over' in capped ? undefined : new TextDecoder().decode(capped.bytes);
+};
+
 /**
- * Reads `robotsUrl`, or answers `undefined` — which the gate reads as "no restrictions", the
- * standard's own answer for a file it cannot obtain. A deadline that fires, a body past the cap
- * and a 404 are all the same answer on purpose: none of them is evidence of a rule.
+ * Reads `robotsUrl`. Three answers, after RFC 9309 §2.3.1: the text; `undefined` when the file is
+ * UNAVAILABLE — a 4xx, a body past the cap, a redirect off `allowHosts` or past the hop cap, a
+ * host the pin refuses — which the gate reads as no rules; and `{ unreachable }` for a 5xx, a 429,
+ * a deadline, a cancellation or a network error, which the gate reads as complete disallow. A
+ * server that cannot answer is not a server that has no rules.
  */
 export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
   const call: ScrapeFetch = init.fetch ?? fetch;
-  // Both bounds are screened HERE, at construction, and both floors are 1 — because every way this
-  // read can fail is the same answer, `undefined`, which the gate reads as "no restrictions". A
+  const resolve = init.resolve ?? resolveHost;
+  // Both bounds are screened HERE, at construction, and both floors are 1 — because a bad bound
+  // turns into a failed read on every origin, and a read that fails is never evidence of a rule. A
   // `NaN` deadline throws a bare `TypeError` out of `AbortSignal.timeout` (measured: "Value NaN is
   // outside the range [0, 9007199254740991]") straight into the gate's own `.catch`, and a `NaN`
   // cap makes `readWithinLimit` refuse after the request already left. A zero of either is the
-  // same outcome spelled deliberately: an expired deadline and a cap every file is over. Robots
-  // enforcement off, for the whole run, with nothing in the log.
+  // same outcome spelled deliberately: an expired deadline and a cap every file is over — every
+  // origin refused, or (the cap) every origin unrestricted, for the whole run.
   const limit = finiteCount(
     'robotsFetcher',
     'maxBytes',
@@ -75,7 +128,7 @@ export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
     init.timeoutMs ?? DEFAULT_ROBOTS_TIMEOUT_MS,
     1,
   );
-  return async (robotsUrl: string): Promise<string | undefined> => {
+  return async (robotsUrl: string): Promise<RobotsAnswer> => {
     // Armed per read, not per gate: the gate is long-lived and reads once per origin, so a
     // deadline created alongside it would already have expired by the second origin.
     const deadline = AbortSignal.timeout(timeoutMs);
@@ -83,18 +136,49 @@ export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
     // Resolved here, at the read, because the session that owns the exit did not exist when this
     // fetcher was built. An empty string is not an exit and is dropped with the absent one.
     const proxy = init.proxy?.();
+    const dialled = proxy === undefined || proxy === '' ? undefined : proxy;
     try {
-      const response = await call(robotsUrl, {
-        signal,
-        ...(proxy === undefined || proxy === '' ? {} : { proxy }),
-      });
-      if (!response.ok) return undefined;
-      // Counted as it arrives rather than `.text()`, which materialises the whole body first: a
-      // multi-gigabyte robots.txt is a heap the worker never gets back.
-      const capped = await readWithinLimit(response.body, limit);
-      return 'over' in capped ? undefined : new TextDecoder().decode(capped.bytes);
-    } catch {
-      return undefined;
+      let url = robotsUrl;
+      for (let hops = 0; ; hops += 1) {
+        if (init.allowHosts !== undefined && !onList(url, init.allowHosts)) return undefined;
+        // Pinned exactly as the HTTP leg pins (`pinned-host.ts`), and for its reason skipped
+        // through a proxy. A name resolving inward throws, which this read answers as "no robots".
+        const dial =
+          init.allowHosts === undefined || dialled !== undefined
+            ? { url }
+            : await dialTarget(url, init.allowHosts, resolve);
+        // `manual`, never the platform default: `follow` dials wherever the site points before
+        // anything here can ask whether that host is on the list.
+        const response = await call(dial.url, {
+          signal,
+          redirect: 'manual',
+          ...(dialled === undefined ? {} : { proxy: dialled }),
+          ...(dial.host === undefined ? {} : { headers: { host: dial.host } }),
+          ...(dial.serverName === undefined ? {} : { tls: { serverName: dial.serverName } }),
+        });
+        const next = redirectHop(
+          response.status,
+          response.headers.get('location'),
+          url,
+          'GET',
+          undefined,
+        );
+        if (next === undefined) return await answerOf(response, limit);
+        await response.body?.cancel().catch(() => undefined);
+        if (hops >= MAX_ROBOTS_REDIRECTS) return undefined;
+        // With no list to ask, only a hop that stays on the host the read started on is
+        // followed — an `http:` → `https:` upgrade or a moved path. Another host is a host
+        // nobody screened, and is never dialled.
+        if (init.allowHosts === undefined && !sameHost(robotsUrl, next.url)) return undefined;
+        url = next.url;
+      }
+    } catch (error) {
+      // The pin refusing an inward name is a POLICY answer about the host, and the navigation is
+      // refused by the same rule — it is not evidence the origin is down.
+      if (isUltimateError(error) && error.code === 'X_SCRAPE_HOST_BLOCKED') return undefined;
+      if (deadline.aborted) return { unreachable: 'timeout' };
+      if (init.signal?.aborted === true) return { unreachable: 'cancelled' };
+      return { unreachable: 'network' };
     }
   };
 }

package/src/robots.ts

@@ -126,7 +126,20 @@ export interface RobotsGate {
   readonly ignoredBecause: string | undefined;
 }
 
-export type RobotsFetch = (robotsUrl: string) => Promise<string | undefined>;
+/**
+ * RFC 9309 §2.3.1.4: the file could not be obtained because of a server or network error — a 5xx,
+ * a 429, a refused connection, a deadline. The standard's answer is COMPLETE DISALLOW, and it is
+ * a different answer from `undefined` (a 4xx: the file is unavailable, so there are no rules).
+ * `unreachable` says why, for the log; nothing branches on its text.
+ */
+export interface RobotsUnreachable {
+  readonly unreachable: string;
+}
+
+/** The file's text, `undefined` for no file (no rules), or unreachable (no access). */
+export type RobotsAnswer = string | undefined | RobotsUnreachable;
+
+export type RobotsFetch = (robotsUrl: string) => Promise<RobotsAnswer>;
 
 export interface RobotsGateInit extends RobotsFetchInit {
   readonly policy: RobotsPolicy;
@@ -135,20 +148,32 @@ export interface RobotsGateInit extends RobotsFetchInit {
   readonly fetchText?: RobotsFetch | undefined;
 }
 
+/** What the gate holds per origin: the rules, or the standard's complete disallow. */
+type Verdict = RobotsRules | 'unreachable';
+
+const verdictOf = (answer: RobotsAnswer, agent: string): Verdict => {
+  if (answer === undefined) return { rules: [] };
+  if (typeof answer === 'string') return parseRobots(answer, agent);
+  return 'unreachable';
+};
+
 /**
  * One fetch of `/robots.txt` per ORIGIN per run, cached. A gate that re-fetched per navigation
  * would triple the request count of every scrape it protects.
  *
- * An origin whose robots.txt cannot be read is ALLOWED. That is the standard's own answer — a
- * missing file means no restrictions — and the alternative fails every run behind a flaky CDN.
+ * A MISSING robots.txt (a 4xx, a redirect chain past the cap) is no restrictions — the standard's
+ * own answer. An UNREACHABLE one (5xx, 429, a network error, a deadline, a read that rejects) is
+ * complete disallow (RFC 9309 §2.3.1.4), and is NOT cached: the next navigation to the origin asks
+ * again, so a run behind a flaky CDN recovers the moment the file answers instead of failing for
+ * the rest of the run — and an origin cannot shed its own rules by answering 503 once.
  */
-export function createRobotsGate(init: RobotsGateInit): RobotsGate {
+export function robotsGate(init: RobotsGateInit): RobotsGate {
   if (init.policy !== 'obey') {
     return { assertAllowed: () => Promise.resolve(), ignoredBecause: init.policy.ignore };
   }
   const agent = init.agent ?? DEFAULT_ROBOTS_AGENT;
   const read = init.fetchText ?? robotsFetcher(init);
-  const cache = new Map<string, Promise<RobotsRules>>();
+  const cache = new Map<string, Promise<Verdict>>();
   return {
     ignoredBecause: undefined,
     async assertAllowed(url: string): Promise<void> {
@@ -160,14 +185,27 @@ export function createRobotsGate(init: RobotsGateInit): RobotsGate {
       }
       if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return;
       const origin = parsed.origin;
-      let rules = cache.get(origin);
-      if (rules === undefined) {
-        rules = read(`${origin}/robots.txt`)
-          .then((text) => (text === undefined ? { rules: [] } : parseRobots(text, agent)))
-          .catch(() => ({ rules: [] }));
-        cache.set(origin, rules);
+      let pending = cache.get(origin);
+      if (pending === undefined) {
+        // A caller-supplied read that rejects has not produced a file, and "could not ask" is
+        // the unreachable case, not the missing one.
+        const fresh = read(`${origin}/robots.txt`).then(
+          (answer) => verdictOf(answer, agent),
+          (): Verdict => 'unreachable',
+        );
+        pending = fresh;
+        cache.set(origin, fresh);
+        // Concurrent navigations still share the one in-flight read; only a settled unreachable
+        // verdict is evicted, and only if no later read has replaced it.
+        void fresh.then((verdict) => {
+          if (verdict === 'unreachable' && cache.get(origin) === fresh) cache.delete(origin);
+        });
       }
-      if (!robotsAllows(await rules, `${parsed.pathname}${parsed.search}`)) {
+      const verdict = await pending;
+      if (
+        verdict === 'unreachable' ||
+        !robotsAllows(verdict, `${parsed.pathname}${parsed.search}`)
+      ) {
         throw robotsDisallowed(url, agent);
       }
     },

package/src/scrape-run.ts

@@ -12,15 +12,15 @@
 import { finiteCount, finiteOption } from '@ultimat3/core';
 import type { JobRunArgs } from '@ultimat3/jobs';
 import { parse } from '@ultimat3/schema';
-import { createArtifactWriter } from './artifacts';
+import { artifactWriter } from './artifacts';
 import type { AuthPlanInput } from './auth';
 import {
   burnSession,
-  createPrompt,
   ensureAuthenticated,
   markRefused,
+  observeSession,
   persistSession,
-  restorableSession,
+  scrapePrompt,
 } from './auth';
 import { scrapeClock } from './clock';
 import type { ScrapeSession } from './driver';
@@ -31,16 +31,16 @@ import type { ScrapeEventFields } from './events';
 import { scrapeLogger, withStepEvent } from './events';
 import { guardYield } from './expect';
 import { burnsSession, errorCode, neverRetried } from './failures';
-import { createPacer, DEFAULT_NAVIGATION_RATE } from './rate';
+import { DEFAULT_NAVIGATION_RATE, scrapePacer } from './rate';
 import { runRecovery } from './recover';
-import { createRobotsGate } from './robots';
+import { robotsGate } from './robots';
 import type { ScrapeDefinition, ScrapeReport } from './scrape';
 import { containsSecret } from './secret-scan';
-import { createSecretBag, MIN_REDACTABLE_LENGTH } from './secrets';
+import { MIN_REDACTABLE_LENGTH, secretBag } from './secrets';
 import { sessionKeyFor } from './session-state';
 import { hasCredentials, splitCredentials, urlSecretValues } from './url-secrets';
 import type { ScrapeUsage } from './usage';
-import { createUsageMeter, rememberFailedUsage } from './usage';
+import { rememberFailedUsage, usageMeter } from './usage';
 
 /** `ctx.actor` is a structural read: this package never imports the auth types (tier 2). */
 const orgOf = (ctx: unknown): string | undefined => {
@@ -99,7 +99,7 @@ export async function runScrape<I, Row>(
     attempt: args.attempt,
     driver: driver.name,
   });
-  const secrets = createSecretBag(definition.secrets ?? []);
+  const secrets = secretBag(definition.secrets ?? []);
   // The run's exit, resolved HERE — in the worker, under the job's tenant — so what the queue row
   // carries is the id of whatever holds it. An empty answer is "none", the same as absent: a
   // driver handed `''` would dial `--proxy-server=` and call that a decision.
@@ -120,11 +120,11 @@ export async function runScrape<I, Row>(
   // refuses the DECLARATION and never sees a definition assembled by hand, which `runScrape` is
   // exported to accept. `finiteOption` and not `finiteCount` — a rate of 0.5 is one navigation
   // every two seconds, and `scrape()` owns the "greater than zero" half.
-  const pace = createPacer(
+  const pace = scrapePacer(
     finiteOption('the scrape definition', 'rate', definition.rate ?? DEFAULT_NAVIGATION_RATE),
     clock,
   );
-  const artifact = createArtifactWriter({
+  const artifact = artifactWriter({
     storage: definition.artifacts?.storage,
     scrape: definition.name,
     runId: args.runId,
@@ -147,7 +147,8 @@ export async function runScrape<I, Row>(
 
   // Read BEFORE the browser opens: a refused credential must not reach a login form again, and
   // opening a session first would already have spent an identity on a run that cannot succeed.
-  const restored = await restorableSession(plan);
+  const observed = await observeSession(plan);
+  const restored = observed.restorable;
   const pageTimeoutMs = toMillis(definition.pageTimeout, DEFAULT_PAGE_TIMEOUT_MS, 'pageTimeout');
   // The exit the session dials, readable only AFTER `driver.open()` — a driver with no run exit
   // falls back to its own option, and the gate below is an argument to `open()`, so the gate asks
@@ -155,7 +156,7 @@ export async function runScrape<I, Row>(
   // during a navigation, which is after this is assigned.
   let sessionProxy: string | undefined;
   // Before `open()`: the time a rented browser takes to arrive is time it was held.
-  const usage = createUsageMeter(clock);
+  const usage = usageMeter(clock);
   const session = await driver.open({
     name: definition.name,
     logger,
@@ -172,8 +173,10 @@ export async function runScrape<I, Row>(
     // unreachable by `ctx.signal`; without the third the read leaves from a different IP than
     // every page load, and an origin reachable only through the proxy answers nothing — which
     // this gate reads as "no restrictions".
-    robots: createRobotsGate({
+    robots: robotsGate({
       policy: definition.robots ?? 'obey',
+      // Every hop of the read screened by the run's own host rule, as the HTTP leg's are.
+      allowHosts: rules.allowHosts,
       timeoutMs: pageTimeoutMs,
       signal: args.ctx.signal,
       proxy: () => sessionProxy,
@@ -184,6 +187,11 @@ export async function runScrape<I, Row>(
     watchdog: definition.watchdog,
   });
   sessionProxy = session.proxy;
+  // The VERSION of the stored record this run is on — the one it found (restorable or not), or the
+  // one it persisted after its own login. A burn or a tombstone on the way out compares against
+  // it, so a record another run on the key saved meanwhile is not destroyed by this run's failure
+  // (`auth.ts`'s `stillSeen`).
+  let seen = observed.seen;
 
   try {
     if (definition.auth !== undefined) {
@@ -197,7 +205,7 @@ export async function runScrape<I, Row>(
             page: session.page,
             secrets,
             restored,
-            prompt: createPrompt({
+            prompt: scrapePrompt({
               scrape: definition.name,
               handler: definition.prompt,
               input: args.input,
@@ -212,7 +220,15 @@ export async function runScrape<I, Row>(
       );
       // Persisted after a LOGIN only, never after a reuse: rewriting the record on every run
       // refreshes `savedAt` without refreshing the session, so `maxAge` would never expire it.
-      if (loggedIn) await persistSession(plan, session.page);
+      //
+      // And a failure to SAVE never fails the attempt: the login happened, the body can run, and
+      // failing here made the retry log in again — one more arrival at the login form, and on a
+      // store that stays down, one per attempt. Logged, like the burn and the tombstone below.
+      if (loggedIn) {
+        await recordSessionOutcome('session.save', logger, async () => {
+          seen = await persistSession(plan, session.page);
+        });
+      }
     }
 
     const raw = await bodyWithRecovery(definition, args, session, logger, artifact, secrets);
@@ -252,9 +268,9 @@ export async function runScrape<I, Row>(
     // Kept for `onSettled`, and only when somebody declared one to read it.
     if (definition.onSettled !== undefined) rememberFailedUsage(args.runId, used);
     if (errorCode(thrown) === 'X_SCRAPE_AUTH_FAILED') {
-      await recordSessionOutcome('session.refuse', logger, () => markRefused(plan));
+      await recordSessionOutcome('session.refuse', logger, () => markRefused(plan, seen));
     } else if (burnsSession(thrown)) {
-      await recordSessionOutcome('session.burn', logger, () => burnSession(plan));
+      await recordSessionOutcome('session.burn', logger, () => burnSession(plan, seen));
     }
     if (definition.artifacts?.onFailure !== false) await saveFailureArtifact(session, artifact);
     throw thrown;
@@ -274,8 +290,8 @@ const usageFields = (used: ScrapeUsage): ScrapeEventFields => ({
 });
 
 /**
- * The tombstone or the burn, on the way out — best effort, and it may NEVER replace the failure
- * that caused it. `markRefused` reaches `store.save()` reaches `storage.put()`, so an S3 503 or an
+ * The save after a login, and the tombstone or the burn on the way out — best effort, and none
+ * may ever replace the run's own outcome. `markRefused` reaches `store.save()` reaches `storage.put()`, so an S3 503 or an
  * `X_STORAGE_PATH_UNSAFE` from a tenant whose key sanitises to nothing used to propagate out of
  * the catch and REPLACE a terminal `X_SCRAPE_AUTH_FAILED` with a retryable one. Attempt 2 then
  * found no tombstone — the save is what failed — and walked the same rejected password back to
@@ -283,15 +299,16 @@ const usageFields = (used: ScrapeUsage): ScrapeEventFields => ({
  * reason: the run's own error is the one the reader needs.
  */
 async function recordSessionOutcome(
-  step: 'session.refuse' | 'session.burn',
+  step: 'session.save' | 'session.refuse' | 'session.burn',
   logger: ReturnType<typeof scrapeLogger>,
   write: () => Promise<void>,
 ): Promise<void> {
   try {
     await write();
   } catch (thrown) {
-    // Logged rather than swallowed silently: the tombstone is missing, so the NEXT attempt will
-    // re-probe rather than refuse cheaply, and that is a fact an operator has to be able to see.
+    // Logged rather than swallowed silently: a missing tombstone means the NEXT attempt re-probes
+    // rather than refusing cheaply, a missing save means it logs in again, and an operator has to
+    // be able to see either.
     logger.error('scrape.session.write_failed', { step, code: errorCode(thrown) });
   }
 }
@@ -306,8 +323,8 @@ async function bodyWithRecovery<I, Row>(
   args: JobRunArgs<I>,
   session: ScrapeSession,
   logger: ReturnType<typeof scrapeLogger>,
-  artifact: ReturnType<typeof createArtifactWriter>,
-  secrets: ReturnType<typeof createSecretBag>,
+  artifact: ReturnType<typeof artifactWriter>,
+  secrets: ReturnType<typeof secretBag>,
 ): Promise<readonly unknown[]> {
   const body = (): Promise<readonly unknown[]> =>
     definition.run({
@@ -348,7 +365,7 @@ async function bodyWithRecovery<I, Row>(
  */
 async function saveFailureArtifact(
   session: ScrapeSession,
-  artifact: ReturnType<typeof createArtifactWriter>,
+  artifact: ReturnType<typeof artifactWriter>,
 ): Promise<void> {
   try {
     await artifact.save('page.html', await session.page.html());

package/src/scrape.ts

@@ -9,7 +9,7 @@
 // the account. So this file is a FACTORY over `job()`, and a scrape inherits `.enqueue()`, the
 // worker's cancellation, the dead-letter path, `x jobs show` and its manifest row for free.
 
-import type { Ctx } from '@ultimat3/core';
+import type { Ctx, HostRule } from '@ultimat3/core';
 import { assert } from '@ultimat3/core';
 import type {
   JobCompleted,
@@ -30,7 +30,7 @@ import type { ScrapeClock } from './clock';
 import type { ScrapeDriver } from './driver';
 import { yieldHistoryMissing } from './error-throws';
 import type { YieldExpectation, YieldHistory } from './expect';
-import type { HostRule } from './hosts';
+import { maxDropFraction } from './expect';
 import type { ScrapeHttp } from './http';
 import type { ScrapePage } from './page';
 import type { Recovery } from './recover';
@@ -198,8 +198,9 @@ export function scrape<I, Row>(definition: ScrapeDefinition<I, Row>): JobHandle<
   // Two halves that must be set together. `maxDrop` is a fraction of a trailing median and only
   // `history:` can supply one, so declaring it alone is an alarm that cannot fire — refused here,
   // where it is written, rather than discovered as a scrape that never once went red.
-  if (definition.expect?.maxDrop !== undefined && definition.history === undefined) {
-    throw yieldHistoryMissing(definition.name);
+  if (definition.expect?.maxDrop !== undefined) {
+    if (definition.history === undefined) throw yieldHistoryMissing(definition.name);
+    maxDropFraction(definition.name, definition.expect.maxDrop);
   }
   const onSettled = definition.onSettled?.bind(definition);
   return job<I, ScrapeReport<Row>>({

package/src/secrets.ts

@@ -7,7 +7,7 @@
 // logger. The third is not, because pixels cannot be redacted after the fact.
 
 import type { Secret } from '@ultimat3/core';
-import { revealSecret, secret, UltimateError } from '@ultimat3/core';
+import { escapeHtml, revealSecret, secret, UltimateError } from '@ultimat3/core';
 import type { ConsoleLine, NetworkEntry, PageError } from './rings';
 
 export interface ScrapeSecrets {
@@ -33,7 +33,7 @@ export type SecretResolver = (name: string) => string | undefined;
 
 const fromEnvironment: SecretResolver = (name) => Bun.env[name];
 
-export function createSecretBag(
+export function secretBag(
   names: readonly string[],
   resolve: SecretResolver = fromEnvironment,
 ): ScrapeSecrets {
@@ -95,6 +95,56 @@ export const SECRET_PLACEHOLDER = '[redacted]';
  */
 export const MIN_REDACTABLE_LENGTH = 4;
 
+/**
+ * The SUBSET a serialiser escaped, each character spelt by core's one table — a page escapes fewer
+ * characters than `escapeHtml` does (a browser's text serialiser leaves quotes alone), so the
+ * spelling is the table applied to that serialiser's set, never a second table.
+ */
+const escapeOnly = (value: string, chars: RegExp): string => value.replace(chars, escapeHtml);
+
+/**
+ * Every spelling a page or a URL writes one value in. The raw value alone missed the case
+ * `safeNetwork` exists for: a password in a query string arrives percent-encoded, or form-encoded
+ * (`+` for a space, `'` and `(` escaped), and a server-rendered page writes it HTML-escaped — as
+ * text (`& < >`, what a browser's serialiser emits), as an attribute (`& "`),
+ * or through a template that escapes all five (`'` or `'` for the apostrophe).
+ * Each spelling is at least as long as the value, so the length floor is decided on the value.
+ */
+function spellingsOf(value: string): readonly string[] {
+  const text = escapeOnly(value, /[&<>]/g);
+  const attribute = escapeOnly(value, /[&"]/g);
+  const full = escapeOnly(value, /[&<>"]/g);
+  return [
+    value,
+    encodeURIComponent(value),
+    new URLSearchParams([['', value]]).toString().slice(1),
+    text,
+    attribute,
+    full.replaceAll("'", '&#39;'),
+    full.replaceAll("'", '&#x27;'),
+  ];
+}
+
+const REGEX_SPECIAL = /[.*+?^${}()|[\]\\/]/g;
+
+/**
+ * One spelling as a pattern: literal throughout, except that the hex digits of a `%XX` escape
+ * match in either case — `%2F` and `%2f` are the same byte, and clients emit both. The value's own
+ * letters stay case-sensitive: `Hunter2` is not `hunter2`.
+ */
+const spellingPattern = (spelling: string): RegExp =>
+  new RegExp(
+    spelling
+      .split(/(%[0-9A-Fa-f]{2})/)
+      .map((part, index) =>
+        index % 2 === 1
+          ? part.replace(/[A-Fa-f]/g, (digit) => `[${digit.toUpperCase()}${digit.toLowerCase()}]`)
+          : part.replace(REGEX_SPECIAL, '\\$&'),
+      )
+      .join(''),
+    'g',
+  );
+
 /**
  * Redaction BY VALUE, over the text this package hands back or persists. Name-based redaction only
  * catches a secret travelling under a name somebody remembered to list, and a password pasted into
@@ -111,20 +161,22 @@ export const MIN_REDACTABLE_LENGTH = 4;
  * `MIN_REDACTABLE_LENGTH`, and pixels — which is why a secret TAINTS the page and captures are
  * refused outright.
  *
- * Longest first, so a secret that contains another one does not leave its tail behind.
+ * Every spelling of every value (`spellingsOf`), longest first, so a secret that contains another
+ * one does not leave its tail behind.
  */
 export function redactSecrets(text: string, secrets: ScrapeSecrets | undefined): string {
   if (secrets === undefined) return text;
   const learned = concealed.get(secrets);
   if (secrets.names.length === 0 && (learned === undefined || learned.size === 0)) return text;
   const values = [
-    ...secrets.names.map((name) => revealSecret(secrets.get(name))),
-    ...(learned ?? []),
-  ]
-    .filter((value) => value.length >= MIN_REDACTABLE_LENGTH)
-    .sort((a, b) => b.length - a.length);
+    ...new Set(
+      [...secrets.names.map((name) => revealSecret(secrets.get(name))), ...(learned ?? [])]
+        .filter((value) => value.length >= MIN_REDACTABLE_LENGTH)
+        .flatMap(spellingsOf),
+    ),
+  ].sort((a, b) => b.length - a.length);
   let out = text;
-  for (const value of values) out = out.split(value).join(SECRET_PLACEHOLDER);
+  for (const value of values) out = out.replace(spellingPattern(value), SECRET_PLACEHOLDER);
   return out;
 }
 

package/src/session-state.ts

@@ -38,6 +38,13 @@ export interface SessionState extends SessionSnapshot {
   readonly key: string;
   /** ISO 8601. What a `maxAge` on reuse is measured against. */
   readonly savedAt: string;
+  /**
+   * Unique per save — what a burn or a tombstone compares to decide whether the record is still
+   * the one a run saw (`auth.ts`'s `stillSeen`). Never `savedAt`: two saves in one millisecond, or
+   * on one frozen clock, share it. Absent on a record written before it existed; `recordVersion`
+   * reads `savedAt` for that one.
+   */
+  readonly version?: string | undefined;
   /**
    * ISO 8601, set when this site REFUSED these credentials — and the reason the record survives
    * the failure instead of being deleted with it.
@@ -309,6 +316,7 @@ export function parseSessionState(raw: unknown, key: string): SessionState | und
   return {
     key,
     savedAt: value.savedAt,
+    ...(typeof value.version === 'string' ? { version: value.version } : {}),
     ...(typeof value.refusedAt === 'string' ? { refusedAt: value.refusedAt } : {}),
     cookies: value.cookies.flatMap((cookie: unknown) => toCookie(cookie) ?? []),
     headers: value.headers ?? {},
@@ -317,3 +325,6 @@ export function parseSessionState(raw: unknown, key: string): SessionState | und
     origin: value.origin ?? '',
   };
 }
+
+/** The identity a conditional burn or tombstone compares: the record's version, else its `savedAt`. */
+export const recordVersion = (state: SessionState): string => state.version ?? state.savedAt;