@ultimat3/scraping 23.0.0 → 24.0.0

package/src/robots-fetch.ts

@@ -9,7 +9,12 @@
 // `driver.open()`, and the proxy is a driver option the session only reports on the way back out.
 
 import { finiteCount, readWithinLimit } from '@ultimat3/core';
+import type { HostRule } from './hosts';
 import type { ScrapeFetch } from './http';
+import { redirectHop } from './http-redirect';
+import { interceptVerdict } from './intercept';
+import type { HostResolve } from './pinned-host';
+import { dialTarget, resolveHost } from './pinned-host';
 import type { RobotsFetch } from './robots';
 
 /**
@@ -23,6 +28,9 @@ export const DEFAULT_ROBOTS_TIMEOUT_MS = 10_000;
 /** Google's own documented ceiling for the file, and generous for a list of path prefixes. */
 export const DEFAULT_ROBOTS_MAX_BYTES = 500 * 1024;
 
+/** RFC 9309 §2.3.1.2: a crawler SHOULD follow at least five consecutive redirects for robots.txt. */
+export const MAX_ROBOTS_REDIRECTS = 5;
+
 export interface RobotsFetchInit {
   /** Per-read wall clock. Defaults to `DEFAULT_ROBOTS_TIMEOUT_MS`. */
   readonly timeoutMs?: number | undefined;
@@ -41,6 +49,15 @@ export interface RobotsFetchInit {
    */
   readonly proxy?: (() => string | undefined) | undefined;
   readonly maxBytes?: number | undefined;
+  /**
+   * The run's `allowHosts`, asked about EVERY hop — the first URL included — through
+   * `interceptVerdict`, the one host rule. A hop off the list is never requested and the read
+   * answers "no robots". With none given, only a redirect that stays on the starting HOSTNAME is
+   * followed (a scheme upgrade, a moved path), still capped; another host is never dialled.
+   */
+  readonly allowHosts?: readonly HostRule[] | undefined;
+  /** The resolver a wildcard-admitted name is checked and pinned with. Defaults to `Bun.dns`. */
+  readonly resolve?: HostResolve | undefined;
   /**
    * The platform `fetch`, injectable so the default path itself is testable. `ScrapeFetch` and not
    * `typeof fetch`: the latter also carries `preconnect`, so nothing a caller can write satisfies
@@ -49,13 +66,38 @@ export interface RobotsFetchInit {
   readonly fetch?: ScrapeFetch | undefined;
 }
 
+const onList = (url: string, allowHosts: readonly HostRule[]): boolean =>
+  interceptVerdict(url, 'fetch', { allowHosts }) === 'allow';
+
+/** Same hostname over http(s) — the port may change with the scheme. Unparseable is not same. */
+const sameHost = (from: string, to: string): boolean => {
+  try {
+    const next = new URL(to);
+    if (next.protocol !== 'http:' && next.protocol !== 'https:') return false;
+    return new URL(from).hostname === next.hostname;
+  } catch {
+    return false;
+  }
+};
+
+/** The final hop's body under the cap, or `undefined` for a non-2xx answer. */
+const answerOf = async (response: Response, limit: number): Promise<string | undefined> => {
+  if (!response.ok) return undefined;
+  // Counted as it arrives rather than `.text()`, which materialises the whole body first: a
+  // multi-gigabyte robots.txt is a heap the worker never gets back.
+  const capped = await readWithinLimit(response.body, limit);
+  return 'over' in capped ? undefined : new TextDecoder().decode(capped.bytes);
+};
+
 /**
  * Reads `robotsUrl`, or answers `undefined` — which the gate reads as "no restrictions", the
- * standard's own answer for a file it cannot obtain. A deadline that fires, a body past the cap
- * and a 404 are all the same answer on purpose: none of them is evidence of a rule.
+ * standard's own answer for a file it cannot obtain. A deadline that fires, a body past the cap,
+ * a 404 and a redirect off `allowHosts` are all the same answer on purpose: none of them is
+ * evidence of a rule.
  */
 export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
   const call: ScrapeFetch = init.fetch ?? fetch;
+  const resolve = init.resolve ?? resolveHost;
   // Both bounds are screened HERE, at construction, and both floors are 1 — because every way this
   // read can fail is the same answer, `undefined`, which the gate reads as "no restrictions". A
   // `NaN` deadline throws a bare `TypeError` out of `AbortSignal.timeout` (measured: "Value NaN is
@@ -83,16 +125,42 @@ export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
     // Resolved here, at the read, because the session that owns the exit did not exist when this
     // fetcher was built. An empty string is not an exit and is dropped with the absent one.
     const proxy = init.proxy?.();
+    const dialled = proxy === undefined || proxy === '' ? undefined : proxy;
     try {
-      const response = await call(robotsUrl, {
-        signal,
-        ...(proxy === undefined || proxy === '' ? {} : { proxy }),
-      });
-      if (!response.ok) return undefined;
-      // Counted as it arrives rather than `.text()`, which materialises the whole body first: a
-      // multi-gigabyte robots.txt is a heap the worker never gets back.
-      const capped = await readWithinLimit(response.body, limit);
-      return 'over' in capped ? undefined : new TextDecoder().decode(capped.bytes);
+      let url = robotsUrl;
+      for (let hops = 0; ; hops += 1) {
+        if (init.allowHosts !== undefined && !onList(url, init.allowHosts)) return undefined;
+        // Pinned exactly as the HTTP leg pins (`pinned-host.ts`), and for its reason skipped
+        // through a proxy. A name resolving inward throws, which this read answers as "no robots".
+        const dial =
+          init.allowHosts === undefined || dialled !== undefined
+            ? { url }
+            : await dialTarget(url, init.allowHosts, resolve);
+        // `manual`, never the platform default: `follow` dials wherever the site points before
+        // anything here can ask whether that host is on the list.
+        const response = await call(dial.url, {
+          signal,
+          redirect: 'manual',
+          ...(dialled === undefined ? {} : { proxy: dialled }),
+          ...(dial.host === undefined ? {} : { headers: { host: dial.host } }),
+          ...(dial.serverName === undefined ? {} : { tls: { serverName: dial.serverName } }),
+        });
+        const next = redirectHop(
+          response.status,
+          response.headers.get('location'),
+          url,
+          'GET',
+          undefined,
+        );
+        if (next === undefined) return await answerOf(response, limit);
+        await response.body?.cancel().catch(() => undefined);
+        if (hops >= MAX_ROBOTS_REDIRECTS) return undefined;
+        // With no list to ask, only a hop that stays on the host the read started on is
+        // followed — an `http:` → `https:` upgrade or a moved path. Another host is a host
+        // nobody screened, and is never dialled.
+        if (init.allowHosts === undefined && !sameHost(robotsUrl, next.url)) return undefined;
+        url = next.url;
+      }
     } catch {
       return undefined;
     }

package/src/scrape-run.ts

@@ -19,8 +19,8 @@ import {
   createPrompt,
   ensureAuthenticated,
   markRefused,
+  observeSession,
   persistSession,
-  restorableSession,
 } from './auth';
 import { scrapeClock } from './clock';
 import type { ScrapeSession } from './driver';
@@ -147,7 +147,8 @@ export async function runScrape<I, Row>(
 
   // Read BEFORE the browser opens: a refused credential must not reach a login form again, and
   // opening a session first would already have spent an identity on a run that cannot succeed.
-  const restored = await restorableSession(plan);
+  const observed = await observeSession(plan);
+  const restored = observed.restorable;
   const pageTimeoutMs = toMillis(definition.pageTimeout, DEFAULT_PAGE_TIMEOUT_MS, 'pageTimeout');
   // The exit the session dials, readable only AFTER `driver.open()` — a driver with no run exit
   // falls back to its own option, and the gate below is an argument to `open()`, so the gate asks
@@ -174,6 +175,8 @@ export async function runScrape<I, Row>(
     // this gate reads as "no restrictions".
     robots: createRobotsGate({
       policy: definition.robots ?? 'obey',
+      // Every hop of the read screened by the run's own host rule, as the HTTP leg's are.
+      allowHosts: rules.allowHosts,
       timeoutMs: pageTimeoutMs,
       signal: args.ctx.signal,
       proxy: () => sessionProxy,
@@ -184,6 +187,11 @@ export async function runScrape<I, Row>(
     watchdog: definition.watchdog,
   });
   sessionProxy = session.proxy;
+  // The VERSION of the stored record this run is on — the one it found (restorable or not), or the
+  // one it persisted after its own login. A burn or a tombstone on the way out compares against
+  // it, so a record another run on the key saved meanwhile is not destroyed by this run's failure
+  // (`auth.ts`'s `stillSeen`).
+  let seen = observed.seen;
 
   try {
     if (definition.auth !== undefined) {
@@ -212,7 +220,15 @@ export async function runScrape<I, Row>(
       );
       // Persisted after a LOGIN only, never after a reuse: rewriting the record on every run
       // refreshes `savedAt` without refreshing the session, so `maxAge` would never expire it.
-      if (loggedIn) await persistSession(plan, session.page);
+      //
+      // And a failure to SAVE never fails the attempt: the login happened, the body can run, and
+      // failing here made the retry log in again — one more arrival at the login form, and on a
+      // store that stays down, one per attempt. Logged, like the burn and the tombstone below.
+      if (loggedIn) {
+        await recordSessionOutcome('session.save', logger, async () => {
+          seen = await persistSession(plan, session.page);
+        });
+      }
     }
 
     const raw = await bodyWithRecovery(definition, args, session, logger, artifact, secrets);
@@ -252,9 +268,9 @@ export async function runScrape<I, Row>(
     // Kept for `onSettled`, and only when somebody declared one to read it.
     if (definition.onSettled !== undefined) rememberFailedUsage(args.runId, used);
     if (errorCode(thrown) === 'X_SCRAPE_AUTH_FAILED') {
-      await recordSessionOutcome('session.refuse', logger, () => markRefused(plan));
+      await recordSessionOutcome('session.refuse', logger, () => markRefused(plan, seen));
     } else if (burnsSession(thrown)) {
-      await recordSessionOutcome('session.burn', logger, () => burnSession(plan));
+      await recordSessionOutcome('session.burn', logger, () => burnSession(plan, seen));
     }
     if (definition.artifacts?.onFailure !== false) await saveFailureArtifact(session, artifact);
     throw thrown;
@@ -274,8 +290,8 @@ const usageFields = (used: ScrapeUsage): ScrapeEventFields => ({
 });
 
 /**
- * The tombstone or the burn, on the way out — best effort, and it may NEVER replace the failure
- * that caused it. `markRefused` reaches `store.save()` reaches `storage.put()`, so an S3 503 or an
+ * The save after a login, and the tombstone or the burn on the way out — best effort, and none
+ * may ever replace the run's own outcome. `markRefused` reaches `store.save()` reaches `storage.put()`, so an S3 503 or an
  * `X_STORAGE_PATH_UNSAFE` from a tenant whose key sanitises to nothing used to propagate out of
  * the catch and REPLACE a terminal `X_SCRAPE_AUTH_FAILED` with a retryable one. Attempt 2 then
  * found no tombstone — the save is what failed — and walked the same rejected password back to
@@ -283,15 +299,16 @@ const usageFields = (used: ScrapeUsage): ScrapeEventFields => ({
  * reason: the run's own error is the one the reader needs.
  */
 async function recordSessionOutcome(
-  step: 'session.refuse' | 'session.burn',
+  step: 'session.save' | 'session.refuse' | 'session.burn',
   logger: ReturnType<typeof scrapeLogger>,
   write: () => Promise<void>,
 ): Promise<void> {
   try {
     await write();
   } catch (thrown) {
-    // Logged rather than swallowed silently: the tombstone is missing, so the NEXT attempt will
-    // re-probe rather than refuse cheaply, and that is a fact an operator has to be able to see.
+    // Logged rather than swallowed silently: a missing tombstone means the NEXT attempt re-probes
+    // rather than refusing cheaply, a missing save means it logs in again, and an operator has to
+    // be able to see either.
     logger.error('scrape.session.write_failed', { step, code: errorCode(thrown) });
   }
 }

package/src/scrape.ts

@@ -30,6 +30,7 @@ import type { ScrapeClock } from './clock';
 import type { ScrapeDriver } from './driver';
 import { yieldHistoryMissing } from './error-throws';
 import type { YieldExpectation, YieldHistory } from './expect';
+import { maxDropFraction } from './expect';
 import type { HostRule } from './hosts';
 import type { ScrapeHttp } from './http';
 import type { ScrapePage } from './page';
@@ -198,8 +199,9 @@ export function scrape<I, Row>(definition: ScrapeDefinition<I, Row>): JobHandle<
   // Two halves that must be set together. `maxDrop` is a fraction of a trailing median and only
   // `history:` can supply one, so declaring it alone is an alarm that cannot fire — refused here,
   // where it is written, rather than discovered as a scrape that never once went red.
-  if (definition.expect?.maxDrop !== undefined && definition.history === undefined) {
-    throw yieldHistoryMissing(definition.name);
+  if (definition.expect?.maxDrop !== undefined) {
+    if (definition.history === undefined) throw yieldHistoryMissing(definition.name);
+    maxDropFraction(definition.name, definition.expect.maxDrop);
   }
   const onSettled = definition.onSettled?.bind(definition);
   return job<I, ScrapeReport<Row>>({

package/src/secrets.ts

@@ -7,7 +7,7 @@
 // logger. The third is not, because pixels cannot be redacted after the fact.
 
 import type { Secret } from '@ultimat3/core';
-import { revealSecret, secret, UltimateError } from '@ultimat3/core';
+import { escapeHtml, revealSecret, secret, UltimateError } from '@ultimat3/core';
 import type { ConsoleLine, NetworkEntry, PageError } from './rings';
 
 export interface ScrapeSecrets {
@@ -95,6 +95,56 @@ export const SECRET_PLACEHOLDER = '[redacted]';
  */
 export const MIN_REDACTABLE_LENGTH = 4;
 
+/**
+ * The SUBSET a serialiser escaped, each character spelt by core's one table — a page escapes fewer
+ * characters than `escapeHtml` does (a browser's text serialiser leaves quotes alone), so the
+ * spelling is the table applied to that serialiser's set, never a second table.
+ */
+const escapeOnly = (value: string, chars: RegExp): string => value.replace(chars, escapeHtml);
+
+/**
+ * Every spelling a page or a URL writes one value in. The raw value alone missed the case
+ * `safeNetwork` exists for: a password in a query string arrives percent-encoded, or form-encoded
+ * (`+` for a space, `'` and `(` escaped), and a server-rendered page writes it HTML-escaped — as
+ * text (`& < >`, what a browser's serialiser emits), as an attribute (`& "`),
+ * or through a template that escapes all five (`'` or `'` for the apostrophe).
+ * Each spelling is at least as long as the value, so the length floor is decided on the value.
+ */
+function spellingsOf(value: string): readonly string[] {
+  const text = escapeOnly(value, /[&<>]/g);
+  const attribute = escapeOnly(value, /[&"]/g);
+  const full = escapeOnly(value, /[&<>"]/g);
+  return [
+    value,
+    encodeURIComponent(value),
+    new URLSearchParams([['', value]]).toString().slice(1),
+    text,
+    attribute,
+    full.replaceAll("'", '&#39;'),
+    full.replaceAll("'", '&#x27;'),
+  ];
+}
+
+const REGEX_SPECIAL = /[.*+?^${}()|[\]\\/]/g;
+
+/**
+ * One spelling as a pattern: literal throughout, except that the hex digits of a `%XX` escape
+ * match in either case — `%2F` and `%2f` are the same byte, and clients emit both. The value's own
+ * letters stay case-sensitive: `Hunter2` is not `hunter2`.
+ */
+const spellingPattern = (spelling: string): RegExp =>
+  new RegExp(
+    spelling
+      .split(/(%[0-9A-Fa-f]{2})/)
+      .map((part, index) =>
+        index % 2 === 1
+          ? part.replace(/[A-Fa-f]/g, (digit) => `[${digit.toUpperCase()}${digit.toLowerCase()}]`)
+          : part.replace(REGEX_SPECIAL, '\\$&'),
+      )
+      .join(''),
+    'g',
+  );
+
 /**
  * Redaction BY VALUE, over the text this package hands back or persists. Name-based redaction only
  * catches a secret travelling under a name somebody remembered to list, and a password pasted into
@@ -111,20 +161,22 @@ export const MIN_REDACTABLE_LENGTH = 4;
  * `MIN_REDACTABLE_LENGTH`, and pixels — which is why a secret TAINTS the page and captures are
  * refused outright.
  *
- * Longest first, so a secret that contains another one does not leave its tail behind.
+ * Every spelling of every value (`spellingsOf`), longest first, so a secret that contains another
+ * one does not leave its tail behind.
  */
 export function redactSecrets(text: string, secrets: ScrapeSecrets | undefined): string {
   if (secrets === undefined) return text;
   const learned = concealed.get(secrets);
   if (secrets.names.length === 0 && (learned === undefined || learned.size === 0)) return text;
   const values = [
-    ...secrets.names.map((name) => revealSecret(secrets.get(name))),
-    ...(learned ?? []),
-  ]
-    .filter((value) => value.length >= MIN_REDACTABLE_LENGTH)
-    .sort((a, b) => b.length - a.length);
+    ...new Set(
+      [...secrets.names.map((name) => revealSecret(secrets.get(name))), ...(learned ?? [])]
+        .filter((value) => value.length >= MIN_REDACTABLE_LENGTH)
+        .flatMap(spellingsOf),
+    ),
+  ].sort((a, b) => b.length - a.length);
   let out = text;
-  for (const value of values) out = out.split(value).join(SECRET_PLACEHOLDER);
+  for (const value of values) out = out.replace(spellingPattern(value), SECRET_PLACEHOLDER);
   return out;
 }
 

package/src/session-state.ts

@@ -38,6 +38,13 @@ export interface SessionState extends SessionSnapshot {
   readonly key: string;
   /** ISO 8601. What a `maxAge` on reuse is measured against. */
   readonly savedAt: string;
+  /**
+   * Unique per save — what a burn or a tombstone compares to decide whether the record is still
+   * the one a run saw (`auth.ts`'s `stillSeen`). Never `savedAt`: two saves in one millisecond, or
+   * on one frozen clock, share it. Absent on a record written before it existed; `recordVersion`
+   * reads `savedAt` for that one.
+   */
+  readonly version?: string | undefined;
   /**
    * ISO 8601, set when this site REFUSED these credentials — and the reason the record survives
    * the failure instead of being deleted with it.
@@ -309,6 +316,7 @@ export function parseSessionState(raw: unknown, key: string): SessionState | und
   return {
     key,
     savedAt: value.savedAt,
+    ...(typeof value.version === 'string' ? { version: value.version } : {}),
     ...(typeof value.refusedAt === 'string' ? { refusedAt: value.refusedAt } : {}),
     cookies: value.cookies.flatMap((cookie: unknown) => toCookie(cookie) ?? []),
     headers: value.headers ?? {},
@@ -317,3 +325,6 @@ export function parseSessionState(raw: unknown, key: string): SessionState | und
     origin: value.origin ?? '',
   };
 }
+
+/** The identity a conditional burn or tombstone compares: the record's version, else its `savedAt`. */
+export const recordVersion = (state: SessionState): string => state.version ?? state.savedAt;

package/src/url-secrets.ts

@@ -51,13 +51,61 @@ export function splitCredentials(url: string): {
   return { bare: parsed.toString().replace(/\/$/, ''), username, password };
 }
 
+/**
+ * The words that name a credential, matched against a query key's COMPONENTS — split on `_`, `-`,
+ * `.` and camelCase — never as substrings: `keyboard` holds `key` and `monkey` holds it too, and
+ * concealing their values blanked ordinary words out of every artifact. Read off the key and not
+ * the value, because a provider's token can be short and the key is the URL's own statement.
+ */
+const CREDENTIAL_WORDS = new Set([
+  'token',
+  'key',
+  'apikey',
+  'secret',
+  'auth',
+  'authorization',
+  'sig',
+  'signature',
+  'pass',
+  'password',
+  'passwd',
+  'pwd',
+  'cred',
+  'credential',
+  'credentials',
+  'session',
+  'sessionid',
+  'sid',
+  'jwt',
+  'bearer',
+]);
+
+const credentialKey = (key: string): boolean =>
+  key
+    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+    .toLowerCase()
+    .split(/[\s_.-]+/)
+    .some((part) => CREDENTIAL_WORDS.has(part));
+
+/**
+ * A value that is a capability whatever it is called: long, one unbroken run, letters and digits
+ * mixed — a browser id, an access token. A word (`residential`, `true`, `eu`) is none of those.
+ */
+const credentialShaped = (value: string): boolean =>
+  value.length >= 16 && !/\s/.test(value) && /[0-9]/.test(value) && /[a-z]/i.test(value);
+
 /**
  * Every value in the URL that is, or may be, a credential — for the run's redaction set.
  *
- * The whole URL first, then the parts that can be quoted on their own: a launcher's error says
- * `401 for /devtools/browser/<id>?token=…` as readily as it repeats the URL. The path counts
- * because a browser id in a CDP path IS the capability. Encoded and decoded forms both, since a
- * message may carry either. The host is left out on purpose: it is what an error is allowed to say.
+ * The whole URL first, then the parts a message quotes on their own: a launcher's error says
+ * `401 for /devtools/browser/<id>?token=…` as readily as it repeats the URL. Encoded and decoded
+ * forms both, since a message may carry either. The host is left out on purpose: it is what an
+ * error is allowed to say.
+ *
+ * A query VALUE or a path SEGMENT only when it is credential-shaped or sits under a
+ * credential-named key — never every value. Concealing `stealth=true&proxy=residential` blanked
+ * every `true` and `residential` in the page, the console and the network ring of the run: the
+ * unreadable artifact `MIN_REDACTABLE_LENGTH` exists to prevent.
  */
 export function urlSecretValues(url: string): readonly string[] {
   const values = new Set<string>([url]);
@@ -67,9 +115,22 @@ export function urlSecretValues(url: string): readonly string[] {
     // `admin` by value would blank the word out of every page artifact.
     values.add(parsed.password);
     values.add(decoded(parsed.password));
-    if (parsed.pathname !== '/') values.add(`${parsed.pathname}${parsed.search}`);
-    if (parsed.search !== '') values.add(parsed.search.slice(1));
-    for (const value of parsed.searchParams.values()) values.add(value);
+    if (parsed.search !== '') {
+      values.add(`${parsed.pathname}${parsed.search}`);
+      values.add(parsed.search.slice(1));
+    }
+    const segments = parsed.pathname.split('/');
+    for (const [index, segment] of segments.entries()) {
+      // By POSITION as well as by shape: Chrome's `/devtools/<browser|page>/<id>` id IS the
+      // capability, and an id of letters alone is not credential-shaped.
+      const capability = index >= 2 && segments[index - 2] === 'devtools';
+      if (capability || credentialShaped(decoded(segment))) {
+        values.add(decoded(segment)).add(segment);
+      }
+    }
+    for (const [key, value] of parsed.searchParams) {
+      if (credentialKey(key) || credentialShaped(value)) values.add(value);
+    }
   } catch {
     // Not a URL: the whole string above is all there is to redact.
   }