@ultimat3/scraping 19.2.0 → 19.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/scraping",
-  "version": "19.2.0",
+  "version": "19.3.2",
   "description": "Browser automation as a job: scrape() returns a JobHandle",
   "license": "MIT",
   "type": "module",
@@ -30,9 +30,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "19.2.0",
-    "@ultimat3/jobs": "19.2.0",
-    "@ultimat3/schema": "19.2.0",
-    "@ultimat3/storage": "19.2.0"
+    "@ultimat3/core": "19.3.2",
+    "@ultimat3/jobs": "19.3.2",
+    "@ultimat3/schema": "19.3.2",
+    "@ultimat3/storage": "19.3.2"
   }
 }

package/src/error-throws.ts

@@ -3,7 +3,13 @@
 // a browser is rendered through core's `renderThrowable`, which is the rule `bun run error-render`
 // enforces.
 
-import { isRetryableStatus, renderThrowable, UltimateError } from '@ultimat3/core';
+import {
+  isFixShellSafe,
+  isRetryableStatus,
+  renderFixShellArg,
+  renderThrowable,
+  UltimateError,
+} from '@ultimat3/core';
 import type { CaptureClip } from './capture-clip';
 import { ScrapeError } from './errors';
 
@@ -99,7 +105,18 @@ export const profileLocked = (profileDir: string): ScrapeError =>
   new ScrapeError({
     code: 'X_SCRAPE_PROFILE_LOCKED',
     cause: `another browser process holds the profile at ${profileDir}`,
-    fix: `rm -f ${profileDir}/SingletonLock once no browser is using it, or give this run its own localBrowser({ profileDir })`,
+    // `profileDir` is the app's — `localBrowser({ profileDir })`, and a multi-tenant run composes
+    // it from a tenant id — and this is the one line in this package that leads with `rm`.
+    //
+    // A COMMAND or PROSE, never a mixture of the two, and both halves were wrong. The trailing
+    // sentence was bare text after an `rm` operand, so pasting the line deleted `once`, `no`,
+    // `browser` and four more names out of the current directory; and for a path a shell would
+    // read, the placeholder that replaced it — `<the profile directory …>` — is redirection
+    // syntax rather than a path, so the "command" could not run at all. So the safe branch puts
+    // the explanation behind a `#` and the unsafe one drops the command entirely.
+    fix: isFixShellSafe(profileDir)
+      ? `rm -f ${renderFixShellArg(profileDir, '')}/SingletonLock   # only once no browser is using it, or give this run its own localBrowser({ profileDir })`
+      : 'delete the SingletonLock file inside the profile directory the cause names, once no browser is using it — or give this run its own localBrowser({ profileDir })',
     meta: { profileDir },
   });
 
@@ -320,6 +337,21 @@ export const bodyTooLarge = (url: string, readBytes: number, maxBytes: number):
     meta: { url, readBytes, maxBytes },
   });
 
+/**
+ * The chain never settled. Its own code rather than `X_SCRAPE_HTTP_FAILED`, because the two send
+ * their reader to different places: that one means the site answered and said no, this one means
+ * the site kept pointing somewhere else and the leg stopped asking. `hops` is quoted so the cause
+ * is falsifiable against the ring — `page.network()` holds one entry per hop, each with the URL
+ * that was really requested rather than the one the caller wrote.
+ */
+export const redirectLoop = (url: string, lastUrl: string, hops: number): ScrapeError =>
+  new ScrapeError({
+    code: 'X_SCRAPE_REDIRECT_LOOP',
+    cause: `${url} redirected ${String(hops)} times without answering; the last hop pointed at ${lastUrl}`,
+    fix: 'request the URL the chain settles on — page.network() lists every hop this leg took — or repair the session: a chain that never ends is a login redirecting to a page that redirects back to the login',
+    meta: { url, lastUrl, hops },
+  });
+
 /**
  * TERMINAL, and the retry table cannot be talked out of it. A site that locks an account after
  * three wrong attempts turns a retrying framework into the thing that destroys the user's

package/src/errors.ts

@@ -33,6 +33,7 @@ export const SCRAPE_OWNED_ERROR_CODES = [
   'X_SCRAPE_CAPTURE_INVALID',
   'X_SCRAPE_HTTP_FAILED',
   'X_SCRAPE_BODY_TOO_LARGE',
+  'X_SCRAPE_REDIRECT_LOOP',
   'X_SCRAPE_AUTH_FAILED',
   'X_SCRAPE_SESSION_EXPIRED',
   'X_SCRAPE_PROMPT_UNANSWERED',
@@ -83,6 +84,7 @@ export const SCRAPE_ERROR_TITLES: Readonly<Record<ScrapeOwnedErrorCode, string>>
   X_SCRAPE_CAPTURE_INVALID: 'the capture names a framing no picture can be taken with',
   X_SCRAPE_HTTP_FAILED: 'the site answered the HTTP leg with a non-2xx status',
   X_SCRAPE_BODY_TOO_LARGE: 'the HTTP response body passed its byte cap',
+  X_SCRAPE_REDIRECT_LOOP: 'the HTTP leg followed its hop limit of redirects without an answer',
   X_SCRAPE_AUTH_FAILED: 'the credentials were rejected',
   X_SCRAPE_SESSION_EXPIRED: 'the restored session is no longer valid and nothing can renew it',
   X_SCRAPE_PROMPT_UNANSWERED: 'a login step asked for a code and nothing answered',
@@ -150,6 +152,11 @@ export const SCRAPE_ERROR_RETRY = {
   // A response size is a property of the endpoint, not of the moment: attempt 2 buffers the same
   // gigabyte and dies the same way. The fix is a number on the request, so a human decides it.
   X_SCRAPE_BODY_TOO_LARGE: 'terminal',
+  // A chain that does not settle is a property of the endpoint and of the session that reaches it,
+  // not of the moment: attempt 2 restores the same cookies and walks the same ten hops. The
+  // reachable case is a login redirecting to a page that redirects back to the login, and the
+  // repair is a credential or a URL — a human's edit, not a queue's second try.
+  X_SCRAPE_REDIRECT_LOOP: 'terminal',
   X_SCRAPE_YIELD_COLLAPSED: 'terminal',
   // A declaration error, raised by `scrape()` before any attempt exists — there is no run to
   // retry, and the same definition would refuse identically forever.

package/src/hosts.ts

@@ -12,7 +12,20 @@ export const ANY_HOST: HostRule = '*';
  * Schemes with no host to match. `about:blank` is where every browser starts, `data:` and `blob:`
  * never leave the process — refusing them would refuse the first page load of every run.
  */
-const HOSTLESS_SCHEMES = new Set(['about:', 'data:', 'blob:', 'javascript:']);
+const HOSTLESS_SCHEMES = new Set(['about:', 'data:', 'blob:']);
+
+/**
+ * Hostless AND refused, which is why it is not on the line above — it sat there until 2026-09.
+ * `javascript:` has no host for the same reason `data:` has none, and that is the whole
+ * resemblance: the other three are inert content this process renders, while this one is code
+ * EXECUTED in the current document's origin, with the session's cookies and the session's
+ * `localStorage` already in scope. `allowHosts` cannot say anything about a URL with no host to
+ * name, so "no host, therefore allowed" was the allow list opting itself out of the one navigation
+ * that needs no host to exfiltrate through — `javascript:fetch('/admin').then(post_elsewhere)` is
+ * a same-origin read on an allow-listed site. Fail closed; there is no legitimate scrape verb that
+ * needs it (`page.eval` is the declared seam).
+ */
+const REFUSED_SCHEMES = new Set(['javascript:']);
 
 export interface HostDecision {
   readonly allowed: boolean;
@@ -44,6 +57,7 @@ export function hostMatches(host: string, rule: HostRule): boolean {
  */
 export function hostDecision(url: string, allowHosts: readonly HostRule[]): HostDecision {
   const scheme = url.slice(0, Math.max(0, url.indexOf(':') + 1)).toLowerCase();
+  if (REFUSED_SCHEMES.has(scheme)) return { allowed: false, host: '' };
   if (HOSTLESS_SCHEMES.has(scheme)) return { allowed: true, host: '' };
   let host: string;
   try {

package/src/http-redirect.ts

@@ -0,0 +1,78 @@
+// The redirect chain as a decision this leg makes hop by hop, rather than one `fetch` follows on
+// its own. A followed redirect is a request to a URL nobody screened: `allowHosts`, robots and the
+// rate limit are all asked about the url the caller wrote, and the answer comes back from wherever
+// the site pointed. This file is the arithmetic — which status is a hop, where it goes, and what
+// method carries — so `http.ts` re-applies the same three gates per hop and nothing else changes.
+
+/**
+ * The five statuses that carry a `Location` a client may follow. `300` is deliberately absent:
+ * multiple choices has no single target, and picking one for the caller is guessing.
+ */
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+
+/**
+ * The hop ceiling, and a NAMED one so the refusal can quote it. Ten is the number every major
+ * client settled on (curl's `--max-redirs`, the fetch spec's own counter) — high enough that no
+ * honest CDN chain reaches it, low enough that a login bouncing to a page that bounces back costs
+ * ten requests instead of a worker.
+ */
+export const MAX_REDIRECT_HOPS = 10;
+
+/** What the next hop is asked with: a redirect can change the method and drop the body. */
+export interface RedirectHop {
+  readonly url: string;
+  readonly method: string;
+  readonly body: string | undefined;
+}
+
+/**
+ * `303` means "look over there with a GET", and `301`/`302` after a POST mean the same thing in
+ * every browser and every HTTP client shipped since — the spec permits the rewrite and universal
+ * practice performs it. `307`/`308` exist precisely to say "re-send exactly what you sent", so
+ * they keep both. Getting this wrong is not cosmetic: re-POSTing an order body at the URL a `303`
+ * points to is a second order.
+ *
+ * The rewrite is per METHOD as well as per status, which "anything that is not a GET or a HEAD
+ * loses its body" got wrong in the other direction: the fetch standard rewrites `301`/`302` for
+ * POST alone, so a `PUT` or a `DELETE` re-asked as a bodyless GET is the caller's write silently
+ * not happening — one read, a 200, and nothing changed at the target.
+ */
+const carriesBody = (status: number, method: string): boolean => {
+  if (status === 307 || status === 308) return true;
+  // Case-folded because `fetch` normalises a standard method on the way out — `post` leaves as
+  // `POST` — so a decision keyed on the caller's spelling would rewrite one and keep the other.
+  const verb = method.toUpperCase();
+  // `303` is the status that rewrites every method BUT those two, and it spares `HEAD` on purpose:
+  // a HEAD promoted to a GET fetches the body the caller said it did not want.
+  if (status === 303) return verb === 'GET' || verb === 'HEAD';
+  return verb !== 'POST';
+};
+
+/**
+ * The next hop, or `undefined` when this response IS the answer. A `Location` that will not parse
+ * against the hop it came from is `undefined` too, and that is the fail-closed side: a URL this
+ * package cannot resolve is a URL it cannot screen, so it is never requested — the 3xx becomes the
+ * returned response and `X_SCRAPE_HTTP_FAILED` reports it with its own status.
+ */
+export function redirectHop(
+  status: number,
+  location: string | null,
+  from: string,
+  method: string,
+  body: string | undefined,
+): RedirectHop | undefined {
+  if (!REDIRECT_STATUSES.has(status)) return undefined;
+  if (location === null || location.trim() === '') return undefined;
+  let resolved: string;
+  try {
+    resolved = new URL(location, from).toString();
+  } catch {
+    return undefined;
+  }
+  const keep = carriesBody(status, method);
+  return {
+    url: resolved,
+    method: keep ? method : 'GET',
+    body: keep ? body : undefined,
+  };
+}

package/src/http.ts

@@ -15,7 +15,9 @@ import type { StandardSchemaV1 } from '@ultimat3/schema';
 import { parse } from '@ultimat3/schema';
 import type { ScrapeClock } from './clock';
 import { cookieHeaderFor } from './cookie-scope';
-import { bodyTooLarge, hostBlocked, httpFailed, scrapeTimeout } from './error-throws';
+import { bodyTooLarge, hostBlocked, httpFailed, redirectLoop, scrapeTimeout } from './error-throws';
+import type { RedirectHop } from './http-redirect';
+import { MAX_REDIRECT_HOPS, redirectHop } from './http-redirect';
 import type { InterceptRules } from './intercept';
 import { interceptVerdict } from './intercept';
 import type { NetworkRing } from './rings';
@@ -163,13 +165,93 @@ export function responseOver(
   };
 }
 
+/**
+ * A hop's body is thrown away — nothing reads a redirect's — and an unread stream holds its socket
+ * open until the collector gets to it. `cancel()` on a body that already errored rejects, and that
+ * rejection is not this request's failure: the hop is over and the next one is what the caller is
+ * waiting for.
+ */
+const discardHopBody = async (response: Response): Promise<void> => {
+  try {
+    await response.body?.cancel();
+  } catch {
+    // Discarded bytes cannot fail a request.
+  }
+};
+
+/**
+ * The credentials a CALLER set, which are scoped to the origin they were set for — the platform's
+ * own `follow` deleted these on a cross-origin hop (step 13 of the fetch standard's HTTP-redirect
+ * fetch) and this file took the chain over, so this file owns the strip. `cookie` is here for the
+ * hand-written header only: the jar's own value is computed per hop by `cookieHeaderFor`, which
+ * has always been scoped to the host being dialled.
+ */
+const CROSS_ORIGIN_STRIPPED = new Set(['authorization', 'proxy-authorization', 'cookie']);
+
+const withoutCredentials = (headers: Readonly<Record<string, string>>): Record<string, string> =>
+  Object.fromEntries(
+    Object.entries(headers).filter(([name]) => !CROSS_ORIGIN_STRIPPED.has(name.toLowerCase())),
+  );
+
+/** Unparseable is not same-origin: a URL this package cannot read is one it cannot vouch for. */
+const sameOrigin = (left: string, right: string): boolean => {
+  try {
+    return new URL(left).origin === new URL(right).origin;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * The final answer, read under the cap. Counted as it arrives rather than `.text()`, which
+ * materialises first and checks never: a 30s stream at 50MB/s is a 1.5GB allocation the worker
+ * does not get back, and it takes every other job on that worker with it. The same read
+ * `robots-fetch.ts` performs.
+ */
+const readResponse = async (
+  url: string,
+  response: Response,
+  maxBytes: number,
+  secrets: ScrapeSecrets | undefined,
+): Promise<ScrapeResponse> => {
+  const capped = await readWithinLimit(response.body, maxBytes);
+  if ('over' in capped) throw bodyTooLarge(url, capped.over, maxBytes);
+  const body = new TextDecoder().decode(capped.bytes);
+  return responseOver(
+    url,
+    response.status,
+    headerRecord(response.headers),
+    () => Promise.resolve(body),
+    secrets,
+  );
+};
+
 /**
  * The real transport. Every guarantee the page makes is re-applied here, in the same order and
  * through the same functions — `interceptVerdict` is the one host rule, `RobotsGate` is the one
  * robots rule, and neither is re-implemented for the second leg.
+ *
+ * REDIRECTS ARE FOLLOWED BY THIS FILE, hop by hop, and not by `fetch`. Until 2026-09 the call
+ * carried the platform default `redirect: 'follow'`, so an allow-listed endpoint answering
+ * `302 -> http://169.254.169.254/…` read the metadata service THROUGH the allow list: the three
+ * gates above had all been asked about the URL the caller wrote, `res.url` and the network ring
+ * both reported that URL, and nothing ever asked robots about where the body came from. The CDP
+ * leg never had the hole — interception fires per hop there — and this file's header claims parity
+ * with it.
  */
 export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
   const call: ScrapeFetch = init.fetch ?? fetch;
+  /**
+   * The three gates, in one place, so the initial URL and hop seven are screened by the same code
+   * in the same order. A second copy for redirects is how the two drift.
+   */
+  const screen = async (target: string): Promise<void> => {
+    if (interceptVerdict(target, 'fetch', init.rules) !== 'allow') {
+      throw hostBlocked(target, init.rules.allowHosts);
+    }
+    await init.robots?.assertAllowed(target);
+    await init.pace?.(init.signal);
+  };
   return {
     async request(url: string, request: HttpRequestInit = {}): Promise<ScrapeResponse> {
       // Screened FIRST — before the activity touch, before the robots read this method performs
@@ -196,52 +278,84 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
         1,
       );
       init.onActivity?.();
-      if (interceptVerdict(url, 'fetch', init.rules) !== 'allow') {
-        throw hostBlocked(url, init.rules.allowHosts);
-      }
-      await init.robots?.assertAllowed(url);
-      await init.pace?.(init.signal);
+      await screen(url);
       const session = await init.session();
-      const cookies = cookieHeaderFor(session.cookies, url);
       // `AbortSignal.timeout` and NOT `clock.sleep`: this is a deadline handed to the platform's
       // own fetch, not a wait this package performs — and under a test clock a slept deadline
       // would fire on the microtask after it was armed, cancelling every request instantly.
       // The offline transport (`http-recorded.ts`) is what a test runs, and it has no deadline.
+      //
+      // ONE deadline for the whole chain, not one per hop: the caller declared a budget for
+      // getting an answer, and ten hops each allowed the full budget is ten times the wait.
       const deadlineSignal = AbortSignal.timeout(timeoutMs);
       const signals = init.signal === undefined ? [deadlineSignal] : [deadlineSignal, init.signal];
       try {
-        const response = await call(url, {
-          method: request.method ?? 'GET',
-          headers: {
-            ...session.headers,
-            ...(session.userAgent === '' ? {} : { 'user-agent': session.userAgent }),
-            ...(cookies === undefined ? {} : { cookie: cookies }),
-            ...request.headers,
-          },
-          ...(request.body === undefined ? {} : { body: request.body }),
-          signal: AbortSignal.any(signals),
-          ...(init.proxy === undefined ? {} : { proxy: init.proxy }),
-        });
-        init.network.push({
-          method: request.method ?? 'GET',
-          url,
-          status: response.status,
-          resourceType: 'fetch',
-          at: init.clock.now().getTime(),
-        });
-        // Counted as it arrives rather than `.text()`, which materialises first and checks never:
-        // a 30s stream at 50MB/s is a 1.5GB allocation the worker does not get back, and it takes
-        // every other job on that worker with it. The same read `robots-fetch.ts` performs.
-        const capped = await readWithinLimit(response.body, maxBytes);
-        if ('over' in capped) throw bodyTooLarge(url, capped.over, maxBytes);
-        const body = new TextDecoder().decode(capped.bytes);
-        return responseOver(
+        let hop: RedirectHop = {
           url,
-          response.status,
-          headerRecord(response.headers),
-          () => Promise.resolve(body),
-          init.secrets,
-        );
+          method: request.method ?? 'GET',
+          body: request.body,
+        };
+        // Latched off at the first cross-origin hop and never back on: the fetch standard DELETES
+        // the header from the request rather than re-deciding per hop, so `A -> B -> A` does not
+        // hand A's bearer back on the way home.
+        let credentialsInScope = true;
+        for (let followed = 0; ; followed += 1) {
+          // Per hop, because the jar is every domain the browser touched: a cookie computed for
+          // the first host and re-sent to the second is the leak `cookie-scope.ts` exists to
+          // prevent, arriving through the one door that never asked it twice.
+          const cookies = cookieHeaderFor(session.cookies, hop.url);
+          // Both header sources re-scoped the way the jar above already is, in the SAME precedence
+          // they had before: an `authorization` minted for the first host was the one credential
+          // that still rode along to wherever a `302` pointed.
+          const carried = credentialsInScope
+            ? session.headers
+            : withoutCredentials(session.headers);
+          const declared = credentialsInScope
+            ? request.headers
+            : withoutCredentials(request.headers ?? {});
+          const response = await call(hop.url, {
+            method: hop.method,
+            headers: {
+              ...carried,
+              ...(session.userAgent === '' ? {} : { 'user-agent': session.userAgent }),
+              ...(cookies === undefined ? {} : { cookie: cookies }),
+              ...declared,
+            },
+            ...(hop.body === undefined ? {} : { body: hop.body }),
+            signal: AbortSignal.any(signals),
+            // The whole point: the platform's own `follow` is what made the four lines above
+            // decorative, because it dials the target itself and hands back one Response.
+            redirect: 'manual',
+            ...(init.proxy === undefined ? {} : { proxy: init.proxy }),
+          });
+          // The URL that was REQUESTED, hop by hop. A chain reported under the caller's URL sends
+          // its reader hunting for a request the site never answered.
+          init.network.push({
+            method: hop.method,
+            url: hop.url,
+            status: response.status,
+            resourceType: 'fetch',
+            at: init.clock.now().getTime(),
+          });
+          const next = redirectHop(
+            response.status,
+            response.headers.get('location'),
+            hop.url,
+            hop.method,
+            hop.body,
+          );
+          if (next === undefined)
+            return await readResponse(hop.url, response, maxBytes, init.secrets);
+          // Discarded BEFORE the refusal, not after it: a throw over an unread stream holds that
+          // hop's socket until the collector reaches it, and the refusal path is exactly the one a
+          // hostile chain drives ten times per request.
+          await discardHopBody(response);
+          if (followed >= MAX_REDIRECT_HOPS) throw redirectLoop(url, next.url, followed + 1);
+          init.onActivity?.();
+          await screen(next.url);
+          if (!sameOrigin(hop.url, next.url)) credentialsInScope = false;
+          hop = next;
+        }
       } catch (thrown) {
         // A deadline that fired is this package's own timeout, with its own code and fix — never
         // the platform's bare `TimeoutError` reaching a job's retry classifier unclassified.

package/src/index.ts

@@ -68,6 +68,7 @@ export {
   profileLocked,
   promptUnanswered,
   recoverRefused,
+  redirectLoop,
   remoteRequired,
   robotsDisallowed,
   scrapeNotImplemented,
@@ -112,6 +113,8 @@ export type { HttpRequestInit, HttpTransportInit, ScrapeHttp, ScrapeResponse } f
 export { DEFAULT_HTTP_MAX_BYTES, httpOverFetch, responseOver } from './http';
 export type { HttpRecordingLookup, RecordedHttpInit } from './http-recorded';
 export { httpRecordingFilename, httpRecordingsOf, recordedHttp } from './http-recorded';
+export type { RedirectHop } from './http-redirect';
+export { MAX_REDIRECT_HOPS, redirectHop } from './http-redirect';
 export type { InterceptRules, InterceptVerdict } from './intercept';
 export { interceptVerdict, refusalEntry } from './intercept';
 export type { OfflineSessionInit } from './offline-session';