@ultimat3/scraping 29.3.1 → 29.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/scraping",
-  "version": "29.3.1",
+  "version": "29.3.2",
   "description": "Browser automation as a job: scrape() returns a JobHandle",
   "license": "MIT",
   "type": "module",
@@ -31,9 +31,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "29.3.1",
-    "@ultimat3/jobs": "29.3.1",
-    "@ultimat3/schema": "29.3.1",
-    "@ultimat3/storage": "29.3.1"
+    "@ultimat3/core": "29.3.2",
+    "@ultimat3/jobs": "29.3.2",
+    "@ultimat3/schema": "29.3.2",
+    "@ultimat3/storage": "29.3.2"
   }
 }

package/src/cdp-arm.ts

@@ -16,7 +16,7 @@ import type {
   PageErrorRing,
   ResourceType,
 } from './rings';
-import { pageErrorEntry, RESOURCE_TYPES } from './rings';
+import { clamped, pageErrorEntry, RESOURCE_TYPES } from './rings';
 
 /** What `arm()` reads: the page it subscribes to, the rules it judges by, the clock it stamps with. */
 export interface CdpArmInit {
@@ -134,7 +134,7 @@ export async function arm(init: CdpArmInit, sinks: CdpSinks): Promise<void> {
     const verdict = interceptVerdict(url, type, init.rules);
     const at = init.clock.now().getTime();
     if (verdict === 'allow') {
-      network.push({ method, url, resourceType: type, at });
+      network.push({ method, url: clamped(url), resourceType: type, at });
       // Caught, never floated: both reject when the target closed mid-request (and on any other
       // protocol error), and a floated rejection is one Bun ends the process on. The request is
       // already recorded; there is nothing left to do for a target that is gone.
@@ -147,7 +147,8 @@ export async function arm(init: CdpArmInit, sinks: CdpSinks): Promise<void> {
   init.page.on('console', (payload) => {
     console_.push({
       level: consoleLevel(readStringFrom(payload, 'type') ?? ''),
-      text: readStringFrom(payload, 'text') ?? '',
+      // Clamped: the ring bounds the count, and one `console.log(JSON.stringify(state))` is megabytes.
+      text: clamped(readStringFrom(payload, 'text') ?? ''),
       at: init.clock.now().getTime(),
     });
   });

package/src/clock.ts

@@ -131,3 +131,18 @@ export function deadline(clock: Clock, totalMs: number, subject = 'a scrape wait
   const remainingMs = (): number => Math.max(0, budgetMs - (clock.monotonic() - startedAt));
   return { totalMs: budgetMs, remainingMs, expired: () => remainingMs() <= 0 };
 }
+
+/**
+ * The timer a request DEADLINE is armed on (`deadline-signal.ts`) — not a wait this package
+ * performs, but a timer all the same, so it lives here. Cleared when the request settles, where
+ * `AbortSignal.timeout` stayed armed for its whole span after the answer had landed.
+ */
+export interface DeadlineTimer {
+  set(fire: () => void, ms: number): unknown;
+  clear(handle: unknown): void;
+}
+
+export const platformDeadlineTimer: DeadlineTimer = Object.freeze({
+  set: (fire: () => void, ms: number): unknown => setTimeout(fire, ms),
+  clear: (handle: unknown): void => clearTimeout(handle as ReturnType<typeof setTimeout>),
+});

package/src/deadline-signal.ts

@@ -0,0 +1,46 @@
+// One request's signal: a deadline plus the caller's own signal, undone when the request settles.
+// `AbortSignal.any([AbortSignal.timeout(ms), outer])` composed the same thing and could not be
+// undone — every read hung one listener (and one composite) on a session-long `outer` and armed a
+// timer that outlived the answer. The shape is `@ultimat3/jobs`' `createRunSignal`.
+
+import { type DeadlineTimer, platformDeadlineTimer } from './clock';
+
+export type { DeadlineTimer } from './clock';
+
+export interface DeadlineSignal extends Disposable {
+  /** Aborts at the deadline (a `TimeoutError`, as `AbortSignal.timeout`'s) or with `outer`. */
+  readonly signal: AbortSignal;
+  /** Whether the DEADLINE fired — not the caller cancelling. What the timeout code is chosen by. */
+  timedOut(): boolean;
+}
+
+/**
+ * `ms` is the caller's already-screened deadline (a positive finite count). `[Symbol.dispose]`
+ * clears the timer and detaches from `outer`; idempotent, and it never aborts.
+ */
+export function deadlineSignal(
+  ms: number,
+  outer?: AbortSignal,
+  timer: DeadlineTimer = platformDeadlineTimer,
+): DeadlineSignal {
+  const controller = new AbortController();
+  let fired = false;
+  const handle = timer.set(() => {
+    fired = true;
+    controller.abort(new DOMException('The operation timed out.', 'TimeoutError'));
+  }, ms);
+  const forward = (): void => controller.abort(outer?.reason);
+  if (outer?.aborted === true) forward();
+  else outer?.addEventListener('abort', forward, { once: true });
+  let open = true;
+  return {
+    signal: controller.signal,
+    timedOut: () => fired,
+    [Symbol.dispose]: () => {
+      if (!open) return;
+      open = false;
+      timer.clear(handle);
+      outer?.removeEventListener('abort', forward);
+    },
+  };
+}

package/src/http.ts

@@ -15,6 +15,7 @@ import type { StandardSchemaV1 } from '@ultimat3/schema';
 import { parse } from '@ultimat3/schema';
 import type { ScrapeClock } from './clock';
 import { cookieHeaderFor } from './cookie-scope';
+import { type DeadlineTimer, deadlineSignal } from './deadline-signal';
 import { bodyTooLarge, hostBlocked, httpFailed, redirectLoop, scrapeTimeout } from './error-throws';
 import type { RedirectHop } from './http-redirect';
 import { MAX_REDIRECT_HOPS, redirectHop } from './http-redirect';
@@ -123,6 +124,8 @@ export interface HttpTransportInit {
   readonly fetch?: ScrapeFetch | undefined;
   /** The resolver a wildcard-admitted name is checked and pinned with. Defaults to `Bun.dns`. */
   readonly resolve?: HostResolve | undefined;
+  /** The deadline's timer. Defaults to the platform's; a test hands its own (`deadline-signal.ts`). */
+  readonly timer?: DeadlineTimer | undefined;
 }
 
 /**
@@ -320,15 +323,16 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
       init.onActivity?.();
       let dial = await screen(url);
       const session = await init.session();
-      // `AbortSignal.timeout` and NOT `clock.sleep`: this is a deadline handed to the platform's
+      // A platform timer and NOT `clock.sleep`: this is a deadline handed to the platform's
       // own fetch, not a wait this package performs — and under a test clock a slept deadline
       // would fire on the microtask after it was armed, cancelling every request instantly.
       // The offline transport (`http-recorded.ts`) is what a test runs, and it has no deadline.
       //
       // ONE deadline for the whole chain, not one per hop: the caller declared a budget for
       // getting an answer, and ten hops each allowed the full budget is ten times the wait.
-      const deadlineSignal = AbortSignal.timeout(timeoutMs);
-      const signals = init.signal === undefined ? [deadlineSignal] : [deadlineSignal, init.signal];
+      // Disposed when the request settles: the timer cleared and the session-long `init.signal`
+      // let go — `AbortSignal.any` per hop hung a listener on it for every hop of every request.
+      using deadline = deadlineSignal(timeoutMs, init.signal, init.timer);
       try {
         let hop: RedirectHop = {
           url,
@@ -365,7 +369,7 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
             ),
             ...(dial.serverName === undefined ? {} : { tls: { serverName: dial.serverName } }),
             ...(hop.body === undefined ? {} : { body: hop.body }),
-            signal: AbortSignal.any(signals),
+            signal: deadline.signal,
             // The whole point: the platform's own `follow` is what made the four lines above
             // decorative, because it dials the target itself and hands back one Response.
             redirect: 'manual',
@@ -404,7 +408,7 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
       } catch (thrown) {
         // A deadline that fired is this package's own timeout, with its own code and fix — never
         // the platform's bare `TimeoutError` reaching a job's retry classifier unclassified.
-        if (deadlineSignal.aborted) throw scrapeTimeout(`http ${url}`, timeoutMs);
+        if (deadline.timedOut()) throw scrapeTimeout(`http ${url}`, timeoutMs);
         throw thrown;
       }
     },

package/src/intercept.ts

@@ -8,6 +8,7 @@
 import type { HostRule } from '@ultimat3/core';
 import { hostDecision } from '@ultimat3/core';
 import type { NetworkEntry, ResourceType } from './rings';
+import { clamped } from './rings';
 
 export interface InterceptRules {
   readonly allowHosts: readonly HostRule[];
@@ -38,4 +39,4 @@ export const refusalEntry = (
   verdict: Exclude<InterceptVerdict, 'allow'>,
   at: number,
   method = 'GET',
-): NetworkEntry => ({ method, url, resourceType, at, refused: verdict });
+): NetworkEntry => ({ method, url: clamped(url), resourceType, at, refused: verdict });

package/src/rings.ts

@@ -44,10 +44,13 @@ export interface PageError {
  * The per-entry cap, which the ring itself cannot give: a ring bounds the COUNT, and one
  * `RangeError: Maximum call stack size exceeded` carries thousands of frames — 200 of those is
  * megabytes held per page, which is the same OOM this file exists to prevent, one level down.
+ * The same cap holds a console line's text and a network entry's URL: a page that logs a
+ * serialised state blob, or requests a `data:` URL, is the same megabytes per entry.
  */
 export const MAX_PAGE_ERROR_CHARS = 4_000;
 
-const clamped = (text: string): string =>
+/** One entry's text, held to `MAX_PAGE_ERROR_CHARS` — every ring entry a page controls goes through it. */
+export const clamped = (text: string): string =>
   text.length <= MAX_PAGE_ERROR_CHARS ? text : `${text.slice(0, MAX_PAGE_ERROR_CHARS - 1)}…`;
 
 /**

package/src/robots-fetch.ts

@@ -10,6 +10,7 @@
 
 import type { HostRule } from '@ultimat3/core';
 import { finiteCount, isUltimateError, readWithinLimit } from '@ultimat3/core';
+import { type DeadlineTimer, deadlineSignal } from './deadline-signal';
 import type { ScrapeFetch } from './http';
 import { redirectHop } from './http-redirect';
 import { interceptVerdict } from './intercept';
@@ -58,6 +59,8 @@ export interface RobotsFetchInit {
   readonly allowHosts?: readonly HostRule[] | undefined;
   /** The resolver a wildcard-admitted name is checked and pinned with. Defaults to `Bun.dns`. */
   readonly resolve?: HostResolve | undefined;
+  /** The deadline's timer. Defaults to the platform's; a test hands its own (`deadline-signal.ts`). */
+  readonly timer?: DeadlineTimer | undefined;
   /**
    * The platform `fetch`, injectable so the default path itself is testable. `ScrapeFetch` and not
    * `typeof fetch`: the latter also carries `preconnect`, so nothing a caller can write satisfies
@@ -92,7 +95,12 @@ const answerOf = async (response: Response, limit: number): Promise<RobotsAnswer
     await response.body?.cancel().catch(() => undefined);
     return { unreachable: `status ${String(response.status)}` };
   }
-  if (!response.ok) return undefined;
+  if (!response.ok) {
+    // A 4xx's body is never read: cancelled, so its connection is not held until collection (the
+    // deadline that used to abort it is disposed with the read).
+    await response.body?.cancel().catch(() => undefined);
+    return undefined;
+  }
   // Counted as it arrives rather than `.text()`, which materialises the whole body first: a
   // multi-gigabyte robots.txt is a heap the worker never gets back.
   const capped = await readWithinLimit(response.body, limit);
@@ -131,8 +139,10 @@ export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
   return async (robotsUrl: string): Promise<RobotsAnswer> => {
     // Armed per read, not per gate: the gate is long-lived and reads once per origin, so a
     // deadline created alongside it would already have expired by the second origin.
-    const deadline = AbortSignal.timeout(timeoutMs);
-    const signal = init.signal === undefined ? deadline : AbortSignal.any([deadline, init.signal]);
+    // Disposed when the read settles: the timer cleared and the run-long `init.signal` let go.
+    // `AbortSignal.any` here hung one listener on it per origin read, for the life of the run.
+    using deadline = deadlineSignal(timeoutMs, init.signal, init.timer);
+    const signal = deadline.signal;
     // Resolved here, at the read, because the session that owns the exit did not exist when this
     // fetcher was built. An empty string is not an exit and is dropped with the absent one.
     const proxy = init.proxy?.();
@@ -176,7 +186,7 @@ export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
       // The pin refusing an inward name is a POLICY answer about the host, and the navigation is
       // refused by the same rule — it is not evidence the origin is down.
       if (isUltimateError(error) && error.code === 'X_SCRAPE_HOST_BLOCKED') return undefined;
-      if (deadline.aborted) return { unreachable: 'timeout' };
+      if (deadline.timedOut()) return { unreachable: 'timeout' };
       if (init.signal?.aborted === true) return { unreachable: 'cancelled' };
       return { unreachable: 'network' };
     }