@ultimat3/scraping 5.0.1 → 7.0.0

package/README.md

@@ -84,6 +84,29 @@ Both legs replay from **one** fixture directory (`fixtureBrowser(dir)`), so a hy
 login, session handoff, HTTP bulk fetch — is tested end to end. Both legs apply the same robots
 gate, offline included.
 
+## What the page reports back, and the one thing a picture cannot say
+
+Three bounded rings, read off `ScrapePage`. Bounded because a ten-thousand-page run that kept
+every line holds the whole browsing history in the worker's heap — and each read says how much the
+bound threw away, so a count taken from one is never quietly a floor.
+
+| Read | Answers |
+|---|---|
+| `page.console()` | `ConsoleLine[]` — what the page LOGGED |
+| `page.pageErrors()` / `page.pageErrorsDropped()` | `PageError[]` — what the page THREW and nobody caught, with the `stack` when the exception carried one |
+| `page.network()` / `page.networkDropped()` | `NetworkEntry[]` — every request, refusals included |
+
+`console()` and `pageErrors()` are separate streams because they are separate events: an island
+that throws during hydration calls no console method, so a scrape reading the console alone sees a
+page that looks silent and is broken — and a screenshot of it is a picture of the server-rendered
+markup, indistinguishable from one that worked.
+
+Empty is a legitimate answer, never a missing method: the offline drivers parse markup and execute
+none of it, so nothing there can throw. `ScrapeTarget.pageErrors` is a **required** ring for the
+same reason — a third-party driver that could omit it would be silent about errors it can see.
+Entries are built with `pageErrorEntry()`, which truncates at `MAX_PAGE_ERROR_CHARS`: the ring
+bounds the count, and one `Maximum call stack size exceeded` is thousands of frames.
+
 ## What it owns
 
 | Module | Owns |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/scraping",
-  "version": "5.0.1",
+  "version": "7.0.0",
   "description": "Browser automation as a job: scrape() returns a JobHandle",
   "license": "MIT",
   "type": "module",
@@ -30,9 +30,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "5.0.1",
-    "@ultimat3/jobs": "5.0.1",
-    "@ultimat3/schema": "5.0.1",
-    "@ultimat3/storage": "5.0.1"
+    "@ultimat3/core": "7.0.0",
+    "@ultimat3/jobs": "7.0.0",
+    "@ultimat3/schema": "7.0.0",
+    "@ultimat3/storage": "7.0.0"
   }
 }

package/src/cdp-fake.ts

@@ -36,6 +36,12 @@ type Handlers = Map<string, ((payload: unknown) => void)[]>;
 export interface FakeCdpBrowser extends CdpBrowserLike {
   /** Fire a request event, as a real browser would when the page fetches a subresource. */
   emitRequest(url: string, resourceType: string): void;
+  /**
+   * Fire `pageerror` — the page threw and nothing caught it. `payload` is whatever the library
+   * would hand a handler, `unknown` on purpose: a page can throw a string as easily as an `Error`,
+   * and the target reads it defensively either way.
+   */
+  emitPageError(payload: unknown): void;
   readonly aborted: readonly string[];
   readonly closed: boolean;
 }
@@ -130,6 +136,9 @@ export function fakeCdpBrowser(init: FakeCdpPageInit): FakeCdpBrowser {
         });
       }
     },
+    emitPageError(payload: unknown): void {
+      for (const handler of handlers.get('pageerror') ?? []) handler(payload);
+    },
     aborted,
     get closed(): boolean {
       return closed;

package/src/cdp-port.ts

@@ -37,6 +37,17 @@ export interface CdpPageLike {
   screenshot(options: { readonly fullPage?: boolean }): Promise<Uint8Array | string>;
   pdf(options?: Record<string, unknown>): Promise<Uint8Array>;
   setRequestInterception(enabled: boolean): Promise<void>;
+  /**
+   * `event` stays a bare `string` — a union of the four names this package subscribes to would be
+   * this file naming somebody else's event vocabulary, which is the thing it exists not to do, and
+   * a launcher whose emitter is wider would then fail to satisfy the port for no reason.
+   *
+   * Those four, and the pair that is easy to confuse: `request`, `console`, `pageerror` — an
+   * uncaught exception INSIDE the page, which leaves the session perfectly usable — and `error`,
+   * which is the renderer CRASHING and is what `X_SCRAPE_PAGE_CRASHED` is raised from. Every
+   * payload arrives `unknown` and is read defensively in `cdp-target.ts`; nothing here may name
+   * the library's own types for them.
+   */
   on(event: string, handler: (payload: unknown) => void): unknown;
   frames(): readonly CdpFrameLike[];
   close(): Promise<void>;

package/src/cdp-target.ts

@@ -10,8 +10,16 @@ import type { ScrapeClock } from './clock';
 import { browserUnreachable, pageCrashed, scrapeNotImplemented } from './error-throws';
 import type { InterceptRules } from './intercept';
 import { interceptVerdict, refusalEntry } from './intercept';
-import type { ConsoleLine, NetworkEntry, ResourceType } from './rings';
-import { createRing, RESOURCE_TYPES } from './rings';
+import type {
+  ConsoleLine,
+  ConsoleRing,
+  NetworkEntry,
+  NetworkRing,
+  PageError,
+  PageErrorRing,
+  ResourceType,
+} from './rings';
+import { createRing, pageErrorEntry, RESOURCE_TYPES } from './rings';
 import type { SessionSnapshot } from './session-state';
 import type {
   CaptureOptions,
@@ -88,6 +96,25 @@ const readStringFrom = (owner: unknown, key: string): string | undefined => {
   return typeof answer === 'string' ? answer : undefined;
 };
 
+/**
+ * A `pageerror` payload, read defensively — never cast, and never assumed to be an `Error`.
+ *
+ * `readStringFrom`, the same reader the console handler uses, because the payload has the same
+ * problem: `message` and `stack` are an own property on one build and an accessor on another, and
+ * a schema parse cannot call an accessor. A page can also `throw 'a string'` or throw a frozen
+ * object with no `message` at all — both reach here, and both are recorded as SOMETHING having
+ * thrown, because an entry with a poor message is still the difference between "the island threw"
+ * and silence.
+ */
+const readPageError = (payload: unknown, at: number): PageError => {
+  if (typeof payload === 'string') return pageErrorEntry({ message: payload, at });
+  return pageErrorEntry({
+    message: readStringFrom(payload, 'message') ?? '',
+    stack: readStringFrom(payload, 'stack'),
+    at,
+  });
+};
+
 /**
  * The one failure `guard()` must NOT re-label, and the line is drawn at exactly one code.
  *
@@ -116,17 +143,25 @@ export interface CdpTargetInit {
   readonly ringCapacity?: number | undefined;
 }
 
+/**
+ * Everything `arm()` writes into. Named rather than positional: three rings of near-identical
+ * type plus a latch is a call site nobody can read, and swapping two of them is a mistake the
+ * compiler cannot catch.
+ */
+interface CdpSinks {
+  readonly network: NetworkRing;
+  readonly console: ConsoleRing;
+  readonly pageErrors: PageErrorRing;
+  readonly crashed: { value: string | undefined };
+}
+
 /**
  * Interception is armed BEFORE the first navigation and refuses at the request, not after the
  * response — an `allowHosts` that reported afterwards would be a log line about bytes that
  * already left the container.
  */
-async function arm(
-  init: CdpTargetInit,
-  network: ReturnType<typeof createRing<NetworkEntry>>,
-  console_: ReturnType<typeof createRing<ConsoleLine>>,
-  crashed: { value: string | undefined },
-): Promise<void> {
+async function arm(init: CdpTargetInit, sinks: CdpSinks): Promise<void> {
+  const { network, console: console_, pageErrors, crashed } = sinks;
   await init.page.setRequestInterception(true);
   init.page.on('request', (payload) => {
     const request = asRequest(payload);
@@ -154,6 +189,23 @@ async function arm(
       at: init.clock.now().getTime(),
     });
   });
+  /**
+   * The page threw and nothing caught it. THE gap this ring closes: a screenshot of an island
+   * that threw during hydration is a picture of the server-rendered markup, indistinguishable
+   * from a page that worked — and `console` does not carry it, because throwing calls no console
+   * method. Subscribed here, beside the others, so a target is observing before its first
+   * navigation: an exception raised during load has no second chance to be recorded.
+   *
+   * NOT the same event as `error` below, and the difference is the whole reason this is a
+   * separate handler: puppeteer's `pageerror` is "an uncaught exception happens within the page"
+   * and its `error` is "the page crashes" (`PageEvent.PageError` / `PageEvent.Error`). One is the
+   * app being broken and the session is fine; the other is the tab being gone. Recording a
+   * `pageerror` into `crashed` would make every scrape of a page with one bad island answer
+   * X_SCRAPE_PAGE_CRASHED — a code registered `terminal` — for a page still perfectly usable.
+   */
+  init.page.on('pageerror', (payload) => {
+    pageErrors.push(readPageError(payload, init.clock.now().getTime()));
+  });
   // A renderer that dies must be a CODE, not a hang: every later call answers X_SCRAPE_PAGE_CRASHED
   // instead of waiting out its own timeout against a tab that is gone.
   init.page.on('error', (payload) => {
@@ -164,8 +216,9 @@ async function arm(
 export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
   const console_ = createRing<ConsoleLine>(init.ringCapacity);
   const network = createRing<NetworkEntry>(init.ringCapacity);
+  const pageErrors = createRing<PageError>(init.ringCapacity);
   const crashed: { value: string | undefined } = { value: undefined };
-  await arm(init, network, console_, crashed);
+  await arm(init, { network, console: console_, pageErrors, crashed });
   let pendingStorage: SessionSnapshot | undefined;
 
   const originOf = (url: string): string => {
@@ -233,6 +286,9 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
     driver: CDP_DRIVER,
     console: console_,
     network,
+    // Shared with every frame target below, through the spread: an exception is the PAGE's, and a
+    // per-frame ring would hide a throw from an iframe behind whichever handle the caller held.
+    pageErrors,
     url: () => init.page.url(),
     goto: (url: string, options: GotoOptions) =>
       guard('goto', async () => {

package/src/html-target.ts

@@ -13,7 +13,7 @@ import type { InterceptRules } from './intercept';
 import { interceptVerdict, refusalEntry } from './intercept';
 import type { PageRecording } from './recording';
 import { splitDownload } from './recording';
-import type { ConsoleLine, NetworkEntry } from './rings';
+import type { ConsoleLine, NetworkEntry, PageError } from './rings';
 import { createRing } from './rings';
 import type { SessionSnapshot } from './session-state';
 import { EMPTY_SESSION } from './session-state';
@@ -74,6 +74,12 @@ const keyOf = (selector: string, element: ElementSnapshot | undefined): string =
 export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
   const consoleRing = createRing<ConsoleLine>();
   const networkRing = createRing<NetworkEntry>();
+  // Built and never pushed to, deliberately: this target parses markup and executes none of it, so
+  // there is no uncaught exception for it to have. It ANSWERS rather than omitting the ring —
+  // `page.pageErrors()` returning `[]` here is the honest "nothing threw, and nothing could",
+  // where a missing ring would be a page method that throws on two of the three drivers.
+  // `driver-parity.test.ts` pins the divergence, beside the box/hit-target one it already carries.
+  const pageErrorRing = createRing<PageError>();
   const overlay = new Map<string, string>();
   let page: PageRecording = init.start ?? EMPTY;
   let armed: string | undefined;
@@ -164,6 +170,7 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
     driver: init.driver,
     console: consoleRing,
     network: networkRing,
+    pageErrors: pageErrorRing,
     url: () => page.url,
     goto: (url: string, _options: GotoOptions): Promise<void> => navigate(url),
     content: (): Promise<string> => {

package/src/index.ts

@@ -137,10 +137,18 @@ export type {
   ConsoleRing,
   NetworkEntry,
   NetworkRing,
+  PageError,
+  PageErrorRing,
   ResourceType,
   Ring,
 } from './rings';
-export { createRing, DEFAULT_RING_CAPACITY, RESOURCE_TYPES } from './rings';
+export {
+  createRing,
+  DEFAULT_RING_CAPACITY,
+  MAX_PAGE_ERROR_CHARS,
+  pageErrorEntry,
+  RESOURCE_TYPES,
+} from './rings';
 export type { RobotsFetch, RobotsGate, RobotsGateInit, RobotsPolicy, RobotsRules } from './robots';
 export { createRobotsGate, DEFAULT_ROBOTS_AGENT, parseRobots, robotsAllows } from './robots';
 export type { RobotsFetchInit } from './robots-fetch';

package/src/page-over-target.ts

@@ -212,7 +212,9 @@ export function pageOverTarget(target: ScrapeTarget, ctx: PageContext): ScrapePa
     cookies: (): Promise<readonly ScrapeCookie[]> => target.cookies(),
     session: () => target.session(),
     console: () => target.console.entries(),
+    pageErrors: () => target.pageErrors.entries(),
     network: () => target.network.entries(),
     networkDropped: () => target.network.dropped,
+    pageErrorsDropped: () => target.pageErrors.dropped,
   };
 }

package/src/page.ts

@@ -8,7 +8,7 @@
 
 import type { Secret } from '@ultimat3/core';
 import type { ActionabilityState } from './actionability';
-import type { ConsoleLine, NetworkEntry } from './rings';
+import type { ConsoleLine, NetworkEntry, PageError } from './rings';
 import type { SessionSnapshot } from './session-state';
 import type { ElementSnapshot, ScrapeCookie, ScrapeDownloadFile } from './target';
 
@@ -93,6 +93,13 @@ export interface ScrapePage extends ScrapeFrame {
   session(): Promise<SessionSnapshot>;
   /** The bounded tail. Bounded because a long run's full history is an OOM, not a log. */
   console(): readonly ConsoleLine[];
+  /**
+   * The uncaught exceptions the page threw, which `console()` does NOT carry: an island that
+   * throws calls no console method, so a scrape reading console alone sees a page that looks
+   * silent and is broken. Empty on a driver with no JS engine — the offline drivers parse HTML
+   * and never execute it, so nothing there can throw.
+   */
+  pageErrors(): readonly PageError[];
   network(): readonly NetworkEntry[];
   /**
    * How many entries the bound above threw away. It is the same honesty `Ring.dropped` carries:
@@ -100,4 +107,10 @@ export interface ScrapePage extends ScrapeFrame {
    * is non-zero, and a scrape that blocked 5,000 images otherwise reports 200 with no hint.
    */
   networkDropped(): number;
+  /**
+   * The same honesty for the errors, and it is the count a verdict gates on: an island throwing
+   * inside a `requestAnimationFrame` produces thousands, and "3 page errors" read off a bounded
+   * tail of 200 would be a number a reader trusts and should not.
+   */
+  pageErrorsDropped(): number;
 }

package/src/rings.ts

@@ -1,6 +1,6 @@
-// Bounded console and network history. BOUNDED is the whole point: a scrape of ten thousand
-// pages that kept every console line and every request holds the run's entire browsing history in
-// the worker's heap, and the incident is an OOM two hours in rather than a scraper bug.
+// Bounded console, page-error and network history. BOUNDED is the whole point: a scrape of ten
+// thousand pages that kept every console line and every request holds the run's entire browsing
+// history in the worker's heap, and the incident is an OOM two hours in rather than a scraper bug.
 
 export interface ConsoleLine {
   readonly level: 'log' | 'info' | 'warn' | 'error' | 'debug';
@@ -19,6 +19,55 @@ export interface NetworkEntry {
   readonly refused?: 'blocked' | 'host' | 'robots' | undefined;
 }
 
+/**
+ * An uncaught exception the PAGE threw — the half a picture cannot carry.
+ *
+ * Its own entry type, and not a `ConsoleLine` with `level: 'error'`, because it is a different
+ * event: a page that threw called no console method, so folding one into the console stream
+ * reports a `console.error` that never happened — and `ConsoleLine` has nowhere to keep the stack,
+ * which is the field that says WHICH island threw.
+ */
+export interface PageError {
+  /** The exception's message. `''` when the payload carried none — never a fabricated one. */
+  readonly message: string;
+  /**
+   * The JS stack, when the exception carried one. Absent means the payload had none, never that
+   * the throw had no origin: a `throw 'a string'` in the page reaches here with a message alone.
+   */
+  readonly stack?: string | undefined;
+  readonly at: number;
+}
+
+/**
+ * The per-entry cap, which the ring itself cannot give: a ring bounds the COUNT, and one
+ * `RangeError: Maximum call stack size exceeded` carries thousands of frames — 200 of those is
+ * megabytes held per page, which is the same OOM this file exists to prevent, one level down.
+ */
+export const MAX_PAGE_ERROR_CHARS = 4_000;
+
+const clamped = (text: string): string =>
+  text.length <= MAX_PAGE_ERROR_CHARS ? text : `${text.slice(0, MAX_PAGE_ERROR_CHARS - 1)}…`;
+
+/**
+ * The ONE way a `PageError` is built. A driver that assembled the object literal itself would be
+ * a driver whose stacks are unbounded, and the truncation would then be a rule per driver rather
+ * than a property of the type.
+ */
+export function pageErrorEntry(input: {
+  readonly message: string;
+  readonly stack?: string | undefined;
+  readonly at: number;
+}): PageError {
+  // An empty stack is ABSENT, not empty: `stack: ''` prints as a blank stack, which reads as
+  // "the exception had no origin" rather than "the payload did not carry one".
+  const stack = input.stack === undefined || input.stack === '' ? undefined : clamped(input.stack);
+  return {
+    message: clamped(input.message),
+    ...(stack === undefined ? {} : { stack }),
+    at: input.at,
+  };
+}
+
 export const RESOURCE_TYPES = [
   'document',
   'stylesheet',
@@ -48,6 +97,7 @@ export interface Ring<T> {
 
 export type ConsoleRing = Ring<ConsoleLine>;
 export type NetworkRing = Ring<NetworkEntry>;
+export type PageErrorRing = Ring<PageError>;
 
 export function createRing<T>(capacity: number = DEFAULT_RING_CAPACITY): Ring<T> {
   const items: T[] = [];

package/src/target.ts

@@ -3,7 +3,7 @@
 // lets `page-over-target.ts` be the ONE implementation of `ScrapePage` for the real browser, the
 // fixture replayer and the fake alike.
 
-import type { ConsoleRing, NetworkRing } from './rings';
+import type { ConsoleRing, NetworkRing, PageErrorRing } from './rings';
 import type { SessionSnapshot } from './session-state';
 
 /**
@@ -96,6 +96,15 @@ export interface ScrapeTarget {
   readonly driver: string;
   readonly console: ConsoleRing;
   readonly network: NetworkRing;
+  /**
+   * REQUIRED, and empty is a legitimate answer. A driver with no JS engine has no uncaught
+   * exception to report and answers an empty ring; an optional member would instead let a driver
+   * be silent about errors it CAN see, which is precisely the blind spot this ring closes — and
+   * `ScrapePage.pageErrors()` would then be a method that answers nothing on an unknown subset of
+   * drivers. A third-party driver added before 2026-08-21 gets a type error naming this field,
+   * which is the whole enforcement.
+   */
+  readonly pageErrors: PageErrorRing;
   url(): string;
   goto(url: string, options: GotoOptions): Promise<void>;
   /** Serialised HTML of THIS target — the document for a page, the subtree for a frame. */