@ultimat3/scraping 30.9.0 → 31.0.0

package/src/cdp-connection.ts

@@ -0,0 +1,290 @@
+// One responsibility: a Chrome DevTools Protocol connection — request framing, response
+// correlation, the per-call deadline — over a TRANSPORT that moves whole messages: a launched
+// Chrome's pipe (`cdp-pipe.ts`) or a remote browser's WebSocket (`cdpConnect`). It knows neither
+// the launch (`cdp-launch.ts`) nor a page surface (testing's `cdp-e2e-page.ts`, `pdf-render.ts`).
+
+import { assert, stringField } from '@ultimat3/core';
+import { CdpCallFailedError, CdpTimeoutError } from './cdp-errors';
+import { wireWatch } from './cdp-wire-watch';
+
+// **No library, and that is the point rather than an economy.** `cdp-port.ts` declares a ~25-method
+// port because `ScrapePage` is a full scraping surface, and its intended implementation is
+// `puppeteer-core`. `E2eBrowserPage` is FIVE methods, and CDP's wire format is one JSON object with
+// an `id` — so the whole thing an e2e driver needs is this file plus two small ones, on Bun's
+// native `WebSocket`, with no dependency to add to a repo whose first non-negotiable is that Bun's
+// natives replace most of them.
+
+/** One CDP result. `unknown` because every payload here is somebody else's JSON. */
+export interface CdpResult {
+  readonly result?: Record<string, unknown> | undefined;
+}
+
+export interface CdpConnection {
+  /**
+   * Send one command. `sessionId` targets an attached page rather than the browser itself.
+   * `deadlineMs` replaces the connection's per-call deadline for THIS call — the launcher's first
+   * call is a cold start, not a call, and is the only caller that passes one.
+   */
+  send(
+    method: string,
+    params?: Record<string, unknown>,
+    sessionId?: string,
+    deadlineMs?: number,
+  ): Promise<CdpResult>;
+  /**
+   * Wait for the next occurrence of one CDP **event**, or for the deadline. Answers `true` when the
+   * event arrived and `false` when it did not — it never throws, because every caller has a better
+   * assertion to fail on than "the event was late".
+   *
+   * It exists because a command's reply is not always the signal. `Page.navigate`'s reply is
+   * DROPPED whenever the navigation swaps the render process — measured on Chrome 150: the page
+   * loads, the server is hit, a later `Runtime.evaluate` answers, and the navigate frame never
+   * comes back at all. A driver that treated the reply as the completion signal waits out its full
+   * deadline on the most ordinary navigation there is.
+   */
+  once(method: string, sessionId: string | undefined, timeoutMs: number): Promise<boolean>;
+  /**
+   * Subscribe to every occurrence of one CDP event, until the returned function is called.
+   *
+   * `once` cannot express what this is for: a target that attaches AFTER the driver stopped
+   * listening is a service worker whose network conditions nobody set, which is an `offline()`
+   * that silently does nothing to the one thing serving the page.
+   *
+   * The listener is also handed the event's `sessionId` (`undefined` for a browser-level event), so
+   * a subscriber that owns ONE page can ignore the frames every other attached target sends.
+   */
+  on(method: string, listener: CdpEventListener): () => void;
+  close(): void;
+}
+
+/** One event's params, and the session it arrived on. */
+export type CdpEventListener = (
+  params: Record<string, unknown>,
+  sessionId: string | undefined,
+) => void;
+
+interface Pending {
+  readonly resolve: (value: CdpResult) => void;
+  readonly reject: (reason: Error) => void;
+  readonly timer: ReturnType<typeof setTimeout>;
+}
+
+/** A CDP error frame: `{ error: { code, message } }`, both fields somebody else's. */
+const errorText = (frame: Record<string, unknown>): string | undefined => {
+  const error = frame['error'];
+  if (typeof error !== 'object' || error === null) return undefined;
+  const message = (error as Record<string, unknown>)['message'];
+  return typeof message === 'string' ? message : 'the browser refused the call';
+};
+
+/**
+ * What a connection needs of the wire: whole messages out, whole messages in, and a close. Framing
+ * is the transport's — a WebSocket frames per message, the pipe splits on NUL — so correlation and
+ * deadlines are written once, over either.
+ */
+export interface CdpTransport {
+  send(text: string): void;
+  close(): void;
+  /** Installed exactly once, by the connection, before the first `send`. */
+  listen(handlers: {
+    readonly message: (text: string) => void;
+    readonly closed: (reason: string) => void;
+  }): void;
+}
+
+export interface CdpConnectionOptions {
+  readonly endpoint: string;
+  /** Per-call deadline. A CDP call that never answers is a suite that never finishes. */
+  readonly timeoutMs: number;
+}
+
+/**
+ * What went wrong, in the socket's own words when it has any. Bun hands `onerror` an `ErrorEvent`
+ * whose `message` is the only place the reason lives — "Failed to connect", "Expected 101 status
+ * code" — and a refusal that drops it names the step and not the cause.
+ */
+const failure = (event: unknown, what: string): string => {
+  const message = stringField(event, 'message');
+  return message === undefined || message === '' ? what : `${what}: ${message}`;
+};
+
+/**
+ * A remote browser over its WebSocket url. NOT the e2e driver's wire: measured in the dummy's
+ * `offline-feed` suite on Bun 1.4.0, Bun's WebSocket client handed `onmessage` text spliced out of
+ * several frames — 64 unparseable frames in one run, one of them the reply to a `Runtime.evaluate`
+ * that then waited out its whole deadline. A launched Chrome is driven over `cdp-pipe.ts` instead.
+ */
+export async function cdpConnect(options: CdpConnectionOptions): Promise<CdpConnection> {
+  assert(
+    options.endpoint.startsWith('ws://') || options.endpoint.startsWith('wss://'),
+    `the CDP endpoint is ${options.endpoint === '' ? 'empty' : 'not a WebSocket url'}`,
+    'pass the `webSocketDebuggerUrl` Chrome prints on stderr, or the one /json/version answers',
+  );
+  const socket = new WebSocket(options.endpoint);
+  await new Promise<void>((resolve, reject) => {
+    // The socket is CLOSED on both failures: a handshake that timed out otherwise kept dialling
+    // in the background, holding a descriptor and a pending connection for a caller that gave up.
+    const timer = setTimeout(() => {
+      socket.close();
+      reject(new CdpTimeoutError({ method: 'connect', timeoutMs: options.timeoutMs }));
+    }, options.timeoutMs);
+    socket.onopen = (): void => {
+      clearTimeout(timer);
+      resolve();
+    };
+    // `onerror` is the handshake's only until `listen` below reassigns it: a failure BEFORE open
+    // has no pending call to abandon, and rejecting is the only way the caller hears about it.
+    socket.onerror = (event: Event): void => {
+      clearTimeout(timer);
+      socket.close();
+      reject(
+        new CdpCallFailedError({
+          method: 'connect',
+          detail: failure(event, 'the browser refused the connection'),
+        }),
+      );
+    };
+  });
+
+  const transport: CdpTransport = {
+    send: (text) => socket.send(text),
+    close: () => socket.close(),
+    listen(handlers) {
+      socket.onmessage = (event: MessageEvent): void => {
+        handlers.message(typeof event.data === 'string' ? event.data : '');
+      };
+      socket.onclose = (): void => handlers.closed('the browser closed the CDP connection');
+      socket.onerror = (event: Event): void =>
+        handlers.closed(failure(event, 'the CDP connection failed'));
+    },
+  };
+  return cdpConnectOver(transport, options.timeoutMs);
+}
+
+/** A connection over any transport — correlation, deadlines and events, written once. */
+export function cdpConnectOver(transport: CdpTransport, timeoutMs: number): CdpConnection {
+  const pending = new Map<number, Pending>();
+  const waiters = new Set<(method: string, sessionId: string | undefined) => void>();
+  const listeners = new Map<string, Set<CdpEventListener>>();
+  let nextId = 0;
+  let closed = false;
+  // What a call that is never answered reports instead of a bare "did not answer".
+  const watch = wireWatch();
+
+  // Every in-flight call is settled on close. Without this a suite whose browser died waits out
+  // one full deadline per call and reports a timeout, where the true fault is a dead browser.
+  const abandon = (reason: string): void => {
+    closed = true;
+    for (const [, one] of pending) {
+      clearTimeout(one.timer);
+      one.reject(new CdpCallFailedError({ method: 'the connection', detail: reason }));
+    }
+    pending.clear();
+    // A waiter is a "did this happen" question, and on a dead connection the answer is no. Its
+    // own timer settles it, so nothing is left hanging; clearing the set only stops a late frame
+    // from resolving a waiter whose connection has gone.
+    waiters.clear();
+  };
+
+  const message = (raw: string): void => {
+    let frame: Record<string, unknown>;
+    try {
+      frame = JSON.parse(raw) as Record<string, unknown>;
+    } catch {
+      // An unparseable frame is the browser's, not ours, and there is no call to fail with it:
+      // correlation is by `id`, and a frame we cannot read has none. COUNTED, though: the call
+      // whose reply it was waits out its deadline, and its timeout says frames were dropped.
+      watch.drop();
+      return;
+    }
+    watch.frame(frame);
+    const id = frame['id'];
+    if (typeof id !== 'number') {
+      const method = frame['method'];
+      if (typeof method !== 'string') return;
+      const on = typeof frame['sessionId'] === 'string' ? frame['sessionId'] : undefined;
+      for (const waiter of [...waiters]) waiter(method, on);
+      const subscribed = listeners.get(method);
+      if (subscribed !== undefined) {
+        const params = frame['params'];
+        const payload: Record<string, unknown> =
+          typeof params === 'object' && params !== null ? (params as Record<string, unknown>) : {};
+        // A copy, because a listener may unsubscribe itself while this loop is running.
+        for (const listener of [...subscribed]) listener(payload, on);
+      }
+      return;
+    }
+    const one = pending.get(id);
+    if (one === undefined) return;
+    pending.delete(id);
+    clearTimeout(one.timer);
+    const failed = errorText(frame);
+    if (failed !== undefined) {
+      one.reject(new CdpCallFailedError({ method: `call ${String(id)}`, detail: failed }));
+      return;
+    }
+    const result = frame['result'];
+    one.resolve({
+      result:
+        typeof result === 'object' && result !== null
+          ? (result as Record<string, unknown>)
+          : undefined,
+    });
+  };
+  transport.listen({ message, closed: abandon });
+
+  return {
+    send(method, params = {}, sessionId, deadlineMs = timeoutMs): Promise<CdpResult> {
+      if (closed) {
+        return Promise.reject(
+          new CdpCallFailedError({ method, detail: 'the CDP connection is already closed' }),
+        );
+      }
+      nextId += 1;
+      const id = nextId;
+      return new Promise<CdpResult>((resolve, reject) => {
+        const sentAt = watch.mark(sessionId);
+        const timer = setTimeout(() => {
+          pending.delete(id);
+          const observed = watch.observe(sentAt, sessionId, !closed);
+          reject(new CdpTimeoutError({ method, timeoutMs: deadlineMs, observed }));
+        }, deadlineMs);
+        pending.set(id, { resolve, reject, timer });
+        transport.send(
+          JSON.stringify({ id, method, params, ...(sessionId === undefined ? {} : { sessionId }) }),
+        );
+      });
+    },
+    once(method, sessionId, timeoutMs): Promise<boolean> {
+      if (closed) return Promise.resolve(false);
+      return new Promise<boolean>((resolve) => {
+        const waiter = (seen: string, on: string | undefined): void => {
+          if (seen !== method) return;
+          if (sessionId !== undefined && on !== sessionId) return;
+          clearTimeout(timer);
+          waiters.delete(waiter);
+          resolve(true);
+        };
+        const timer = setTimeout(() => {
+          waiters.delete(waiter);
+          resolve(false);
+        }, timeoutMs);
+        waiters.add(waiter);
+      });
+    },
+    on(method, listener): () => void {
+      const subscribed = listeners.get(method) ?? new Set();
+      subscribed.add(listener);
+      listeners.set(method, subscribed);
+      return () => {
+        subscribed.delete(listener);
+        if (subscribed.size === 0) listeners.delete(method);
+      };
+    },
+    close(): void {
+      listeners.clear();
+      abandon('the driver closed the CDP connection');
+      transport.close();
+    },
+  };
+}

package/src/cdp-error-codes.ts

@@ -0,0 +1,28 @@
+// The registry for the raw-CDP browser's four `X_CDP_*` codes — the launcher, the connection and
+// the calls over it. Owned here since 31.0.0, when the launcher moved from `@ultimat3/testing` to
+// its runtime home; `@ultimat3/testing` re-reads these titles and registers only its own `X_E2E_*`.
+import { registerErrorCodes } from '@ultimat3/core';
+
+// Four and not one, because the four repairs differ: install a browser, read the browser's own
+// stderr, look at the page, raise a deadline.
+export const CDP_ERROR_CODES = [
+  'X_CDP_BROWSER_MISSING',
+  'X_CDP_LAUNCH_FAILED',
+  'X_CDP_CALL_FAILED',
+  'X_CDP_TIMEOUT',
+] as const;
+
+export type CdpErrorCode = (typeof CDP_ERROR_CODES)[number];
+
+export const CDP_ERROR_TITLES = Object.freeze<Record<CdpErrorCode, string>>({
+  X_CDP_BROWSER_MISSING: 'no Chrome or Chromium is installed for the e2e driver to launch',
+  X_CDP_LAUNCH_FAILED: 'the browser started and never announced a DevTools endpoint',
+  X_CDP_CALL_FAILED: 'the browser refused a DevTools call',
+  X_CDP_TIMEOUT: 'a DevTools call did not answer inside its deadline',
+});
+
+// Owned here and borrowed by nobody, so unconditional: a second package claiming one must fail as
+// X_ERROR_CODE_DUPLICATE rather than quietly keep whichever title registered first.
+registerErrorCodes(
+  Object.fromEntries(Object.entries(CDP_ERROR_TITLES).map(([code, title]) => [code, { title }])),
+);

package/src/cdp-errors.ts

@@ -0,0 +1,179 @@
+// One constructor per way the raw-CDP browser refuses — the e2e driver's, `x shot`'s and `pdf()`'s.
+// Every cause quotes a value that came out of a BROWSER or off a spawned process's stderr, so
+// every one is rendered rather than interpolated.
+
+import { isFixShellSafe, renderCauseValue, UltimateError } from '@ultimat3/core';
+// Bare: the titles these constructors' codes render with are registered there.
+import './cdp-error-codes';
+
+/**
+ * No browser to drive. This is the one an author meets first, so its fix names the two ways out:
+ * point the driver at a binary, or accept that this machine cannot run the check.
+ */
+export class CdpBrowserMissingError extends UltimateError {
+  constructor(input: { readonly tried: readonly string[] }) {
+    super({
+      code: 'X_CDP_BROWSER_MISSING',
+      cause: `no Chrome or Chromium executable was found — tried ${renderCauseValue(input.tried)}`,
+      fix: 'export CHROME_PATH=/usr/bin/google-chrome   # any Chrome or Chromium binary (GitHub-hosted ubuntu runners ship this one); unset, a browser-backed e2e suite skips',
+    });
+  }
+}
+
+/** One start of the browser that ended without an answer — what `cdp-launch-attempt.ts` reports. */
+export interface CdpLaunchAttempt {
+  /**
+   * `deadline`: alive and silent for `waitedMs`. `closed`: its wire ended, or its dial was
+   * refused, before any answer. `spawn`: the binary could not be started at all — it never ran.
+   */
+  readonly why: 'deadline' | 'closed' | 'spawn';
+  readonly waitedMs: number;
+  /** The code it exited with on its own, or `null` when the launcher had to kill it. */
+  readonly exitCode: number | null;
+  /** The last lines it wrote to stderr; empty when it wrote none. */
+  readonly stderr: string;
+  /**
+   * What the LAUNCHER saw end the attempt, rendered: the refused dial and its endpoint, the call
+   * that timed out, the spawn's own error. Absent on a record that predates it.
+   */
+  readonly detail?: string | undefined;
+}
+
+const HAND_FLAGS =
+  '--headless=new --no-sandbox --disable-dev-shm-usage --disable-gpu --remote-debugging-port=0 about:blank';
+
+const endedText = (attempt: CdpLaunchAttempt): string => {
+  const after = `after ${String(attempt.waitedMs)}ms`;
+  if (attempt.why === 'spawn') return 'could not be started';
+  if (attempt.why === 'deadline')
+    return `no answer inside ${String(attempt.waitedMs)}ms, so it was killed`;
+  return attempt.exitCode === null
+    ? `its DevTools connection ended ${after} without an answer, and it was killed`
+    : `exited with code ${String(attempt.exitCode)} ${after} without answering`;
+};
+
+const attemptText = (attempt: CdpLaunchAttempt, index: number): string => {
+  const saw =
+    attempt.detail === undefined || attempt.detail === ''
+      ? ''
+      : `: ${renderCauseValue(attempt.detail)}`;
+  const said = attempt.stderr === '' ? 'it printed nothing' : renderCauseValue(attempt.stderr);
+  return `launch ${String(index + 1)}: ${endedText(attempt)}${saw}; stderr: ${said}`;
+};
+
+/**
+ * The binary ran and never answered a DevTools call — a crash, a bad flag, a sandbox refusal, or a
+ * machine too loaded to start it inside the launch deadline. Every attempt is in the cause with its
+ * own stderr, because Chrome's start-up noise reads the same in a healthy browser and a dead one:
+ * what tells them apart is whether it EXITED or was still starting when the deadline passed.
+ */
+export class CdpLaunchFailedError extends UltimateError {
+  constructor(input: {
+    readonly executable: string;
+    readonly attempts: readonly CdpLaunchAttempt[];
+  }) {
+    // Run by hand, a healthy Chrome prints `DevTools listening on ws://…` and a broken one prints
+    // why. A path that is not shell-inert is the operator's own CHROME_PATH — every candidate is.
+    const binary = isFixShellSafe(input.executable) ? input.executable : '"$CHROME_PATH"';
+    super({
+      code: 'X_CDP_LAUNCH_FAILED',
+      cause: `${renderCauseValue(input.executable)} did not announce a DevTools endpoint: it answered no DevTools call in ${String(input.attempts.length)} ${input.attempts.length === 1 ? 'launch' : 'launches, each on its own fresh profile'} — ${input.attempts.map(attemptText).join(' · ')}`,
+      fix: `${binary} ${HAND_FLAGS}`,
+      meta: { executable: input.executable, attempts: input.attempts },
+    });
+  }
+}
+
+/** A CDP call answered with an error frame, or the connection died under it. */
+export class CdpCallFailedError extends UltimateError {
+  constructor(input: { readonly method: string; readonly detail: string }) {
+    super({
+      code: 'X_CDP_CALL_FAILED',
+      cause: `the browser refused ${renderCauseValue(input.method)}: ${renderCauseValue(input.detail)}`,
+      fix: 'await page.evaluate(() => document.body.innerHTML)   # what the page had; a call that refuses the same way means the browser itself is gone, so read the launch above it',
+    });
+  }
+}
+
+/** How a call's target went away: its session detached, its target was destroyed, or it crashed. */
+export type CdpTargetGone = 'detached' | 'destroyed' | 'crashed';
+
+/** What the wire did between a call being sent and its deadline (`cdp-wire-watch.ts`). */
+export interface CdpTimeoutObservation {
+  /** The session the call was sent on; `null` for a browser-level call. */
+  readonly sessionId: string | null;
+  /** Frames that arrived and parsed after the call was sent — replies to other calls, and events. */
+  readonly framesArrived: number;
+  /** The last of those, oldest first, bounded: `reply <id>`, or `<method> @<session>`. */
+  readonly lastFrames: readonly string[];
+  /** Frames that arrived after the call and could not be parsed. One of them may be its reply. */
+  readonly framesDropped: number;
+  readonly transportOpen: boolean;
+  /** `detached`, `destroyed` or `crashed` when the call's target went away; `null` while it is there. */
+  readonly targetGone: CdpTargetGone | null;
+  /** Main-frame navigations on the call's session since it was sent. */
+  readonly navigations: number;
+}
+
+/** The one-word verdict a reader — or a script over `--json` — branches on. */
+export type CdpTimeoutReading = 'target-gone' | 'lost-in-transport' | 'no-answer';
+
+export const readTimeout = (seen: CdpTimeoutObservation): CdpTimeoutReading =>
+  seen.targetGone !== null
+    ? 'target-gone'
+    : seen.framesDropped > 0
+      ? 'lost-in-transport'
+      : 'no-answer';
+
+const READING: Record<CdpTimeoutReading, (seen: CdpTimeoutObservation) => string> = {
+  'target-gone': (seen) =>
+    `its target is gone (${String(seen.targetGone)}), so nothing was left to answer`,
+  'lost-in-transport': (seen) =>
+    `${String(seen.framesDropped)} frame(s) arrived unparseable after it was sent and were dropped, so its reply may have been lost in transport (${String(seen.framesArrived)} other frame(s) arrived intact)`,
+  'no-answer': (seen) => {
+    if (seen.framesArrived === 0) {
+      return `nothing at all arrived after the call was sent, on a transport that is ${seen.transportOpen ? 'still open' : 'closed'} — the whole browser went silent`;
+    }
+    const moved =
+      seen.navigations === 0
+        ? ''
+        : `; the page navigated ${String(seen.navigations)} time(s) since the call`;
+    return `the target did not answer: ${String(seen.framesArrived)} other frame(s) arrived intact after the call, the last ${renderCauseValue(seen.lastFrames)}${moved}`;
+  },
+};
+
+/**
+ * A call that never answered. Its own code rather than `X_TIMEOUT`, because the actionable half is
+ * WHICH call: a hung `Page.navigate` is an app that never finishes responding, and a hung
+ * `Runtime.evaluate` is an expression that never settles.
+ *
+ * `observed` is what the connection saw meanwhile, and it is what tells three failures apart that
+ * all read "did not answer": the target went away, the reply was dropped on the wire, or the target
+ * was there and silent. A deadline the connection did not keep (`waitFor`, the handshake) has none.
+ */
+export class CdpTimeoutError extends UltimateError {
+  constructor(input: {
+    readonly method: string;
+    readonly timeoutMs: number;
+    readonly observed?: CdpTimeoutObservation | undefined;
+  }) {
+    const { observed } = input;
+    const base = `${renderCauseValue(input.method)} did not answer inside ${String(input.timeoutMs)}ms`;
+    const reading = observed === undefined ? undefined : readTimeout(observed);
+    super({
+      code: 'X_CDP_TIMEOUT',
+      cause:
+        observed === undefined || reading === undefined
+          ? base
+          : `${base} — ${READING[reading](observed)}`,
+      fix:
+        reading === undefined || reading === 'no-answer'
+          ? 'installE2eDriver({ page, baseUrl, timeoutMs: 60_000 })   # a larger deadline — or find the request the page still waits on: a navigation that never settles is an app that never finishes its response'
+          : // `--json`: the rerun's `meta.reading` and `meta.framesDropped` are only printed there.
+            'x test e2e --json',
+      ...(observed === undefined
+        ? {}
+        : { meta: { method: input.method, timeoutMs: input.timeoutMs, reading, ...observed } }),
+    });
+  }
+}