@ultimat3/scraping 22.15.0 → 24.0.0

package/src/cdp-fake-target.ts

@@ -0,0 +1,67 @@
+// The browser TARGET of `cdp-fake.ts`'s browser: one raw session that records what the driver
+// enabled and answers `Fetch.requestPaused` the way Chrome does — a request from a target this
+// package never created (a popup) pauses here and nowhere else. Split out because `cdp-fake.ts`
+// stands near its size ceiling.
+
+import type { CdpBrowserSessionLike, CdpBrowserTargetLike } from './cdp-port';
+
+export interface FakeBrowserTarget {
+  readonly target: CdpBrowserTargetLike;
+  /** A request from another target — a popup — pausing at the browser-level `Fetch` domain. */
+  emitTargetRequest(url: string, resourceType: string, method?: string): void;
+  /** What browser-level interception decided, and whether it was enabled at all. */
+  readonly browserFetch: {
+    readonly enabled: boolean;
+    readonly continued: readonly string[];
+    readonly failed: readonly string[];
+  };
+}
+
+/** `refuseEnable`: the browser answers `Fetch.enable` with a protocol error, as a locked-down provider might. */
+export function fakeBrowserTarget(
+  options: { readonly refuseEnable?: boolean } = {},
+): FakeBrowserTarget {
+  const listeners: ((payload: unknown) => void)[] = [];
+  const byId = new Map<string, string>();
+  const continued: string[] = [];
+  const failed: string[] = [];
+  let enabled = false;
+  const session: CdpBrowserSessionLike = {
+    send: (method, params) => {
+      if (method === 'Fetch.enable') {
+        if (options.refuseEnable === true) {
+          return Promise.reject(new Error("'Fetch.enable' wasn't found"));
+        }
+        enabled = true;
+      }
+      const id = typeof params?.['requestId'] === 'string' ? params['requestId'] : '';
+      if (method === 'Fetch.continueRequest') continued.push(byId.get(id) ?? id);
+      if (method === 'Fetch.failRequest') failed.push(byId.get(id) ?? id);
+      return Promise.resolve({});
+    },
+    detach: () => Promise.resolve(),
+    on: (event, handler) => {
+      if (event === 'Fetch.requestPaused') listeners.push(handler);
+      return undefined;
+    },
+  };
+  return {
+    target: { createCDPSession: () => Promise.resolve(session) },
+    emitTargetRequest(url, resourceType, method = 'GET'): void {
+      // Paused only while enabled, as in Chrome: before `Fetch.enable` the request just leaves.
+      if (!enabled) return;
+      const requestId = `interception-${String(byId.size + 1)}`;
+      byId.set(requestId, url);
+      for (const listener of listeners) {
+        listener({ requestId, resourceType, request: { url, method } });
+      }
+    },
+    browserFetch: {
+      get enabled(): boolean {
+        return enabled;
+      },
+      continued,
+      failed,
+    },
+  };
+}

package/src/cdp-fake.ts

@@ -8,6 +8,8 @@
 //
 // Precedent: `packages/storage/src/driver-s3-fixture.ts` ships the same way.
 
+import type { FakeBrowserTarget } from './cdp-fake-target';
+import { fakeBrowserTarget } from './cdp-fake-target';
 import type {
   CdpBrowserLike,
   CdpFrameLike,
@@ -72,7 +74,9 @@ export interface FakeCdpPageInit {
 
 type Handlers = Map<string, ((payload: unknown) => void)[]>;
 
-export interface FakeCdpBrowser extends CdpBrowserLike {
+export interface FakeCdpBrowser
+  extends CdpBrowserLike,
+    Pick<FakeBrowserTarget, 'emitTargetRequest' | 'browserFetch'> {
   /** Fire a request event, as a real browser would when the page fetches a subresource. */
   emitRequest(url: string, resourceType: string): void;
   /**
@@ -158,6 +162,7 @@ export function fakeCdpBrowser(init: FakeCdpPageInit): FakeCdpBrowser {
   const prepared: string[] = [];
   const sessions = { created: 0, detached: 0 };
   const axBySelector = init.accessibility ?? {};
+  const browserTarget = fakeBrowserTarget();
 
   /**
    * A raw session answering the four commands `cdp-a11y.ts` sends, in CDP's own wire shapes —
@@ -364,6 +369,9 @@ export function fakeCdpBrowser(init: FakeCdpPageInit): FakeCdpBrowser {
 
   return {
     newPage: () => Promise.resolve(page),
+    target: () => browserTarget.target,
+    emitTargetRequest: (...args) => browserTarget.emitTargetRequest(...args),
+    browserFetch: browserTarget.browserFetch,
     cookies: () => Promise.resolve(cookies),
     setCookie: (...next: readonly unknown[]) => {
       cookies = next as readonly ScrapeCookie[];

package/src/cdp-port.ts

@@ -143,6 +143,16 @@ export interface CdpPageLike {
    * reason, and refused by name when absent.
    */
   createCDPSession?(): Promise<CdpSessionLike>;
+  /**
+   * `page.authenticate(credentials)`: answers the PROXY's own auth challenge. It is the only way a
+   * launched browser dials a credentialed exit — `--proxy-server` takes no userinfo, and one
+   * written there would sit in the process list. OPTIONAL for `keyboard`'s reason; a credentialed
+   * exit on a launcher without it is `X_SCRAPE_EGRESS_UNSUPPORTED`, never a direct connection.
+   */
+  authenticate?(credentials: {
+    readonly username: string;
+    readonly password: string;
+  }): Promise<void>;
   frames(): readonly CdpFrameLike[];
   close(): Promise<void>;
 }
@@ -157,8 +167,29 @@ export interface CdpFrameLike {
   select(selector: string, ...values: string[]): Promise<string[]>;
 }
 
+/**
+ * A raw session on the BROWSER target — `browser.target().createCDPSession()` under puppeteer —
+ * with the event half `CdpSessionLike` does not need: `Fetch.requestPaused` arrives as an event.
+ * `event` stays a bare `string` for `CdpPageLike.on`'s reason.
+ */
+export interface CdpBrowserSessionLike extends CdpSessionLike {
+  on(event: string, handler: (payload: unknown) => void): unknown;
+}
+
+/** The browser's own target — `browser.target()` under puppeteer. */
+export interface CdpBrowserTargetLike {
+  createCDPSession(): Promise<CdpBrowserSessionLike>;
+}
+
 export interface CdpBrowserLike {
   newPage(): Promise<CdpPageLike>;
+  /**
+   * REQUIRED, unlike the page's optional verbs: it is how `allowHosts` reaches every target the
+   * browser opens — a popup, a `target=_blank` link — and not only the one page this package
+   * created. Page-level interception never sees those, and a rule that holds on one tab of a
+   * browser a scraped site controls is advisory. A launcher without it does not compile.
+   */
+  target(): CdpBrowserTargetLike;
   /** Present on a browser that owns a cookie jar. Absent ones answer `X_NOT_IMPLEMENTED`. */
   cookies?(): Promise<unknown>;
   setCookie?(...cookies: readonly unknown[]): Promise<void>;

package/src/cdp-resolver.ts

@@ -0,0 +1,81 @@
+// A browser RENTED per session: `remoteBrowser({ cdpUrl })` takes a function instead of a string,
+// and what the function hands back is acquired once and released exactly once. The framework
+// ships no vendor — a provider wrapper is about twenty lines of app code that returns this shape.
+
+import { isUltimateError } from '@ultimat3/core';
+import type { MoneyValue } from '@ultimat3/schema';
+import { parse, t } from '@ultimat3/schema';
+import { cdpResolveFailed } from './error-throws';
+
+export interface CdpResolverRequest {
+  /** The scrape's name. */
+  readonly scrape: string;
+  /** The job run asking — what a provider session is tagged with. Absent outside a job. */
+  readonly runId: string | undefined;
+  /** The exit this run dials, when it declared one: rent the browser ON it. May carry credentials. */
+  readonly egress: string | undefined;
+  /** The run's cancellation. A rental still in flight when it aborts is the resolver's to cancel. */
+  readonly signal: AbortSignal | undefined;
+}
+
+export interface CdpResolution {
+  /** The connect URL. Usually carries the provider's access token: it is redacted, never printed. */
+  readonly cdpUrl: string;
+  /**
+   * Hand the browser back. Runs ONCE — in the session's `close()` on success, failure and abort,
+   * and straight away when the attach itself fails. A rejection is swallowed: it runs on the way
+   * out of a run whose own outcome is the one the reader needs.
+   */
+  release?(): Promise<void>;
+  /** What the rental cost, when the provider says so up front. `Money`, never a float. */
+  readonly cost?: MoneyValue | undefined;
+}
+
+export type CdpResolver = (request: CdpResolverRequest) => Promise<CdpResolution>;
+
+export interface AcquiredCdp {
+  readonly cdpUrl: string;
+  readonly cost: MoneyValue | undefined;
+  /** The resolution's `release`, latched: the second call does nothing. Never throws. */
+  release(): Promise<void>;
+}
+
+/** A fixed URL and a resolver, as one shape — so the driver has one attach path, not two. */
+export async function acquireCdp(
+  source: string | CdpResolver,
+  request: CdpResolverRequest,
+): Promise<AcquiredCdp> {
+  if (typeof source === 'string') {
+    return { cdpUrl: source, cost: undefined, release: () => Promise.resolve() };
+  }
+  let resolution: CdpResolution;
+  try {
+    resolution = await source(request);
+  } catch (thrown) {
+    // The resolver's own coded failure keeps its code and its classification; anything else is
+    // "no browser to attach to", which is this package's retryable attach failure.
+    throw isUltimateError(thrown) ? thrown : cdpResolveFailed(request.scrape, thrown);
+  }
+  let released = false;
+  const release = async (): Promise<void> => {
+    if (released) return;
+    released = true;
+    try {
+      await resolution.release?.();
+    } catch {
+      // See `CdpResolution.release`: a failed hand-back never replaces the run's own outcome.
+    }
+  };
+  try {
+    return {
+      cdpUrl: resolution.cdpUrl,
+      // Parsed, not trusted: the type says `minor: number` and `0.35` satisfies it. A float cost
+      // is refused here, before it is added to anything.
+      cost: resolution.cost === undefined ? undefined : parse(t.money, resolution.cost),
+      release,
+    };
+  } catch (thrown) {
+    await release();
+    throw thrown;
+  }
+}

package/src/cdp-target.ts

@@ -1,13 +1,13 @@
 // `ScrapeTarget` over a real browser, through the structural CDP port. Everything driver-specific
 // in this package lives here and in `driver-cdp.ts`; the vocabulary above it does not change.
 
-import { isUltimateError } from '@ultimat3/core';
+import { isUltimateError, NotImplementedError } from '@ultimat3/core';
 import type { StandardSchemaV1 } from '@ultimat3/schema';
 import { parse, t } from '@ultimat3/schema';
 import { browserRecord } from './browser-record';
 import { axNodesFor } from './cdp-a11y';
 import type { CdpArmInit } from './cdp-arm';
-import { arm } from './cdp-arm';
+import { arm, armBrowser } from './cdp-arm';
 import type { CdpBrowserLike, CdpFrameLike } from './cdp-port';
 import {
   clearExpression,
@@ -16,7 +16,7 @@ import {
   snapshotExpression,
 } from './cdp-snapshot';
 import { type ColorScheme, colorSchemeFeatures } from './color-scheme';
-import { browserUnreachable, pageCrashed, scrapeNotImplemented } from './error-throws';
+import { browserUnreachable, pageCrashed } from './error-throws';
 import { parseKeyChord } from './key-chord';
 import type { ConsoleLine, NetworkEntry, PageError } from './rings';
 import { createRing } from './rings';
@@ -54,7 +54,7 @@ const cookieSchema = t.array(
  *
  * | Code | Why a retry cannot change it |
  * |---|---|
- * | `X_NOT_IMPLEMENTED` | a fact about the launcher's own shape. A browser cannot produce it; only this file's `scrapeNotImplemented()` can, and the method is still missing on attempt five |
+ * | `X_NOT_IMPLEMENTED` | a fact about the launcher's own shape. A browser cannot produce it; only this file's own `NotImplementedError` refusals can, and the method is still missing on attempt five |
  * | `X_VALIDATION_FAILED` | the browser ANSWERED, and the answer did not match the shape this driver reads it with. The page is what it is; attempt five reads the same DOM |
  *
  * Both were re-labelled `X_SCRAPE_BROWSER_UNREACHABLE`, which `errors.ts` registers `retryable`,
@@ -99,6 +99,7 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
   const pageErrors = createRing<PageError>();
   const crashed: { value: string | undefined } = { value: undefined };
   await arm(init, { network, console: console_, pageErrors, crashed });
+  await armBrowser(init, network);
   let pendingStorage: SessionSnapshot | undefined;
 
   const originOf = (url: string): string => {
@@ -183,10 +184,11 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
     // rather than answered from the PARENT document, which is what the spread alone would do.
     accessibility: (_selector, _max): Promise<readonly AxNode[]> =>
       Promise.reject(
-        scrapeNotImplemented(
-          'accessibility() on a frame of the puppeteer driver',
-          'read the accessibility tree through the page — page.accessibility(selector) — which covers the top-level document only',
-        ),
+        new NotImplementedError({
+          cause:
+            'accessibility() on a frame of the puppeteer driver is not implemented by this driver',
+          fix: 'read the accessibility tree through the page — page.accessibility(selector) — which covers the top-level document only',
+        }),
       ),
   });
 
@@ -228,10 +230,10 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
       await guard('press', async () => {
         const keyboard = init.page.keyboard;
         if (keyboard === undefined) {
-          throw scrapeNotImplemented(
-            'press() on a CDP page with no keyboard',
-            'upgrade the launcher to a puppeteer-core that exposes page.keyboard, or dispatch the key from the app under test instead of the browser',
-          );
+          throw new NotImplementedError({
+            cause: 'press() on a CDP page with no keyboard is not implemented by this driver',
+            fix: 'upgrade the launcher to a puppeteer-core that exposes page.keyboard, or dispatch the key from the app under test instead of the browser',
+          });
         }
         // Down in the order written, the key, then up in REVERSE — the order a hand releases
         // them, and the order the browser's own `press` with modifiers performs. `.call`-free:
@@ -248,10 +250,10 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
       guard('focus', async () => {
         const focus = init.page.focus;
         if (typeof focus !== 'function') {
-          throw scrapeNotImplemented(
-            'focus() on a CDP page with no focus() method',
-            'upgrade the launcher to a puppeteer-core that exposes page.focus(), or click the element instead',
-          );
+          throw new NotImplementedError({
+            cause: 'focus() on a CDP page with no focus() method is not implemented by this driver',
+            fix: 'upgrade the launcher to a puppeteer-core that exposes page.focus(), or click the element instead',
+          });
         }
         // `.call`, for `setColorScheme`'s reason: the member is read off the object.
         await focus.call(init.page, selector);
@@ -260,10 +262,11 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
       guard('accessibility', async () => {
         const createSession = init.page.createCDPSession;
         if (typeof createSession !== 'function') {
-          throw scrapeNotImplemented(
-            'accessibility() on a CDP page with no createCDPSession() method',
-            'upgrade the launcher to a puppeteer-core that exposes page.createCDPSession(), or assert on the markup with page.query() — which reads attributes, not what the browser computed',
-          );
+          throw new NotImplementedError({
+            cause:
+              'accessibility() on a CDP page with no createCDPSession() method is not implemented by this driver',
+            fix: 'upgrade the launcher to a puppeteer-core that exposes page.createCDPSession(), or assert on the markup with page.query() — which reads attributes, not what the browser computed',
+          });
         }
         const session = await createSession.call(init.page);
         try {
@@ -276,10 +279,11 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
       guard('setOfflineMode', async () => {
         const source = init.page as { setOfflineMode?: (value: boolean) => Promise<void> };
         if (typeof source.setOfflineMode !== 'function') {
-          throw scrapeNotImplemented(
-            'setOfflineMode() on a CDP page with no setOfflineMode() method',
-            'upgrade the launcher to a puppeteer-core that exposes page.setOfflineMode(), or drive the condition from the app under test instead of the browser',
-          );
+          throw new NotImplementedError({
+            cause:
+              'setOfflineMode() on a CDP page with no setOfflineMode() method is not implemented by this driver',
+            fix: 'upgrade the launcher to a puppeteer-core that exposes page.setOfflineMode(), or drive the condition from the app under test instead of the browser',
+          });
         }
         await source.setOfflineMode(enabled);
       }),
@@ -291,10 +295,11 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
         // older spelling.
         const emulate = init.page.emulateMediaFeatures;
         if (typeof emulate !== 'function') {
-          throw scrapeNotImplemented(
-            'setColorScheme() on a CDP page with no emulateMediaFeatures() method',
-            'upgrade the launcher to a puppeteer-core that exposes page.emulateMediaFeatures(), or have the page under test set its own theme',
-          );
+          throw new NotImplementedError({
+            cause:
+              'setColorScheme() on a CDP page with no emulateMediaFeatures() method is not implemented by this driver',
+            fix: 'upgrade the launcher to a puppeteer-core that exposes page.emulateMediaFeatures(), or have the page under test set its own theme',
+          });
         }
         // `.call`, because the member is read off the object: an unbound method loses `this` and
         // puppeteer's own `Page` needs it.
@@ -305,10 +310,11 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
         // Read off the object and called through it, for `setColorScheme`'s reason.
         const evaluateOnNewDocument = init.page.evaluateOnNewDocument;
         if (typeof evaluateOnNewDocument !== 'function') {
-          throw scrapeNotImplemented(
-            'prepare() on a CDP page with no evaluateOnNewDocument() method',
-            'upgrade the launcher to a puppeteer-core that exposes page.evaluateOnNewDocument(), or seed the state from the app under test instead of the browser',
-          );
+          throw new NotImplementedError({
+            cause:
+              'prepare() on a CDP page with no evaluateOnNewDocument() method is not implemented by this driver',
+            fix: 'upgrade the launcher to a puppeteer-core that exposes page.evaluateOnNewDocument(), or seed the state from the app under test instead of the browser',
+          });
         }
         await evaluateOnNewDocument.call(init.page, expression);
       }),
@@ -333,10 +339,11 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
       guard('cookies', async () => {
         const source = init.browser as { cookies?: () => Promise<unknown> };
         if (typeof source.cookies !== 'function') {
-          throw scrapeNotImplemented(
-            'cookies() on a CDP browser with no cookies() method',
-            'upgrade the launcher to a puppeteer-core that exposes browser.cookies(), or read them with page.evaluate("document.cookie")',
-          );
+          throw new NotImplementedError({
+            cause:
+              'cookies() on a CDP browser with no cookies() method is not implemented by this driver',
+            fix: 'upgrade the launcher to a puppeteer-core that exposes browser.cookies(), or read them with page.evaluate("document.cookie")',
+          });
         }
         return parse(cookieSchema, await source.cookies());
       }),
@@ -349,10 +356,10 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
     // `try` — a synchronous `throw` jumps over the first of those entirely.
     download: (_options): Promise<ScrapeDownloadFile> =>
       Promise.reject(
-        scrapeNotImplemented(
-          'download() on the puppeteer driver',
-          'fetch the file inside the page — page.evaluate("fetch(url).then(r => r.text())") — or run this scrape on fixtureBrowser(), whose download() is complete',
-        ),
+        new NotImplementedError({
+          cause: 'download() on the puppeteer driver is not implemented by this driver',
+          fix: 'fetch the file inside the page — page.evaluate("fetch(url).then(r => r.text())") — or run this scrape on fixtureBrowser(), whose download() is complete',
+        }),
       ),
     frames: () =>
       guard('frames', () =>
@@ -378,10 +385,19 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
         } catch {
           origin = '';
         }
-        const source = init.browser as { cookies?: () => Promise<unknown> };
+        // Refused, never `[]`: the HTTP leg reads its cookies from here on every request and
+        // `persistSession` stores what this answers, so an empty jar went out logged-out and was
+        // saved as the session — both silently, while every other missing port method refuses.
+        if (typeof init.browser.cookies !== 'function') {
+          throw new NotImplementedError({
+            cause:
+              'session() on a CDP browser with no cookies() method is not implemented by this driver',
+            fix: 'upgrade the launcher to a puppeteer-core that exposes browser.cookies() — the HTTP leg and a persisted session both read the jar through it',
+          });
+        }
+        const jar = parse(cookieSchema, await init.browser.cookies());
         return {
-          cookies:
-            typeof source.cookies === 'function' ? parse(cookieSchema, await source.cookies()) : [],
+          cookies: jar,
           headers: {},
           // `browserRecord` and not `t.record()`: a page may legitimately hold a key called
           // `constructor`, and the schema refuses that name outright — see `browser-record.ts`.
@@ -394,11 +410,17 @@ export async function cdpTarget(init: CdpTargetInit): Promise<ScrapeTarget> {
       guard('restore', async () => {
         // Two halves, because they belong to two different moments: cookies are the browser's and
         // can be put back now, storage is an ORIGIN's and cannot exist until one is loaded.
-        const source = init.browser as {
-          setCookie?: (...cookies: readonly unknown[]) => Promise<void>;
-        };
-        if (typeof source.setCookie === 'function' && session.cookies.length > 0) {
-          await source.setCookie(...session.cookies);
+        if (session.cookies.length > 0) {
+          // Refused rather than skipped: a restore that dropped the cookies opened the session
+          // logged-out while the run believed it restored one.
+          if (typeof init.browser.setCookie !== 'function') {
+            throw new NotImplementedError({
+              cause:
+                'restore() of cookies on a CDP browser with no setCookie() method is not implemented by this driver',
+              fix: 'upgrade the launcher to a puppeteer-core that exposes browser.setCookie(), or set auth: { reuse: false } on the scrape() definition so no session is restored',
+            });
+          }
+          await init.browser.setCookie(...session.cookies);
         }
         pendingStorage = Object.keys(session.storage).length > 0 ? session : undefined;
         await applyPendingStorage();

package/src/clock.ts

@@ -48,6 +48,40 @@ export const systemScrapeClock: ScrapeClock = Object.freeze({
   },
 });
 
+/**
+ * The system clock with the waiting taken out: `now()` and `monotonic()` are real, and every
+ * `sleep` is ONE turn of the event loop whatever was asked for. For a test whose run waits on
+ * something that really happens in the same process — an answer the test publishes, a job another
+ * worker holds — where `testClock()`'s "sleeping is advancing" would run a five-minute budget out
+ * in microtasks before the test's next line. A deadline is still a real one.
+ */
+export const noWaitClock: ScrapeClock = Object.freeze({
+  now: () => systemScrapeClock.now(),
+  monotonic: () => systemScrapeClock.monotonic(),
+  sleep: (_ms: number, signal?: AbortSignal) => systemScrapeClock.sleep(0, signal),
+});
+
+let ambient: ScrapeClock | undefined;
+
+/**
+ * The clock every run in this process waits on, unless its definition pins one — the counterpart
+ * of `setScrapeDriver()`, and the seam a TEST takes: `setScrapeClock(noWaitClock)` in a
+ * `beforeEach`, and a declaration under test needs no `clock:` field written for the test's sake.
+ */
+export function setScrapeClock(clock: ScrapeClock): void {
+  ambient = clock;
+}
+
+/** The process's scrape clock: the installed one, or the system's. */
+export function scrapeClock(): ScrapeClock {
+  return ambient ?? systemScrapeClock;
+}
+
+/** Test seam: back to the system clock, so the next file runs on real time again. */
+export function resetScrapeClock(): void {
+  ambient = undefined;
+}
+
 /** The abort reason, unwrapped — a caller's `AbortSignal.reason` is whatever they put there. */
 export function throwIfAborted(signal: AbortSignal): void {
   if (signal.aborted) throw signal.reason;