@ultimat3/scraping 30.8.0 → 31.0.0

package/src/pdf-render.ts

@@ -0,0 +1,241 @@
+// One page printed to PDF over raw CDP: open a target, fence its requests to the route's origin,
+// navigate, wait for the load event and the fonts, refuse an error page, `Page.printToPDF`. The
+// browser is `cdp-launch.ts`'s — one launch per render, at most `maxConcurrent` alive, killed when
+// the request is aborted, closed in a `finally` — and injectable, so `bun test` needs no Chrome.
+
+import type { SleepClock } from '@ultimat3/core';
+import { ambientClock, finiteCount, stringField } from '@ultimat3/core';
+import type { CdpConnection } from './cdp-connection';
+import { CdpCallFailedError, CdpTimeoutError } from './cdp-errors';
+import type { FoundChromeOptions } from './cdp-launch';
+import { LAUNCH_TIMEOUT_MS, launchFoundChrome } from './cdp-launch';
+import { pdfRouteFailed } from './error-throws-pdf';
+import { printToPdfParams } from './pdf-cdp-params';
+import { raisePdfFence } from './pdf-fence';
+import type { PdfLayout } from './pdf-request';
+import { pdfSlots } from './pdf-slots';
+
+/** One render's every deadline: the launch's first answer, each CDP call, the load event. */
+export const DEFAULT_PDF_TIMEOUT_MS = 30_000;
+
+/**
+ * Sequential deadlines one render spends after the launch: create, attach, `Page.enable`, the two
+ * header calls, the load, the settled read, the print, the close — and one spare.
+ */
+export const PDF_RENDER_DEADLINES = 10;
+
+/** The launched browser's OS-level bound: a cold start, then every deadline a render spends. */
+export const pdfBrowserLifetimeMs = (timeoutMs: number): number =>
+  Math.max(timeoutMs, LAUNCH_TIMEOUT_MS) + PDF_RENDER_DEADLINES * timeoutMs;
+
+/**
+ * Live print browsers per renderer, by default. Each Chrome holds 150-300 MB; two keeps a burst of
+ * prints (or a retry storm) from taking the web role's memory with it, and queues the rest.
+ */
+export const DEFAULT_PDF_MAX_CONCURRENT = 2;
+
+/**
+ * How a production render launches. The SANDBOX STAYS ON: a print renders the app's own pages, and
+ * those carry user content — the sandbox is what stands between a renderer exploit and the web
+ * role. The test launcher's `--no-sandbox` is for CI runners; the framework's runtime image is
+ * distroless and ships no Chrome, so the app's own image supplies it and runs it as non-root, where
+ * Chrome's namespace sandbox needs unprivileged user namespaces (or its setuid helper). A container
+ * that cannot grant them fails the launch with `X_CDP_LAUNCH_FAILED` quoting Chrome's "No usable
+ * sandbox"; that operator opts out in one visible line, `chromePdfRenderer({ sandbox: false })`.
+ */
+export const pdfLaunchOptions = (timeoutMs: number, sandbox = true): FoundChromeOptions => ({
+  lifetimeMs: pdfBrowserLifetimeMs(timeoutMs),
+  sandbox,
+});
+
+export interface PdfRenderRequest {
+  /** Absolute. `pdf()` builds it from the app's own origin and a route path, never a caller's URL. */
+  readonly url: string;
+  readonly layout: PdfLayout;
+  /**
+   * Added to the requests the page makes to the ROUTE'S OWN ORIGIN — how a private route learns
+   * who is printing it. Never sent anywhere else: every other origin is blocked (`pdf-fence.ts`).
+   */
+  readonly headers?: Readonly<Record<string, string>> | undefined;
+  /** The request's cancellation: aborted, a queued render leaves the queue, a running one its browser. */
+  readonly signal?: AbortSignal | undefined;
+}
+
+/** The seam `pdf()` prints through. `chromePdfRenderer()` is the one this package ships. */
+export interface PdfRenderer {
+  render(request: PdfRenderRequest): Promise<Uint8Array>;
+}
+
+/** A browser one render owns: its connection, and the close that ends it. `LaunchedBrowser` fits. */
+export interface PdfBrowser {
+  readonly connection: CdpConnection;
+  close(): Promise<void>;
+}
+
+export interface ChromePdfRendererOptions {
+  /** Defaults to `launchFoundChrome` with `pdfLaunchOptions`: `CHROME_PATH`, then the known paths. */
+  readonly launch?: (() => Promise<PdfBrowser>) | undefined;
+  readonly timeoutMs?: number | undefined;
+  /** Browsers alive at once. Default `DEFAULT_PDF_MAX_CONCURRENT`. */
+  readonly maxConcurrent?: number | undefined;
+  /** How long a render waits for a slot before `X_SCRAPE_PDF_BUSY`. Default `timeoutMs`. */
+  readonly queueTimeoutMs?: number | undefined;
+  /** What the queue waits on. Default `ambientClock()`, read per wait. */
+  readonly clock?: SleepClock | undefined;
+  /** Chrome's sandbox for the default launch — on unless the container cannot grant it. */
+  readonly sandbox?: boolean | undefined;
+}
+
+/**
+ * Read in the page once it loaded: the fonts settled (a PDF printed before a webfont arrives prints
+ * the fallback face), then the document's own HTTP status — `0` where the browser cannot say.
+ */
+const SETTLED_STATUS = `(async () => {
+  await document.fonts.ready;
+  const entry = performance.getEntriesByType('navigation')[0];
+  return entry && typeof entry.responseStatus === 'number' ? entry.responseStatus : 0;
+})()`;
+
+const required = (value: string | undefined, method: string, field: string): string => {
+  if (value === undefined || value === '') {
+    throw new CdpCallFailedError({ method, detail: `the browser answered no ${field}` });
+  }
+  return value;
+};
+
+/** Navigate and wait for `load` — the event, not the reply, which Chrome drops on a process swap. */
+async function navigate(
+  connection: CdpConnection,
+  sessionId: string,
+  url: string,
+  timeoutMs: number,
+): Promise<void> {
+  const loaded = connection.once('Page.loadEventFired', sessionId, timeoutMs);
+  const refused = connection.send('Page.navigate', { url }, sessionId).then(
+    (answer) => {
+      const failed = answer.result?.['errorText'];
+      return typeof failed === 'string' && failed !== '' ? failed : undefined;
+    },
+    () => undefined,
+  );
+  const failed = await Promise.race([loaded.then(() => undefined), refused]);
+  if (failed !== undefined) {
+    throw new CdpCallFailedError({ method: `Page.navigate to ${url}`, detail: failed });
+  }
+  if (!(await loaded)) throw new CdpTimeoutError({ method: 'Page.loadEventFired', timeoutMs });
+}
+
+async function printOnce(
+  connection: CdpConnection,
+  request: PdfRenderRequest,
+  timeoutMs: number,
+): Promise<Uint8Array> {
+  const created = await connection.send('Target.createTarget', { url: 'about:blank' });
+  const targetId = required(
+    stringField(created.result, 'targetId'),
+    'Target.createTarget',
+    'targetId',
+  );
+  try {
+    const attached = await connection.send('Target.attachToTarget', { targetId, flatten: true });
+    const sessionId = required(
+      stringField(attached.result, 'sessionId'),
+      'Target.attachToTarget',
+      'sessionId',
+    );
+    await connection.send('Page.enable', {}, sessionId);
+    await raisePdfFence({
+      connection,
+      sessionId,
+      origin: new URL(request.url).origin,
+      headers: request.headers,
+    });
+    await navigate(connection, sessionId, request.url, timeoutMs);
+    const settled = await connection.send(
+      'Runtime.evaluate',
+      { expression: SETTLED_STATUS, awaitPromise: true, returnByValue: true },
+      sessionId,
+    );
+    const inner = settled.result?.['result'];
+    const status =
+      typeof inner === 'object' && inner !== null && 'value' in inner ? inner.value : 0;
+    if (typeof status === 'number' && status >= 400) throw pdfRouteFailed(request.url, status);
+    const printed = await connection.send(
+      'Page.printToPDF',
+      printToPdfParams(request.layout),
+      sessionId,
+    );
+    return Uint8Array.fromBase64(
+      required(stringField(printed.result, 'data'), 'Page.printToPDF', 'data'),
+    );
+  } finally {
+    await connection.send('Target.closeTarget', { targetId }).catch(() => undefined);
+  }
+}
+
+/** One render on a browser this call owns: killed the moment the request is aborted. */
+async function renderOn(
+  browser: PdfBrowser,
+  request: PdfRenderRequest,
+  timeoutMs: number,
+): Promise<Uint8Array> {
+  const { signal } = request;
+  const kill = (): void => {
+    void browser.close();
+  };
+  signal?.addEventListener('abort', kill, { once: true });
+  try {
+    signal?.throwIfAborted();
+    return await printOnce(browser.connection, request, timeoutMs);
+  } catch (error) {
+    // A call cut off by the kill reports the dead socket; the reason the request was dropped is
+    // the truth, and is what the caller's own catch expects.
+    throw signal?.aborted === true ? signal.reason : error;
+  } finally {
+    signal?.removeEventListener('abort', kill);
+    await browser.close();
+  }
+}
+
+/**
+ * Prints through a Chrome launched for the one render and closed after it, success or not: no
+ * browser outlives a request, and nothing is shared between two tenants' documents. At most
+ * `maxConcurrent` are alive; the rest queue, bounded. A launch costs a second or two; a pool is the
+ * measured next step, not a guess (axiom 9).
+ */
+export function chromePdfRenderer(options: ChromePdfRendererOptions = {}): PdfRenderer {
+  const timeoutMs = finiteCount(
+    'chromePdfRenderer',
+    'timeoutMs',
+    options.timeoutMs ?? DEFAULT_PDF_TIMEOUT_MS,
+    1,
+  );
+  const slots = pdfSlots({
+    maxConcurrent: finiteCount(
+      'chromePdfRenderer',
+      'maxConcurrent',
+      options.maxConcurrent ?? DEFAULT_PDF_MAX_CONCURRENT,
+      1,
+    ),
+    queueTimeoutMs: finiteCount(
+      'chromePdfRenderer',
+      'queueTimeoutMs',
+      options.queueTimeoutMs ?? timeoutMs,
+      0,
+    ),
+    clock: () => options.clock ?? ambientClock(),
+  });
+  const launch =
+    options.launch ??
+    (() => launchFoundChrome(process.env, timeoutMs, pdfLaunchOptions(timeoutMs, options.sandbox)));
+  return {
+    async render(request) {
+      const release = await slots.acquire(request.signal);
+      try {
+        return await renderOn(await launch(), request, timeoutMs);
+      } finally {
+        release();
+      }
+    },
+  };
+}

package/src/pdf-slots.ts

@@ -0,0 +1,68 @@
+// The bound on live print browsers: a counting semaphore, first come first served. A queued render
+// waits on the CLOCK (never a timer of its own — `clock-discipline.test.ts`) for a bounded time and
+// then refuses with a code; an aborted request leaves the queue at once and takes no slot.
+
+import type { SleepClock } from '@ultimat3/core';
+import { pdfBusy } from './error-throws-pdf';
+
+export interface PdfSlotsOptions {
+  readonly maxConcurrent: number;
+  readonly queueTimeoutMs: number;
+  /** Read per wait, so an `installClock` made after the renderer still governs it. */
+  readonly clock: () => SleepClock;
+}
+
+export interface PdfSlots {
+  /** Answers the release — idempotent — once a slot is this caller's. */
+  acquire(signal?: AbortSignal): Promise<() => void>;
+}
+
+interface Waiter {
+  readonly grant: () => void;
+}
+
+export function pdfSlots(options: PdfSlotsOptions): PdfSlots {
+  let busy = 0;
+  const queue: Waiter[] = [];
+  const releaser = (): (() => void) => {
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      const next = queue.shift();
+      // The slot passes straight to the next waiter: `busy` never dips, so no newcomer jumps it.
+      if (next === undefined) busy -= 1;
+      else next.grant();
+    };
+  };
+  return {
+    async acquire(signal) {
+      signal?.throwIfAborted();
+      if (busy < options.maxConcurrent) {
+        busy += 1;
+        return releaser();
+      }
+      const granted = new AbortController();
+      const waiter: Waiter = { grant: () => granted.abort() };
+      queue.push(waiter);
+      const leave = (): void => {
+        const at = queue.indexOf(waiter);
+        if (at !== -1) queue.splice(at, 1);
+      };
+      try {
+        const wake =
+          signal === undefined ? granted.signal : AbortSignal.any([granted.signal, signal]);
+        await options.clock().sleep(options.queueTimeoutMs, wake);
+      } catch (error) {
+        // Woken: either the slot is ours, or the caller gave up — the request's reason, verbatim.
+        if (granted.signal.aborted) return releaser();
+        leave();
+        throw signal?.aborted === true ? signal.reason : error;
+      }
+      // Slept the whole deadline. A grant racing the wake-up still owns the slot: hand it back.
+      if (granted.signal.aborted) return releaser();
+      leave();
+      throw pdfBusy(options);
+    },
+  };
+}

package/src/rate.ts

@@ -2,14 +2,14 @@
 // scraper that looks like a denial of service to the site it depends on, and the ban is charged
 // to whoever owns the IP — usually the whole fleet, not the one run.
 
-import type { ScrapeClock } from './clock';
+import type { SleepClock } from '@ultimat3/core';
 
 /** Deliberately slow. Raising it is a decision; there is no way to turn pacing off. */
 export const DEFAULT_NAVIGATION_RATE = 1;
 
 export type Pacer = (signal?: AbortSignal) => Promise<void>;
 
-export function scrapePacer(rate: number, clock: ScrapeClock): Pacer {
+export function scrapePacer(rate: number, clock: SleepClock): Pacer {
   const intervalMs = 1_000 / rate;
   let nextAt = 0;
   return async (signal?: AbortSignal): Promise<void> => {

package/src/scrape-run.ts

@@ -9,7 +9,7 @@
 // A step record saying "logged in" would be a checkpoint asserting something about a session that
 // may have expired an hour ago.
 
-import { finiteCount, finiteOption } from '@ultimat3/core';
+import { ambientClock, finiteCount, finiteOption } from '@ultimat3/core';
 import type { JobRunArgs } from '@ultimat3/jobs';
 import { parse } from '@ultimat3/schema';
 import { artifactWriter } from './artifacts';
@@ -22,7 +22,6 @@ import {
   persistSession,
   scrapePrompt,
 } from './auth';
-import { scrapeClock } from './clock';
 import type { ScrapeSession } from './driver';
 import { scrapeDriver } from './driver';
 import { driverUnknown, outputInvalid } from './error-throws';
@@ -88,7 +87,7 @@ export async function runScrape<I, Row>(
   definition: ScrapeDefinition<I, Row>,
   args: JobRunArgs<I>,
 ): Promise<ScrapeReport<Row>> {
-  const clock = definition.clock ?? scrapeClock();
+  const clock = definition.clock ?? ambientClock();
   const driver = definition.driver ?? scrapeDriver();
   // The scrape's name goes in the SCRAPE slot: there is no driver here to name, which is the
   // whole failure.
@@ -342,7 +341,7 @@ async function bodyWithRecovery<I, Row>(
     });
   try {
     return await withStepEvent(
-      { name: 'body', logger, clock: definition.clock ?? scrapeClock(), attempt: args.attempt },
+      { name: 'body', logger, clock: definition.clock ?? ambientClock(), attempt: args.attempt },
       body,
     );
   } catch (thrown) {

package/src/scrape.ts

@@ -9,7 +9,7 @@
 // the account. So this file is a FACTORY over `job()`, and a scrape inherits `.enqueue()`, the
 // worker's cancellation, the dead-letter path, `x jobs show` and its manifest row for free.
 
-import type { Ctx, HostRule } from '@ultimat3/core';
+import type { Ctx, HostRule, SleepClock } from '@ultimat3/core';
 import { assert } from '@ultimat3/core';
 import type {
   JobCompleted,
@@ -26,7 +26,6 @@ import type { StandardSchemaV1 } from '@ultimat3/schema';
 import type { StorageDriver } from '@ultimat3/storage';
 import type { ArtifactWriter } from './artifacts';
 import type { PromptHandler, ScrapeAuth } from './auth';
-import type { ScrapeClock } from './clock';
 import type { ScrapeDriver } from './driver';
 import { yieldHistoryMissing } from './error-throws';
 import type { YieldExpectation, YieldHistory } from './expect';
@@ -140,10 +139,11 @@ export interface ScrapeDefinition<I, Row> {
   readonly concurrency?: JobConcurrency<I> | undefined;
   readonly queue?: string | undefined;
   /**
-   * Pins this definition's clock. Omit it: a run waits on the process's (`scrapeClock()`), which
-   * is the system clock and what a test replaces with `setScrapeClock()`.
+   * Pins this definition's clock. Omit it: a run waits on the process's (`ambientClock()`), which
+   * is the system clock unless `clock` in `apps/<app>/runtime.ts` (or a test's
+   * `installClock`, `@ultimat3/core/adapter`) replaced it.
    */
-  readonly clock?: ScrapeClock | undefined;
+  readonly clock?: SleepClock | undefined;
   run(args: ScrapeRunArgs<I>): Promise<readonly unknown[]>;
   /**
    * The job's own `onSettled`, with what a scrape adds: a `completed` run hands over its

package/src/target.ts

@@ -118,7 +118,7 @@ export interface GotoOptions {
  * so `page.screenshot({ timeout })` was a documented deadline that bounded nothing. Deleted
  * rather than implemented: the CDP port's own `screenshot({ fullPage })` has no timeout slot to
  * forward it to, and a deadline enforced in `page-over-target.ts` would have to race
- * `ScrapeClock.sleep`, which under `testScrapeClock` resolves on the first microtask and would time
+ * `SleepClock.sleep`, which under `frozenClock` resolves on the first microtask and would time
  * out every capture in every test. A driver's own default is the honest bound.
  */
 export type CaptureOptions = CaptureFraming;

package/src/usage.ts

@@ -2,8 +2,8 @@
 // and nothing else: no quota, no plan, no ledger — what a number here is worth, and what happens
 // when it is too high, is the app's (`docs/idea/19-mechanism-not-convention.md`).
 
+import type { SleepClock } from '@ultimat3/core';
 import type { MoneyValue } from '@ultimat3/schema';
-import type { ScrapeClock } from './clock';
 
 export interface ScrapeUsage {
   /** Milliseconds the browser session was held, from before `driver.open()` to this report. */
@@ -43,7 +43,7 @@ export interface RunUsageMeter extends UsageMeter {
 }
 
 /** Built BEFORE `driver.open()`, so the time a rented browser took to arrive is time it was held. */
-export function usageMeter(clock: ScrapeClock): RunUsageMeter {
+export function usageMeter(clock: SleepClock): RunUsageMeter {
   const startedAt = clock.monotonic();
   let navigations = 0;
   let httpRequests = 0;

package/src/watchdog.ts

@@ -10,8 +10,8 @@
 // socket die, which is what turns an infinite await into a catchable `X_SCRAPE_WEDGED` — and a
 // graceful-quit CEILING on shutdown, past which the same kill runs.
 
+import type { SleepClock } from '@ultimat3/core';
 import { finiteCount } from '@ultimat3/core';
-import type { ScrapeClock } from './clock';
 import { watchdogStopped, wedged } from './error-throws';
 
 export const DEFAULT_IDLE_MS = 120_000;
@@ -19,7 +19,7 @@ export const DEFAULT_GRACE_MS = 5_000;
 const WATCH_POLL_MS = 250;
 
 export interface WedgeGuardInit {
-  readonly clock: ScrapeClock;
+  readonly clock: SleepClock;
   /** Longest gap between two browser operations before the process is killed. */
   readonly idleMs?: number | undefined;
   /** How long a graceful quit may take before the same kill runs. */
@@ -129,7 +129,7 @@ export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
       }
     }
   };
-  // Never floating. `ScrapeClock` is a seam an app implements and `init.kill()` comes from a
+  // Never floating. `SleepClock` is a seam an app implements and `init.kill()` comes from a
   // driver, so this loop can reject on somebody else's code — and a bare `void` turned that into
   // an unhandled rejection, a process-level event belonging to no run, with the guard silently
   // dead behind it. A guard that stopped measuring is incident #1 with nothing armed, so the run