@ultimat3/scraping 30.9.0 → 31.1.0

package/src/index.ts

@@ -14,7 +14,6 @@ export type {
   ScrapeAuth,
 } from './auth';
 export { burnSession, ensureAuthenticated, restorableSession, scrapePrompt } from './auth';
-export { browserRecord } from './browser-record';
 export type { CaptureClip, CaptureFraming } from './capture-clip';
 export { assertCaptureFraming } from './capture-clip';
 export type {
@@ -26,92 +25,46 @@ export type {
   CdpLauncherLike,
   CdpPageLike,
   CdpPdfOptions,
-  CdpRequestLike,
   CdpScreenshotOptions,
   CdpSessionLike,
 } from './cdp-port';
 export type { CdpResolution, CdpResolver, CdpResolverRequest } from './cdp-resolver';
 export { snapshotExpression } from './cdp-snapshot';
 export type { CdpTargetInit } from './cdp-target';
-export { CDP_DRIVER, cdpTarget } from './cdp-target';
-export type { Deadline, ScrapeClock, TestScrapeClock } from './clock';
-export {
-  deadline,
-  noWaitClock,
-  resetScrapeClock,
-  scrapeClock,
-  setScrapeClock,
-  systemScrapeClock,
-  testScrapeClock,
-} from './clock';
+export { cdpTarget } from './cdp-target';
+export type { Deadline } from './clock';
+export { deadline } from './clock';
 export type { ColorScheme } from './color-scheme';
-export { COLOR_SCHEMES, isColorScheme } from './color-scheme';
 export { cookieHeaderFor } from './cookie-scope';
 export type { ScrapeDriver, ScrapeSession, SessionInit } from './driver';
-export { resetScrapeDriver, scrapeDriver, setScrapeDriver } from './driver';
+export { scrapeDriver } from './driver';
 export type { BrowserOptions, LocalBrowserOptions, RemoteBrowserOptions } from './driver-cdp';
 export { localBrowser, remoteBrowser } from './driver-cdp';
 export type { FakeBrowserOptions, FakePageOptions, FakePages } from './driver-fake';
-export { FAKE_DRIVER, FAKE_PAGE_URL, fakeBrowser, fakePage, recordingsOf } from './driver-fake';
+export { fakeBrowser, fakePage } from './driver-fake';
 export type { FixtureBrowserOptions } from './driver-recorded';
-export { FIXTURE_DRIVER, fixtureBrowser } from './driver-recorded';
+export { fixtureBrowser } from './driver-recorded';
 export {
-  authFailed,
   blocked,
-  bodyTooLarge,
   browserUnreachable,
-  cdpAttachFailed,
-  cdpResolveFailed,
-  downloadTimeout,
   driverUnknown,
-  fixtureMissing,
-  fixtureStale,
-  hostBlocked,
   httpFailed,
-  keyInvalid,
-  notActionable,
-  outputInvalid,
-  pageCrashed,
-  profileLocked,
   promptUnanswered,
-  recoverRefused,
-  redirectLoop,
   remoteRequired,
-  robotsDisallowed,
   scrapeSessionExpired,
-  scrapeTimeout,
-  secretExposed,
-  selectorMissing,
   wedged,
-  yieldCollapsed,
 } from './error-throws';
+export type { ScrapeErrorCode, ScrapeErrorInit } from './errors';
 export {
-  egressInPayload,
-  egressUnsupported,
-  promptAnswerInvalid,
-  promptTimedOut,
-} from './error-throws-session';
-export type { ScrapeErrorCode, ScrapeErrorInit, ScrapeOwnedErrorCode } from './errors';
-export {
-  isRetryableScrapeError,
   isScrapeError,
-  SCRAPE_BORROWED_ERROR_CODES,
   SCRAPE_ERROR_CODES,
   SCRAPE_ERROR_RETRY,
-  SCRAPE_ERROR_TITLES,
   SCRAPE_OWNED_ERROR_CODES,
   ScrapeError,
 } from './errors';
 export type { AnswerPromptInput, EventPromptOptions } from './event-prompt';
-export {
-  answerPrompt,
-  DEFAULT_PROMPT_ANSWER_TTL_MS,
-  DEFAULT_PROMPT_POLL_MS,
-  eventPrompt,
-  promptEventName,
-} from './event-prompt';
-export type { ScrapeEventFields, StepEvent } from './events';
-export type { YieldCheck, YieldExpectation, YieldGuardInput, YieldHistory } from './expect';
+export { answerPrompt, eventPrompt, promptEventName } from './event-prompt';
+export type { YieldExpectation, YieldGuardInput, YieldHistory } from './expect';
 export {
   guardYield,
   MIN_BASELINE_RUNS,
@@ -120,21 +73,14 @@ export {
 } from './expect';
 export { BURNS_SESSION, errorCode, NEVER_RETRIED } from './failures';
 export { queryHtml } from './html-query';
-export type { MarkupRequest } from './html-requests';
-export type { HtmlTargetInit, RecordingLookup } from './html-target';
-export { htmlTarget } from './html-target';
 export type { HttpRequestInit, HttpTransportInit, ScrapeHttp, ScrapeResponse } from './http';
 export { DEFAULT_HTTP_MAX_BYTES, httpOverFetch, responseOver } from './http';
 export type { HttpRecordingLookup, RecordedHttpInit } from './http-recorded';
-export { httpRecordingsOf, recordedHttp } from './http-recorded';
-export type { RedirectHop } from './http-redirect';
 export { MAX_REDIRECT_HOPS } from './http-redirect';
 export type { InterceptRules, InterceptVerdict } from './intercept';
 export { interceptVerdict } from './intercept';
 export type { KeyChord, KeyModifier } from './key-chord';
 export { KEY_MODIFIERS, parseKeyChord } from './key-chord';
-export type { OfflineSessionInit } from './offline-session';
-export { openOfflineSession } from './offline-session';
 export type {
   AccessibilityOptions,
   CaptureRequest,
@@ -146,19 +92,21 @@ export type {
 } from './page';
 export type { PageContext } from './page-over-target';
 export { pageOverTarget } from './page-over-target';
+export type { PdfActionArgs, PdfActionOptions, PdfActionOutput } from './pdf-action';
+export { pdf } from './pdf-action';
+export type {
+  ChromePdfRendererOptions,
+  PdfBrowser,
+  PdfRenderer,
+  PdfRenderRequest,
+} from './pdf-render';
+export { chromePdfRenderer } from './pdf-render';
 export type { PdfFormat, PdfLayout, PdfMarginMm, PdfRequest } from './pdf-request';
 export { PDF_FORMATS } from './pdf-request';
 export type { Pacer } from './rate';
 export { scrapePacer } from './rate';
 export type { HttpRecording, PageRecording } from './recording';
-export {
-  httpRecordingSchema,
-  pageRecordingSchema,
-  parseHttpRecording,
-  parseRecording,
-} from './recording';
 export type { RecoveryAttempt, RecoveryHook } from './recover';
-export { runRecovery } from './recover';
 export type {
   ConsoleLine,
   ConsoleRing,
@@ -181,10 +129,9 @@ export type {
   RobotsGate,
   RobotsGateInit,
   RobotsPolicy,
-  RobotsRules,
   RobotsUnreachable,
 } from './robots';
-export { parseRobots, robotsAllows, robotsGate } from './robots';
+export { robotsGate } from './robots';
 export type { RobotsFetchInit } from './robots-fetch';
 export {
   DEFAULT_ROBOTS_MAX_BYTES,
@@ -220,7 +167,6 @@ export {
   memorySessionStore,
   parseSessionState,
   recordVersion,
-  SESSION_SEAL_PURPOSE,
   sessionDigest,
   sessionKeyFor,
   storageSessionStore,
@@ -238,8 +184,8 @@ export type {
   ScrapeDownloadFile,
   ScrapeTarget,
 } from './target';
-export { endpointLabel, urlSecretValues } from './url-secrets';
+export { urlSecretValues } from './url-secrets';
 export type { RunUsageMeter, ScrapeUsage, UsageMeter } from './usage';
 export { usageMeter } from './usage';
-export type { WedgeGuard, WedgeGuardInit } from './watchdog';
+export type { WedgeGuard } from './watchdog';
 export { DEFAULT_GRACE_MS } from './watchdog';

package/src/launch-args.ts

@@ -17,6 +17,9 @@ const ROUTE_SWITCHES = new Set([
   'no-proxy-server',
 ]);
 
+/** Removes Chrome's implicit loopback bypass, so loopback goes through the exit like any host. */
+const PROXY_LOOPBACK_TOO = '--proxy-bypass-list=<-loopback>';
+
 const switchName = (arg: string): string | undefined => {
   const match = /^--?([^=]+)/.exec(arg);
   return match?.[1]?.toLowerCase();
@@ -46,6 +49,12 @@ export function launchArgs(
     }
   }
   const args = [...((callerArgs as readonly string[] | undefined) ?? [])];
-  if (proxyArg !== undefined) args.push(`--proxy-server=${proxyArg}`);
+  // `<-loopback>` beside every exit: Chrome ≥ 72 sends localhost, `*.localhost` and loopback
+  // literals around `--proxy-server` by default, so a page reached the worker's own loopback
+  // services DIRECT while the session reported an exit — and the browser leg's DNS screen stands
+  // down for a proxied session. Subtracting the implicit rule sends them through the exit too.
+  if (proxyArg !== undefined) {
+    args.push(`--proxy-server=${proxyArg}`, PROXY_LOOPBACK_TOO);
+  }
   return args.length === 0 ? undefined : args;
 }

package/src/page-over-target.ts

@@ -3,12 +3,11 @@
 // re-resolution, host enforcement and secret taint are written once and cannot drift between the
 // driver a test uses and the driver production uses.
 
-import type { Secret } from '@ultimat3/core';
+import type { Secret, SleepClock } from '@ultimat3/core';
 import { finiteCount, hostDecision, isSecret, revealSecret } from '@ultimat3/core';
 import type { ActionabilityState } from './actionability';
 import { awaitActionable } from './actionability';
 import { assertCaptureFraming } from './capture-clip';
-import type { ScrapeClock } from './clock';
 import { deadline } from './clock';
 import type { ColorScheme } from './color-scheme';
 import { hostBlocked, secretExposed, selectorMissing } from './error-throws';
@@ -39,7 +38,7 @@ import { ROOT_SELECTOR } from './target';
 import type { UsageMeter } from './usage';
 
 export interface PageContext {
-  readonly clock: ScrapeClock;
+  readonly clock: SleepClock;
   /**
    * Called before EVERY operation, poll included. This is what the wedge watchdog measures: the
    * gap between two of these is the definition of "the browser stopped answering", and putting

package/src/pdf-action.ts

@@ -0,0 +1,116 @@
+// `pdf()` — a factory over ACTION (`PRIMITIVE_FACTORIES`), never a ninth primitive: render one of
+// the app's own routes to PDF through the CDP browser, write it to a disk, answer its key. Input
+// schema, policy, rate limit, meter and audit are the action's own, so an invoice PDF is gated
+// exactly like the invoice: HTTP, OpenAPI, MCP, `.job()` and `.as()` come with it.
+
+import type { Action, ActionMeter, ActionPolicy, ActionRateLimit } from '@ultimat3/action';
+import { action } from '@ultimat3/action';
+import type { Ctx } from '@ultimat3/core';
+import type { InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
+import { t } from '@ultimat3/schema';
+import type { StorageDriver } from '@ultimat3/storage';
+import { pdfOriginMissing, pdfRouteInvalid } from './error-throws-pdf';
+import type { PdfRenderer } from './pdf-render';
+import { chromePdfRenderer } from './pdf-render';
+import type { PdfLayout } from './pdf-request';
+import { pdfLayoutOf } from './pdf-request';
+
+export const PDF_CONTENT_TYPE = 'application/pdf';
+
+/** What every `pdf()` answers: where the document is, and how big. Never the bytes themselves. */
+const pdfOutput = () => t.object({ key: t.string, bytes: t.number, contentType: t.string });
+
+export type PdfActionOutput = ReturnType<typeof pdfOutput>;
+
+/** What `route`, `key` and `headers` are handed: the parsed input and the call's context. */
+export interface PdfActionArgs<TInput extends StandardSchemaV1> {
+  readonly input: InferOutput<TInput>;
+  readonly ctx: Ctx;
+}
+
+export interface PdfActionOptions<TInput extends StandardSchemaV1, TRow = unknown> {
+  readonly input: TInput;
+  readonly policy: ActionPolicy<TRow>;
+  /** A PATH on this app (`/invoices/42/print`). Another origin, or a `//` path, is refused. */
+  route(args: PdfActionArgs<TInput>): string;
+  /** Where the document lands on `storage`. Tenant-scoped by the app: the key is its to choose. */
+  key(args: PdfActionArgs<TInput>): string;
+  /** A THUNK, read per call: the app's disks exist after boot, the action at module evaluation. */
+  readonly storage: () => StorageDriver;
+  /** Checked once, here, by `pdfLayoutOf` — a layout no engine honours never reaches a browser. */
+  readonly layout?: PdfLayout | undefined;
+  /**
+   * Headers the browser sends with every request the page makes — how a private route learns who
+   * is printing it (a short-lived token the app signs). Absent: the route must be readable bare.
+   */
+  readonly headers?:
+    | ((args: PdfActionArgs<TInput>) => Readonly<Record<string, string>>)
+    | undefined;
+  /** The origin the route is served on. Defaults to `APP_URL`, read per call. */
+  readonly origin?: (() => string | undefined) | undefined;
+  /**
+   * Defaults to one `chromePdfRenderer()` the whole process shares — its `maxConcurrent` bounds every
+   * `pdf()` that names none. A test hands in one over a fake CDP connection.
+   */
+  readonly renderer?: PdfRenderer | undefined;
+  readonly row?: ((args: PdfActionArgs<TInput>) => TRow | null | Promise<TRow | null>) | undefined;
+  readonly rateLimit?: ActionRateLimit | undefined;
+  readonly meter?: ActionMeter | undefined;
+  readonly audit?: boolean | undefined;
+}
+
+const appOrigin = (): string | undefined => process.env['APP_URL'];
+
+let shared: PdfRenderer | undefined;
+/**
+ * ONE default renderer per process, built on first use: its `maxConcurrent` bounds every `pdf()`
+ * that names none, so ten declarations do not mean twenty live Chromes.
+ */
+const sharedRenderer = (): PdfRenderer => {
+  shared ??= chromePdfRenderer();
+  return shared;
+};
+
+/** The route joined onto the origin, refused unless the result is still that origin. */
+function routeUrl(origin: string, path: string): string {
+  if (!path.startsWith('/') || path.startsWith('//') || path.includes('\\')) {
+    throw pdfRouteInvalid(path);
+  }
+  const base = new URL(origin);
+  const url = new URL(path, base);
+  if (url.origin !== base.origin) throw pdfRouteInvalid(path);
+  return url.href;
+}
+
+export function pdf<TInput extends StandardSchemaV1, TRow = unknown>(
+  options: PdfActionOptions<TInput, TRow>,
+): Action<TInput, PdfActionOutput> {
+  const layout = pdfLayoutOf(options.layout ?? {});
+  const renderer = options.renderer ?? sharedRenderer();
+  const origin = options.origin ?? appOrigin;
+  const { row } = options;
+  return action<TInput, PdfActionOutput, TRow>({
+    input: options.input,
+    output: pdfOutput(),
+    policy: options.policy,
+    ...(row === undefined ? {} : { row: (args) => row(args) }),
+    ...(options.rateLimit === undefined ? {} : { rateLimit: options.rateLimit }),
+    ...(options.meter === undefined ? {} : { meter: options.meter }),
+    ...(options.audit === undefined ? {} : { audit: options.audit }),
+    async handle(args) {
+      const served = origin()?.trim();
+      if (served === undefined || served === '') throw pdfOriginMissing();
+      const url = routeUrl(served, options.route(args));
+      const bytes = await renderer.render({
+        url,
+        layout,
+        // A client that went away, or a request past its deadline, frees its slot and its browser.
+        signal: args.ctx.signal,
+        ...(options.headers === undefined ? {} : { headers: options.headers(args) }),
+      });
+      const key = options.key(args);
+      await options.storage().put(key, bytes, { contentType: PDF_CONTENT_TYPE });
+      return { key, bytes: bytes.byteLength, contentType: PDF_CONTENT_TYPE };
+    },
+  });
+}

package/src/pdf-cdp-params.ts

@@ -0,0 +1,65 @@
+// A checked PDF layout in the PROTOCOL's own words — `Page.printToPDF`'s parameters — for the raw-CDP
+// printer `pdf()` drives. `pdf-print.ts` is the same layout in puppeteer's words; the two exist
+// because the two callers speak to different things. Inches, because the protocol counts in them.
+
+import type { PdfFormat, PdfLayout } from './pdf-request';
+
+const MM_PER_INCH = 25.4;
+
+/** Width × height in millimetres, portrait — ISO 216 and the ANSI sizes, as every engine has them. */
+const PAPER_MM: ReadonlyMap<PdfFormat, readonly [number, number]> = new Map([
+  ['letter', [215.9, 279.4]],
+  ['legal', [215.9, 355.6]],
+  ['tabloid', [279.4, 431.8]],
+  ['ledger', [431.8, 279.4]],
+  ['a0', [841, 1189]],
+  ['a1', [594, 841]],
+  ['a2', [420, 594]],
+  ['a3', [297, 420]],
+  ['a4', [210, 297]],
+  ['a5', [148, 210]],
+  ['a6', [105, 148]],
+]);
+
+/** An empty band rather than an absent one, for `pdf-print.ts`'s reason: Chrome's own date and URL. */
+const BLANK_TEMPLATE = '<span></span>';
+
+const inches = (mm: number): number => mm / MM_PER_INCH;
+
+/** The paper, in millimetres, when the layout names one. `undefined` is the engine's letter. */
+const paperOf = (layout: PdfLayout): readonly [number, number] | undefined => {
+  if (layout.format !== undefined) return PAPER_MM.get(layout.format);
+  if (layout.widthMm !== undefined && layout.heightMm !== undefined) {
+    return [layout.widthMm, layout.heightMm];
+  }
+  return undefined;
+};
+
+/**
+ * `Page.printToPDF`'s params for a layout `pdfLayoutOf` already checked. Only the fields the caller
+ * gave are sent; the bytes come back base64 in the reply (`transferMode: 'ReturnAsBase64'`).
+ */
+export function printToPdfParams(layout: PdfLayout): Record<string, unknown> {
+  const paper = paperOf(layout);
+  const margin = layout.marginMm;
+  const band = layout.headerHtml !== undefined || layout.footerHtml !== undefined;
+  return {
+    transferMode: 'ReturnAsBase64',
+    ...(paper === undefined ? {} : { paperWidth: inches(paper[0]), paperHeight: inches(paper[1]) }),
+    ...(margin?.top === undefined ? {} : { marginTop: inches(margin.top) }),
+    ...(margin?.right === undefined ? {} : { marginRight: inches(margin.right) }),
+    ...(margin?.bottom === undefined ? {} : { marginBottom: inches(margin.bottom) }),
+    ...(margin?.left === undefined ? {} : { marginLeft: inches(margin.left) }),
+    ...(layout.printBackground === undefined ? {} : { printBackground: layout.printBackground }),
+    ...(layout.preferCssPageSize === undefined
+      ? {}
+      : { preferCSSPageSize: layout.preferCssPageSize }),
+    ...(band
+      ? {
+          displayHeaderFooter: true,
+          headerTemplate: layout.headerHtml ?? BLANK_TEMPLATE,
+          footerTemplate: layout.footerHtml ?? BLANK_TEMPLATE,
+        }
+      : {}),
+  };
+}

package/src/pdf-fence.ts

@@ -0,0 +1,86 @@
+// The print's request fence, on the page's own session: every request pauses (`Fetch`); one to the
+// route's ORIGIN continues with the app's headers, every other — an off-origin redirect too — is
+// failed before it leaves. Never `Network.setExtraHTTPHeaders`: it sent the print token to any
+// `<img src>` a post carried, and let the page dial internal hosts.
+
+import type { CdpConnection } from './cdp-connection';
+
+export interface PdfFenceInit {
+  readonly connection: CdpConnection;
+  readonly sessionId: string;
+  /** `new URL(route).origin` — scheme, host and port, compared whole. */
+  readonly origin: string;
+  readonly headers?: Readonly<Record<string, string>> | undefined;
+}
+
+interface Paused {
+  readonly requestId: string;
+  readonly url: string;
+  readonly headers: Readonly<Record<string, unknown>>;
+}
+
+const readPaused = (params: Record<string, unknown>): Paused | undefined => {
+  const request = params['request'];
+  const requestId = params['requestId'];
+  if (typeof requestId !== 'string' || typeof request !== 'object' || request === null) {
+    return undefined;
+  }
+  const url = 'url' in request && typeof request.url === 'string' ? request.url : '';
+  const headers =
+    'headers' in request && typeof request.headers === 'object' && request.headers !== null
+      ? Object.fromEntries(Object.entries(request.headers))
+      : {};
+  return { requestId, url, headers };
+};
+
+const originOf = (url: string): string | undefined => {
+  try {
+    return new URL(url).origin;
+  } catch {
+    return undefined;
+  }
+};
+
+/** The request's own headers with the app's laid over them, case-insensitively, as CDP's list. */
+function mergedHeaders(
+  own: Readonly<Record<string, unknown>>,
+  extra: Readonly<Record<string, string>>,
+): { readonly name: string; readonly value: string }[] {
+  const overridden = new Set(Object.keys(extra).map((name) => name.toLowerCase()));
+  const kept = Object.entries(own)
+    .filter(([name, value]) => typeof value === 'string' && !overridden.has(name.toLowerCase()))
+    .map(([name, value]) => ({ name, value: String(value) }));
+  return [...kept, ...Object.entries(extra).map(([name, value]) => ({ name, value }))];
+}
+
+/**
+ * Raise the fence before the page navigates. Answers its removal. A decision the browser refuses
+ * (the target already gone) is dropped: the page's own load then fails, which is the error to read.
+ */
+export async function raisePdfFence(init: PdfFenceInit): Promise<() => void> {
+  const { connection, sessionId } = init;
+  const extra = init.headers ?? {};
+  const decide = (paused: Paused): Promise<unknown> => {
+    if (originOf(paused.url) !== init.origin) {
+      return connection.send(
+        'Fetch.failRequest',
+        { requestId: paused.requestId, errorReason: 'BlockedByClient' },
+        sessionId,
+      );
+    }
+    return connection.send(
+      'Fetch.continueRequest',
+      Object.keys(extra).length === 0
+        ? { requestId: paused.requestId }
+        : { requestId: paused.requestId, headers: mergedHeaders(paused.headers, extra) },
+      sessionId,
+    );
+  };
+  const off = connection.on('Fetch.requestPaused', (params, from) => {
+    if (from !== sessionId) return;
+    const paused = readPaused(params);
+    if (paused !== undefined) void decide(paused).catch(() => undefined);
+  });
+  await connection.send('Fetch.enable', { patterns: [{ urlPattern: '*' }] }, sessionId);
+  return off;
+}