@ultimat3/scraping 2.0.0

package/src/page.ts

@@ -0,0 +1,103 @@
+// The vocabulary a `run()` body writes against — small, declarative, and driver-blind. Fourteen
+// verbs, chosen because every scraper in the audit re-implemented these and nothing else; a
+// fifteenth would be a second way to do something on this list.
+//
+// Nothing here mentions puppeteer, CDP, a frame handle or a locator. That is the seam: a run body
+// written against this file runs unchanged on a real browser, on a recorded fixture and on a
+// parsed HTML string, which is what makes `bun test` need no Chrome.
+
+import type { Secret } from '@ultimat3/core';
+import type { ActionabilityState } from './actionability';
+import type { ConsoleLine, NetworkEntry } from './rings';
+import type { SessionSnapshot } from './session-state';
+import type { ElementSnapshot, ScrapeCookie, ScrapeDownloadFile } from './target';
+
+export interface WaitOptions {
+  readonly state?: ActionabilityState | undefined;
+  /** Milliseconds. Falls back to the scrape's own `timeout`. */
+  readonly timeout?: number | undefined;
+}
+
+export interface ElementValue {
+  readonly tag: string;
+  readonly text: string;
+  /** The control's value, `''` for anything that is not one. */
+  readonly value: string;
+  readonly attrs: Readonly<Record<string, string>>;
+}
+
+/**
+ * A frame, or the document — everything both can do. Held as a LAZY handle: every method
+ * re-resolves its underlying frame at call time, so a handle taken before a navigation still
+ * addresses the right frame afterwards rather than a detached one.
+ */
+export interface ScrapeFrame {
+  url(): string;
+  /** Blocks until the element reaches `state` (default `actionable`), then answers its snapshot. */
+  waitFor(selector: string, options?: WaitOptions): Promise<ElementSnapshot>;
+  /**
+   * Waits for the element to be visible, enabled, unobstructed and STILL before clicking. A raw
+   * driver's click waits for the selector alone, which is why every app that uses one grows a
+   * `waitForTimeout(800)` somewhere above it.
+   */
+  click(selector: string, options?: WaitOptions): Promise<void>;
+  /** Appends. A `Secret` marks this page as carrying one, which refuses later pixel captures. */
+  type(selector: string, text: string | Secret, options?: WaitOptions): Promise<void>;
+  /** Clears first, then types — the spelling a login form wants. */
+  fill(selector: string, text: string | Secret, options?: WaitOptions): Promise<void>;
+  select(selector: string, values: readonly string[], options?: WaitOptions): Promise<void>;
+  /** Every match, as values. Row assembly is the app's business, never the framework's. */
+  values(selector: string): Promise<readonly ElementValue[]>;
+  /** The first match's text, or `''`. */
+  text(selector?: string): Promise<string>;
+  /** Serialised HTML, redacted by value and with password fields blanked. */
+  html(): Promise<string>;
+  count(selector: string): Promise<number>;
+  /** The expression's result, `unknown` — parse it with a schema, never cast it. */
+  evaluate(expression: string): Promise<unknown>;
+  /**
+   * A child frame by name, `id`, or `<iframe>` selector. RE-RESOLVED on every call made through
+   * the returned handle: a frame handle captured before a re-navigation is the single biggest
+   * correctness trap in this whole vocabulary, and the type cannot express "stale" — so nothing
+   * here holds one.
+   */
+  frame(nameOrSelector: string): ScrapeFrame;
+}
+
+export interface CaptureRequest {
+  readonly fullPage?: boolean | undefined;
+  readonly timeout?: number | undefined;
+}
+
+export interface DownloadRequest {
+  readonly timeout?: number | undefined;
+}
+
+export interface ScrapePage extends ScrapeFrame {
+  /**
+   * Navigate. Refused before a byte leaves when the host is not in `allowHosts`
+   * (`X_SCRAPE_HOST_BLOCKED`) or robots.txt disallows the path (`X_SCRAPE_ROBOTS_DISALLOWED`).
+   */
+  goto(url: string, options?: { readonly timeout?: number | undefined }): Promise<void>;
+  /** PNG bytes. REFUSED once a secret has been typed into this page — see `secrets.ts`. */
+  screenshot(options?: CaptureRequest): Promise<Uint8Array>;
+  /** PDF bytes. Refused on the same condition, for the same reason. */
+  pdf(options?: CaptureRequest): Promise<Uint8Array>;
+  /** The file the last click produced, or `X_SCRAPE_DOWNLOAD_TIMEOUT`. */
+  download(options?: DownloadRequest): Promise<ScrapeDownloadFile>;
+  cookies(): Promise<readonly ScrapeCookie[]>;
+  /**
+   * The handoff, made explicit: what the HTTP leg will send, as a value an author can inspect and
+   * a fixture can assert on. `http` uses it automatically — this is for seeing what carried over.
+   */
+  session(): Promise<SessionSnapshot>;
+  /** The bounded tail. Bounded because a long run's full history is an OOM, not a log. */
+  console(): readonly ConsoleLine[];
+  network(): readonly NetworkEntry[];
+  /**
+   * How many entries the bound above threw away. It is the same honesty `Ring.dropped` carries:
+   * any count taken from `network()` — the run's `refused` total included — is a floor once this
+   * is non-zero, and a scrape that blocked 5,000 images otherwise reports 200 with no hint.
+   */
+  networkDropped(): number;
+}

package/src/rate.ts

@@ -0,0 +1,23 @@
+// Navigations per second, paced against the injected clock. A scraper with no pacing is a
+// scraper that looks like a denial of service to the site it depends on, and the ban is charged
+// to whoever owns the IP — usually the whole fleet, not the one run.
+
+import type { ScrapeClock } from './clock';
+
+/** Deliberately slow. Raising it is a decision; there is no way to turn pacing off. */
+export const DEFAULT_NAVIGATION_RATE = 1;
+
+export type Pacer = (signal?: AbortSignal) => Promise<void>;
+
+export function createPacer(rate: number, clock: ScrapeClock): Pacer {
+  const intervalMs = 1_000 / rate;
+  let nextAt = 0;
+  return async (signal?: AbortSignal): Promise<void> => {
+    const now = clock.monotonic();
+    const waitMs = Math.max(0, nextAt - now);
+    // Booked BEFORE the wait, so two concurrent navigations queue behind each other instead of
+    // both reading the same free slot and leaving together.
+    nextAt = Math.max(now, nextAt) + intervalMs;
+    if (waitMs > 0) await clock.sleep(waitMs, signal);
+  };
+}

package/src/recording.ts

@@ -0,0 +1,68 @@
+// What an offline driver replays: one page, as data. The same shape whether it was written by
+// hand for `fakeBrowser()` or read off disk by `fixtureBrowser()`, so a test that outgrows an
+// inline fixture moves to a directory without a single edit to the run body.
+
+import type { StandardSchemaV1 } from '@ultimat3/schema';
+import { parse, t } from '@ultimat3/schema';
+
+export interface PageRecording {
+  readonly url: string;
+  readonly html: string;
+  /**
+   * `expression -> JSON`. `evaluate()` answers `unknown` on every driver, so a recording holds
+   * the JSON text and the target parses it — an expression with no entry is an UNRECORDED
+   * request and throws, exactly like an unrecorded navigation.
+   */
+  readonly evaluate?: Readonly<Record<string, string>> | undefined;
+  /** `<iframe>` name or `src` -> that frame's HTML. */
+  readonly frames?: Readonly<Record<string, string>> | undefined;
+  /** Click-target selector -> the file that click produces, as `filename:contents`. */
+  readonly downloads?: Readonly<Record<string, string>> | undefined;
+  /** ISO 8601. Absent means "no age", and `fixtureBrowser({ maxAge })` cannot judge it. */
+  readonly recordedAt?: string | undefined;
+}
+
+export const pageRecordingSchema: StandardSchemaV1<unknown, PageRecording> = t.object({
+  url: t.string,
+  html: t.string,
+  evaluate: t.optional(t.record(t.string)),
+  frames: t.optional(t.record(t.string)),
+  downloads: t.optional(t.record(t.string)),
+  recordedAt: t.optional(t.string),
+}) as unknown as StandardSchemaV1<unknown, PageRecording>;
+
+/** On-disk JSON is `unknown` and is PARSED, never cast — a fixture is somebody's edited file. */
+export const parseRecording = (raw: unknown): PageRecording => parse(pageRecordingSchema, raw);
+
+/**
+ * The HTTP leg's half of the SAME fixture directory. One format covers both transports, because a
+ * hybrid scrape — browser login, session handoff, HTTP bulk fetch — is only testable if the whole
+ * run replays from one place. Split them across two mechanisms and the hybrid path, which is where
+ * the real code lives, becomes the untestable path.
+ */
+export interface HttpRecording {
+  readonly url: string;
+  readonly method: string;
+  readonly status: number;
+  readonly body: string;
+  readonly headers?: Readonly<Record<string, string>> | undefined;
+  readonly recordedAt?: string | undefined;
+}
+
+export const httpRecordingSchema: StandardSchemaV1<unknown, HttpRecording> = t.object({
+  url: t.string,
+  method: t.string,
+  status: t.number,
+  body: t.string,
+  headers: t.optional(t.record(t.string)),
+  recordedAt: t.optional(t.string),
+}) as unknown as StandardSchemaV1<unknown, HttpRecording>;
+
+export const parseHttpRecording = (raw: unknown): HttpRecording => parse(httpRecordingSchema, raw);
+
+/** `report.csv:a,b,c` -> the two halves. A value with no colon is all contents, no name. */
+export function splitDownload(value: string): { filename: string; contents: string } {
+  const colon = value.indexOf(':');
+  if (colon === -1) return { filename: 'download', contents: value };
+  return { filename: value.slice(0, colon), contents: value.slice(colon + 1) };
+}

package/src/recover.ts

@@ -0,0 +1,51 @@
+// The recovery seam: what happens when a selector has moved and the run failed on it.
+//
+// Two shapes. A FUNCTION is complete and shipped — an app writes its own fallback, gets the page
+// and the failure, and answers whether the attempt should be retried. `'agent'` is the seam for
+// letting a model re-derive the selector from the page, and it is an honest
+// `X_NOT_IMPLEMENTED` today, in the shape `packages/jobs/src/driver-redis.ts` uses: correct types
+// so an app can be written against it, one labelled throw, no silent no-op.
+
+import { recoverRefused, scrapeNotImplemented } from './error-throws';
+import type { ScrapePage } from './page';
+
+export interface RecoveryAttempt {
+  readonly scrape: string;
+  readonly page: ScrapePage;
+  /** The failure, as thrown. `unknown` — it is whatever the body raised. */
+  readonly failure: unknown;
+  readonly attempt: number;
+}
+
+/**
+ * `true` retries the body once more in the same session; `false` re-raises. A hook that returns
+ * `false` is not an error — declining is a decision. `X_SCRAPE_RECOVER_REFUSED` is for the hook
+ * that cannot even judge, and it carries the hook's own reason.
+ */
+export type RecoveryHook = (attempt: RecoveryAttempt) => Promise<boolean> | boolean;
+
+export type Recovery = RecoveryHook | 'agent';
+
+/** What the agent recovery will need when it lands. Declared now so the seam is typed, not free. */
+export interface AgentRecovery {
+  readonly kind: 'agent';
+  /** The model call. Deliberately not typed against `@ultimat3/ai` yet — see the throw below. */
+  readonly instruction?: string | undefined;
+}
+
+const AGENT_FIX =
+  "replace recover: 'agent' with a function — recover: ({ page }) => page.waitFor('<the moved selector>').then(() => true) — which is complete today";
+
+export async function runRecovery(recovery: Recovery, attempt: RecoveryAttempt): Promise<boolean> {
+  if (recovery === 'agent') {
+    // The agent path lands with `@ultimat3/ai`'s `llm()` behind it: page HTML in, a selector out,
+    // re-run once. Until it does, this throws rather than answering `false` — a recovery that
+    // silently declines is indistinguishable from one that was never configured.
+    throw scrapeNotImplemented("recover: 'agent'", AGENT_FIX);
+  }
+  const verdict: unknown = await recovery(attempt);
+  if (typeof verdict !== 'boolean') {
+    throw recoverRefused(attempt.scrape, 'the hook answered something other than true or false');
+  }
+  return verdict;
+}

package/src/rings.ts

@@ -0,0 +1,73 @@
+// Bounded console and network history. BOUNDED is the whole point: a scrape of ten thousand
+// pages that kept every console line and every request holds the run's entire browsing history in
+// the worker's heap, and the incident is an OOM two hours in rather than a scraper bug.
+
+export interface ConsoleLine {
+  readonly level: 'log' | 'info' | 'warn' | 'error' | 'debug';
+  readonly text: string;
+  readonly at: number;
+}
+
+export interface NetworkEntry {
+  readonly method: string;
+  readonly url: string;
+  /** Absent while the request is still in flight or was aborted by interception. */
+  readonly status?: number | undefined;
+  readonly resourceType: ResourceType;
+  readonly at: number;
+  /** Set when interception refused it — `blocked` for `block:`, `host` for `allowHosts`. */
+  readonly refused?: 'blocked' | 'host' | 'robots' | undefined;
+}
+
+export const RESOURCE_TYPES = [
+  'document',
+  'stylesheet',
+  'image',
+  'media',
+  'font',
+  'script',
+  'xhr',
+  'fetch',
+  'websocket',
+  'other',
+] as const;
+
+export type ResourceType = (typeof RESOURCE_TYPES)[number];
+
+export const DEFAULT_RING_CAPACITY = 200;
+
+export interface Ring<T> {
+  readonly capacity: number;
+  push(entry: T): void;
+  /** Oldest first. A copy — a caller holding the ring's own array would see it mutate. */
+  entries(): readonly T[];
+  /** How many were dropped to keep the bound. Non-zero is the honest "you are not seeing it all". */
+  readonly dropped: number;
+  clear(): void;
+}
+
+export type ConsoleRing = Ring<ConsoleLine>;
+export type NetworkRing = Ring<NetworkEntry>;
+
+export function createRing<T>(capacity: number = DEFAULT_RING_CAPACITY): Ring<T> {
+  const items: T[] = [];
+  let dropped = 0;
+  return {
+    capacity,
+    push(entry: T): void {
+      items.push(entry);
+      while (items.length > capacity) {
+        items.shift();
+        dropped += 1;
+      }
+    },
+    entries: () => [...items],
+    get dropped(): number {
+      return dropped;
+    },
+    clear(): void {
+      items.length = 0;
+      dropped = 0;
+    },
+  };
+}

package/src/robots.ts

@@ -0,0 +1,144 @@
+// robots.txt, obeyed by DEFAULT, and ignorable only with a written reason.
+//
+// Ultimate's users are not all scraping their own accounts. `robots: 'obey'` being the default
+// makes the polite thing the thing that happens when nobody decides; `robots: { ignore: reason }`
+// makes the impolite thing a sentence somebody has to write, in the diff, next to their name.
+// There is no boolean, because `robots: false` is a decision with no author.
+
+import { robotsDisallowed } from './error-throws';
+
+export type RobotsPolicy = 'obey' | { readonly ignore: string };
+
+export const DEFAULT_ROBOTS_AGENT = 'ultimate-scraper';
+
+export interface RobotsRule {
+  readonly allow: boolean;
+  readonly path: string;
+}
+
+export interface RobotsRules {
+  readonly rules: readonly RobotsRule[];
+}
+
+/**
+ * The `User-agent` groups that apply to `agent`: its own, plus `*` when the file names no group
+ * for it. A file that addresses this agent specifically REPLACES the wildcard group, which is
+ * what the standard says and what every operator writing a targeted rule expects.
+ */
+export function parseRobots(text: string, agent: string): RobotsRules {
+  const wanted = agent.toLowerCase();
+  const groups = new Map<string, RobotsRule[]>();
+  let current: string[] = [];
+  let inGroup = false;
+  for (const raw of text.split('\n')) {
+    const line = raw.split('#')[0]?.trim() ?? '';
+    if (line === '') continue;
+    const colon = line.indexOf(':');
+    if (colon === -1) continue;
+    const field = line.slice(0, colon).trim().toLowerCase();
+    const value = line.slice(colon + 1).trim();
+    if (field === 'user-agent') {
+      if (inGroup) current = [];
+      inGroup = false;
+      current.push(value.toLowerCase());
+      continue;
+    }
+    if (field !== 'allow' && field !== 'disallow') continue;
+    inGroup = true;
+    for (const name of current) {
+      const rules = groups.get(name) ?? [];
+      // An empty `Disallow:` is the documented spelling of "allow everything", not a rule about
+      // the empty path — read as a path it would match every URL and refuse the whole site.
+      if (value !== '' || field === 'allow') rules.push({ allow: field === 'allow', path: value });
+      groups.set(name, rules);
+    }
+  }
+  return { rules: groups.get(wanted) ?? groups.get('*') ?? [] };
+}
+
+const escaped = (literal: string): string => literal.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+/** `/private/*.pdf$` — the two wildcards robots.txt defines, and no others. */
+const patternMatches = (pattern: string, path: string): boolean => {
+  const anchored = pattern.endsWith('$');
+  const body = anchored ? pattern.slice(0, -1) : pattern;
+  const source = body.split('*').map(escaped).join('.*');
+  return new RegExp(`^${source}${anchored ? '$' : ''}`).test(path);
+};
+
+/**
+ * Longest match wins, and a tie goes to `Allow` — the rule every major crawler implements. A
+ * naive "first disallow wins" reading refuses paths an operator explicitly re-allowed.
+ */
+export function robotsAllows(rules: RobotsRules, path: string): boolean {
+  let verdict = true;
+  let best = -1;
+  for (const rule of rules.rules) {
+    if (!patternMatches(rule.path, path)) continue;
+    const weight = rule.path.length;
+    if (weight > best || (weight === best && rule.allow)) {
+      best = weight;
+      verdict = rule.allow;
+    }
+  }
+  return verdict;
+}
+
+export interface RobotsGate {
+  /** Throws `X_SCRAPE_ROBOTS_DISALLOWED`, or returns. Called before every navigation. */
+  assertAllowed(url: string): Promise<void>;
+  /** The reason an `ignore` gate carries, for the run's own record. `undefined` when obeying. */
+  readonly ignoredBecause: string | undefined;
+}
+
+export type RobotsFetch = (robotsUrl: string) => Promise<string | undefined>;
+
+const fetchRobots: RobotsFetch = async (robotsUrl) => {
+  const response = await fetch(robotsUrl);
+  return response.ok ? await response.text() : undefined;
+};
+
+export interface RobotsGateInit {
+  readonly policy: RobotsPolicy;
+  readonly agent?: string | undefined;
+  readonly fetchText?: RobotsFetch | undefined;
+}
+
+/**
+ * One fetch of `/robots.txt` per ORIGIN per run, cached. A gate that re-fetched per navigation
+ * would triple the request count of every scrape it protects.
+ *
+ * An origin whose robots.txt cannot be read is ALLOWED. That is the standard's own answer — a
+ * missing file means no restrictions — and the alternative fails every run behind a flaky CDN.
+ */
+export function createRobotsGate(init: RobotsGateInit): RobotsGate {
+  if (init.policy !== 'obey') {
+    return { assertAllowed: () => Promise.resolve(), ignoredBecause: init.policy.ignore };
+  }
+  const agent = init.agent ?? DEFAULT_ROBOTS_AGENT;
+  const read = init.fetchText ?? fetchRobots;
+  const cache = new Map<string, Promise<RobotsRules>>();
+  return {
+    ignoredBecause: undefined,
+    async assertAllowed(url: string): Promise<void> {
+      let parsed: URL;
+      try {
+        parsed = new URL(url);
+      } catch {
+        return;
+      }
+      if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return;
+      const origin = parsed.origin;
+      let rules = cache.get(origin);
+      if (rules === undefined) {
+        rules = read(`${origin}/robots.txt`)
+          .then((text) => (text === undefined ? { rules: [] } : parseRobots(text, agent)))
+          .catch(() => ({ rules: [] }));
+        cache.set(origin, rules);
+      }
+      if (!robotsAllows(await rules, `${parsed.pathname}${parsed.search}`)) {
+        throw robotsDisallowed(url, agent);
+      }
+    },
+  };
+}