@ultimat3/scraping 2.0.0

package/src/target.ts

@@ -0,0 +1,118 @@
+// The driver-blind currency: what every browser driver hands back, and the only shape the page
+// vocabulary reads. No puppeteer type, no CDP type and no `any` crosses this line — which is what
+// lets `page-over-target.ts` be the ONE implementation of `ScrapePage` for the real browser, the
+// fixture replayer and the fake alike.
+
+import type { ConsoleRing, NetworkRing } from './rings';
+import type { SessionSnapshot } from './session-state';
+
+/**
+ * The document root, as a selector every target must answer. `text()` with no argument is "all
+ * the text on this page", and a driver-specific spelling of that would be the first thing to
+ * diverge between the fake and the real browser.
+ */
+export const ROOT_SELECTOR = 'html';
+
+export interface ElementBox {
+  readonly x: number;
+  readonly y: number;
+  readonly width: number;
+  readonly height: number;
+}
+
+/**
+ * One element, as of one observation. Everything the actionability rule needs, and nothing that
+ * would let a caller hold a live handle: a snapshot is a VALUE, so it cannot go stale behind the
+ * caller's back the way a `frameLocator` handle does — it is simply old, and `page-over-target.ts`
+ * takes a fresh one before every act.
+ */
+export interface ElementSnapshot {
+  readonly tag: string;
+  readonly attrs: Readonly<Record<string, string>>;
+  readonly text: string;
+  /** The form value for a control, `''` for anything else. Never `undefined` — absent is ''. */
+  readonly value: string;
+  readonly visible: boolean;
+  readonly enabled: boolean;
+  /**
+   * Absent means THIS TARGET HAS NO LAYOUT ENGINE (the fake and the fixture parse HTML; nothing
+   * is laid out). Never a fabricated zero box: `driver-parity.test.ts` pins the divergence in one
+   * place, and a `{ x: 0, y: 0 }` invented here would make a covered button test green on the
+   * only driver that could have caught it.
+   */
+  readonly box?: ElementBox | undefined;
+  /** Whether this element is what a click at its own centre would hit. Absent: no layout. */
+  readonly hitTarget?: boolean | undefined;
+}
+
+export interface ScrapeCookie {
+  readonly name: string;
+  readonly value: string;
+  readonly domain: string;
+  readonly path: string;
+  readonly expires?: number | undefined;
+  readonly httpOnly: boolean;
+  readonly secure: boolean;
+}
+
+export interface ScrapeDownloadFile {
+  readonly filename: string;
+  readonly bytes: Uint8Array;
+}
+
+/** A child frame, addressed by whatever the page can see of it. */
+export interface FrameRef {
+  readonly name: string;
+  readonly url: string;
+  /** The `<iframe>` selector in the PARENT document, when the driver can tell. */
+  readonly selector?: string | undefined;
+  readonly target: ScrapeTarget;
+}
+
+export interface GotoOptions {
+  readonly timeoutMs: number;
+  readonly signal?: AbortSignal | undefined;
+}
+
+export interface CaptureOptions {
+  readonly fullPage?: boolean | undefined;
+  readonly timeoutMs: number;
+}
+
+/**
+ * The port. Twelve methods, every one of them something a browser genuinely does and nothing a
+ * scraper's vocabulary should be re-deriving per driver.
+ */
+export interface ScrapeTarget {
+  /** `puppeteer` | `fixture` | `fake`. Appears in every error cause raised against it. */
+  readonly driver: string;
+  readonly console: ConsoleRing;
+  readonly network: NetworkRing;
+  url(): string;
+  goto(url: string, options: GotoOptions): Promise<void>;
+  /** Serialised HTML of THIS target — the document for a page, the subtree for a frame. */
+  content(): Promise<string>;
+  query(selector: string): Promise<readonly ElementSnapshot[]>;
+  click(selector: string, index: number): Promise<void>;
+  /** Appends, exactly as typing does. Clearing first is `fill`'s job at the page level. */
+  type(selector: string, text: string): Promise<void>;
+  clear(selector: string): Promise<void>;
+  select(selector: string, values: readonly string[]): Promise<void>;
+  /** The expression runs in the page. The result is `unknown` and is parsed by the caller. */
+  evaluate(expression: string): Promise<unknown>;
+  screenshot(options: CaptureOptions): Promise<Uint8Array>;
+  pdf(options: CaptureOptions): Promise<Uint8Array>;
+  cookies(): Promise<readonly ScrapeCookie[]>;
+  /** Whatever the last click/navigation produced as a file, or a `X_SCRAPE_DOWNLOAD_TIMEOUT`. */
+  download(options: { readonly timeoutMs: number }): Promise<ScrapeDownloadFile>;
+  frames(): Promise<readonly FrameRef[]>;
+  /**
+   * Everything that makes this client THIS client: cookies, storage, the headers the site now
+   * expects, the user agent. It is the browser-to-HTTP handoff, and it is the thing a session
+   * store persists — so it is credential material and is summarised, never logged.
+   */
+  session(): Promise<SessionSnapshot>;
+  /** Put a previously captured session back, before the first navigation. */
+  restore(session: SessionSnapshot): Promise<void>;
+  close(): Promise<void>;
+}

package/src/watchdog.ts

@@ -0,0 +1,100 @@
+// The wedge and zombie discipline, which is the difference between a scrape that fails and a
+// scrape that pins a worker.
+//
+// Two real incidents, both from the same missing mechanism. One run held a queue slot for 3h11m
+// and persisted nothing: the browser's socket was open, so every `await` was legitimately waiting
+// and no timeout was armed on anything. Another left a browser process alive after its run ended
+// and the queue could not be drained until a redeploy.
+//
+// So: an INACTIVITY watchdog that kills the OS process — killing it is what makes the blocked
+// socket die, which is what turns an infinite await into a catchable `X_SCRAPE_WEDGED` — and a
+// graceful-quit CEILING on shutdown, past which the same kill runs.
+
+import type { ScrapeClock } from './clock';
+import { wedged } from './error-throws';
+
+export const DEFAULT_IDLE_MS = 120_000;
+export const DEFAULT_GRACE_MS = 5_000;
+const WATCH_POLL_MS = 250;
+
+export interface WedgeGuardInit {
+  readonly clock: ScrapeClock;
+  /** Longest gap between two browser operations before the process is killed. */
+  readonly idleMs?: number | undefined;
+  /** How long a graceful quit may take before the same kill runs. */
+  readonly graceMs?: number | undefined;
+  /** What is being watched, for the cause line. */
+  readonly what: string;
+  /** The polite ending: `browser.close()`. May hang, which is the whole reason for the ceiling. */
+  quit(): Promise<void>;
+  /**
+   * SIGKILL, not SIGTERM. A wedged Chrome ignores the polite signal — that is what "wedged"
+   * means — and a kill that can be ignored is not a ceiling.
+   */
+  kill(): void;
+}
+
+export interface WedgeGuard {
+  /** Aborts with `X_SCRAPE_WEDGED` when the idle budget passes. Handed to every wait. */
+  readonly signal: AbortSignal;
+  /** Called after every browser operation. The watchdog measures the gap between these. */
+  touch(): void;
+  /** Graceful quit under the ceiling, then kill. Idempotent, and never throws. */
+  shutdown(): Promise<void>;
+  /** True once the watchdog fired — a run that ends after this ended because of it. */
+  readonly fired: boolean;
+}
+
+export function createWedgeGuard(init: WedgeGuardInit): WedgeGuard {
+  const idleMs = init.idleMs ?? DEFAULT_IDLE_MS;
+  const graceMs = init.graceMs ?? DEFAULT_GRACE_MS;
+  const controller = new AbortController();
+  let lastTouch = init.clock.monotonic();
+  let stopped = false;
+  let fired = false;
+
+  const watch = async (): Promise<void> => {
+    while (!stopped) {
+      await init.clock.sleep(WATCH_POLL_MS);
+      if (stopped) return;
+      if (init.clock.monotonic() - lastTouch < idleMs) continue;
+      fired = true;
+      stopped = true;
+      // Kill FIRST, abort second. The abort is what the awaiting code sees; the kill is what
+      // makes the socket it is blocked on close. Reversing them leaves the run cancelled and the
+      // browser alive, which is the zombie half of the same incident.
+      init.kill();
+      controller.abort(wedged(init.what, idleMs));
+    }
+  };
+  void watch();
+
+  return {
+    signal: controller.signal,
+    get fired(): boolean {
+      return fired;
+    },
+    touch(): void {
+      lastTouch = init.clock.monotonic();
+    },
+    async shutdown(): Promise<void> {
+      if (stopped) return;
+      stopped = true;
+      let quit = false;
+      // A ceiling, not a hope: whichever finishes first wins, and if it is the clock the process
+      // is killed. `browser.close()` on a wedged renderer never returns.
+      await Promise.race([
+        init.quit().then(
+          () => {
+            quit = true;
+          },
+          () => {
+            quit = false;
+          },
+        ),
+        init.clock.sleep(graceMs),
+      ]);
+      if (!quit) init.kill();
+    },
+  };
+}