@ultimat3/scraping 2.0.0

package/src/failures.ts

@@ -0,0 +1,47 @@
+// Reading a thrown value safely. Every classification decision this package makes — is this
+// terminal, does this burn the session, what code does the event line carry — asks these two
+// questions, and both have to be answerable about a value nobody here constructed.
+
+import { isUltimateError } from '@ultimat3/core';
+
+/** The stable code, or `undefined`. Never the message: a message is whatever a site put in it. */
+export function errorCode(thrown: unknown): string | undefined {
+  if (isUltimateError(thrown)) return thrown.code;
+  // A structural read, not `instanceof`: an error crossing a worker or a subprocess arrives as a
+  // plain object, and refusing to recognise it there is how a terminal failure gets retried.
+  if (typeof thrown === 'object' && thrown !== null) {
+    const code: unknown = (thrown as { code?: unknown }).code;
+    if (typeof code === 'string' && code.startsWith('X_')) return code;
+  }
+  return undefined;
+}
+
+/**
+ * Codes after which the persisted identity must not be reused. `X_SCRAPE_BLOCKED` is the whole
+ * reason this list exists: a flagged profile stays flagged, so the retry has to arrive as
+ * somebody else or it fails identically, forever.
+ */
+export const BURNS_SESSION: ReadonlySet<string> = new Set([
+  'X_SCRAPE_BLOCKED',
+  'X_SCRAPE_SESSION_EXPIRED',
+]);
+
+/**
+ * Codes no retry may ever follow, whatever the retry policy says. `X_SCRAPE_AUTH_FAILED` is the
+ * one that matters: a site that locks an account after three wrong attempts turns a retrying
+ * framework into the thing that destroys the user's account.
+ */
+export const NEVER_RETRIED: ReadonlySet<string> = new Set([
+  'X_SCRAPE_AUTH_FAILED',
+  'X_SCRAPE_PROMPT_UNANSWERED',
+]);
+
+export const burnsSession = (thrown: unknown): boolean => {
+  const code = errorCode(thrown);
+  return code !== undefined && BURNS_SESSION.has(code);
+};
+
+export const neverRetried = (thrown: unknown): boolean => {
+  const code = errorCode(thrown);
+  return code !== undefined && NEVER_RETRIED.has(code);
+};

package/src/hosts.ts

@@ -0,0 +1,56 @@
+// `allowHosts`, as a decision every driver asks before a request leaves — never a note in a
+// README. A headless browser inside your network is the widest SSRF surface an app can own: one
+// injected `<img src="http://169.254.169.254/…">` on a page you do not control is a credential
+// read, and no amount of "we only visit example.com" in prose intercepts it.
+
+export type HostRule = string;
+
+/** The one spelling that means "every host", written out so it is visible in review. */
+export const ANY_HOST: HostRule = '*';
+
+/**
+ * Schemes with no host to match. `about:blank` is where every browser starts, `data:` and `blob:`
+ * never leave the process — refusing them would refuse the first page load of every run.
+ */
+const HOSTLESS_SCHEMES = new Set(['about:', 'data:', 'blob:', 'javascript:']);
+
+export interface HostDecision {
+  readonly allowed: boolean;
+  /** The host the URL resolved to, `''` for a hostless scheme. */
+  readonly host: string;
+}
+
+/**
+ * `example.com` matches that host EXACTLY. `*.example.com` matches any subdomain and NOT the
+ * apex — the two are written separately on purpose: an allow list that silently included every
+ * subdomain would let a `cdn-user-content.example.com` (whose contents somebody else controls)
+ * through a rule an author wrote for the apex.
+ */
+export function hostMatches(host: string, rule: HostRule): boolean {
+  if (rule === ANY_HOST) return true;
+  const normalised = host.toLowerCase();
+  const cleaned = rule.trim().toLowerCase();
+  if (cleaned.startsWith('*.')) {
+    const suffix = cleaned.slice(1);
+    return normalised.endsWith(suffix) && normalised.length > suffix.length;
+  }
+  return normalised === cleaned;
+}
+
+/**
+ * Fails CLOSED: a URL that cannot be parsed is refused. A driver handed a malformed request has
+ * no way to know where it would have gone, and "we could not tell, so we let it through" is the
+ * decision that makes the whole list advisory.
+ */
+export function hostDecision(url: string, allowHosts: readonly HostRule[]): HostDecision {
+  const scheme = url.slice(0, Math.max(0, url.indexOf(':') + 1)).toLowerCase();
+  if (HOSTLESS_SCHEMES.has(scheme)) return { allowed: true, host: '' };
+  let host: string;
+  try {
+    host = new URL(url).hostname;
+  } catch {
+    return { allowed: false, host: '' };
+  }
+  if (host === '') return { allowed: false, host };
+  return { allowed: allowHosts.some((rule) => hostMatches(host, rule)), host };
+}

package/src/html-query.ts

@@ -0,0 +1,119 @@
+// CSS selectors over an HTML string, on Bun's own `HTMLRewriter` — no jsdom, no cheerio, no
+// dependency at all. This is what makes the fake and the fixture drivers real enough to be worth
+// testing against: they answer the same `query()` the browser driver does, from markup, offline.
+//
+// The selector subset is `HTMLRewriter`'s (lol-html): type, `#id`, `.class`, `[attr]`,
+// `[attr=value]`, descendant and child combinators. A selector it refuses is refused loudly.
+
+import type { ElementSnapshot } from './target';
+import { ROOT_SELECTOR } from './target';
+
+/** Elements with no end tag — `onEndTag` is not available for them, so they close on open. */
+const VOID_TAGS = new Set([
+  'area',
+  'base',
+  'br',
+  'col',
+  'embed',
+  'hr',
+  'img',
+  'input',
+  'link',
+  'meta',
+  'source',
+  'track',
+  'wbr',
+]);
+
+const HIDDEN_STYLE = /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden)\s*(?:;|$)/i;
+
+/** What "rendered" can mean without a layout engine: the markup says so, or it does not. */
+export const markupVisible = (tag: string, attrs: Readonly<Record<string, string>>): boolean => {
+  if ('hidden' in attrs) return false;
+  if (tag === 'input' && attrs['type']?.toLowerCase() === 'hidden') return false;
+  const style = attrs['style'];
+  if (style !== undefined && HIDDEN_STYLE.test(style)) return false;
+  return attrs['aria-hidden'] !== 'true';
+};
+
+export const markupEnabled = (attrs: Readonly<Record<string, string>>): boolean =>
+  !('disabled' in attrs) && attrs['aria-disabled'] !== 'true';
+
+interface Open {
+  readonly tag: string;
+  readonly attrs: Record<string, string>;
+  readonly parts: string[];
+}
+
+const finish = (open: Open): ElementSnapshot => ({
+  tag: open.tag,
+  attrs: open.attrs,
+  text: open.parts.join('').replaceAll(/\s+/g, ' ').trim(),
+  value: open.attrs['value'] ?? '',
+  visible: markupVisible(open.tag, open.attrs),
+  enabled: markupEnabled(open.attrs),
+  // No box and no hit-target: THIS TARGET HAS NO LAYOUT ENGINE. Fabricating either is how an
+  // offline suite reports a covered button as clickable. See `actionability.ts`.
+});
+
+/**
+ * Every element matching `selector`, in document order, with its text content flattened.
+ *
+ * `ROOT_SELECTOR` answers even for a fragment with no `<html>` element — a fixture is usually a
+ * body snippet, and `page.text()` returning `''` for one would be a silent wrong answer rather
+ * than a missing element.
+ */
+export async function queryHtml(
+  html: string,
+  selector: string,
+): Promise<readonly ElementSnapshot[]> {
+  const done: ElementSnapshot[] = [];
+  const open: Open[] = [];
+  const allText: string[] = [];
+  const rewriter = new HTMLRewriter()
+    .on('*', {
+      text(chunk): void {
+        allText.push(chunk.text);
+      },
+    })
+    .on(selector, {
+      element(element): void {
+        const tag = element.tagName.toLowerCase();
+        const attrs: Record<string, string> = {};
+        for (const [name, value] of element.attributes) attrs[name.toLowerCase()] = value;
+        const record: Open = { tag, attrs, parts: [] };
+        if (VOID_TAGS.has(tag)) {
+          done.push(finish(record));
+          return;
+        }
+        open.push(record);
+        element.onEndTag(() => {
+          const index = open.lastIndexOf(record);
+          if (index !== -1) open.splice(index, 1);
+          done.push(finish(record));
+        });
+      },
+      text(chunk): void {
+        // Every OPEN match, not just the innermost: a `<div>` and the `<span>` inside it both
+        // legitimately match `div, span`, and both contain the text.
+        for (const record of open) record.parts.push(chunk.text);
+      },
+    });
+  await rewriter.transform(new Response(html)).text();
+  // Anything still open never saw its end tag (malformed markup). It is still an element that
+  // matched, so it is reported rather than dropped — a scraper's input is somebody else's HTML.
+  for (const record of open) done.push(finish(record));
+  if (done.length === 0 && selector === ROOT_SELECTOR) {
+    return [
+      {
+        tag: 'html',
+        attrs: {},
+        text: allText.join('').replaceAll(/\s+/g, ' ').trim(),
+        value: '',
+        visible: true,
+        enabled: true,
+      },
+    ];
+  }
+  return done;
+}

package/src/html-requests.ts

@@ -0,0 +1,45 @@
+// The subresource requests a document would make, read out of its markup. The offline drivers
+// have no network stack, so this is how they still exercise interception: an `<img>` pointing at
+// a host `allowHosts` does not list produces the same refused network entry offline that a real
+// browser produces on the wire.
+
+import { queryHtml } from './html-query';
+import type { ResourceType } from './rings';
+
+export interface MarkupRequest {
+  readonly url: string;
+  readonly resourceType: ResourceType;
+}
+
+/** Selector, attribute and the type a browser would classify it as. `<a href>` is not a request. */
+const SOURCES: readonly (readonly [string, string, ResourceType])[] = [
+  ['img', 'src', 'image'],
+  ['script', 'src', 'script'],
+  ['link', 'href', 'stylesheet'],
+  ['iframe', 'src', 'document'],
+  ['source', 'src', 'media'],
+  ['video', 'src', 'media'],
+  ['audio', 'src', 'media'],
+];
+
+/** Absolute URLs only: a relative one resolves against `base`, which is the page's own URL. */
+export async function markupRequests(
+  html: string,
+  base: string,
+): Promise<readonly MarkupRequest[]> {
+  const requests: MarkupRequest[] = [];
+  for (const [selector, attribute, resourceType] of SOURCES) {
+    for (const element of await queryHtml(html, selector)) {
+      const raw = element.attrs[attribute];
+      if (raw === undefined || raw === '') continue;
+      // `<link>` covers icons, preloads and manifests too; only a stylesheet is a stylesheet.
+      if (selector === 'link' && (element.attrs['rel'] ?? '') !== 'stylesheet') continue;
+      try {
+        requests.push({ url: new URL(raw, base).toString(), resourceType });
+      } catch {
+        // A specifier no URL parser accepts is not a request any browser would make either.
+      }
+    }
+  }
+  return requests;
+}

package/src/html-target.ts

@@ -0,0 +1,229 @@
+// A `ScrapeTarget` over recorded HTML: no process, no port, no CDP. This is what `bun test` runs
+// against, and it is the reason a scraper's tests need no Chrome.
+//
+// The rule that makes it worth having: an UNRECORDED request THROWS. A driver that quietly fell
+// through to the network would make a green offline suite that is secretly hitting production —
+// the exact failure an offline driver exists to prevent.
+
+import type { ScrapeClock } from './clock';
+import { browserUnreachable, downloadTimeout, fixtureMissing, fixtureStale } from './error-throws';
+import { queryHtml } from './html-query';
+import { markupRequests } from './html-requests';
+import type { InterceptRules } from './intercept';
+import { interceptVerdict, refusalEntry } from './intercept';
+import type { PageRecording } from './recording';
+import { splitDownload } from './recording';
+import type { ConsoleLine, NetworkEntry } from './rings';
+import { createRing } from './rings';
+import type { SessionSnapshot } from './session-state';
+import { EMPTY_SESSION } from './session-state';
+import type {
+  CaptureOptions,
+  ElementSnapshot,
+  FrameRef,
+  GotoOptions,
+  ScrapeCookie,
+  ScrapeDownloadFile,
+  ScrapeTarget,
+} from './target';
+
+/** Deterministic bytes, so an artifact test asserts on a stable digest. Not a real PNG render. */
+const FAKE_PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+const FAKE_PDF = new TextEncoder().encode('%PDF-1.4 offline');
+
+export type RecordingLookup = (url: string) => Promise<PageRecording | undefined>;
+
+export interface HtmlTargetInit {
+  readonly driver: string;
+  readonly lookup: RecordingLookup;
+  readonly rules: InterceptRules;
+  readonly clock: ScrapeClock;
+  /** Where recordings come from, for the cause line: a directory, or `fakeBrowser()`. */
+  readonly source: string;
+  readonly start?: PageRecording | undefined;
+  readonly maxAgeMs?: number | undefined;
+  readonly cookies?: readonly ScrapeCookie[] | undefined;
+  /** What `page.session()` answers, so a fixture can assert on the browser-to-HTTP handoff. */
+  readonly session?: SessionSnapshot | undefined;
+}
+
+const EMPTY: PageRecording = { url: 'about:blank', html: '' };
+
+/** Typed text is an overlay keyed by `id`, then `name`, then the selector used to type it. */
+const keyOf = (selector: string, element: ElementSnapshot | undefined): string => {
+  if (element === undefined) return selector;
+  const id = element.attrs['id'];
+  if (id !== undefined) return `#${id}`;
+  const name = element.attrs['name'];
+  if (name !== undefined) return `[name=${name}]`;
+  return selector;
+};
+
+export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
+  const consoleRing = createRing<ConsoleLine>();
+  const networkRing = createRing<NetworkEntry>();
+  const overlay = new Map<string, string>();
+  let page: PageRecording = init.start ?? EMPTY;
+  let armed: string | undefined;
+  let closed = false;
+  let session: SessionSnapshot = init.session ?? {
+    ...EMPTY_SESSION,
+    cookies: init.cookies ?? [],
+  };
+
+  const live = (): void => {
+    if (closed) throw browserUnreachable(init.driver, 'the offline target is already closed');
+  };
+
+  const withOverlay = (selector: string, elements: readonly ElementSnapshot[]): ElementSnapshot[] =>
+    elements.map((element) => {
+      const typed = overlay.get(keyOf(selector, element));
+      return typed === undefined ? element : { ...element, value: typed };
+    });
+
+  const query = async (selector: string): Promise<readonly ElementSnapshot[]> => {
+    live();
+    return withOverlay(selector, await queryHtml(page.html, selector));
+  };
+
+  const at = (selector: string, index: number): Promise<ElementSnapshot | undefined> =>
+    query(selector).then((elements) => elements[index]);
+
+  /** Interception, offline: every request the markup would make, judged by the same rule. */
+  const intercept = async (recording: PageRecording): Promise<void> => {
+    for (const request of await markupRequests(recording.html, recording.url)) {
+      const verdict = interceptVerdict(request.url, request.resourceType, init.rules);
+      const now = init.clock.now().getTime();
+      networkRing.push(
+        verdict === 'allow'
+          ? {
+              method: 'GET',
+              url: request.url,
+              resourceType: request.resourceType,
+              at: now,
+              status: 200,
+            }
+          : refusalEntry(request.url, request.resourceType, verdict, now),
+      );
+    }
+  };
+
+  const load = async (url: string): Promise<void> => {
+    const found = await init.lookup(url);
+    if (found === undefined) throw fixtureMissing(url, init.source);
+    if (init.maxAgeMs !== undefined && found.recordedAt !== undefined) {
+      const age = init.clock.now().getTime() - new Date(found.recordedAt).getTime();
+      if (age > init.maxAgeMs) throw fixtureStale(url, age, init.maxAgeMs);
+    }
+    page = found;
+    overlay.clear();
+    armed = undefined;
+    networkRing.push({
+      method: 'GET',
+      url,
+      resourceType: 'document',
+      status: 200,
+      at: init.clock.now().getTime(),
+    });
+    await intercept(found);
+  };
+
+  /** Relative specifiers resolve against the current page, exactly as a browser resolves them. */
+  const navigate = async (url: string): Promise<void> => {
+    live();
+    let absolute: string;
+    try {
+      absolute = new URL(url, page.url === 'about:blank' ? undefined : page.url).toString();
+    } catch {
+      throw fixtureMissing(url, init.source);
+    }
+    await load(absolute);
+  };
+
+  const frameTarget = (html: string, url: string): ScrapeTarget => ({
+    ...base,
+    url: () => url,
+    content: () => Promise.resolve(html),
+    query: async (selector) => withOverlay(selector, await queryHtml(html, selector)),
+    frames: () => Promise.resolve([]),
+  });
+
+  const base: ScrapeTarget = {
+    driver: init.driver,
+    console: consoleRing,
+    network: networkRing,
+    url: () => page.url,
+    goto: (url: string, _options: GotoOptions): Promise<void> => navigate(url),
+    content: (): Promise<string> => {
+      live();
+      return Promise.resolve(page.html);
+    },
+    query,
+    async click(selector: string, index: number): Promise<void> {
+      const element = await at(selector, index);
+      if (element === undefined) throw fixtureMissing(`${page.url} ${selector}`, init.source);
+      const download = page.downloads?.[selector] ?? page.downloads?.[element.attrs['id'] ?? ''];
+      if (download !== undefined) armed = download;
+      const href =
+        element.attrs['data-goto'] ?? (element.tag === 'a' ? element.attrs['href'] : undefined);
+      if (href !== undefined && href !== '') await navigate(href);
+    },
+    async type(selector: string, text: string): Promise<void> {
+      const element = await at(selector, 0);
+      const key = keyOf(selector, element);
+      overlay.set(key, `${overlay.get(key) ?? element?.value ?? ''}${text}`);
+    },
+    async clear(selector: string): Promise<void> {
+      overlay.set(keyOf(selector, await at(selector, 0)), '');
+    },
+    async select(selector: string, values: readonly string[]): Promise<void> {
+      overlay.set(keyOf(selector, await at(selector, 0)), values[0] ?? '');
+    },
+    evaluate(expression: string): Promise<unknown> {
+      live();
+      const recorded = page.evaluate?.[expression];
+      // Unrecorded and therefore refused, for the same reason an unrecorded page is: an offline
+      // driver that invented an answer here would make the assertion above it meaningless.
+      if (recorded === undefined)
+        throw fixtureMissing(`${page.url} evaluate(${expression})`, init.source);
+      return Promise.resolve(JSON.parse(recorded) as unknown);
+    },
+    screenshot: (_options: CaptureOptions): Promise<Uint8Array> => Promise.resolve(FAKE_PNG),
+    pdf: (_options: CaptureOptions): Promise<Uint8Array> => Promise.resolve(FAKE_PDF),
+    cookies: (): Promise<readonly ScrapeCookie[]> => Promise.resolve(session.cookies),
+    session: (): Promise<SessionSnapshot> => Promise.resolve(session),
+    restore: (next: SessionSnapshot): Promise<void> => {
+      session = next;
+      return Promise.resolve();
+    },
+    download(options: { readonly timeoutMs: number }): Promise<ScrapeDownloadFile> {
+      if (armed === undefined) throw downloadTimeout(options.timeoutMs, page.url);
+      const { filename, contents } = splitDownload(armed);
+      armed = undefined;
+      return Promise.resolve({ filename, bytes: new TextEncoder().encode(contents) });
+    },
+    async frames(): Promise<readonly FrameRef[]> {
+      const refs: FrameRef[] = [];
+      for (const element of await queryHtml(page.html, 'iframe')) {
+        const name = element.attrs['name'] ?? element.attrs['id'] ?? '';
+        const src = element.attrs['src'] ?? '';
+        const html = page.frames?.[name] ?? page.frames?.[src];
+        if (html === undefined)
+          throw fixtureMissing(`${page.url} iframe ${name || src}`, init.source);
+        const url = src === '' ? page.url : new URL(src, page.url).toString();
+        refs.push({
+          name,
+          url,
+          selector: element.attrs['id'] === undefined ? undefined : `#${element.attrs['id']}`,
+          target: frameTarget(html, url),
+        });
+      }
+      return refs;
+    },
+    close(): Promise<void> {
+      closed = true;
+      return Promise.resolve();
+    },
+  };
+  return base;
+}

package/src/http-recorded.ts

@@ -0,0 +1,85 @@
+// The HTTP leg, offline — the second half of the ONE fixture format. A hybrid scrape replays end
+// to end from a single directory: browser login, session handoff, HTTP bulk fetch.
+//
+// Same rule as the page half: an unrecorded request THROWS. A transport that fell through to the
+// real network would make the hybrid path — which is where the interesting code lives — the one
+// path a test never really covers.
+
+import type { ScrapeClock } from './clock';
+import { fixtureMissing, fixtureStale, hostBlocked } from './error-throws';
+import type { HttpRequestInit, ScrapeHttp, ScrapeResponse } from './http';
+import { responseOver } from './http';
+import type { InterceptRules } from './intercept';
+import { interceptVerdict } from './intercept';
+import type { HttpRecording } from './recording';
+import type { NetworkRing } from './rings';
+import type { RobotsGate } from './robots';
+
+export type HttpRecordingLookup = (
+  method: string,
+  url: string,
+) => Promise<HttpRecording | undefined>;
+
+export interface RecordedHttpInit {
+  readonly lookup: HttpRecordingLookup;
+  readonly rules: InterceptRules;
+  readonly network: NetworkRing;
+  readonly clock: ScrapeClock;
+  readonly source: string;
+  /**
+   * The SAME gate the page leg holds. Without it a hybrid scrape whose JSON endpoint is
+   * `Disallow:`ed replayed green offline and threw terminal `X_SCRAPE_ROBOTS_DISALLOWED` on its
+   * first real attempt — the identical argument the host rule below already makes for itself.
+   */
+  readonly robots?: RobotsGate | undefined;
+  readonly maxAgeMs?: number | undefined;
+}
+
+/** `GET https://api.example.com/v1/orders?page=2` -> `http-get-api-example-com-v1-orders-page-2`. */
+export function httpRecordingFilename(method: string, url: string): string {
+  const slug = `${method}-${url.replace(/^[a-z]+:\/\//i, '')}`
+    .toLowerCase()
+    .replaceAll(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+  return `http-${slug}.json`;
+}
+
+export function recordedHttp(init: RecordedHttpInit): ScrapeHttp {
+  return {
+    async request(url: string, request: HttpRequestInit = {}): Promise<ScrapeResponse> {
+      const method = (request.method ?? 'GET').toUpperCase();
+      // The host rule applies to the offline leg too. Otherwise a scrape that a fixture proves
+      // correct would be the first thing to hit a host `allowHosts` forbids, in production.
+      if (interceptVerdict(url, 'fetch', init.rules) !== 'allow') {
+        throw hostBlocked(url, init.rules.allowHosts);
+      }
+      await init.robots?.assertAllowed(url);
+      const found = await init.lookup(method, url);
+      if (found === undefined) throw fixtureMissing(`${method} ${url}`, init.source);
+      if (init.maxAgeMs !== undefined && found.recordedAt !== undefined) {
+        const age = init.clock.now().getTime() - new Date(found.recordedAt).getTime();
+        if (age > init.maxAgeMs) throw fixtureStale(`${method} ${url}`, age, init.maxAgeMs);
+      }
+      init.network.push({
+        method,
+        url,
+        status: found.status,
+        resourceType: 'fetch',
+        at: init.clock.now().getTime(),
+      });
+      return responseOver(url, found.status, found.headers ?? {}, () =>
+        Promise.resolve(found.body),
+      );
+    },
+  };
+}
+
+/** In-memory recordings, keyed `METHOD url`. What `fakeBrowser({ http })` is built from. */
+export function httpRecordingsOf(recordings: readonly HttpRecording[]): Map<string, HttpRecording> {
+  return new Map(
+    recordings.map((recording) => [
+      `${recording.method.toUpperCase()} ${recording.url}`,
+      recording,
+    ]),
+  );
+}