@ultimat3/scraping 24.0.0 → 25.0.0

package/src/robots-fetch.ts

@@ -8,14 +8,14 @@
 // The exit is a RESOLVER, not a string: `scrape-run.ts` builds this gate as an argument to
 // `driver.open()`, and the proxy is a driver option the session only reports on the way back out.
 
-import { finiteCount, readWithinLimit } from '@ultimat3/core';
-import type { HostRule } from './hosts';
+import type { HostRule } from '@ultimat3/core';
+import { finiteCount, isUltimateError, readWithinLimit } from '@ultimat3/core';
 import type { ScrapeFetch } from './http';
 import { redirectHop } from './http-redirect';
 import { interceptVerdict } from './intercept';
 import type { HostResolve } from './pinned-host';
 import { dialTarget, resolveHost } from './pinned-host';
-import type { RobotsFetch } from './robots';
+import type { RobotsAnswer, RobotsFetch } from './robots';
 
 /**
  * A deadline is applied ALWAYS, proxy or no proxy, session or no session: the failure it prevents
@@ -80,8 +80,18 @@ const sameHost = (from: string, to: string): boolean => {
   }
 };
 
-/** The final hop's body under the cap, or `undefined` for a non-2xx answer. */
-const answerOf = async (response: Response, limit: number): Promise<string | undefined> => {
+/**
+ * RFC 9309 §2.3.1.4 names server errors; a 429 is the same condition spelled as a client status —
+ * the origin is shedding load, not reporting a missing file — and every major crawler reads it so.
+ */
+const isUnreachableStatus = (status: number): boolean => status >= 500 || status === 429;
+
+/** The final hop's body under the cap, unreachable for a 5xx/429, `undefined` for another non-2xx. */
+const answerOf = async (response: Response, limit: number): Promise<RobotsAnswer> => {
+  if (isUnreachableStatus(response.status)) {
+    await response.body?.cancel().catch(() => undefined);
+    return { unreachable: `status ${String(response.status)}` };
+  }
   if (!response.ok) return undefined;
   // Counted as it arrives rather than `.text()`, which materialises the whole body first: a
   // multi-gigabyte robots.txt is a heap the worker never gets back.
@@ -90,21 +100,22 @@ const answerOf = async (response: Response, limit: number): Promise<string | und
 };
 
 /**
- * Reads `robotsUrl`, or answers `undefined` — which the gate reads as "no restrictions", the
- * standard's own answer for a file it cannot obtain. A deadline that fires, a body past the cap,
- * a 404 and a redirect off `allowHosts` are all the same answer on purpose: none of them is
- * evidence of a rule.
+ * Reads `robotsUrl`. Three answers, after RFC 9309 §2.3.1: the text; `undefined` when the file is
+ * UNAVAILABLE — a 4xx, a body past the cap, a redirect off `allowHosts` or past the hop cap, a
+ * host the pin refuses — which the gate reads as no rules; and `{ unreachable }` for a 5xx, a 429,
+ * a deadline, a cancellation or a network error, which the gate reads as complete disallow. A
+ * server that cannot answer is not a server that has no rules.
  */
 export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
   const call: ScrapeFetch = init.fetch ?? fetch;
   const resolve = init.resolve ?? resolveHost;
-  // Both bounds are screened HERE, at construction, and both floors are 1 — because every way this
-  // read can fail is the same answer, `undefined`, which the gate reads as "no restrictions". A
+  // Both bounds are screened HERE, at construction, and both floors are 1 — because a bad bound
+  // turns into a failed read on every origin, and a read that fails is never evidence of a rule. A
   // `NaN` deadline throws a bare `TypeError` out of `AbortSignal.timeout` (measured: "Value NaN is
   // outside the range [0, 9007199254740991]") straight into the gate's own `.catch`, and a `NaN`
   // cap makes `readWithinLimit` refuse after the request already left. A zero of either is the
-  // same outcome spelled deliberately: an expired deadline and a cap every file is over. Robots
-  // enforcement off, for the whole run, with nothing in the log.
+  // same outcome spelled deliberately: an expired deadline and a cap every file is over — every
+  // origin refused, or (the cap) every origin unrestricted, for the whole run.
   const limit = finiteCount(
     'robotsFetcher',
     'maxBytes',
@@ -117,7 +128,7 @@ export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
     init.timeoutMs ?? DEFAULT_ROBOTS_TIMEOUT_MS,
     1,
   );
-  return async (robotsUrl: string): Promise<string | undefined> => {
+  return async (robotsUrl: string): Promise<RobotsAnswer> => {
     // Armed per read, not per gate: the gate is long-lived and reads once per origin, so a
     // deadline created alongside it would already have expired by the second origin.
     const deadline = AbortSignal.timeout(timeoutMs);
@@ -161,8 +172,13 @@ export function robotsFetcher(init: RobotsFetchInit = {}): RobotsFetch {
         if (init.allowHosts === undefined && !sameHost(robotsUrl, next.url)) return undefined;
         url = next.url;
       }
-    } catch {
-      return undefined;
+    } catch (error) {
+      // The pin refusing an inward name is a POLICY answer about the host, and the navigation is
+      // refused by the same rule — it is not evidence the origin is down.
+      if (isUltimateError(error) && error.code === 'X_SCRAPE_HOST_BLOCKED') return undefined;
+      if (deadline.aborted) return { unreachable: 'timeout' };
+      if (init.signal?.aborted === true) return { unreachable: 'cancelled' };
+      return { unreachable: 'network' };
     }
   };
 }

package/src/robots.ts

@@ -126,7 +126,20 @@ export interface RobotsGate {
   readonly ignoredBecause: string | undefined;
 }
 
-export type RobotsFetch = (robotsUrl: string) => Promise<string | undefined>;
+/**
+ * RFC 9309 §2.3.1.4: the file could not be obtained because of a server or network error — a 5xx,
+ * a 429, a refused connection, a deadline. The standard's answer is COMPLETE DISALLOW, and it is
+ * a different answer from `undefined` (a 4xx: the file is unavailable, so there are no rules).
+ * `unreachable` says why, for the log; nothing branches on its text.
+ */
+export interface RobotsUnreachable {
+  readonly unreachable: string;
+}
+
+/** The file's text, `undefined` for no file (no rules), or unreachable (no access). */
+export type RobotsAnswer = string | undefined | RobotsUnreachable;
+
+export type RobotsFetch = (robotsUrl: string) => Promise<RobotsAnswer>;
 
 export interface RobotsGateInit extends RobotsFetchInit {
   readonly policy: RobotsPolicy;
@@ -135,20 +148,32 @@ export interface RobotsGateInit extends RobotsFetchInit {
   readonly fetchText?: RobotsFetch | undefined;
 }
 
+/** What the gate holds per origin: the rules, or the standard's complete disallow. */
+type Verdict = RobotsRules | 'unreachable';
+
+const verdictOf = (answer: RobotsAnswer, agent: string): Verdict => {
+  if (answer === undefined) return { rules: [] };
+  if (typeof answer === 'string') return parseRobots(answer, agent);
+  return 'unreachable';
+};
+
 /**
  * One fetch of `/robots.txt` per ORIGIN per run, cached. A gate that re-fetched per navigation
  * would triple the request count of every scrape it protects.
  *
- * An origin whose robots.txt cannot be read is ALLOWED. That is the standard's own answer — a
- * missing file means no restrictions — and the alternative fails every run behind a flaky CDN.
+ * A MISSING robots.txt (a 4xx, a redirect chain past the cap) is no restrictions — the standard's
+ * own answer. An UNREACHABLE one (5xx, 429, a network error, a deadline, a read that rejects) is
+ * complete disallow (RFC 9309 §2.3.1.4), and is NOT cached: the next navigation to the origin asks
+ * again, so a run behind a flaky CDN recovers the moment the file answers instead of failing for
+ * the rest of the run — and an origin cannot shed its own rules by answering 503 once.
  */
-export function createRobotsGate(init: RobotsGateInit): RobotsGate {
+export function robotsGate(init: RobotsGateInit): RobotsGate {
   if (init.policy !== 'obey') {
     return { assertAllowed: () => Promise.resolve(), ignoredBecause: init.policy.ignore };
   }
   const agent = init.agent ?? DEFAULT_ROBOTS_AGENT;
   const read = init.fetchText ?? robotsFetcher(init);
-  const cache = new Map<string, Promise<RobotsRules>>();
+  const cache = new Map<string, Promise<Verdict>>();
   return {
     ignoredBecause: undefined,
     async assertAllowed(url: string): Promise<void> {
@@ -160,14 +185,27 @@ export function createRobotsGate(init: RobotsGateInit): RobotsGate {
       }
       if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return;
       const origin = parsed.origin;
-      let rules = cache.get(origin);
-      if (rules === undefined) {
-        rules = read(`${origin}/robots.txt`)
-          .then((text) => (text === undefined ? { rules: [] } : parseRobots(text, agent)))
-          .catch(() => ({ rules: [] }));
-        cache.set(origin, rules);
+      let pending = cache.get(origin);
+      if (pending === undefined) {
+        // A caller-supplied read that rejects has not produced a file, and "could not ask" is
+        // the unreachable case, not the missing one.
+        const fresh = read(`${origin}/robots.txt`).then(
+          (answer) => verdictOf(answer, agent),
+          (): Verdict => 'unreachable',
+        );
+        pending = fresh;
+        cache.set(origin, fresh);
+        // Concurrent navigations still share the one in-flight read; only a settled unreachable
+        // verdict is evicted, and only if no later read has replaced it.
+        void fresh.then((verdict) => {
+          if (verdict === 'unreachable' && cache.get(origin) === fresh) cache.delete(origin);
+        });
       }
-      if (!robotsAllows(await rules, `${parsed.pathname}${parsed.search}`)) {
+      const verdict = await pending;
+      if (
+        verdict === 'unreachable' ||
+        !robotsAllows(verdict, `${parsed.pathname}${parsed.search}`)
+      ) {
         throw robotsDisallowed(url, agent);
       }
     },

package/src/scrape-run.ts

@@ -12,15 +12,15 @@
 import { finiteCount, finiteOption } from '@ultimat3/core';
 import type { JobRunArgs } from '@ultimat3/jobs';
 import { parse } from '@ultimat3/schema';
-import { createArtifactWriter } from './artifacts';
+import { artifactWriter } from './artifacts';
 import type { AuthPlanInput } from './auth';
 import {
   burnSession,
-  createPrompt,
   ensureAuthenticated,
   markRefused,
   observeSession,
   persistSession,
+  scrapePrompt,
 } from './auth';
 import { scrapeClock } from './clock';
 import type { ScrapeSession } from './driver';
@@ -31,16 +31,16 @@ import type { ScrapeEventFields } from './events';
 import { scrapeLogger, withStepEvent } from './events';
 import { guardYield } from './expect';
 import { burnsSession, errorCode, neverRetried } from './failures';
-import { createPacer, DEFAULT_NAVIGATION_RATE } from './rate';
+import { DEFAULT_NAVIGATION_RATE, scrapePacer } from './rate';
 import { runRecovery } from './recover';
-import { createRobotsGate } from './robots';
+import { robotsGate } from './robots';
 import type { ScrapeDefinition, ScrapeReport } from './scrape';
 import { containsSecret } from './secret-scan';
-import { createSecretBag, MIN_REDACTABLE_LENGTH } from './secrets';
+import { MIN_REDACTABLE_LENGTH, secretBag } from './secrets';
 import { sessionKeyFor } from './session-state';
 import { hasCredentials, splitCredentials, urlSecretValues } from './url-secrets';
 import type { ScrapeUsage } from './usage';
-import { createUsageMeter, rememberFailedUsage } from './usage';
+import { rememberFailedUsage, usageMeter } from './usage';
 
 /** `ctx.actor` is a structural read: this package never imports the auth types (tier 2). */
 const orgOf = (ctx: unknown): string | undefined => {
@@ -99,7 +99,7 @@ export async function runScrape<I, Row>(
     attempt: args.attempt,
     driver: driver.name,
   });
-  const secrets = createSecretBag(definition.secrets ?? []);
+  const secrets = secretBag(definition.secrets ?? []);
   // The run's exit, resolved HERE — in the worker, under the job's tenant — so what the queue row
   // carries is the id of whatever holds it. An empty answer is "none", the same as absent: a
   // driver handed `''` would dial `--proxy-server=` and call that a decision.
@@ -120,11 +120,11 @@ export async function runScrape<I, Row>(
   // refuses the DECLARATION and never sees a definition assembled by hand, which `runScrape` is
   // exported to accept. `finiteOption` and not `finiteCount` — a rate of 0.5 is one navigation
   // every two seconds, and `scrape()` owns the "greater than zero" half.
-  const pace = createPacer(
+  const pace = scrapePacer(
     finiteOption('the scrape definition', 'rate', definition.rate ?? DEFAULT_NAVIGATION_RATE),
     clock,
   );
-  const artifact = createArtifactWriter({
+  const artifact = artifactWriter({
     storage: definition.artifacts?.storage,
     scrape: definition.name,
     runId: args.runId,
@@ -156,7 +156,7 @@ export async function runScrape<I, Row>(
   // during a navigation, which is after this is assigned.
   let sessionProxy: string | undefined;
   // Before `open()`: the time a rented browser takes to arrive is time it was held.
-  const usage = createUsageMeter(clock);
+  const usage = usageMeter(clock);
   const session = await driver.open({
     name: definition.name,
     logger,
@@ -173,7 +173,7 @@ export async function runScrape<I, Row>(
     // unreachable by `ctx.signal`; without the third the read leaves from a different IP than
     // every page load, and an origin reachable only through the proxy answers nothing — which
     // this gate reads as "no restrictions".
-    robots: createRobotsGate({
+    robots: robotsGate({
       policy: definition.robots ?? 'obey',
       // Every hop of the read screened by the run's own host rule, as the HTTP leg's are.
       allowHosts: rules.allowHosts,
@@ -205,7 +205,7 @@ export async function runScrape<I, Row>(
             page: session.page,
             secrets,
             restored,
-            prompt: createPrompt({
+            prompt: scrapePrompt({
               scrape: definition.name,
               handler: definition.prompt,
               input: args.input,
@@ -323,8 +323,8 @@ async function bodyWithRecovery<I, Row>(
   args: JobRunArgs<I>,
   session: ScrapeSession,
   logger: ReturnType<typeof scrapeLogger>,
-  artifact: ReturnType<typeof createArtifactWriter>,
-  secrets: ReturnType<typeof createSecretBag>,
+  artifact: ReturnType<typeof artifactWriter>,
+  secrets: ReturnType<typeof secretBag>,
 ): Promise<readonly unknown[]> {
   const body = (): Promise<readonly unknown[]> =>
     definition.run({
@@ -365,7 +365,7 @@ async function bodyWithRecovery<I, Row>(
  */
 async function saveFailureArtifact(
   session: ScrapeSession,
-  artifact: ReturnType<typeof createArtifactWriter>,
+  artifact: ReturnType<typeof artifactWriter>,
 ): Promise<void> {
   try {
     await artifact.save('page.html', await session.page.html());

package/src/scrape.ts

@@ -9,7 +9,7 @@
 // the account. So this file is a FACTORY over `job()`, and a scrape inherits `.enqueue()`, the
 // worker's cancellation, the dead-letter path, `x jobs show` and its manifest row for free.
 
-import type { Ctx } from '@ultimat3/core';
+import type { Ctx, HostRule } from '@ultimat3/core';
 import { assert } from '@ultimat3/core';
 import type {
   JobCompleted,
@@ -31,7 +31,6 @@ import type { ScrapeDriver } from './driver';
 import { yieldHistoryMissing } from './error-throws';
 import type { YieldExpectation, YieldHistory } from './expect';
 import { maxDropFraction } from './expect';
-import type { HostRule } from './hosts';
 import type { ScrapeHttp } from './http';
 import type { ScrapePage } from './page';
 import type { Recovery } from './recover';

package/src/secrets.ts

@@ -33,7 +33,7 @@ export type SecretResolver = (name: string) => string | undefined;
 
 const fromEnvironment: SecretResolver = (name) => Bun.env[name];
 
-export function createSecretBag(
+export function secretBag(
   names: readonly string[],
   resolve: SecretResolver = fromEnvironment,
 ): ScrapeSecrets {

package/src/target.ts

@@ -15,6 +15,18 @@ import type { SessionSnapshot } from './session-state';
  */
 export const ROOT_SELECTOR = 'html';
 
+/** What a `query()` may do to the page before it measures. */
+export interface QueryOptions {
+  /**
+   * Scroll the FIRST match into the viewport before measuring. `elementFromPoint` answers nothing
+   * for a point off-screen, so without it every element below the fold read `hitTarget: false` —
+   * "covered" — and each act on it waited out its whole timeout as `X_SCRAPE_NOT_ACTIONABLE`.
+   * Only the `actionable` wait asks (`page-over-target.ts`): a READ must not move the page, or the
+   * next screenshot is of somewhere else. A driver with no layout engine ignores it.
+   */
+  readonly reveal?: boolean | undefined;
+}
+
 export interface ElementBox {
   readonly x: number;
   readonly y: number;
@@ -105,7 +117,7 @@ export interface GotoOptions {
  * so `page.screenshot({ timeout })` was a documented deadline that bounded nothing. Deleted
  * rather than implemented: the CDP port's own `screenshot({ fullPage })` has no timeout slot to
  * forward it to, and a deadline enforced in `page-over-target.ts` would have to race
- * `ScrapeClock.sleep`, which under `testClock` resolves on the first microtask and would time
+ * `ScrapeClock.sleep`, which under `testScrapeClock` resolves on the first microtask and would time
  * out every capture in every test. A driver's own default is the honest bound.
  */
 export type CaptureOptions = CaptureFraming;
@@ -134,7 +146,7 @@ export interface ScrapeTarget {
   goto(url: string, options: GotoOptions): Promise<void>;
   /** Serialised HTML of THIS target — the document for a page, the subtree for a frame. */
   content(): Promise<string>;
-  query(selector: string): Promise<readonly ElementSnapshot[]>;
+  query(selector: string, options?: QueryOptions): Promise<readonly ElementSnapshot[]>;
   /**
    * Clicks the FIRST match. It took an `index` until 2026-08 that `html-target.ts` honoured and
    * `cdp-target.ts` dropped — its implementations are `click: (selector) => …`, so puppeteer

package/src/usage.ts

@@ -43,7 +43,7 @@ export interface RunUsageMeter extends UsageMeter {
 }
 
 /** Built BEFORE `driver.open()`, so the time a rented browser took to arrive is time it was held. */
-export function createUsageMeter(clock: ScrapeClock): RunUsageMeter {
+export function usageMeter(clock: ScrapeClock): RunUsageMeter {
   const startedAt = clock.monotonic();
   let navigations = 0;
   let httpRequests = 0;

package/src/cdp-fake-target.ts

@@ -1,67 +0,0 @@
-// The browser TARGET of `cdp-fake.ts`'s browser: one raw session that records what the driver
-// enabled and answers `Fetch.requestPaused` the way Chrome does — a request from a target this
-// package never created (a popup) pauses here and nowhere else. Split out because `cdp-fake.ts`
-// stands near its size ceiling.
-
-import type { CdpBrowserSessionLike, CdpBrowserTargetLike } from './cdp-port';
-
-export interface FakeBrowserTarget {
-  readonly target: CdpBrowserTargetLike;
-  /** A request from another target — a popup — pausing at the browser-level `Fetch` domain. */
-  emitTargetRequest(url: string, resourceType: string, method?: string): void;
-  /** What browser-level interception decided, and whether it was enabled at all. */
-  readonly browserFetch: {
-    readonly enabled: boolean;
-    readonly continued: readonly string[];
-    readonly failed: readonly string[];
-  };
-}
-
-/** `refuseEnable`: the browser answers `Fetch.enable` with a protocol error, as a locked-down provider might. */
-export function fakeBrowserTarget(
-  options: { readonly refuseEnable?: boolean } = {},
-): FakeBrowserTarget {
-  const listeners: ((payload: unknown) => void)[] = [];
-  const byId = new Map<string, string>();
-  const continued: string[] = [];
-  const failed: string[] = [];
-  let enabled = false;
-  const session: CdpBrowserSessionLike = {
-    send: (method, params) => {
-      if (method === 'Fetch.enable') {
-        if (options.refuseEnable === true) {
-          return Promise.reject(new Error("'Fetch.enable' wasn't found"));
-        }
-        enabled = true;
-      }
-      const id = typeof params?.['requestId'] === 'string' ? params['requestId'] : '';
-      if (method === 'Fetch.continueRequest') continued.push(byId.get(id) ?? id);
-      if (method === 'Fetch.failRequest') failed.push(byId.get(id) ?? id);
-      return Promise.resolve({});
-    },
-    detach: () => Promise.resolve(),
-    on: (event, handler) => {
-      if (event === 'Fetch.requestPaused') listeners.push(handler);
-      return undefined;
-    },
-  };
-  return {
-    target: { createCDPSession: () => Promise.resolve(session) },
-    emitTargetRequest(url, resourceType, method = 'GET'): void {
-      // Paused only while enabled, as in Chrome: before `Fetch.enable` the request just leaves.
-      if (!enabled) return;
-      const requestId = `interception-${String(byId.size + 1)}`;
-      byId.set(requestId, url);
-      for (const listener of listeners) {
-        listener({ requestId, resourceType, request: { url, method } });
-      }
-    },
-    browserFetch: {
-      get enabled(): boolean {
-        return enabled;
-      },
-      continued,
-      failed,
-    },
-  };
-}