@ultimat3/scraping 2.0.0

package/src/http.ts

@@ -0,0 +1,158 @@
+// The second transport, and the one that makes a scraper fast: drive the BROWSER through login,
+// 2FA and navigation, then reverse-engineer the site's own JSON endpoints and pull the bulk over
+// plain HTTP. Two hundred paginated pages clicked through is minutes and a hundred chances to
+// break; the same data off the endpoint behind them is seconds, and a JSON endpoint changes far
+// less often than a DOM.
+//
+// It is SESSION-BOUND, never a bare `fetch`. The browser's cookies, the browser's headers, the
+// browser's proxy, the same `allowHosts`, the same robots gate, the same rate limit, the same
+// cancellation. A second transport that quietly had none of those would be a hole in every
+// guarantee the page vocabulary makes — and a different exit IP mid-session is exactly what
+// anti-bot systems look for.
+
+import type { StandardSchemaV1 } from '@ultimat3/schema';
+import { parse } from '@ultimat3/schema';
+import type { ScrapeClock } from './clock';
+import { cookieHeaderFor } from './cookie-scope';
+import { hostBlocked, httpFailed, scrapeTimeout } from './error-throws';
+import type { InterceptRules } from './intercept';
+import { interceptVerdict } from './intercept';
+import type { NetworkRing } from './rings';
+import type { RobotsGate } from './robots';
+import type { SessionSnapshot } from './session-state';
+
+export interface HttpRequestInit {
+  readonly method?: string | undefined;
+  readonly headers?: Readonly<Record<string, string>> | undefined;
+  readonly body?: string | undefined;
+  /** Milliseconds. Falls back to the session's own default. */
+  readonly timeout?: number | undefined;
+}
+
+export interface ScrapeResponse {
+  readonly url: string;
+  readonly status: number;
+  readonly ok: boolean;
+  readonly headers: Readonly<Record<string, string>>;
+  text(): Promise<string>;
+  /** `unknown`, always. A response body is somebody else's JSON until a schema says otherwise. */
+  json(): Promise<unknown>;
+  /**
+   * Parse-or-throw, and the blessed path: a non-2xx answer is `X_SCRAPE_HTTP_FAILED` before the
+   * schema ever runs, so "the endpoint moved" never arrives as "the schema is wrong".
+   */
+  parse<T>(schema: StandardSchemaV1<unknown, T>): Promise<T>;
+}
+
+export interface ScrapeHttp {
+  /** One method. `get`/`post` sugar would be a second way to do the same thing (axiom 1). */
+  request(url: string, init?: HttpRequestInit): Promise<ScrapeResponse>;
+}
+
+export interface HttpTransportInit {
+  readonly rules: InterceptRules;
+  readonly clock: ScrapeClock;
+  readonly timeoutMs: number;
+  readonly network: NetworkRing;
+  /**
+   * Read fresh on every request, and asynchronously because reading a real browser's jar is a
+   * round trip. A snapshot captured when the session opened would be the LOGGED-OUT one forever,
+   * which is precisely the handoff this transport exists to make.
+   */
+  session(): Promise<SessionSnapshot>;
+  readonly robots?: RobotsGate | undefined;
+  readonly pace?: ((signal?: AbortSignal) => Promise<void>) | undefined;
+  readonly signal?: AbortSignal | undefined;
+  readonly onActivity?: (() => void) | undefined;
+  /** The SAME proxy the browser dialled through. A different exit IP is a different client. */
+  readonly proxy?: string | undefined;
+  readonly fetch?: typeof fetch | undefined;
+}
+
+const headerRecord = (headers: Headers): Record<string, string> => {
+  const out: Record<string, string> = {};
+  headers.forEach((value, key) => {
+    out[key] = value;
+  });
+  return out;
+};
+
+export function responseOver(
+  url: string,
+  status: number,
+  headers: Readonly<Record<string, string>>,
+  body: () => Promise<string>,
+): ScrapeResponse {
+  const ok = status >= 200 && status < 300;
+  const text = body;
+  return {
+    url,
+    status,
+    ok,
+    headers,
+    text,
+    json: async (): Promise<unknown> => JSON.parse(await text()) as unknown,
+    async parse<T>(schema: StandardSchemaV1<unknown, T>): Promise<T> {
+      if (!ok) throw httpFailed(url, status, (await text()).slice(0, 200));
+      return parse(schema, JSON.parse(await text()) as unknown);
+    },
+  };
+}
+
+/**
+ * The real transport. Every guarantee the page makes is re-applied here, in the same order and
+ * through the same functions — `interceptVerdict` is the one host rule, `RobotsGate` is the one
+ * robots rule, and neither is re-implemented for the second leg.
+ */
+export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
+  const call = init.fetch ?? fetch;
+  return {
+    async request(url: string, request: HttpRequestInit = {}): Promise<ScrapeResponse> {
+      init.onActivity?.();
+      if (interceptVerdict(url, 'fetch', init.rules) !== 'allow') {
+        throw hostBlocked(url, init.rules.allowHosts);
+      }
+      await init.robots?.assertAllowed(url);
+      await init.pace?.(init.signal);
+      const session = await init.session();
+      const cookies = cookieHeaderFor(session.cookies, url);
+      const timeoutMs = request.timeout ?? init.timeoutMs;
+      // `AbortSignal.timeout` and NOT `clock.sleep`: this is a deadline handed to the platform's
+      // own fetch, not a wait this package performs — and under a test clock a slept deadline
+      // would fire on the microtask after it was armed, cancelling every request instantly.
+      // The offline transport (`http-recorded.ts`) is what a test runs, and it has no deadline.
+      const deadlineSignal = AbortSignal.timeout(timeoutMs);
+      const signals = init.signal === undefined ? [deadlineSignal] : [deadlineSignal, init.signal];
+      try {
+        const response = await call(url, {
+          method: request.method ?? 'GET',
+          headers: {
+            ...session.headers,
+            ...(session.userAgent === '' ? {} : { 'user-agent': session.userAgent }),
+            ...(cookies === undefined ? {} : { cookie: cookies }),
+            ...request.headers,
+          },
+          ...(request.body === undefined ? {} : { body: request.body }),
+          signal: AbortSignal.any(signals),
+          ...(init.proxy === undefined ? {} : { proxy: init.proxy }),
+        } as RequestInit);
+        init.network.push({
+          method: request.method ?? 'GET',
+          url,
+          status: response.status,
+          resourceType: 'fetch',
+          at: init.clock.now().getTime(),
+        });
+        const body = await response.text();
+        return responseOver(url, response.status, headerRecord(response.headers), () =>
+          Promise.resolve(body),
+        );
+      } catch (thrown) {
+        // A deadline that fired is this package's own timeout, with its own code and fix — never
+        // the platform's bare `TimeoutError` reaching a job's retry classifier unclassified.
+        if (deadlineSignal.aborted) throw scrapeTimeout(`http ${url}`, timeoutMs);
+        throw thrown;
+      }
+    },
+  };
+}

package/src/index.ts

@@ -0,0 +1,178 @@
+// Public API of @ultimat3/scraping. Explicit re-exports only — and complete enough that a third
+// party can implement `ScrapeDriver` from this list alone. If a driver author needs a deep import,
+// the seam is not a seam.
+
+export type { ActionabilityState, ActionabilityWait } from './actionability';
+export { actionabilityProblem, awaitActionable, DEFAULT_POLL_MS, isStable } from './actionability';
+export type { ArtifactRef, ArtifactWriter, ArtifactWriterInit } from './artifacts';
+export { contentTypeFor, createArtifactWriter, DEFAULT_ARTIFACT_PREFIX } from './artifacts';
+export type {
+  AuthContext,
+  PromptHandler,
+  PromptRequest,
+  ScrapeAuth,
+} from './auth';
+export { burnSession, createPrompt, ensureAuthenticated, restorableSession } from './auth';
+export type {
+  CdpBrowserLike,
+  CdpFrameLike,
+  CdpLauncherLike,
+  CdpPageLike,
+  CdpRequestLike,
+} from './cdp-port';
+export { parseSnapshots, snapshotExpression } from './cdp-snapshot';
+export type { CdpTargetInit } from './cdp-target';
+export { CDP_DRIVER, cdpTarget } from './cdp-target';
+export type { Deadline, ScrapeClock, TestScrapeClock } from './clock';
+export { deadline, systemScrapeClock, testClock, throwIfAborted } from './clock';
+export {
+  cookieDomainMatches,
+  cookieHeaderFor,
+  cookiePathMatches,
+  cookiesForUrl,
+} from './cookie-scope';
+export type { ScrapeDriver, ScrapeSession, SessionInit } from './driver';
+export { resetScrapeDriver, scrapeDriver, setScrapeDriver } from './driver';
+export type { BrowserOptions, LocalBrowserOptions, RemoteBrowserOptions } from './driver-cdp';
+export { localBrowser, remoteBrowser } from './driver-cdp';
+export type { FakeBrowserOptions, FakePageOptions, FakePages } from './driver-fake';
+export { FAKE_DRIVER, FAKE_PAGE_URL, fakeBrowser, fakePage, recordingsOf } from './driver-fake';
+export type { FixtureBrowserOptions } from './driver-fixture';
+export { FIXTURE_DRIVER, fixtureBrowser, recordingFilename } from './driver-fixture';
+export {
+  authFailed,
+  blocked,
+  browserUnreachable,
+  cdpAttachFailed,
+  downloadTimeout,
+  driverUnknown,
+  fixtureMissing,
+  fixtureStale,
+  hostBlocked,
+  httpFailed,
+  notActionable,
+  outputInvalid,
+  pageCrashed,
+  profileLocked,
+  promptUnanswered,
+  recoverRefused,
+  remoteRequired,
+  robotsDisallowed,
+  scrapeNotImplemented,
+  scrapeTimeout,
+  secretExposed,
+  selectorMissing,
+  sessionExpired,
+  wedged,
+  yieldCollapsed,
+} from './error-throws';
+export type { ScrapeErrorCode, ScrapeErrorInit, ScrapeOwnedErrorCode } from './errors';
+export {
+  isRetryableScrapeError,
+  isScrapeError,
+  SCRAPE_BORROWED_ERROR_CODES,
+  SCRAPE_ERROR_CODES,
+  SCRAPE_ERROR_RETRY,
+  SCRAPE_ERROR_TITLES,
+  SCRAPE_OWNED_ERROR_CODES,
+  ScrapeError,
+} from './errors';
+export type { ScrapeEventFields, StepEvent } from './events';
+export { scrapeLogger, withStepEvent } from './events';
+export type { YieldCheck, YieldExpectation, YieldGuardInput, YieldHistory } from './expect';
+export {
+  DEFAULT_YIELD_WINDOW,
+  guardYield,
+  MIN_BASELINE_RUNS,
+  median,
+  memoryYieldHistory,
+  yieldProblem,
+} from './expect';
+export { BURNS_SESSION, burnsSession, errorCode, NEVER_RETRIED, neverRetried } from './failures';
+export type { HostDecision, HostRule } from './hosts';
+export { ANY_HOST, hostDecision, hostMatches } from './hosts';
+export { markupEnabled, markupVisible, queryHtml } from './html-query';
+export type { MarkupRequest } from './html-requests';
+export { markupRequests } from './html-requests';
+export type { HtmlTargetInit, RecordingLookup } from './html-target';
+export { htmlTarget } from './html-target';
+export type { HttpRequestInit, HttpTransportInit, ScrapeHttp, ScrapeResponse } from './http';
+export { httpOverFetch, responseOver } from './http';
+export type { HttpRecordingLookup, RecordedHttpInit } from './http-recorded';
+export { httpRecordingFilename, httpRecordingsOf, recordedHttp } from './http-recorded';
+export type { InterceptRules, InterceptVerdict } from './intercept';
+export { interceptVerdict, refusalEntry } from './intercept';
+export type { OfflineSessionInit } from './offline-session';
+export { openOfflineSession } from './offline-session';
+export type {
+  CaptureRequest,
+  DownloadRequest,
+  ElementValue,
+  ScrapeFrame,
+  ScrapePage,
+  WaitOptions,
+} from './page';
+export type { PageContext } from './page-over-target';
+export { pageOverTarget } from './page-over-target';
+export type { Pacer } from './rate';
+export { createPacer, DEFAULT_NAVIGATION_RATE } from './rate';
+export type { HttpRecording, PageRecording } from './recording';
+export {
+  httpRecordingSchema,
+  pageRecordingSchema,
+  parseHttpRecording,
+  parseRecording,
+  splitDownload,
+} from './recording';
+export type { AgentRecovery, Recovery, RecoveryAttempt, RecoveryHook } from './recover';
+export { runRecovery } from './recover';
+export type {
+  ConsoleLine,
+  ConsoleRing,
+  NetworkEntry,
+  NetworkRing,
+  ResourceType,
+  Ring,
+} from './rings';
+export { createRing, DEFAULT_RING_CAPACITY, RESOURCE_TYPES } from './rings';
+export type { RobotsFetch, RobotsGate, RobotsGateInit, RobotsPolicy, RobotsRules } from './robots';
+export { createRobotsGate, DEFAULT_ROBOTS_AGENT, parseRobots, robotsAllows } from './robots';
+export type {
+  ScrapeArtifacts,
+  ScrapeDefinition,
+  ScrapeReport,
+  ScrapeRunArgs,
+} from './scrape';
+export { scrape } from './scrape';
+export { DEFAULT_PAGE_TIMEOUT_MS, runScrape } from './scrape-run';
+export type { ScrapeSecrets, SecretResolver } from './secrets';
+export {
+  blankPasswordFields,
+  createSecretBag,
+  redactSecrets,
+  SECRET_PLACEHOLDER,
+  safeHtml,
+} from './secrets';
+export type { ScrapeSessionStore, SessionSnapshot, SessionState } from './session-state';
+export {
+  DEFAULT_SESSION_PREFIX,
+  EMPTY_SESSION,
+  memorySessionStore,
+  parseSessionState,
+  sessionDigest,
+  sessionKeyFor,
+  storageSessionStore,
+} from './session-state';
+export type {
+  CaptureOptions,
+  ElementBox,
+  ElementSnapshot,
+  FrameRef,
+  GotoOptions,
+  ScrapeCookie,
+  ScrapeDownloadFile,
+  ScrapeTarget,
+} from './target';
+export { ROOT_SELECTOR } from './target';
+export type { WedgeGuard, WedgeGuardInit } from './watchdog';
+export { createWedgeGuard, DEFAULT_GRACE_MS, DEFAULT_IDLE_MS } from './watchdog';

package/src/intercept.ts

@@ -0,0 +1,41 @@
+// One decision, asked by every driver before a request leaves: allow it, refuse it because
+// `allowHosts` does not list the host, or refuse it because `block` names the resource type.
+//
+// It lives in its own file so the fake, the fixture and the real browser cannot answer it
+// differently — `driver-parity.test.ts` asserts the same refusal on all three, and a rule
+// re-implemented per driver is a rule that holds only on the driver nobody ships.
+
+import type { HostRule } from './hosts';
+import { hostDecision } from './hosts';
+import type { NetworkEntry, ResourceType } from './rings';
+
+export interface InterceptRules {
+  readonly allowHosts: readonly HostRule[];
+  /** Resource types never fetched. `['image', 'media', 'font']` is most scrapes' whole config. */
+  readonly block?: readonly ResourceType[] | undefined;
+}
+
+export type InterceptVerdict = 'allow' | 'host' | 'blocked';
+
+export function interceptVerdict(
+  url: string,
+  resourceType: ResourceType,
+  rules: InterceptRules,
+): InterceptVerdict {
+  // Type first: a blocked image on an allowed host and a blocked image on a foreign host are both
+  // "never fetched", and reporting the cheaper reason keeps `allowHosts` findings meaningful.
+  if (rules.block?.includes(resourceType) === true) return 'blocked';
+  return hostDecision(url, rules.allowHosts).allowed ? 'allow' : 'host';
+}
+
+/** The ring entry a refusal earns. Refusals are RECORDED, never silent — a scrape that came back
+ * empty is diagnosed from this list, and a blocked POST reported as a GET sends its reader hunting
+ * for a request the page never made. `method` is last and optional because a driver that cannot
+ * read one (a parsed document's `<img src>` is a GET by construction) says so by omitting it. */
+export const refusalEntry = (
+  url: string,
+  resourceType: ResourceType,
+  verdict: Exclude<InterceptVerdict, 'allow'>,
+  at: number,
+  method = 'GET',
+): NetworkEntry => ({ method, url, resourceType, at, refused: verdict });

package/src/offline-session.ts

@@ -0,0 +1,67 @@
+// One assembly for both offline drivers: the recorded page target, the recorded HTTP transport
+// and the page vocabulary over them. `fakeBrowser()` and `fixtureBrowser()` differ only in where
+// the recordings come from — and a second copy of this wiring is how the two would drift apart on
+// the day one of them learns about sessions and the other does not.
+
+import type { ScrapeSession, SessionInit } from './driver';
+import type { RecordingLookup } from './html-target';
+import { htmlTarget } from './html-target';
+import type { HttpRecordingLookup } from './http-recorded';
+import { recordedHttp } from './http-recorded';
+import { pageOverTarget } from './page-over-target';
+import type { PageRecording } from './recording';
+import type { SessionSnapshot } from './session-state';
+import type { ScrapeCookie } from './target';
+
+export interface OfflineSessionInit {
+  readonly driver: string;
+  readonly source: string;
+  readonly lookup: RecordingLookup;
+  readonly http: HttpRecordingLookup;
+  readonly session: SessionInit;
+  readonly start?: PageRecording | undefined;
+  readonly maxAgeMs?: number | undefined;
+  readonly cookies?: readonly ScrapeCookie[] | undefined;
+  readonly snapshot?: SessionSnapshot | undefined;
+}
+
+export async function openOfflineSession(init: OfflineSessionInit): Promise<ScrapeSession> {
+  const target = htmlTarget({
+    driver: init.driver,
+    lookup: init.lookup,
+    rules: init.session.rules,
+    clock: init.session.clock,
+    source: init.source,
+    start: init.start,
+    maxAgeMs: init.maxAgeMs,
+    cookies: init.cookies,
+    session: init.snapshot,
+  });
+  // Restored BEFORE the first navigation, exactly as a real driver must: a session put back after
+  // the first request is a first request made logged out.
+  if (init.session.restore !== undefined) await target.restore(init.session.restore);
+  return {
+    driver: init.driver,
+    page: pageOverTarget(target, {
+      clock: init.session.clock,
+      allowHosts: init.session.rules.allowHosts,
+      defaultTimeoutMs: init.session.timeoutMs,
+      secrets: init.session.secrets,
+      robots: init.session.robots,
+      signal: init.session.signal,
+      onActivity: init.session.onActivity,
+      pace: init.session.pace,
+    }),
+    http: recordedHttp({
+      lookup: init.http,
+      rules: init.session.rules,
+      network: target.network,
+      clock: init.session.clock,
+      source: init.source,
+      // The same gate the page above holds, from the same field: two legs, one robots decision.
+      robots: init.session.robots,
+      maxAgeMs: init.maxAgeMs,
+    }),
+    close: () => target.close(),
+  };
+}

package/src/page-over-target.ts

@@ -0,0 +1,215 @@
+// The ONE implementation of the page vocabulary, over the `ScrapeTarget` port. Real browser,
+// recorded fixture and parsed HTML all reach `run()` through this file — so actionability, frame
+// re-resolution, host enforcement and secret taint are written once and cannot drift between the
+// driver a test uses and the driver production uses.
+
+import type { Secret } from '@ultimat3/core';
+import { isSecret, revealSecret } from '@ultimat3/core';
+import type { ActionabilityState } from './actionability';
+import { awaitActionable } from './actionability';
+import type { ScrapeClock } from './clock';
+import { deadline } from './clock';
+import { hostBlocked, secretExposed, selectorMissing } from './error-throws';
+import { hostDecision } from './hosts';
+import type {
+  CaptureRequest,
+  DownloadRequest,
+  ElementValue,
+  ScrapeFrame,
+  ScrapePage,
+  WaitOptions,
+} from './page';
+import type { RobotsGate } from './robots';
+import type { ScrapeSecrets } from './secrets';
+import { safeHtml } from './secrets';
+import type { ElementSnapshot, ScrapeCookie, ScrapeDownloadFile, ScrapeTarget } from './target';
+import { ROOT_SELECTOR } from './target';
+
+export interface PageContext {
+  readonly clock: ScrapeClock;
+  /**
+   * Called before EVERY operation, poll included. This is what the wedge watchdog measures: the
+   * gap between two of these is the definition of "the browser stopped answering", and putting
+   * the hook here rather than in a wrapper means no page method can forget to report.
+   */
+  readonly onActivity?: (() => void) | undefined;
+  /** Awaited before each navigation. `rate.ts` builds it; omitted means unpaced. */
+  readonly pace?: ((signal?: AbortSignal) => Promise<void>) | undefined;
+  readonly allowHosts: readonly string[];
+  readonly defaultTimeoutMs: number;
+  readonly secrets?: ScrapeSecrets | undefined;
+  readonly robots?: RobotsGate | undefined;
+  readonly signal?: AbortSignal | undefined;
+}
+
+/** Mutable, shared by the page and every frame under it: a taint is a property of the SESSION. */
+interface PageState {
+  tainted: boolean;
+}
+
+type Resolve = () => Promise<ScrapeTarget>;
+
+/** Every operation resolves its target through this, so every operation reports activity. */
+const watched = (resolve: Resolve, ctx: PageContext): Resolve => {
+  return () => {
+    ctx.onActivity?.();
+    return resolve();
+  };
+};
+
+const plainText = (text: string | Secret): string => (isSecret(text) ? revealSecret(text) : text);
+
+const toValue = (snapshot: ElementSnapshot): ElementValue => ({
+  tag: snapshot.tag,
+  text: snapshot.text,
+  value: snapshot.value,
+  attrs: snapshot.attrs,
+});
+
+async function first(resolve: Resolve, selector: string): Promise<ElementSnapshot | undefined> {
+  const target = await resolve();
+  return (await target.query(selector))[0];
+}
+
+function frameOver(
+  rawResolve: Resolve,
+  ctx: PageContext,
+  state: PageState,
+  seed: string,
+): ScrapeFrame {
+  const resolve = watched(rawResolve, ctx);
+  let lastUrl = seed;
+  const timeoutFor = (options: WaitOptions | undefined): number =>
+    options?.timeout ?? ctx.defaultTimeoutMs;
+
+  const wait = async (
+    selector: string,
+    options: WaitOptions | undefined,
+    fallback: ActionabilityState,
+  ): Promise<ElementSnapshot> => {
+    const target = await resolve();
+    lastUrl = target.url();
+    return awaitActionable({
+      selector,
+      url: lastUrl,
+      state: options?.state ?? fallback,
+      timeoutMs: timeoutFor(options),
+      clock: ctx.clock,
+      signal: ctx.signal,
+      snapshot: () => first(resolve, selector),
+    });
+  };
+
+  return {
+    url: () => lastUrl,
+    waitFor: (selector, options) => wait(selector, options, 'actionable'),
+    async click(selector, options): Promise<void> {
+      await wait(selector, options, 'actionable');
+      await (await resolve()).click(selector, 0);
+    },
+    async type(selector, text, options): Promise<void> {
+      await wait(selector, options, 'actionable');
+      if (isSecret(text)) state.tainted = true;
+      await (await resolve()).type(selector, plainText(text));
+    },
+    async fill(selector, text, options): Promise<void> {
+      await wait(selector, options, 'actionable');
+      const target = await resolve();
+      await target.clear(selector);
+      if (isSecret(text)) state.tainted = true;
+      await target.type(selector, plainText(text));
+    },
+    async select(selector, values, options): Promise<void> {
+      await wait(selector, options, 'actionable');
+      await (await resolve()).select(selector, values);
+    },
+    async values(selector): Promise<readonly ElementValue[]> {
+      return (await (await resolve()).query(selector)).map(toValue);
+    },
+    async text(selector): Promise<string> {
+      const target = await resolve();
+      if (selector === undefined) return (await target.query(ROOT_SELECTOR))[0]?.text ?? '';
+      return (await target.query(selector))[0]?.text ?? '';
+    },
+    async html(): Promise<string> {
+      return safeHtml(await (await resolve()).content(), ctx.secrets);
+    },
+    async count(selector): Promise<number> {
+      return (await (await resolve()).query(selector)).length;
+    },
+    async evaluate(expression): Promise<unknown> {
+      return (await resolve()).evaluate(expression);
+    },
+    frame(nameOrSelector): ScrapeFrame {
+      // The resolver, not the frame. Every call through the returned handle runs this again, so
+      // a handle taken before a re-navigation addresses the CURRENT frame with that name and
+      // never a detached one — the trap `frameLocator`-style handles set for every caller.
+      const resolveChild = async (): Promise<ScrapeTarget> => {
+        const budget = deadline(ctx.clock, ctx.defaultTimeoutMs);
+        for (;;) {
+          const parent = await resolve();
+          const found = (await parent.frames()).find(
+            (ref) =>
+              ref.name === nameOrSelector ||
+              ref.selector === nameOrSelector ||
+              ref.url === nameOrSelector,
+          );
+          if (found !== undefined) return found.target;
+          if (budget.expired()) {
+            throw selectorMissing(nameOrSelector, parent.url(), ctx.defaultTimeoutMs);
+          }
+          await ctx.clock.sleep(Math.min(50, budget.remainingMs()), ctx.signal);
+        }
+      };
+      return frameOver(resolveChild, ctx, state, lastUrl);
+    },
+  };
+}
+
+/**
+ * Refused BEFORE the navigation, not reported after it. An `allowHosts` consulted afterwards is a
+ * log line about a request that already left the container.
+ */
+async function guardNavigation(url: string, ctx: PageContext): Promise<void> {
+  const decision = hostDecision(url, ctx.allowHosts);
+  if (!decision.allowed) throw hostBlocked(url, ctx.allowHosts);
+  await ctx.robots?.assertAllowed(url);
+}
+
+export function pageOverTarget(target: ScrapeTarget, ctx: PageContext): ScrapePage {
+  const state: PageState = { tainted: false };
+  const frame = frameOver(() => Promise.resolve(target), ctx, state, target.url());
+  const capture = async (
+    kind: 'screenshot' | 'pdf',
+    options: CaptureRequest | undefined,
+  ): Promise<Uint8Array> => {
+    // The leak nobody remembers: a screenshot of a filled login form IS the password, in pixels,
+    // in object storage, forever. Refused rather than masked — a mask over pixels is a guess
+    // about layout, and `page.html()` already gives a redacted artifact that is exact.
+    if (state.tainted) throw secretExposed(kind, target.url());
+    const timeoutMs = options?.timeout ?? ctx.defaultTimeoutMs;
+    const request = { fullPage: options?.fullPage, timeoutMs };
+    return kind === 'screenshot' ? target.screenshot(request) : target.pdf(request);
+  };
+  return {
+    ...frame,
+    async goto(url, options): Promise<void> {
+      await guardNavigation(url, ctx);
+      await ctx.pace?.(ctx.signal);
+      ctx.onActivity?.();
+      await target.goto(url, {
+        timeoutMs: options?.timeout ?? ctx.defaultTimeoutMs,
+        signal: ctx.signal,
+      });
+    },
+    screenshot: (options) => capture('screenshot', options),
+    pdf: (options) => capture('pdf', options),
+    download: (options?: DownloadRequest): Promise<ScrapeDownloadFile> =>
+      target.download({ timeoutMs: options?.timeout ?? ctx.defaultTimeoutMs }),
+    cookies: (): Promise<readonly ScrapeCookie[]> => target.cookies(),
+    session: () => target.session(),
+    console: () => target.console.entries(),
+    network: () => target.network.entries(),
+    networkDropped: () => target.network.dropped,
+  };
+}