@ultimat3/scraping 20.2.1 → 22.0.0

package/README.md

@@ -298,6 +298,15 @@ Every code carries a cause, a runnable `fix:` and a retry classification — see
 `X_SCRAPE_YIELD_COLLAPSED`, `X_SCRAPE_AUTH_FAILED`, `X_SCRAPE_BLOCKED` and `X_SCRAPE_PAGE_CRASHED`
 are the four worth knowing by heart.
 
+### Error classes
+
+Every error class `src/index.ts` exports, for `instanceof` inside one process. Across a wire or
+a job boundary the class is gone and the `code` is what survives — match on that.
+
+| Class | Code | Declared in |
+|---|---|---|
+| `ScrapeError` | any `ScrapeErrorCode` — `SCRAPE_ERROR_CODES` | `src/errors.ts` |
+
 ## Boundary
 
 Tier 5. May import tiers 0-4 only — enforced by `bun run scripts/boundaries.ts`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/scraping",
-  "version": "20.2.1",
+  "version": "22.0.0",
   "description": "Browser automation as a job: scrape() returns a JobHandle",
   "license": "MIT",
   "type": "module",
@@ -30,9 +30,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "20.2.1",
-    "@ultimat3/jobs": "20.2.1",
-    "@ultimat3/schema": "20.2.1",
-    "@ultimat3/storage": "20.2.1"
+    "@ultimat3/core": "22.0.0",
+    "@ultimat3/jobs": "22.0.0",
+    "@ultimat3/schema": "22.0.0",
+    "@ultimat3/storage": "22.0.0"
   }
 }

package/src/cdp-arm.ts

@@ -135,11 +135,14 @@ export async function arm(init: CdpArmInit, sinks: CdpSinks): Promise<void> {
     const at = init.clock.now().getTime();
     if (verdict === 'allow') {
       network.push({ method, url, resourceType: type, at });
-      void request.continue();
+      // Caught, never floated: both reject when the target closed mid-request (and on any other
+      // protocol error), and a floated rejection is one Bun ends the process on. The request is
+      // already recorded; there is nothing left to do for a target that is gone.
+      request.continue().catch(() => undefined);
       return;
     }
     network.push(refusalEntry(url, type, verdict, at, method));
-    void request.abort();
+    request.abort().catch(() => undefined);
   });
   init.page.on('console', (payload) => {
     console_.push({

package/src/hosts.ts

@@ -1,70 +1,4 @@
-// `allowHosts`, as a decision every driver asks before a request leaves — never a note in a
-// README. A headless browser inside your network is the widest SSRF surface an app can own: one
-// injected `<img src="http://169.254.169.254/…">` on a page you do not control is a credential
-// read, and no amount of "we only visit example.com" in prose intercepts it.
-
-export type HostRule = string;
-
-/** The one spelling that means "every host", written out so it is visible in review. */
-export const ANY_HOST: HostRule = '*';
-
-/**
- * Schemes with no host to match. `about:blank` is where every browser starts, `data:` and `blob:`
- * never leave the process — refusing them would refuse the first page load of every run.
- */
-const HOSTLESS_SCHEMES = new Set(['about:', 'data:', 'blob:']);
-
-/**
- * Hostless AND refused, which is why it is not on the line above — it sat there until 2026-09.
- * `javascript:` has no host for the same reason `data:` has none, and that is the whole
- * resemblance: the other three are inert content this process renders, while this one is code
- * EXECUTED in the current document's origin, with the session's cookies and the session's
- * `localStorage` already in scope. `allowHosts` cannot say anything about a URL with no host to
- * name, so "no host, therefore allowed" was the allow list opting itself out of the one navigation
- * that needs no host to exfiltrate through — `javascript:fetch('/admin').then(post_elsewhere)` is
- * a same-origin read on an allow-listed site. Fail closed; there is no legitimate scrape verb that
- * needs it (`page.eval` is the declared seam).
- */
-const REFUSED_SCHEMES = new Set(['javascript:']);
-
-export interface HostDecision {
-  readonly allowed: boolean;
-  /** The host the URL resolved to, `''` for a hostless scheme. */
-  readonly host: string;
-}
-
-/**
- * `example.com` matches that host EXACTLY. `*.example.com` matches any subdomain and NOT the
- * apex — the two are written separately on purpose: an allow list that silently included every
- * subdomain would let a `cdn-user-content.example.com` (whose contents somebody else controls)
- * through a rule an author wrote for the apex.
- */
-export function hostMatches(host: string, rule: HostRule): boolean {
-  if (rule === ANY_HOST) return true;
-  const normalised = host.toLowerCase();
-  const cleaned = rule.trim().toLowerCase();
-  if (cleaned.startsWith('*.')) {
-    const suffix = cleaned.slice(1);
-    return normalised.endsWith(suffix) && normalised.length > suffix.length;
-  }
-  return normalised === cleaned;
-}
-
-/**
- * Fails CLOSED: a URL that cannot be parsed is refused. A driver handed a malformed request has
- * no way to know where it would have gone, and "we could not tell, so we let it through" is the
- * decision that makes the whole list advisory.
- */
-export function hostDecision(url: string, allowHosts: readonly HostRule[]): HostDecision {
-  const scheme = url.slice(0, Math.max(0, url.indexOf(':') + 1)).toLowerCase();
-  if (REFUSED_SCHEMES.has(scheme)) return { allowed: false, host: '' };
-  if (HOSTLESS_SCHEMES.has(scheme)) return { allowed: true, host: '' };
-  let host: string;
-  try {
-    host = new URL(url).hostname;
-  } catch {
-    return { allowed: false, host: '' };
-  }
-  if (host === '') return { allowed: false, host };
-  return { allowed: allowHosts.some((rule) => hostMatches(host, rule)), host };
-}
+// `allowHosts` matching is `@ultimat3/core`'s (`host-rules.ts`), shared with `x shot`'s browser;
+// re-exported here so this package's own modules and its public names are unchanged.
+export type { HostDecision, HostRule } from '@ultimat3/core';
+export { ANY_HOST, hostDecision, hostMatches } from '@ultimat3/core';

package/src/index.ts

@@ -3,12 +3,10 @@
 // the seam is not a seam.
 
 export type { ActionabilityState, ActionabilityWait } from './actionability';
-export { actionabilityProblem, awaitActionable, DEFAULT_POLL_MS, isStable } from './actionability';
+export { awaitActionable } from './actionability';
 export type { ArtifactRef, ArtifactWriter, ArtifactWriterInit } from './artifacts';
 export {
-  contentTypeFor,
   createArtifactWriter,
-  DEFAULT_ARTIFACT_PREFIX,
   DEFAULT_CONTENT_TYPE,
 } from './artifacts';
 export type {
@@ -21,7 +19,6 @@ export { burnSession, createPrompt, ensureAuthenticated, restorableSession } fro
 export { browserRecord } from './browser-record';
 export type { CaptureClip, CaptureFraming } from './capture-clip';
 export { assertCaptureFraming } from './capture-clip';
-export { axNodesFor } from './cdp-a11y';
 export type {
   CdpBrowserLike,
   CdpFrameLike,
@@ -32,24 +29,14 @@ export type {
   CdpScreenshotOptions,
   CdpSessionLike,
 } from './cdp-port';
-export {
-  clearExpression,
-  focusExpression,
-  parseSnapshots,
-  snapshotExpression,
-} from './cdp-snapshot';
+export { snapshotExpression } from './cdp-snapshot';
 export type { CdpTargetInit } from './cdp-target';
 export { CDP_DRIVER, cdpTarget } from './cdp-target';
 export type { Deadline, ScrapeClock, TestScrapeClock } from './clock';
 export { deadline, systemScrapeClock, testClock, throwIfAborted } from './clock';
 export type { ColorScheme } from './color-scheme';
-export { COLOR_SCHEME_FEATURE, COLOR_SCHEMES, isColorScheme } from './color-scheme';
-export {
-  cookieDomainMatches,
-  cookieHeaderFor,
-  cookiePathMatches,
-  cookiesForUrl,
-} from './cookie-scope';
+export { COLOR_SCHEMES, isColorScheme } from './color-scheme';
+export { cookieHeaderFor } from './cookie-scope';
 export type { ScrapeDriver, ScrapeSession, SessionInit } from './driver';
 export { resetScrapeDriver, scrapeDriver, setScrapeDriver } from './driver';
 export type { BrowserOptions, LocalBrowserOptions, RemoteBrowserOptions } from './driver-cdp';
@@ -57,7 +44,7 @@ export { localBrowser, remoteBrowser } from './driver-cdp';
 export type { FakeBrowserOptions, FakePageOptions, FakePages } from './driver-fake';
 export { FAKE_DRIVER, FAKE_PAGE_URL, fakeBrowser, fakePage, recordingsOf } from './driver-fake';
 export type { FixtureBrowserOptions } from './driver-fixture';
-export { FIXTURE_DRIVER, fixtureBrowser, recordingFilename } from './driver-fixture';
+export { FIXTURE_DRIVER, fixtureBrowser } from './driver-fixture';
 export {
   authFailed,
   blocked,
@@ -100,32 +87,28 @@ export {
   ScrapeError,
 } from './errors';
 export type { ScrapeEventFields, StepEvent } from './events';
-export { scrapeLogger, withStepEvent } from './events';
 export type { YieldCheck, YieldExpectation, YieldGuardInput, YieldHistory } from './expect';
 export {
-  DEFAULT_YIELD_WINDOW,
   guardYield,
   MIN_BASELINE_RUNS,
   median,
   memoryYieldHistory,
-  yieldProblem,
 } from './expect';
-export { BURNS_SESSION, burnsSession, errorCode, NEVER_RETRIED, neverRetried } from './failures';
+export { BURNS_SESSION, errorCode, NEVER_RETRIED } from './failures';
 export type { HostDecision, HostRule } from './hosts';
 export { ANY_HOST, hostDecision, hostMatches } from './hosts';
-export { markupEnabled, markupVisible, queryHtml } from './html-query';
+export { queryHtml } from './html-query';
 export type { MarkupRequest } from './html-requests';
-export { markupRequests } from './html-requests';
 export type { HtmlTargetInit, RecordingLookup } from './html-target';
 export { htmlTarget } from './html-target';
 export type { HttpRequestInit, HttpTransportInit, ScrapeHttp, ScrapeResponse } from './http';
 export { DEFAULT_HTTP_MAX_BYTES, httpOverFetch, responseOver } from './http';
 export type { HttpRecordingLookup, RecordedHttpInit } from './http-recorded';
-export { httpRecordingFilename, httpRecordingsOf, recordedHttp } from './http-recorded';
+export { httpRecordingsOf, recordedHttp } from './http-recorded';
 export type { RedirectHop } from './http-redirect';
-export { MAX_REDIRECT_HOPS, redirectHop } from './http-redirect';
+export { MAX_REDIRECT_HOPS } from './http-redirect';
 export type { InterceptRules, InterceptVerdict } from './intercept';
-export { interceptVerdict, refusalEntry } from './intercept';
+export { interceptVerdict } from './intercept';
 export type { KeyChord, KeyModifier } from './key-chord';
 export { KEY_MODIFIERS, parseKeyChord } from './key-chord';
 export type { OfflineSessionInit } from './offline-session';
@@ -139,18 +122,16 @@ export type {
   ScrapePage,
   WaitOptions,
 } from './page';
-export { DEFAULT_ACCESSIBILITY_MAX } from './page';
 export type { PageContext } from './page-over-target';
 export { pageOverTarget } from './page-over-target';
 export type { Pacer } from './rate';
-export { createPacer, DEFAULT_NAVIGATION_RATE } from './rate';
+export { createPacer } from './rate';
 export type { HttpRecording, PageRecording } from './recording';
 export {
   httpRecordingSchema,
   pageRecordingSchema,
   parseHttpRecording,
   parseRecording,
-  splitDownload,
 } from './recording';
 export type { AgentRecovery, Recovery, RecoveryAttempt, RecoveryHook } from './recover';
 export { runRecovery } from './recover';
@@ -166,13 +147,12 @@ export type {
 } from './rings';
 export {
   createRing,
-  DEFAULT_RING_CAPACITY,
   MAX_PAGE_ERROR_CHARS,
   pageErrorEntry,
   RESOURCE_TYPES,
 } from './rings';
 export type { RobotsFetch, RobotsGate, RobotsGateInit, RobotsPolicy, RobotsRules } from './robots';
-export { createRobotsGate, DEFAULT_ROBOTS_AGENT, parseRobots, robotsAllows } from './robots';
+export { createRobotsGate, parseRobots, robotsAllows } from './robots';
 export type { RobotsFetchInit } from './robots-fetch';
 export {
   DEFAULT_ROBOTS_MAX_BYTES,
@@ -189,11 +169,9 @@ export { scrape } from './scrape';
 export { DEFAULT_PAGE_TIMEOUT_MS, runScrape } from './scrape-run';
 export type { ScrapeSecrets, SecretResolver } from './secrets';
 export {
-  blankPasswordFields,
   createSecretBag,
   MIN_REDACTABLE_LENGTH,
   redactSecrets,
-  SECRET_PLACEHOLDER,
   safeConsole,
   safeHtml,
   safeNetwork,
@@ -201,8 +179,6 @@ export {
 } from './secrets';
 export type { ScrapeSessionStore, SessionSnapshot, SessionState } from './session-state';
 export {
-  DEFAULT_SESSION_PREFIX,
-  EMPTY_SESSION,
   memorySessionStore,
   parseSessionState,
   sessionDigest,
@@ -220,6 +196,5 @@ export type {
   ScrapeDownloadFile,
   ScrapeTarget,
 } from './target';
-export { ROOT_SELECTOR } from './target';
 export type { WedgeGuard, WedgeGuardInit } from './watchdog';
-export { createWedgeGuard, DEFAULT_GRACE_MS, DEFAULT_IDLE_MS } from './watchdog';
+export { DEFAULT_GRACE_MS } from './watchdog';