@ultimat3/scraping 23.0.0 → 25.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,16 +5,11 @@
5
5
  // through to the network would make a green offline suite that is secretly hitting production —
6
6
  // the exact failure an offline driver exists to prevent.
7
7
 
8
+ import { NotImplementedError } from '@ultimat3/core';
8
9
  import type { CaptureClip } from './capture-clip';
9
10
  import type { ScrapeClock } from './clock';
10
11
  import type { ColorScheme } from './color-scheme';
11
- import {
12
- browserUnreachable,
13
- downloadTimeout,
14
- fixtureMissing,
15
- fixtureStale,
16
- scrapeNotImplemented,
17
- } from './error-throws';
12
+ import { browserUnreachable, downloadTimeout, fixtureMissing, fixtureStale } from './error-throws';
18
13
  import { queryHtml } from './html-query';
19
14
  import { markupRequests } from './html-requests';
20
15
  import type { InterceptRules } from './intercept';
@@ -23,7 +18,7 @@ import { parseKeyChord } from './key-chord';
23
18
  import type { PageRecording } from './recording';
24
19
  import { splitDownload } from './recording';
25
20
  import type { ConsoleLine, NetworkEntry, PageError } from './rings';
26
- import { createRing } from './rings';
21
+ import { boundedRing } from './rings';
27
22
  import type { SessionSnapshot } from './session-state';
28
23
  import { EMPTY_SESSION } from './session-state';
29
24
  import type {
@@ -124,14 +119,14 @@ interface OfflineDocument {
124
119
  }
125
120
 
126
121
  export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
127
- const consoleRing = createRing<ConsoleLine>();
128
- const networkRing = createRing<NetworkEntry>();
122
+ const consoleRing = boundedRing<ConsoleLine>();
123
+ const networkRing = boundedRing<NetworkEntry>();
129
124
  // Built and never pushed to, deliberately: this target parses markup and executes none of it, so
130
125
  // there is no uncaught exception for it to have. It ANSWERS rather than omitting the ring —
131
126
  // `page.pageErrors()` returning `[]` here is the honest "nothing threw, and nothing could",
132
127
  // where a missing ring would be a page method that throws on two of the three drivers.
133
128
  // `driver-parity.test.ts` pins the divergence, beside the box/hit-target one it already carries.
134
- const pageErrorRing = createRing<PageError>();
129
+ const pageErrorRing = boundedRing<PageError>();
135
130
  const overlay = new Map<string, string>();
136
131
  /**
137
132
  * One overlay per FRAME, held here rather than in the frame target, because `frames()` builds a
@@ -358,10 +353,10 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
358
353
  * attribute would pass every test against the element a screen reader cannot reach.
359
354
  */
360
355
  async accessibility(_selector: string, _max: number): Promise<readonly AxNode[]> {
361
- throw scrapeNotImplemented(
362
- `accessibility() on the ${init.driver} driver`,
363
- 'run this assertion on localBrowser()/remoteBrowser(), whose accessibility() reads the tree the browser computed — an offline driver parses markup and computes no role',
364
- );
356
+ throw new NotImplementedError({
357
+ cause: `accessibility() on the ${init.driver} driver is not implemented by this driver`,
358
+ fix: 'run this assertion on localBrowser()/remoteBrowser(), whose accessibility() reads the tree the browser computed — an offline driver parses markup and computes no role',
359
+ });
365
360
  },
366
361
  evaluate(expression: string): Promise<unknown> {
367
362
  live();
@@ -381,10 +376,10 @@ export function htmlTarget(init: HtmlTargetInit): ScrapeTarget {
381
376
  // `async`, so the refusal REJECTS: the method is typed `Promise<void>` and a synchronous
382
377
  // `throw` from one jumps straight over `page.offline(true).catch(…)` at every caller.
383
378
  async setOfflineMode(_enabled: boolean): Promise<void> {
384
- throw scrapeNotImplemented(
385
- `setOfflineMode() on the ${init.driver} driver`,
386
- 'run this assertion on localBrowser()/remoteBrowser(), whose setOfflineMode() reaches a real browser — an offline driver has no network to cut, so it cannot prove an offline behaviour',
387
- );
379
+ throw new NotImplementedError({
380
+ cause: `setOfflineMode() on the ${init.driver} driver is not implemented by this driver`,
381
+ fix: 'run this assertion on localBrowser()/remoteBrowser(), whose setOfflineMode() reaches a real browser — an offline driver has no network to cut, so it cannot prove an offline behaviour',
382
+ });
388
383
  },
389
384
  /**
390
385
  * ACCEPTED and recorded, where `setOfflineMode` refuses — and the line between them is which
@@ -64,7 +64,9 @@ export function recordedHttp(init: RecordedHttpInit): ScrapeHttp {
64
64
  if (found === undefined) throw fixtureMissing(`${method} ${url}`, init.source);
65
65
  if (init.maxAgeMs !== undefined && found.recordedAt !== undefined) {
66
66
  const age = init.clock.now().getTime() - new Date(found.recordedAt).getTime();
67
- if (age > init.maxAgeMs) throw fixtureStale(`${method} ${url}`, age, init.maxAgeMs);
67
+ // `!(age <= limit)`, never `age > limit`: an unparseable `recordedAt` makes `age` NaN, and
68
+ // `NaN > limit` is false — a recording of unknown age replayed as fresh. `auth.ts` the same.
69
+ if (!(age <= init.maxAgeMs)) throw fixtureStale(`${method} ${url}`, age, init.maxAgeMs);
68
70
  }
69
71
  init.network.push({
70
72
  method,
package/src/http.ts CHANGED
@@ -20,6 +20,8 @@ import type { RedirectHop } from './http-redirect';
20
20
  import { MAX_REDIRECT_HOPS, redirectHop } from './http-redirect';
21
21
  import type { InterceptRules } from './intercept';
22
22
  import { interceptVerdict } from './intercept';
23
+ import type { DialTarget, HostResolve } from './pinned-host';
24
+ import { dialTarget, resolveHost } from './pinned-host';
23
25
  import type { NetworkRing } from './rings';
24
26
  import type { RobotsGate } from './robots';
25
27
  import type { ScrapeSecrets } from './secrets';
@@ -43,6 +45,11 @@ export type ScrapeFetch = (input: string, init: ScrapeFetchInit) => Promise<Resp
43
45
  export interface ScrapeFetchInit extends RequestInit {
44
46
  /** The session's exit. A different exit IP mid-session is a different client to an anti-bot. */
45
47
  readonly proxy?: string | undefined;
48
+ /**
49
+ * The NAME a pinned connection proves (`pinned-host.ts`): the url names the approved address,
50
+ * and Bun's `fetch` verifies the certificate against this instead.
51
+ */
52
+ readonly tls?: { readonly serverName?: string | undefined } | undefined;
46
53
  }
47
54
 
48
55
  export interface HttpRequestInit {
@@ -114,6 +121,8 @@ export interface HttpTransportInit {
114
121
  /** The run's meter: one `httpRequest` per hop on the wire, with the body bytes read for it. */
115
122
  readonly usage?: UsageMeter | undefined;
116
123
  readonly fetch?: ScrapeFetch | undefined;
124
+ /** The resolver a wildcard-admitted name is checked and pinned with. Defaults to `Bun.dns`. */
125
+ readonly resolve?: HostResolve | undefined;
117
126
  }
118
127
 
119
128
  /**
@@ -196,6 +205,22 @@ const withoutCredentials = (headers: Readonly<Record<string, string>>): Record<s
196
205
  Object.entries(headers).filter(([name]) => !CROSS_ORIGIN_STRIPPED.has(name.toLowerCase())),
197
206
  );
198
207
 
208
+ /**
209
+ * The request's headers from every source, LATER sources winning, keyed by the lower-cased name.
210
+ * HTTP names are case-insensitive and `Headers` APPENDS two spellings of one name: a caller's
211
+ * `User-Agent: Mine` beside the session's `user-agent` went out as `BrowserUA, Mine`, and a
212
+ * declared `Cookie` was joined to the jar's instead of replacing it.
213
+ */
214
+ const composeHeaders = (
215
+ ...sources: readonly Readonly<Record<string, string>>[]
216
+ ): Record<string, string> => {
217
+ const out = new Map<string, string>();
218
+ for (const source of sources) {
219
+ for (const [name, value] of Object.entries(source)) out.set(name.toLowerCase(), value);
220
+ }
221
+ return Object.fromEntries(out);
222
+ };
223
+
199
224
  /** Unparseable is not same-origin: a URL this package cannot read is one it cannot vouch for. */
200
225
  const sameOrigin = (left: string, right: string): boolean => {
201
226
  try {
@@ -251,12 +276,21 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
251
276
  * The three gates, in one place, so the initial URL and hop seven are screened by the same code
252
277
  * in the same order. A second copy for redirects is how the two drift.
253
278
  */
254
- const screen = async (target: string): Promise<void> => {
279
+ const resolve = init.resolve ?? resolveHost;
280
+ const screen = async (target: string): Promise<DialTarget> => {
255
281
  if (interceptVerdict(target, 'fetch', init.rules) !== 'allow') {
256
282
  throw hostBlocked(target, init.rules.allowHosts);
257
283
  }
284
+ // Through a proxy the PROXY resolves the name, from its own network, so an answer from this
285
+ // box's resolver says nothing about where the request lands — and pinning would hand the
286
+ // proxy an address where it expects the name.
287
+ const dial =
288
+ init.proxy === undefined
289
+ ? await dialTarget(target, init.rules.allowHosts, resolve)
290
+ : { url: target };
258
291
  await init.robots?.assertAllowed(target);
259
292
  await init.pace?.(init.signal);
293
+ return dial;
260
294
  };
261
295
  return {
262
296
  async request(url: string, request: HttpRequestInit = {}): Promise<ScrapeResponse> {
@@ -284,7 +318,7 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
284
318
  1,
285
319
  );
286
320
  init.onActivity?.();
287
- await screen(url);
321
+ let dial = await screen(url);
288
322
  const session = await init.session();
289
323
  // `AbortSignal.timeout` and NOT `clock.sleep`: this is a deadline handed to the platform's
290
324
  // own fetch, not a wait this package performs — and under a test clock a slept deadline
@@ -319,14 +353,17 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
319
353
  const declared = credentialsInScope
320
354
  ? request.headers
321
355
  : withoutCredentials(request.headers ?? {});
322
- const response = await call(hop.url, {
356
+ const response = await call(dial.url, {
323
357
  method: hop.method,
324
- headers: {
325
- ...carried,
326
- ...(session.userAgent === '' ? {} : { 'user-agent': session.userAgent }),
327
- ...(cookies === undefined ? {} : { cookie: cookies }),
328
- ...declared,
329
- },
358
+ headers: composeHeaders(
359
+ carried,
360
+ session.userAgent === '' ? {} : { 'user-agent': session.userAgent },
361
+ cookies === undefined ? {} : { cookie: cookies },
362
+ declared ?? {},
363
+ // Last: a pinned request is reached AS the name that was screened, whatever was declared.
364
+ dial.host === undefined ? {} : { host: dial.host },
365
+ ),
366
+ ...(dial.serverName === undefined ? {} : { tls: { serverName: dial.serverName } }),
330
367
  ...(hop.body === undefined ? {} : { body: hop.body }),
331
368
  signal: AbortSignal.any(signals),
332
369
  // The whole point: the platform's own `follow` is what made the four lines above
@@ -360,7 +397,7 @@ export function httpOverFetch(init: HttpTransportInit): ScrapeHttp {
360
397
  await discardHopBody(response);
361
398
  if (followed >= MAX_REDIRECT_HOPS) throw redirectLoop(url, next.url, followed + 1);
362
399
  init.onActivity?.();
363
- await screen(next.url);
400
+ dial = await screen(next.url);
364
401
  if (!sameOrigin(hop.url, next.url)) credentialsInScope = false;
365
402
  hop = next;
366
403
  }
package/src/index.ts CHANGED
@@ -5,10 +5,7 @@
5
5
  export type { ActionabilityState, ActionabilityWait } from './actionability';
6
6
  export { awaitActionable } from './actionability';
7
7
  export type { ArtifactRef, ArtifactWriter, ArtifactWriterInit } from './artifacts';
8
- export {
9
- createArtifactWriter,
10
- DEFAULT_CONTENT_TYPE,
11
- } from './artifacts';
8
+ export { artifactWriter } from './artifacts';
12
9
  export type {
13
10
  AuthContext,
14
11
  PromptHandler,
@@ -16,12 +13,14 @@ export type {
16
13
  PromptRequest,
17
14
  ScrapeAuth,
18
15
  } from './auth';
19
- export { burnSession, createPrompt, ensureAuthenticated, restorableSession } from './auth';
16
+ export { burnSession, ensureAuthenticated, restorableSession, scrapePrompt } from './auth';
20
17
  export { browserRecord } from './browser-record';
21
18
  export type { CaptureClip, CaptureFraming } from './capture-clip';
22
19
  export { assertCaptureFraming } from './capture-clip';
23
20
  export type {
24
21
  CdpBrowserLike,
22
+ CdpBrowserSessionLike,
23
+ CdpBrowserTargetLike,
25
24
  CdpFrameLike,
26
25
  CdpKeyboardLike,
27
26
  CdpLauncherLike,
@@ -42,8 +41,7 @@ export {
42
41
  scrapeClock,
43
42
  setScrapeClock,
44
43
  systemScrapeClock,
45
- testClock,
46
- throwIfAborted,
44
+ testScrapeClock,
47
45
  } from './clock';
48
46
  export type { ColorScheme } from './color-scheme';
49
47
  export { COLOR_SCHEMES, isColorScheme } from './color-scheme';
@@ -79,11 +77,10 @@ export {
79
77
  redirectLoop,
80
78
  remoteRequired,
81
79
  robotsDisallowed,
82
- scrapeNotImplemented,
80
+ scrapeSessionExpired,
83
81
  scrapeTimeout,
84
82
  secretExposed,
85
83
  selectorMissing,
86
- sessionExpired,
87
84
  wedged,
88
85
  yieldCollapsed,
89
86
  } from './error-throws';
@@ -121,8 +118,6 @@ export {
121
118
  memoryYieldHistory,
122
119
  } from './expect';
123
120
  export { BURNS_SESSION, errorCode, NEVER_RETRIED } from './failures';
124
- export type { HostDecision, HostRule } from './hosts';
125
- export { ANY_HOST, hostDecision, hostMatches } from './hosts';
126
121
  export { queryHtml } from './html-query';
127
122
  export type { MarkupRequest } from './html-requests';
128
123
  export type { HtmlTargetInit, RecordingLookup } from './html-target';
@@ -151,7 +146,7 @@ export type {
151
146
  export type { PageContext } from './page-over-target';
152
147
  export { pageOverTarget } from './page-over-target';
153
148
  export type { Pacer } from './rate';
154
- export { createPacer } from './rate';
149
+ export { scrapePacer } from './rate';
155
150
  export type { HttpRecording, PageRecording } from './recording';
156
151
  export {
157
152
  httpRecordingSchema,
@@ -172,13 +167,21 @@ export type {
172
167
  Ring,
173
168
  } from './rings';
174
169
  export {
175
- createRing,
170
+ boundedRing,
176
171
  MAX_PAGE_ERROR_CHARS,
177
172
  pageErrorEntry,
178
173
  RESOURCE_TYPES,
179
174
  } from './rings';
180
- export type { RobotsFetch, RobotsGate, RobotsGateInit, RobotsPolicy, RobotsRules } from './robots';
181
- export { createRobotsGate, parseRobots, robotsAllows } from './robots';
175
+ export type {
176
+ RobotsAnswer,
177
+ RobotsFetch,
178
+ RobotsGate,
179
+ RobotsGateInit,
180
+ RobotsPolicy,
181
+ RobotsRules,
182
+ RobotsUnreachable,
183
+ } from './robots';
184
+ export { parseRobots, robotsAllows, robotsGate } from './robots';
182
185
  export type { RobotsFetchInit } from './robots-fetch';
183
186
  export {
184
187
  DEFAULT_ROBOTS_MAX_BYTES,
@@ -196,13 +199,13 @@ export { scrape } from './scrape';
196
199
  export { DEFAULT_PAGE_TIMEOUT_MS, runScrape } from './scrape-run';
197
200
  export type { ScrapeSecrets, SecretResolver } from './secrets';
198
201
  export {
199
- createSecretBag,
200
202
  MIN_REDACTABLE_LENGTH,
201
203
  redactSecrets,
202
204
  safeConsole,
203
205
  safeHtml,
204
206
  safeNetwork,
205
207
  safePageErrors,
208
+ secretBag,
206
209
  } from './secrets';
207
210
  export type {
208
211
  ScrapeSessionStore,
@@ -213,6 +216,7 @@ export type {
213
216
  export {
214
217
  memorySessionStore,
215
218
  parseSessionState,
219
+ recordVersion,
216
220
  SESSION_SEAL_PURPOSE,
217
221
  sessionDigest,
218
222
  sessionKeyFor,
@@ -225,12 +229,13 @@ export type {
225
229
  ElementSnapshot,
226
230
  FrameRef,
227
231
  GotoOptions,
232
+ QueryOptions,
228
233
  ScrapeCookie,
229
234
  ScrapeDownloadFile,
230
235
  ScrapeTarget,
231
236
  } from './target';
232
237
  export { endpointLabel, urlSecretValues } from './url-secrets';
233
238
  export type { RunUsageMeter, ScrapeUsage, UsageMeter } from './usage';
234
- export { createUsageMeter } from './usage';
239
+ export { usageMeter } from './usage';
235
240
  export type { WedgeGuard, WedgeGuardInit } from './watchdog';
236
241
  export { DEFAULT_GRACE_MS } from './watchdog';
package/src/intercept.ts CHANGED
@@ -5,8 +5,8 @@
5
5
  // differently — `driver-parity.test.ts` asserts the same refusal on all three, and a rule
6
6
  // re-implemented per driver is a rule that holds only on the driver nobody ships.
7
7
 
8
- import type { HostRule } from './hosts';
9
- import { hostDecision } from './hosts';
8
+ import type { HostRule } from '@ultimat3/core';
9
+ import { hostDecision } from '@ultimat3/core';
10
10
  import type { NetworkEntry, ResourceType } from './rings';
11
11
 
12
12
  export interface InterceptRules {
@@ -0,0 +1,51 @@
1
+ // The argument list a launched browser gets: the caller's own, then the exit. The exit is the
2
+ // session's — the driver's `proxy` or the run's `egress` — so it is appended LAST and a caller
3
+ // switch that would decide the route instead is refused rather than ranked.
4
+
5
+ import { launchArgsInvalid } from './error-throws-session';
6
+
7
+ /**
8
+ * Every switch Chrome reads a proxy decision from. Matched on the switch NAME, case-folded and
9
+ * with either prefix POSIX Chrome accepts (`--` and `-`), so a value that merely mentions a proxy
10
+ * (`--user-agent=proxy-server-test`) is not refused.
11
+ */
12
+ const ROUTE_SWITCHES = new Set([
13
+ 'proxy-server',
14
+ 'proxy-bypass-list',
15
+ 'proxy-pac-url',
16
+ 'proxy-auto-detect',
17
+ 'no-proxy-server',
18
+ ]);
19
+
20
+ const switchName = (arg: string): string | undefined => {
21
+ const match = /^--?([^=]+)/.exec(arg);
22
+ return match?.[1]?.toLowerCase();
23
+ };
24
+
25
+ /**
26
+ * `undefined` when the launch carries no args at all — the launcher's own defaults stay untouched
27
+ * then. Refused with `X_SCRAPE_LAUNCH_ARGS_INVALID` before anything starts: a browser launched on a
28
+ * second route is a session that reports one exit and leaves the worker by another.
29
+ */
30
+ export function launchArgs(
31
+ callerArgs: unknown,
32
+ proxyArg: string | undefined,
33
+ ): string[] | undefined {
34
+ if (callerArgs !== undefined) {
35
+ if (!Array.isArray(callerArgs) || !callerArgs.every((arg) => typeof arg === 'string')) {
36
+ throw launchArgsInvalid('is not an array of strings', false);
37
+ }
38
+ for (const arg of callerArgs as readonly string[]) {
39
+ const name = switchName(arg);
40
+ if (name !== undefined && ROUTE_SWITCHES.has(name)) {
41
+ throw launchArgsInvalid(
42
+ `sets --${name}, which decides the route this session leaves by`,
43
+ true,
44
+ );
45
+ }
46
+ }
47
+ }
48
+ const args = [...((callerArgs as readonly string[] | undefined) ?? [])];
49
+ if (proxyArg !== undefined) args.push(`--proxy-server=${proxyArg}`);
50
+ return args.length === 0 ? undefined : args;
51
+ }
@@ -4,7 +4,7 @@
4
4
  // driver a test uses and the driver production uses.
5
5
 
6
6
  import type { Secret } from '@ultimat3/core';
7
- import { finiteCount, isSecret, revealSecret } from '@ultimat3/core';
7
+ import { finiteCount, hostDecision, isSecret, revealSecret } from '@ultimat3/core';
8
8
  import type { ActionabilityState } from './actionability';
9
9
  import { awaitActionable } from './actionability';
10
10
  import { assertCaptureFraming } from './capture-clip';
@@ -12,7 +12,6 @@ import type { ScrapeClock } from './clock';
12
12
  import { deadline } from './clock';
13
13
  import type { ColorScheme } from './color-scheme';
14
14
  import { hostBlocked, secretExposed, selectorMissing } from './error-throws';
15
- import { hostDecision } from './hosts';
16
15
  import type {
17
16
  AccessibilityOptions,
18
17
  CaptureRequest,
@@ -30,6 +29,7 @@ import type {
30
29
  AxNode,
31
30
  CaptureOptions,
32
31
  ElementSnapshot,
32
+ QueryOptions,
33
33
  ScrapeCookie,
34
34
  ScrapeDownloadFile,
35
35
  ScrapeTarget,
@@ -80,9 +80,13 @@ const toValue = (snapshot: ElementSnapshot): ElementValue => ({
80
80
  attrs: snapshot.attrs,
81
81
  });
82
82
 
83
- async function first(resolve: Resolve, selector: string): Promise<ElementSnapshot | undefined> {
83
+ async function first(
84
+ resolve: Resolve,
85
+ selector: string,
86
+ options?: QueryOptions,
87
+ ): Promise<ElementSnapshot | undefined> {
84
88
  const target = await resolve();
85
- return (await target.query(selector))[0];
89
+ return (await target.query(selector, options))[0];
86
90
  }
87
91
 
88
92
  function frameOver(
@@ -103,14 +107,18 @@ function frameOver(
103
107
  ): Promise<ElementSnapshot> => {
104
108
  const target = await resolve();
105
109
  lastUrl = target.url();
110
+ const state = options?.state ?? fallback;
111
+ // Only `actionable` reads `hitTarget`, so only it pays for moving the page — on every poll,
112
+ // because a layout shift between two can carry the element back out of view.
113
+ const query: QueryOptions = { reveal: state === 'actionable' };
106
114
  return awaitActionable({
107
115
  selector,
108
116
  url: lastUrl,
109
- state: options?.state ?? fallback,
117
+ state,
110
118
  timeoutMs: timeoutFor(options),
111
119
  clock: ctx.clock,
112
120
  signal: ctx.signal,
113
- snapshot: () => first(resolve, selector),
121
+ snapshot: () => first(resolve, selector, query),
114
122
  });
115
123
  };
116
124
 
@@ -0,0 +1,65 @@
1
+ // The DNS half of `allowHosts`, for the two legs this package dials itself — the HTTP leg and the
2
+ // robots read. Core's `hostDecision` is synchronous and judges NAMES: a wildcard never admits an
3
+ // address literal inside the network, but `allowHosts: ['*']` still admits a hostname that
4
+ // resolves inward. So a name admitted by a wildcard alone is resolved here, every address
5
+ // classified by core's one classifier, and the connection PINNED to the approved address — the
6
+ // shape `@ultimat3/jobs`' `webhook-target.ts` takes for a webhook.
7
+
8
+ import type { HostRule } from '@ultimat3/core';
9
+ import { classifyAddress, hostMatches } from '@ultimat3/core';
10
+ import { hostResolvesInward } from './error-throws-session';
11
+
12
+ /** Every address a hostname answers. Injected in tests: the network is sealed in this repo. */
13
+ export type HostResolve = (hostname: string) => Promise<readonly string[]>;
14
+
15
+ /** `Bun.dns.lookup` — the runtime's own resolver, every family, every address. */
16
+ export const resolveHost: HostResolve = async (hostname) =>
17
+ (await Bun.dns.lookup(hostname)).map((entry) => entry.address);
18
+
19
+ /** What the transport connects to: the URL to dial, and the name it proves when pinned. */
20
+ export interface DialTarget {
21
+ readonly url: string;
22
+ /** The original `host[:port]`, sent as the `Host` header of a pinned request. */
23
+ readonly host?: string | undefined;
24
+ /** For `https:` — the certificate is verified against the NAME, not the address. */
25
+ readonly serverName?: string | undefined;
26
+ }
27
+
28
+ const isWildcard = (rule: HostRule): boolean => {
29
+ const cleaned = rule.trim();
30
+ return cleaned === '*' || cleaned.startsWith('*.');
31
+ };
32
+
33
+ /**
34
+ * The connection for `url`, which `interceptVerdict` has ALREADY allowed. Unchanged when an exact
35
+ * rule names the host — the app said that host, internal or not, in a line a reviewer can see —
36
+ * or when the host is an address literal, which core's floor already judged. Otherwise resolved:
37
+ * one address that is not public refuses the request (`X_SCRAPE_HOST_BLOCKED`), and the dial is
38
+ * pinned to the first approved address, so a name that answers public here and private to the
39
+ * socket (DNS rebinding) is never reached. A resolver failure propagates: nothing unscreened is
40
+ * dialled, and the same name would not have resolved for `fetch` either.
41
+ */
42
+ export async function dialTarget(
43
+ url: string,
44
+ allowHosts: readonly HostRule[],
45
+ resolve: HostResolve,
46
+ ): Promise<DialTarget> {
47
+ const parsed = new URL(url);
48
+ if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return { url };
49
+ const hostname = parsed.hostname.replace(/^\[|\]$/g, '');
50
+ if (classifyAddress(hostname) !== undefined) return { url };
51
+ if (allowHosts.some((rule) => !isWildcard(rule) && hostMatches(hostname, rule))) return { url };
52
+ const addresses = await resolve(hostname);
53
+ const inward = addresses.map((address) => classifyAddress(address) ?? 'unclassifiable');
54
+ const refused =
55
+ addresses.length === 0 ? 'unresolvable' : inward.find((kind) => kind !== 'public');
56
+ if (refused !== undefined) throw hostResolvesInward(url, hostname, refused);
57
+ const pinned = addresses[0] ?? hostname;
58
+ const target = new URL(parsed.href);
59
+ target.hostname = pinned.includes(':') ? `[${pinned}]` : pinned;
60
+ return {
61
+ url: target.href,
62
+ host: parsed.host,
63
+ ...(parsed.protocol === 'https:' ? { serverName: hostname } : {}),
64
+ };
65
+ }
package/src/rate.ts CHANGED
@@ -9,7 +9,7 @@ export const DEFAULT_NAVIGATION_RATE = 1;
9
9
 
10
10
  export type Pacer = (signal?: AbortSignal) => Promise<void>;
11
11
 
12
- export function createPacer(rate: number, clock: ScrapeClock): Pacer {
12
+ export function scrapePacer(rate: number, clock: ScrapeClock): Pacer {
13
13
  const intervalMs = 1_000 / rate;
14
14
  let nextAt = 0;
15
15
  return async (signal?: AbortSignal): Promise<void> => {
package/src/recover.ts CHANGED
@@ -3,10 +3,11 @@
3
3
  // Two shapes. A FUNCTION is complete and shipped — an app writes its own fallback, gets the page
4
4
  // and the failure, and answers whether the attempt should be retried. `'agent'` is the seam for
5
5
  // letting a model re-derive the selector from the page, and it is an honest
6
- // `X_NOT_IMPLEMENTED` today, in the shape `packages/jobs/src/driver-redis.ts` uses: correct types
6
+ // `X_NOT_IMPLEMENTED` today: correct types
7
7
  // so an app can be written against it, one labelled throw, no silent no-op.
8
8
 
9
- import { recoverRefused, scrapeNotImplemented } from './error-throws';
9
+ import { NotImplementedError } from '@ultimat3/core';
10
+ import { recoverRefused } from './error-throws';
10
11
  import type { ScrapePage } from './page';
11
12
 
12
13
  export interface RecoveryAttempt {
@@ -41,7 +42,7 @@ export async function runRecovery(recovery: Recovery, attempt: RecoveryAttempt):
41
42
  // The agent path lands with `@ultimat3/ai`'s `llm()` behind it: page HTML in, a selector out,
42
43
  // re-run once. Until it does, this throws rather than answering `false` — a recovery that
43
44
  // silently declines is indistinguishable from one that was never configured.
44
- throw scrapeNotImplemented("recover: 'agent'", AGENT_FIX);
45
+ throw new NotImplementedError({ cause: "recover: 'agent' is not implemented", fix: AGENT_FIX });
45
46
  }
46
47
  const verdict: unknown = await recovery(attempt);
47
48
  if (typeof verdict !== 'boolean') {
package/src/rings.ts CHANGED
@@ -107,18 +107,18 @@ export type PageErrorRing = Ring<PageError>;
107
107
  *
108
108
  * Both wrong values fail in opposite directions and neither is survivable. A NEGATIVE capacity
109
109
  * makes `while (items.length > capacity) items.shift()` spin forever on an already-empty array
110
- * (`0 > -1`): `createRing(-1).push(1)` never returns, and it is a synchronous loop on the worker's
110
+ * (`0 > -1`): `boundedRing(-1).push(1)` never returns, and it is a synchronous loop on the worker's
111
111
  * only thread, inside a `page.on('request')` handler — past `ctx.signal`, past the wedge watchdog
112
112
  * and past the job timeout, which is incident #1 in this file's header. `NaN` fails the other way:
113
113
  * the comparison is false forever, the ring is UNBOUNDED, and the OOM the bound exists to prevent
114
114
  * arrives two hours in. Zero is a ring that discards everything silently, and a fraction is a
115
115
  * bound that is not the number anybody asked for.
116
116
  */
117
- export function createRing<T>(capacity: number = DEFAULT_RING_CAPACITY): Ring<T> {
117
+ export function boundedRing<T>(capacity: number = DEFAULT_RING_CAPACITY): Ring<T> {
118
118
  assert(
119
119
  Number.isSafeInteger(capacity) && capacity > 0,
120
120
  `a ring capacity must be a positive integer, got ${String(capacity)}: a negative one spins forever on push() and NaN makes the ring unbounded`,
121
- `pass a positive capacity — createRing(${String(DEFAULT_RING_CAPACITY)}) — or omit it and take DEFAULT_RING_CAPACITY`,
121
+ `pass a positive capacity — boundedRing(${String(DEFAULT_RING_CAPACITY)}) — or omit it and take DEFAULT_RING_CAPACITY`,
122
122
  );
123
123
  const items: T[] = [];
124
124
  let dropped = 0;