@ultimat3/scraping 30.9.0 → 31.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +73 -8
  2. package/package.json +9 -6
  3. package/src/actionability.ts +2 -2
  4. package/src/adapter.ts +22 -0
  5. package/src/auth.ts +4 -5
  6. package/src/browser-dial.ts +55 -0
  7. package/src/cdp-arm.ts +81 -22
  8. package/src/cdp-connection.ts +290 -0
  9. package/src/cdp-error-codes.ts +28 -0
  10. package/src/cdp-errors.ts +179 -0
  11. package/src/cdp-launch-attempt.ts +234 -0
  12. package/src/cdp-launch-candidates.ts +62 -0
  13. package/src/cdp-launch-probe.ts +75 -0
  14. package/src/cdp-launch-profile.ts +106 -0
  15. package/src/cdp-launch-reap.ts +68 -0
  16. package/src/cdp-launch-wire.ts +171 -0
  17. package/src/cdp-launch.ts +216 -0
  18. package/src/cdp-pipe.ts +77 -0
  19. package/src/cdp-wire-watch.ts +113 -0
  20. package/src/cdp.ts +46 -0
  21. package/src/clock.ts +4 -111
  22. package/src/driver-cdp.ts +3 -0
  23. package/src/driver-fake.ts +4 -5
  24. package/src/driver.ts +16 -11
  25. package/src/error-throws-pdf.ts +45 -0
  26. package/src/error-throws-session.ts +1 -1
  27. package/src/error-throws.ts +4 -4
  28. package/src/errors.ts +17 -1
  29. package/src/event-prompt.ts +10 -4
  30. package/src/events.ts +2 -3
  31. package/src/html-target.ts +2 -2
  32. package/src/http-recorded.ts +2 -2
  33. package/src/http.ts +2 -2
  34. package/src/index.ts +21 -75
  35. package/src/launch-args.ts +10 -1
  36. package/src/page-over-target.ts +2 -3
  37. package/src/pdf-action.ts +116 -0
  38. package/src/pdf-cdp-params.ts +65 -0
  39. package/src/pdf-fence.ts +86 -0
  40. package/src/pdf-render.ts +241 -0
  41. package/src/pdf-slots.ts +68 -0
  42. package/src/rate.ts +2 -2
  43. package/src/scrape-run.ts +3 -4
  44. package/src/scrape.ts +5 -5
  45. package/src/target.ts +1 -1
  46. package/src/usage.ts +2 -2
  47. package/src/watchdog.ts +3 -3
package/README.md CHANGED
@@ -237,21 +237,28 @@ which is why `onSettled` hands a failed run its last attempt's `usage`. No quota
237
237
 
238
238
  ### The clock, and a test's
239
239
 
240
- A run waits on the PROCESS's clock — `scrapeClock()`, the system's — unless its definition pins
241
- one with `clock:`. A test replaces it the way it replaces the queue driver:
240
+ A run waits on the PROCESS's clock — `@ultimat3/core`'s `ambientClock()`, the system's unless
241
+ `clock` in `apps/<app>/runtime.ts` says otherwise — unless its definition pins one with `clock:`.
242
+ Since 31.0.0 the clocks are core's `SleepClock`s; this package has none of its own. A test
243
+ replaces the process clock through core's installer, which answers its release:
242
244
 
243
245
  ```ts
244
- import { noWaitClock, resetScrapeClock, setScrapeClock } from '@ultimat3/scraping';
246
+ import { noWaitClock } from '@ultimat3/core';
247
+ import { installClock } from '@ultimat3/core/adapter';
245
248
 
246
- setScrapeClock(noWaitClock); // in a beforeEach
247
- resetScrapeClock(); // in the afterEach
249
+ const release = installClock(noWaitClock); // in a beforeEach
250
+ release(); // in the afterEach
248
251
  ```
249
252
 
250
- | Clock | Sleeping | For |
253
+ | Clock (`@ultimat3/core`) | Sleeping | For |
251
254
  |---|---|---|
252
- | `systemScrapeClock` | waits | production |
255
+ | `systemClock` | waits | production |
253
256
  | `noWaitClock` | one turn of the event loop, whatever was asked; `now()` and deadlines are real | a test whose run waits on something that really happens in the same process — an answer the test publishes |
254
- | `testScrapeClock()` | advances virtual time and yields a microtask | a test of a timeout: thirty seconds pass in none |
257
+ | `frozenClock()` | advances virtual time and yields a microtask | a test of a timeout: thirty seconds pass in none |
258
+
259
+ The browser is chosen the same way: `scrapeDriver` in `apps/<app>/runtime.ts`, which the boot
260
+ installs through `installScrapeDriver` on `@ultimat3/scraping/adapter` (the `.` barrel has no
261
+ setter); a definition's own `driver:` overrides it.
255
262
 
256
263
  A third-party driver takes part by reading three `SessionInit` fields: dial `proxy` or throw
257
264
  `egressUnsupported()`, pass `usage` to `pageOverTarget` and its HTTP transport, and answer
@@ -389,6 +396,61 @@ The offline drivers honour it, so the crop is provable with no Chrome — `fakeB
389
396
  different deterministic bytes per rectangle, which is what lets a component screenshot be tested
390
397
  on a machine with no browser — the case CI is.
391
398
 
399
+ ## A route printed to PDF — `pdf()`, a factory over `action`
400
+
401
+ ```ts
402
+ import type { Ctx } from '@ultimat3/core';
403
+ import { can } from '@ultimat3/policy';
404
+ import { t } from '@ultimat3/schema';
405
+ import { pdf } from '@ultimat3/scraping';
406
+ import { disk } from '@ultimat3/storage';
407
+
408
+ /** The app's own short-lived signer for the print route — never the framework's. */
409
+ declare function printToken(ctx: Ctx): string;
410
+
411
+ export const invoicePdf = pdf({
412
+ input: t.object({ id: t.uuid }),
413
+ policy: can('invoices:read'),
414
+ route: ({ input }) => `/invoices/${input.id}/print`, // a PATH on this app, never a URL
415
+ key: ({ input, ctx }) => `invoices/${ctx.actor.orgId}/${input.id}.pdf`,
416
+ storage: () => disk('documents'), // a thunk: read per call
417
+ layout: { format: 'a4', marginMm: { top: 12, bottom: 12 }, printBackground: true },
418
+ headers: ({ ctx }) => ({ authorization: `Bearer ${printToken(ctx)}` }), // the app's own token
419
+ });
420
+ // → { key, bytes, contentType: 'application/pdf' } — HTTP, OpenAPI, MCP, .job(), .as() included
421
+ ```
422
+
423
+ | Rule | Detail |
424
+ |---|---|
425
+ | it is an `action` | input schema, policy, `rateLimit`, `meter`, `audit` are the action's: refused before a browser starts (`pdf-action.test.ts`). `PRIMITIVE_FACTORIES` row `pdf` |
426
+ | only the app's own origin | `route()` must answer a path; a URL, `//host` or a backslash is `X_SCRAPE_PDF_ROUTE_INVALID` (500, terminal). The origin is `APP_URL`, read per call, or `origin: () => …`; neither is `X_SCRAPE_PDF_ORIGIN_MISSING` (500, terminal) |
427
+ | an error page is never a document | the page's own status is read after `load` and `document.fonts.ready`: 4xx/5xx is `X_SCRAPE_PDF_ROUTE_FAILED` (502; a 4xx terminal, a 5xx/429 retryable) |
428
+ | the layout is checked where `pdf()` is written | `pdfLayoutOf`, as `page.pdf()`: `X_SCRAPE_PDF_OPTIONS_INVALID` at module load |
429
+ | one browser per render | `chromePdfRenderer()` launches Chrome through `@ultimat3/scraping/cdp`, prints over raw CDP (`Page.printToPDF`, `pdf-render.ts`, `pdf-cdp-params.ts`) and closes it in a `finally`; no browser outlives a request, none is shared between tenants. A pool is a measured next step, not shipped |
430
+ | a bounded number of browsers | at most `maxConcurrent` (default 2 — each Chrome is 150-300 MB) per renderer, and every `pdf()` naming no `renderer` shares ONE. The rest queue for `queueTimeoutMs` (default `timeoutMs`) on the clock, then `X_SCRAPE_PDF_BUSY` (503, retryable) — `pdf-slots.ts`, `pdf-render-slots.test.ts` |
431
+ | a dropped request ends its render | the action's `ctx.signal` reaches the renderer: aborted in the queue, it leaves without a slot; aborted mid-print, its browser is killed and the slot freed, and the request's own reason is thrown |
432
+ | headers reach the route's origin only | `Fetch.enable` on the page's session (`pdf-fence.ts`): a request to the route's exact origin (scheme, host, port) continues with `headers` added; every other — a user's `<img src="https://elsewhere">`, a redirect off-site, `169.254.169.254` — is failed `BlockedByClient` before it leaves. Never `Network.setExtraHTTPHeaders`, which sent the token everywhere (`pdf-render-fence.test.ts`) |
433
+ | the sandbox stays on | the default launch keeps Chrome's sandbox (`pdfLaunchOptions`): a print renders user content. The runtime image ships no Chrome; the app's image supplies it and runs it as non-root, which needs unprivileged user namespaces (or Chrome's setuid helper). A container that cannot grant them fails `X_CDP_LAUNCH_FAILED` quoting "No usable sandbox" — opt out there in one visible line, `chromePdfRenderer({ sandbox: false })`. The e2e driver and `x shot` keep `--no-sandbox` for CI runners |
434
+ | the bytes go to storage | `storage().put(key, bytes, { contentType: 'application/pdf' })`; the answer is the key, never the bytes. Serve it with a signed URL |
435
+ | tested with no Chrome | `renderer:` takes any `PdfRenderer`; `chromePdfRenderer({ launch })` takes any `{ connection, close }` — `pdf-render.test.ts` drives it over a fake CDP connection |
436
+
437
+ Open (`As of 2026-11`): the route must be readable by the printing browser — a private one forwards
438
+ the app's own short-lived credential through `headers`; no session is minted for it.
439
+
440
+ ## The raw-CDP browser — `@ultimat3/scraping/cdp`
441
+
442
+ The launcher and connection the e2e driver and `x shot` have used since 22.0.0, moved here from
443
+ `@ultimat3/testing` in 31.0.0 because `pdf()` needs them at RUNTIME and testing is a test package.
444
+ Its own entry, off `.`, so a caller that only launches loads no job or storage module.
445
+ This is every name's one home: `@ultimat3/testing` no longer re-exports them (31.0.0, `X_HELPER_COPY`); its `E2E_ERROR_CODES` still lists the four codes.
446
+
447
+ | Export | What |
448
+ |---|---|
449
+ | `findChrome(env)` · `launchChrome({ executable, timeoutMs })` · `launchFoundChrome(env, timeoutMs)` | `CHROME_PATH`, then the platform's install paths; a throwaway profile, the pipe wire, one relaunch |
450
+ | `cdpConnect({ endpoint, timeoutMs })` · `cdpConnectOver(transport, timeoutMs)` | a CDP connection over a remote WebSocket or any whole-message transport |
451
+ | `CdpBrowserMissingError` · `CdpLaunchFailedError` · `CdpCallFailedError` · `CdpTimeoutError` | `X_CDP_BROWSER_MISSING` · `X_CDP_LAUNCH_FAILED` · `X_CDP_CALL_FAILED` · `X_CDP_TIMEOUT`, registered by `cdp-error-codes.ts` |
452
+ | `CHROME_CANDIDATES` · `CONTAINER_CHROME_ARGS` · `chromeLaunchFlags` · `LAUNCH_*` · `CLOSE_GRACE_MS` · `LAUNCHED_CLOSE_MS` | the flags and the deadlines, derived, never restated |
453
+
392
454
  ## What it owns
393
455
 
394
456
  | Module | Owns |
@@ -399,6 +461,7 @@ on a machine with no browser — the case CI is.
399
461
  | `driver-cdp.ts` / `cdp-*.ts` | the real browser, over a structural CDP port |
400
462
  | `driver-fake.ts` / `driver-recorded.ts` / `html-*.ts` | the offline drivers, on Bun's `HTMLRewriter` |
401
463
  | `http.ts` / `http-recorded.ts` | the second transport, live and replayed |
464
+ | `intercept.ts` / `pinned-host.ts` / `browser-dial.ts` | `allowHosts`: one name verdict for every driver, then the DNS half for a host a wildcard admitted — resolved and pinned on the HTTP leg and robots read, resolved before release on the browser leg (a private or unresolvable answer is a `refused: 'host'` ring entry). An exact rule and a proxied session are not resolved |
402
465
  | `auth.ts` / `session-state.ts` | acquire → persist → reuse → validate → burn. The session key encodes each part (`<sanitised>.<digest>`), so two account names that differ only outside `[a-zA-Z0-9._-]` are two sessions. A stored session is sealed. `burnSession(plan, seen)` takes `recordVersion(record)` of the record the run used, so a record another run saved since is not burned |
403
466
  | `event-prompt.ts` | `eventPrompt()` and `answerPrompt()`: a prompt answered from another process, over the job event bus, with the browser open |
404
467
  | `cdp-resolver.ts` | a browser rented per session: acquired once, released once |
@@ -406,6 +469,8 @@ on a machine with no browser — the case CI is.
406
469
  | `secrets.ts` / `url-secrets.ts` / `browser-record.ts` | what may leave this package — declared secrets, values learned mid-run (`conceal`), the credential-bearing parts of a URL — and how a browser's own string map is read |
407
470
  | `expect.ts` | the silent-green alarm |
408
471
  | `watchdog.ts` | the wedge and zombie discipline |
472
+ | `pdf-action.ts` / `pdf-render.ts` / `pdf-cdp-params.ts` / `pdf-slots.ts` / `pdf-fence.ts` | `pdf()` over `action`; one page printed over raw CDP; a layout in the protocol's words; the bound on live browsers; the request fence |
473
+ | `cdp.ts` / `cdp-launch*.ts` / `cdp-connection.ts` / `cdp-pipe.ts` / `cdp-wire-watch.ts` / `cdp-errors.ts` | `@ultimat3/scraping/cdp`: the raw-CDP launcher and connection (moved from `@ultimat3/testing` in 31.0.0) |
409
474
  | `capture-clip.ts` | the one framing rule — crop, whole page, or a refusal — checked before any driver sees it |
410
475
  | `errors.ts` / `error-throws.ts` / `error-throws-session.ts` | this package's `X_*` codes and their retry classification |
411
476
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ultimat3/scraping",
3
- "version": "30.9.0",
3
+ "version": "31.1.0",
4
4
  "description": "Browser automation as a job: scrape() returns a JobHandle",
5
5
  "keywords": [
6
6
  "ultimate",
@@ -24,7 +24,9 @@
24
24
  "provenance": true
25
25
  },
26
26
  "exports": {
27
- ".": "./src/index.ts"
27
+ ".": "./src/index.ts",
28
+ "./adapter": "./src/adapter.ts",
29
+ "./cdp": "./src/cdp.ts"
28
30
  },
29
31
  "files": [
30
32
  "src",
@@ -41,9 +43,10 @@
41
43
  "test": "bun test"
42
44
  },
43
45
  "dependencies": {
44
- "@ultimat3/core": "30.9.0",
45
- "@ultimat3/jobs": "30.9.0",
46
- "@ultimat3/schema": "30.9.0",
47
- "@ultimat3/storage": "30.9.0"
46
+ "@ultimat3/action": "31.1.0",
47
+ "@ultimat3/core": "31.1.0",
48
+ "@ultimat3/jobs": "31.1.0",
49
+ "@ultimat3/schema": "31.1.0",
50
+ "@ultimat3/storage": "31.1.0"
48
51
  }
49
52
  }
@@ -4,8 +4,8 @@
4
4
  // `waitForSelector('#aceptar:not([disabled])')`, which is this rule, spelled once, by hand, at
5
5
  // one call site out of forty.
6
6
 
7
+ import type { SleepClock } from '@ultimat3/core';
7
8
  import { finiteCount } from '@ultimat3/core';
8
- import type { ScrapeClock } from './clock';
9
9
  import { deadline } from './clock';
10
10
  import { notActionable, selectorMissing } from './error-throws';
11
11
  import type { ElementSnapshot } from './target';
@@ -71,7 +71,7 @@ export interface ActionabilityWait {
71
71
  readonly url: string;
72
72
  readonly state: ActionabilityState;
73
73
  readonly timeoutMs: number;
74
- readonly clock: ScrapeClock;
74
+ readonly clock: SleepClock;
75
75
  readonly pollMs?: number | undefined;
76
76
  readonly signal?: AbortSignal | undefined;
77
77
  /** Re-read from the live page on EVERY poll. Never a handle captured before the loop. */
package/src/adapter.ts ADDED
@@ -0,0 +1,22 @@
1
+ // `@ultimat3/scraping/adapter` — the `scrapeDriver` port: the interface a browser driver
2
+ // implements, the errors it throws, and the boot's installer. The installer lives here and never on
3
+ // the `.` barrel, so `scrapeDriver` in `apps/<app>/runtime.ts` is the one way an app picks a browser.
4
+
5
+ export type { ScrapeDriver, ScrapeSession, SessionInit } from './driver';
6
+ export { installScrapeDriver, resetScrapeDriver } from './driver';
7
+ export {
8
+ browserUnreachable,
9
+ cdpAttachFailed,
10
+ hostBlocked,
11
+ notActionable,
12
+ pageCrashed,
13
+ profileLocked,
14
+ remoteRequired,
15
+ scrapeTimeout,
16
+ selectorMissing,
17
+ } from './error-throws';
18
+ export { egressUnsupported } from './error-throws-session';
19
+ export { isScrapeError, ScrapeError } from './errors';
20
+ export type { HttpTransportInit, ScrapeHttp } from './http';
21
+ export type { ScrapePage } from './page';
22
+ export type { SecretResolver } from './secrets';
package/src/auth.ts CHANGED
@@ -17,9 +17,8 @@
17
17
  // rejected credential. A rejected credential is `X_SCRAPE_AUTH_FAILED`, terminal, and the refusal
18
18
  // is written into the session record so the NEXT attempt fails before reaching the login form.
19
19
 
20
- import type { Logger } from '@ultimat3/core';
20
+ import type { Logger, SleepClock } from '@ultimat3/core';
21
21
  import { finiteCount } from '@ultimat3/core';
22
- import type { ScrapeClock } from './clock';
23
22
  import { authFailed, promptUnanswered, scrapeSessionExpired } from './error-throws';
24
23
  import type { ScrapePage } from './page';
25
24
  import type { ScrapeSecrets } from './secrets';
@@ -45,7 +44,7 @@ export interface PromptRequest<I = unknown> {
45
44
  /** 1 for the first prompt of this attempt, 2 for the next. */
46
45
  readonly index: number;
47
46
  /** The run's clock. A handler that waits, waits on this — never on a timer of its own. */
48
- readonly clock: ScrapeClock;
47
+ readonly clock: SleepClock;
49
48
  /** The run's cancellation. A handler that waits past it holds a browser nobody is using. */
50
49
  readonly signal: AbortSignal | undefined;
51
50
  /**
@@ -110,7 +109,7 @@ export interface PromptInit<I = unknown> {
110
109
  readonly input: I;
111
110
  readonly page: ScrapePage;
112
111
  readonly runId: string;
113
- readonly clock: ScrapeClock;
112
+ readonly clock: SleepClock;
114
113
  readonly signal?: AbortSignal | undefined;
115
114
  /** The run's secret set. Every answer is concealed in it before the body sees the answer. */
116
115
  readonly secrets?: ScrapeSecrets | undefined;
@@ -156,7 +155,7 @@ export interface AuthPlanInput<I> {
156
155
  readonly scrape: string;
157
156
  readonly auth: ScrapeAuth<I> | undefined;
158
157
  readonly key: string;
159
- readonly clock: ScrapeClock;
158
+ readonly clock: SleepClock;
160
159
  readonly logger: Logger;
161
160
  }
162
161
 
@@ -0,0 +1,55 @@
1
+ // The DNS half of `allowHosts` for the BROWSER leg. The browser dials by itself, so nothing here can
2
+ // pin its socket the way `pinned-host.ts` pins the HTTP leg; what it can do is refuse to release a
3
+ // request whose wildcard-admitted name resolves inward (`10.x`, `169.254.169.254`) — the SSRF a
4
+ // name-only check let through. Chrome's own re-resolution (rebinding) stays open: `wiki/Known-Gaps.md`.
5
+
6
+ import type { HostRule } from '@ultimat3/core';
7
+ import { classifyAddress, hostMatches } from '@ultimat3/core';
8
+ import { screenDial } from '@ultimat3/core/outbound';
9
+ import type { HostResolve } from './pinned-host';
10
+
11
+ /** How the browser leg screens: the resolver, and whether a proxy resolves names instead. */
12
+ export interface BrowserDialScreen {
13
+ readonly allowHosts: readonly HostRule[];
14
+ readonly resolve: HostResolve;
15
+ /** Through a proxy the EXIT resolves the name, so this box's answer says nothing — the HTTP leg's rule. */
16
+ readonly proxied: boolean;
17
+ }
18
+
19
+ /** The schemes the browser opens a socket for by name. A WebSocket is a dial like any other. */
20
+ const DIALLED_SCHEMES = new Set(['http:', 'https:', 'ws:', 'wss:']);
21
+
22
+ const isWildcard = (rule: HostRule): boolean => {
23
+ const cleaned = rule.trim();
24
+ return cleaned === '*' || cleaned.startsWith('*.');
25
+ };
26
+
27
+ /**
28
+ * `undefined` — synchronously — when there is nothing to resolve: a hostless or non-http scheme,
29
+ * an address literal (core's floor already judged it), a host an EXACT rule names (the app said
30
+ * that host in a line a reviewer can see), or a proxied session. Otherwise a promise of whether
31
+ * the request must be REFUSED: one non-public address, or no answer at all, refuses — the browser
32
+ * would not have reached an unresolvable name either, and failing closed is the only safe read.
33
+ * Asked only of a URL `interceptVerdict` has already allowed.
34
+ */
35
+ export function browserDialRefusal(
36
+ url: string,
37
+ screen: BrowserDialScreen,
38
+ ): Promise<boolean> | undefined {
39
+ if (screen.proxied) return undefined;
40
+ let parsed: URL;
41
+ try {
42
+ parsed = new URL(url);
43
+ } catch {
44
+ return undefined;
45
+ }
46
+ if (!DIALLED_SCHEMES.has(parsed.protocol)) return undefined;
47
+ const hostname = parsed.hostname.replace(/^\[|\]$/g, '');
48
+ if (hostname === '' || classifyAddress(hostname) !== undefined) return undefined;
49
+ const exact = screen.allowHosts.some((rule) => !isWildcard(rule) && hostMatches(hostname, rule));
50
+ if (exact) return undefined;
51
+ return screenDial(parsed, { resolve: screen.resolve }).then(
52
+ (verdict) => !('pin' in verdict),
53
+ () => true,
54
+ );
55
+ }
package/src/cdp-arm.ts CHANGED
@@ -1,13 +1,16 @@
1
1
  // Everything `cdpTarget()` arms on the page BEFORE its first navigation, plus the readers that
2
2
  // turn the library's `unknown` event payloads into ring entries. Split out of `cdp-target.ts` on
3
3
  // 2026-09-19 for one reason: that file stood at 478 lines against the 500-line ceiling, and the
4
- // three verbs `press`/`focus`/`accessibility` did not fit under it. The extraction is VERBATIM —
5
- // the request, console, `pageerror` and `error` handlers are the same code, one file over.
4
+ // three verbs `press`/`focus`/`accessibility` did not fit under it. Since 31.0.0 an allowed request
5
+ // is released only after `browser-dial.ts` screens a wildcard-admitted name's addresses.
6
6
 
7
+ import type { SleepClock } from '@ultimat3/core';
8
+ import { resolveDialHost } from '@ultimat3/core/outbound';
9
+ import { type BrowserDialScreen, browserDialRefusal } from './browser-dial';
7
10
  import type { CdpBrowserLike, CdpPageLike, CdpRequestLike } from './cdp-port';
8
- import type { ScrapeClock } from './clock';
9
11
  import type { InterceptRules } from './intercept';
10
12
  import { interceptVerdict, refusalEntry } from './intercept';
13
+ import type { HostResolve } from './pinned-host';
11
14
  import type {
12
15
  ConsoleLine,
13
16
  ConsoleRing,
@@ -18,13 +21,47 @@ import type {
18
21
  } from './rings';
19
22
  import { clamped, pageErrorEntry, RESOURCE_TYPES } from './rings';
20
23
 
24
+ /**
25
+ * What the DNS screen reads (`browser-dial.ts`). `resolve` defaults to `Bun.dns`, the one the HTTP
26
+ * leg uses; `proxied` is true when the session dials through an exit, which resolves names itself.
27
+ */
28
+ export interface CdpDialInit {
29
+ readonly resolve?: HostResolve | undefined;
30
+ readonly proxied?: boolean | undefined;
31
+ }
32
+
21
33
  /** What `arm()` reads: the page it subscribes to, the rules it judges by, the clock it stamps with. */
22
- export interface CdpArmInit {
34
+ export interface CdpArmInit extends CdpDialInit {
23
35
  readonly page: CdpPageLike;
24
36
  readonly rules: InterceptRules;
25
- readonly clock: ScrapeClock;
37
+ readonly clock: SleepClock;
26
38
  }
27
39
 
40
+ const dialScreen = (init: CdpDialInit & { readonly rules: InterceptRules }): BrowserDialScreen => ({
41
+ allowHosts: init.rules.allowHosts,
42
+ resolve: init.resolve ?? resolveDialHost,
43
+ proxied: init.proxied === true,
44
+ });
45
+
46
+ /**
47
+ * Release or refuse, after the DNS screen when there is one. Synchronous when nothing needs
48
+ * resolving — an exact rule's request leaves on the event's own tick, as before this screen.
49
+ * A refusal is a `host` refusal: the name was admitted, the address it answers is not.
50
+ */
51
+ const screened = (
52
+ url: string,
53
+ screen: BrowserDialScreen,
54
+ release: () => void,
55
+ refuse: () => void,
56
+ ): void => {
57
+ const pending = browserDialRefusal(url, screen);
58
+ if (pending === undefined) {
59
+ release();
60
+ return;
61
+ }
62
+ pending.then((refused) => (refused ? refuse() : release()), refuse);
63
+ };
64
+
28
65
  const asResourceType = (raw: string): ResourceType =>
29
66
  (RESOURCE_TYPES as readonly string[]).includes(raw) ? (raw as ResourceType) : 'other';
30
67
 
@@ -121,6 +158,7 @@ export interface CdpSinks {
121
158
  */
122
159
  export async function arm(init: CdpArmInit, sinks: CdpSinks): Promise<void> {
123
160
  const { network, console: console_, pageErrors, crashed } = sinks;
161
+ const screen = dialScreen(init);
124
162
  await init.page.setRequestInterception(true);
125
163
  init.page.on('request', (payload) => {
126
164
  const request = asRequest(payload);
@@ -133,16 +171,26 @@ export async function arm(init: CdpArmInit, sinks: CdpSinks): Promise<void> {
133
171
  const method = readStringFrom(request, 'method') ?? 'GET';
134
172
  const verdict = interceptVerdict(url, type, init.rules);
135
173
  const at = init.clock.now().getTime();
136
- if (verdict === 'allow') {
137
- network.push({ method, url: clamped(url), resourceType: type, at });
138
- // Caught, never floated: both reject when the target closed mid-request (and on any other
139
- // protocol error), and a floated rejection is one Bun ends the process on. The request is
140
- // already recorded; there is nothing left to do for a target that is gone.
141
- request.continue().catch(() => undefined);
174
+ // Caught, never floated: both reject when the target closed mid-request (and on any other
175
+ // protocol error), and a floated rejection is one Bun ends the process on. The request is
176
+ // already recorded; there is nothing left to do for a target that is gone.
177
+ const refuse = (reason: 'host' | 'blocked'): void => {
178
+ network.push(refusalEntry(url, type, reason, at, method));
179
+ request.abort().catch(() => undefined);
180
+ };
181
+ if (verdict !== 'allow') {
182
+ refuse(verdict);
142
183
  return;
143
184
  }
144
- network.push(refusalEntry(url, type, verdict, at, method));
145
- request.abort().catch(() => undefined);
185
+ screened(
186
+ url,
187
+ screen,
188
+ () => {
189
+ network.push({ method, url: clamped(url), resourceType: type, at });
190
+ request.continue().catch(() => undefined);
191
+ },
192
+ () => refuse('host'),
193
+ );
146
194
  });
147
195
  init.page.on('console', (payload) => {
148
196
  console_.push({
@@ -212,13 +260,14 @@ const asPaused = (payload: unknown): PausedRequest | undefined => {
212
260
  * `driver-cdp.ts`'s `opened()` closes it — a session whose rule would hold on one tab is refused.
213
261
  */
214
262
  export async function armBrowser(
215
- init: {
263
+ init: CdpDialInit & {
216
264
  readonly browser: CdpBrowserLike;
217
265
  readonly rules: InterceptRules;
218
- readonly clock: ScrapeClock;
266
+ readonly clock: SleepClock;
219
267
  },
220
268
  network: NetworkRing,
221
269
  ): Promise<void> {
270
+ const screen = dialScreen(init);
222
271
  const session = await init.browser.target().createCDPSession();
223
272
  session.on('Fetch.requestPaused', (payload) => {
224
273
  const paused = asPaused(payload);
@@ -226,16 +275,26 @@ export async function armBrowser(
226
275
  const type = asResourceType(paused.resourceType.toLowerCase());
227
276
  const verdict = interceptVerdict(paused.url, type, init.rules);
228
277
  const requestId = { requestId: paused.requestId };
278
+ const at = init.clock.now().getTime();
229
279
  // Caught, never floated, for `arm`'s reason: the target may be gone by the time this answers.
230
- if (verdict === 'allow') {
231
- session.send('Fetch.continueRequest', requestId).catch(() => undefined);
280
+ const refuse = (reason: 'host' | 'blocked'): void => {
281
+ network.push(refusalEntry(paused.url, type, reason, at, paused.method));
282
+ session
283
+ .send('Fetch.failRequest', { ...requestId, errorReason: 'BlockedByClient' })
284
+ .catch(() => undefined);
285
+ };
286
+ if (verdict !== 'allow') {
287
+ refuse(verdict);
232
288
  return;
233
289
  }
234
- const at = init.clock.now().getTime();
235
- network.push(refusalEntry(paused.url, type, verdict, at, paused.method));
236
- session
237
- .send('Fetch.failRequest', { ...requestId, errorReason: 'BlockedByClient' })
238
- .catch(() => undefined);
290
+ screened(
291
+ paused.url,
292
+ screen,
293
+ () => {
294
+ session.send('Fetch.continueRequest', requestId).catch(() => undefined);
295
+ },
296
+ () => refuse('host'),
297
+ );
239
298
  });
240
299
  await session.send('Fetch.enable', { patterns: [{ urlPattern: '*' }] });
241
300
  }