@ultimat3/scraping 30.9.0 → 31.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -8
- package/package.json +9 -6
- package/src/actionability.ts +2 -2
- package/src/adapter.ts +22 -0
- package/src/auth.ts +4 -5
- package/src/browser-dial.ts +55 -0
- package/src/cdp-arm.ts +81 -22
- package/src/cdp-connection.ts +290 -0
- package/src/cdp-error-codes.ts +28 -0
- package/src/cdp-errors.ts +179 -0
- package/src/cdp-launch-attempt.ts +234 -0
- package/src/cdp-launch-candidates.ts +62 -0
- package/src/cdp-launch-probe.ts +75 -0
- package/src/cdp-launch-profile.ts +106 -0
- package/src/cdp-launch-reap.ts +68 -0
- package/src/cdp-launch-wire.ts +171 -0
- package/src/cdp-launch.ts +216 -0
- package/src/cdp-pipe.ts +77 -0
- package/src/cdp-wire-watch.ts +113 -0
- package/src/cdp.ts +46 -0
- package/src/clock.ts +4 -111
- package/src/driver-cdp.ts +3 -0
- package/src/driver-fake.ts +4 -5
- package/src/driver.ts +16 -11
- package/src/error-throws-pdf.ts +45 -0
- package/src/error-throws-session.ts +1 -1
- package/src/error-throws.ts +4 -4
- package/src/errors.ts +17 -1
- package/src/event-prompt.ts +10 -4
- package/src/events.ts +2 -3
- package/src/html-target.ts +2 -2
- package/src/http-recorded.ts +2 -2
- package/src/http.ts +2 -2
- package/src/index.ts +21 -75
- package/src/launch-args.ts +10 -1
- package/src/page-over-target.ts +2 -3
- package/src/pdf-action.ts +116 -0
- package/src/pdf-cdp-params.ts +65 -0
- package/src/pdf-fence.ts +86 -0
- package/src/pdf-render.ts +241 -0
- package/src/pdf-slots.ts +68 -0
- package/src/rate.ts +2 -2
- package/src/scrape-run.ts +3 -4
- package/src/scrape.ts +5 -5
- package/src/target.ts +1 -1
- package/src/usage.ts +2 -2
- package/src/watchdog.ts +3 -3
package/README.md
CHANGED
|
@@ -237,21 +237,28 @@ which is why `onSettled` hands a failed run its last attempt's `usage`. No quota
|
|
|
237
237
|
|
|
238
238
|
### The clock, and a test's
|
|
239
239
|
|
|
240
|
-
A run waits on the PROCESS's clock — `
|
|
241
|
-
|
|
240
|
+
A run waits on the PROCESS's clock — `@ultimat3/core`'s `ambientClock()`, the system's unless
|
|
241
|
+
`clock` in `apps/<app>/runtime.ts` says otherwise — unless its definition pins one with `clock:`.
|
|
242
|
+
Since 31.0.0 the clocks are core's `SleepClock`s; this package has none of its own. A test
|
|
243
|
+
replaces the process clock through core's installer, which answers its release:
|
|
242
244
|
|
|
243
245
|
```ts
|
|
244
|
-
import { noWaitClock
|
|
246
|
+
import { noWaitClock } from '@ultimat3/core';
|
|
247
|
+
import { installClock } from '@ultimat3/core/adapter';
|
|
245
248
|
|
|
246
|
-
|
|
247
|
-
|
|
249
|
+
const release = installClock(noWaitClock); // in a beforeEach
|
|
250
|
+
release(); // in the afterEach
|
|
248
251
|
```
|
|
249
252
|
|
|
250
|
-
| Clock | Sleeping | For |
|
|
253
|
+
| Clock (`@ultimat3/core`) | Sleeping | For |
|
|
251
254
|
|---|---|---|
|
|
252
|
-
| `
|
|
255
|
+
| `systemClock` | waits | production |
|
|
253
256
|
| `noWaitClock` | one turn of the event loop, whatever was asked; `now()` and deadlines are real | a test whose run waits on something that really happens in the same process — an answer the test publishes |
|
|
254
|
-
| `
|
|
257
|
+
| `frozenClock()` | advances virtual time and yields a microtask | a test of a timeout: thirty seconds pass in none |
|
|
258
|
+
|
|
259
|
+
The browser is chosen the same way: `scrapeDriver` in `apps/<app>/runtime.ts`, which the boot
|
|
260
|
+
installs through `installScrapeDriver` on `@ultimat3/scraping/adapter` (the `.` barrel has no
|
|
261
|
+
setter); a definition's own `driver:` overrides it.
|
|
255
262
|
|
|
256
263
|
A third-party driver takes part by reading three `SessionInit` fields: dial `proxy` or throw
|
|
257
264
|
`egressUnsupported()`, pass `usage` to `pageOverTarget` and its HTTP transport, and answer
|
|
@@ -389,6 +396,61 @@ The offline drivers honour it, so the crop is provable with no Chrome — `fakeB
|
|
|
389
396
|
different deterministic bytes per rectangle, which is what lets a component screenshot be tested
|
|
390
397
|
on a machine with no browser — the case CI is.
|
|
391
398
|
|
|
399
|
+
## A route printed to PDF — `pdf()`, a factory over `action`
|
|
400
|
+
|
|
401
|
+
```ts
|
|
402
|
+
import type { Ctx } from '@ultimat3/core';
|
|
403
|
+
import { can } from '@ultimat3/policy';
|
|
404
|
+
import { t } from '@ultimat3/schema';
|
|
405
|
+
import { pdf } from '@ultimat3/scraping';
|
|
406
|
+
import { disk } from '@ultimat3/storage';
|
|
407
|
+
|
|
408
|
+
/** The app's own short-lived signer for the print route — never the framework's. */
|
|
409
|
+
declare function printToken(ctx: Ctx): string;
|
|
410
|
+
|
|
411
|
+
export const invoicePdf = pdf({
|
|
412
|
+
input: t.object({ id: t.uuid }),
|
|
413
|
+
policy: can('invoices:read'),
|
|
414
|
+
route: ({ input }) => `/invoices/${input.id}/print`, // a PATH on this app, never a URL
|
|
415
|
+
key: ({ input, ctx }) => `invoices/${ctx.actor.orgId}/${input.id}.pdf`,
|
|
416
|
+
storage: () => disk('documents'), // a thunk: read per call
|
|
417
|
+
layout: { format: 'a4', marginMm: { top: 12, bottom: 12 }, printBackground: true },
|
|
418
|
+
headers: ({ ctx }) => ({ authorization: `Bearer ${printToken(ctx)}` }), // the app's own token
|
|
419
|
+
});
|
|
420
|
+
// → { key, bytes, contentType: 'application/pdf' } — HTTP, OpenAPI, MCP, .job(), .as() included
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
| Rule | Detail |
|
|
424
|
+
|---|---|
|
|
425
|
+
| it is an `action` | input schema, policy, `rateLimit`, `meter`, `audit` are the action's: refused before a browser starts (`pdf-action.test.ts`). `PRIMITIVE_FACTORIES` row `pdf` |
|
|
426
|
+
| only the app's own origin | `route()` must answer a path; a URL, `//host` or a backslash is `X_SCRAPE_PDF_ROUTE_INVALID` (500, terminal). The origin is `APP_URL`, read per call, or `origin: () => …`; neither is `X_SCRAPE_PDF_ORIGIN_MISSING` (500, terminal) |
|
|
427
|
+
| an error page is never a document | the page's own status is read after `load` and `document.fonts.ready`: 4xx/5xx is `X_SCRAPE_PDF_ROUTE_FAILED` (502; a 4xx terminal, a 5xx/429 retryable) |
|
|
428
|
+
| the layout is checked where `pdf()` is written | `pdfLayoutOf`, as `page.pdf()`: `X_SCRAPE_PDF_OPTIONS_INVALID` at module load |
|
|
429
|
+
| one browser per render | `chromePdfRenderer()` launches Chrome through `@ultimat3/scraping/cdp`, prints over raw CDP (`Page.printToPDF`, `pdf-render.ts`, `pdf-cdp-params.ts`) and closes it in a `finally`; no browser outlives a request, none is shared between tenants. A pool is a measured next step, not shipped |
|
|
430
|
+
| a bounded number of browsers | at most `maxConcurrent` (default 2 — each Chrome is 150-300 MB) per renderer, and every `pdf()` naming no `renderer` shares ONE. The rest queue for `queueTimeoutMs` (default `timeoutMs`) on the clock, then `X_SCRAPE_PDF_BUSY` (503, retryable) — `pdf-slots.ts`, `pdf-render-slots.test.ts` |
|
|
431
|
+
| a dropped request ends its render | the action's `ctx.signal` reaches the renderer: aborted in the queue, it leaves without a slot; aborted mid-print, its browser is killed and the slot freed, and the request's own reason is thrown |
|
|
432
|
+
| headers reach the route's origin only | `Fetch.enable` on the page's session (`pdf-fence.ts`): a request to the route's exact origin (scheme, host, port) continues with `headers` added; every other — a user's `<img src="https://elsewhere">`, a redirect off-site, `169.254.169.254` — is failed `BlockedByClient` before it leaves. Never `Network.setExtraHTTPHeaders`, which sent the token everywhere (`pdf-render-fence.test.ts`) |
|
|
433
|
+
| the sandbox stays on | the default launch keeps Chrome's sandbox (`pdfLaunchOptions`): a print renders user content. The runtime image ships no Chrome; the app's image supplies it and runs it as non-root, which needs unprivileged user namespaces (or Chrome's setuid helper). A container that cannot grant them fails `X_CDP_LAUNCH_FAILED` quoting "No usable sandbox" — opt out there in one visible line, `chromePdfRenderer({ sandbox: false })`. The e2e driver and `x shot` keep `--no-sandbox` for CI runners |
|
|
434
|
+
| the bytes go to storage | `storage().put(key, bytes, { contentType: 'application/pdf' })`; the answer is the key, never the bytes. Serve it with a signed URL |
|
|
435
|
+
| tested with no Chrome | `renderer:` takes any `PdfRenderer`; `chromePdfRenderer({ launch })` takes any `{ connection, close }` — `pdf-render.test.ts` drives it over a fake CDP connection |
|
|
436
|
+
|
|
437
|
+
Open (`As of 2026-11`): the route must be readable by the printing browser — a private one forwards
|
|
438
|
+
the app's own short-lived credential through `headers`; no session is minted for it.
|
|
439
|
+
|
|
440
|
+
## The raw-CDP browser — `@ultimat3/scraping/cdp`
|
|
441
|
+
|
|
442
|
+
The launcher and connection the e2e driver and `x shot` have used since 22.0.0, moved here from
|
|
443
|
+
`@ultimat3/testing` in 31.0.0 because `pdf()` needs them at RUNTIME and testing is a test package.
|
|
444
|
+
Its own entry, off `.`, so a caller that only launches loads no job or storage module.
|
|
445
|
+
This is every name's one home: `@ultimat3/testing` no longer re-exports them (31.0.0, `X_HELPER_COPY`); its `E2E_ERROR_CODES` still lists the four codes.
|
|
446
|
+
|
|
447
|
+
| Export | What |
|
|
448
|
+
|---|---|
|
|
449
|
+
| `findChrome(env)` · `launchChrome({ executable, timeoutMs })` · `launchFoundChrome(env, timeoutMs)` | `CHROME_PATH`, then the platform's install paths; a throwaway profile, the pipe wire, one relaunch |
|
|
450
|
+
| `cdpConnect({ endpoint, timeoutMs })` · `cdpConnectOver(transport, timeoutMs)` | a CDP connection over a remote WebSocket or any whole-message transport |
|
|
451
|
+
| `CdpBrowserMissingError` · `CdpLaunchFailedError` · `CdpCallFailedError` · `CdpTimeoutError` | `X_CDP_BROWSER_MISSING` · `X_CDP_LAUNCH_FAILED` · `X_CDP_CALL_FAILED` · `X_CDP_TIMEOUT`, registered by `cdp-error-codes.ts` |
|
|
452
|
+
| `CHROME_CANDIDATES` · `CONTAINER_CHROME_ARGS` · `chromeLaunchFlags` · `LAUNCH_*` · `CLOSE_GRACE_MS` · `LAUNCHED_CLOSE_MS` | the flags and the deadlines, derived, never restated |
|
|
453
|
+
|
|
392
454
|
## What it owns
|
|
393
455
|
|
|
394
456
|
| Module | Owns |
|
|
@@ -399,6 +461,7 @@ on a machine with no browser — the case CI is.
|
|
|
399
461
|
| `driver-cdp.ts` / `cdp-*.ts` | the real browser, over a structural CDP port |
|
|
400
462
|
| `driver-fake.ts` / `driver-recorded.ts` / `html-*.ts` | the offline drivers, on Bun's `HTMLRewriter` |
|
|
401
463
|
| `http.ts` / `http-recorded.ts` | the second transport, live and replayed |
|
|
464
|
+
| `intercept.ts` / `pinned-host.ts` / `browser-dial.ts` | `allowHosts`: one name verdict for every driver, then the DNS half for a host a wildcard admitted — resolved and pinned on the HTTP leg and robots read, resolved before release on the browser leg (a private or unresolvable answer is a `refused: 'host'` ring entry). An exact rule and a proxied session are not resolved |
|
|
402
465
|
| `auth.ts` / `session-state.ts` | acquire → persist → reuse → validate → burn. The session key encodes each part (`<sanitised>.<digest>`), so two account names that differ only outside `[a-zA-Z0-9._-]` are two sessions. A stored session is sealed. `burnSession(plan, seen)` takes `recordVersion(record)` of the record the run used, so a record another run saved since is not burned |
|
|
403
466
|
| `event-prompt.ts` | `eventPrompt()` and `answerPrompt()`: a prompt answered from another process, over the job event bus, with the browser open |
|
|
404
467
|
| `cdp-resolver.ts` | a browser rented per session: acquired once, released once |
|
|
@@ -406,6 +469,8 @@ on a machine with no browser — the case CI is.
|
|
|
406
469
|
| `secrets.ts` / `url-secrets.ts` / `browser-record.ts` | what may leave this package — declared secrets, values learned mid-run (`conceal`), the credential-bearing parts of a URL — and how a browser's own string map is read |
|
|
407
470
|
| `expect.ts` | the silent-green alarm |
|
|
408
471
|
| `watchdog.ts` | the wedge and zombie discipline |
|
|
472
|
+
| `pdf-action.ts` / `pdf-render.ts` / `pdf-cdp-params.ts` / `pdf-slots.ts` / `pdf-fence.ts` | `pdf()` over `action`; one page printed over raw CDP; a layout in the protocol's words; the bound on live browsers; the request fence |
|
|
473
|
+
| `cdp.ts` / `cdp-launch*.ts` / `cdp-connection.ts` / `cdp-pipe.ts` / `cdp-wire-watch.ts` / `cdp-errors.ts` | `@ultimat3/scraping/cdp`: the raw-CDP launcher and connection (moved from `@ultimat3/testing` in 31.0.0) |
|
|
409
474
|
| `capture-clip.ts` | the one framing rule — crop, whole page, or a refusal — checked before any driver sees it |
|
|
410
475
|
| `errors.ts` / `error-throws.ts` / `error-throws-session.ts` | this package's `X_*` codes and their retry classification |
|
|
411
476
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/scraping",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "31.0.0",
|
|
4
4
|
"description": "Browser automation as a job: scrape() returns a JobHandle",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ultimate",
|
|
@@ -24,7 +24,9 @@
|
|
|
24
24
|
"provenance": true
|
|
25
25
|
},
|
|
26
26
|
"exports": {
|
|
27
|
-
".": "./src/index.ts"
|
|
27
|
+
".": "./src/index.ts",
|
|
28
|
+
"./adapter": "./src/adapter.ts",
|
|
29
|
+
"./cdp": "./src/cdp.ts"
|
|
28
30
|
},
|
|
29
31
|
"files": [
|
|
30
32
|
"src",
|
|
@@ -41,9 +43,10 @@
|
|
|
41
43
|
"test": "bun test"
|
|
42
44
|
},
|
|
43
45
|
"dependencies": {
|
|
44
|
-
"@ultimat3/
|
|
45
|
-
"@ultimat3/
|
|
46
|
-
"@ultimat3/
|
|
47
|
-
"@ultimat3/
|
|
46
|
+
"@ultimat3/action": "31.0.0",
|
|
47
|
+
"@ultimat3/core": "31.0.0",
|
|
48
|
+
"@ultimat3/jobs": "31.0.0",
|
|
49
|
+
"@ultimat3/schema": "31.0.0",
|
|
50
|
+
"@ultimat3/storage": "31.0.0"
|
|
48
51
|
}
|
|
49
52
|
}
|
package/src/actionability.ts
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
// `waitForSelector('#aceptar:not([disabled])')`, which is this rule, spelled once, by hand, at
|
|
5
5
|
// one call site out of forty.
|
|
6
6
|
|
|
7
|
+
import type { SleepClock } from '@ultimat3/core';
|
|
7
8
|
import { finiteCount } from '@ultimat3/core';
|
|
8
|
-
import type { ScrapeClock } from './clock';
|
|
9
9
|
import { deadline } from './clock';
|
|
10
10
|
import { notActionable, selectorMissing } from './error-throws';
|
|
11
11
|
import type { ElementSnapshot } from './target';
|
|
@@ -71,7 +71,7 @@ export interface ActionabilityWait {
|
|
|
71
71
|
readonly url: string;
|
|
72
72
|
readonly state: ActionabilityState;
|
|
73
73
|
readonly timeoutMs: number;
|
|
74
|
-
readonly clock:
|
|
74
|
+
readonly clock: SleepClock;
|
|
75
75
|
readonly pollMs?: number | undefined;
|
|
76
76
|
readonly signal?: AbortSignal | undefined;
|
|
77
77
|
/** Re-read from the live page on EVERY poll. Never a handle captured before the loop. */
|
package/src/adapter.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// `@ultimat3/scraping/adapter` — the `scrapeDriver` port: the interface a browser driver
|
|
2
|
+
// implements, the errors it throws, and the boot's installer. The installer lives here and never on
|
|
3
|
+
// the `.` barrel, so `scrapeDriver` in `apps/<app>/runtime.ts` is the one way an app picks a browser.
|
|
4
|
+
|
|
5
|
+
export type { ScrapeDriver, ScrapeSession, SessionInit } from './driver';
|
|
6
|
+
export { installScrapeDriver, resetScrapeDriver } from './driver';
|
|
7
|
+
export {
|
|
8
|
+
browserUnreachable,
|
|
9
|
+
cdpAttachFailed,
|
|
10
|
+
hostBlocked,
|
|
11
|
+
notActionable,
|
|
12
|
+
pageCrashed,
|
|
13
|
+
profileLocked,
|
|
14
|
+
remoteRequired,
|
|
15
|
+
scrapeTimeout,
|
|
16
|
+
selectorMissing,
|
|
17
|
+
} from './error-throws';
|
|
18
|
+
export { egressUnsupported } from './error-throws-session';
|
|
19
|
+
export { isScrapeError, ScrapeError } from './errors';
|
|
20
|
+
export type { HttpTransportInit, ScrapeHttp } from './http';
|
|
21
|
+
export type { ScrapePage } from './page';
|
|
22
|
+
export type { SecretResolver } from './secrets';
|
package/src/auth.ts
CHANGED
|
@@ -17,9 +17,8 @@
|
|
|
17
17
|
// rejected credential. A rejected credential is `X_SCRAPE_AUTH_FAILED`, terminal, and the refusal
|
|
18
18
|
// is written into the session record so the NEXT attempt fails before reaching the login form.
|
|
19
19
|
|
|
20
|
-
import type { Logger } from '@ultimat3/core';
|
|
20
|
+
import type { Logger, SleepClock } from '@ultimat3/core';
|
|
21
21
|
import { finiteCount } from '@ultimat3/core';
|
|
22
|
-
import type { ScrapeClock } from './clock';
|
|
23
22
|
import { authFailed, promptUnanswered, scrapeSessionExpired } from './error-throws';
|
|
24
23
|
import type { ScrapePage } from './page';
|
|
25
24
|
import type { ScrapeSecrets } from './secrets';
|
|
@@ -45,7 +44,7 @@ export interface PromptRequest<I = unknown> {
|
|
|
45
44
|
/** 1 for the first prompt of this attempt, 2 for the next. */
|
|
46
45
|
readonly index: number;
|
|
47
46
|
/** The run's clock. A handler that waits, waits on this — never on a timer of its own. */
|
|
48
|
-
readonly clock:
|
|
47
|
+
readonly clock: SleepClock;
|
|
49
48
|
/** The run's cancellation. A handler that waits past it holds a browser nobody is using. */
|
|
50
49
|
readonly signal: AbortSignal | undefined;
|
|
51
50
|
/**
|
|
@@ -110,7 +109,7 @@ export interface PromptInit<I = unknown> {
|
|
|
110
109
|
readonly input: I;
|
|
111
110
|
readonly page: ScrapePage;
|
|
112
111
|
readonly runId: string;
|
|
113
|
-
readonly clock:
|
|
112
|
+
readonly clock: SleepClock;
|
|
114
113
|
readonly signal?: AbortSignal | undefined;
|
|
115
114
|
/** The run's secret set. Every answer is concealed in it before the body sees the answer. */
|
|
116
115
|
readonly secrets?: ScrapeSecrets | undefined;
|
|
@@ -156,7 +155,7 @@ export interface AuthPlanInput<I> {
|
|
|
156
155
|
readonly scrape: string;
|
|
157
156
|
readonly auth: ScrapeAuth<I> | undefined;
|
|
158
157
|
readonly key: string;
|
|
159
|
-
readonly clock:
|
|
158
|
+
readonly clock: SleepClock;
|
|
160
159
|
readonly logger: Logger;
|
|
161
160
|
}
|
|
162
161
|
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// The DNS half of `allowHosts` for the BROWSER leg. The browser dials by itself, so nothing here can
|
|
2
|
+
// pin its socket the way `pinned-host.ts` pins the HTTP leg; what it can do is refuse to release a
|
|
3
|
+
// request whose wildcard-admitted name resolves inward (`10.x`, `169.254.169.254`) — the SSRF a
|
|
4
|
+
// name-only check let through. Chrome's own re-resolution (rebinding) stays open: `wiki/Known-Gaps.md`.
|
|
5
|
+
|
|
6
|
+
import type { HostRule } from '@ultimat3/core';
|
|
7
|
+
import { classifyAddress, hostMatches } from '@ultimat3/core';
|
|
8
|
+
import { screenDial } from '@ultimat3/core/outbound';
|
|
9
|
+
import type { HostResolve } from './pinned-host';
|
|
10
|
+
|
|
11
|
+
/** How the browser leg screens: the resolver, and whether a proxy resolves names instead. */
|
|
12
|
+
export interface BrowserDialScreen {
|
|
13
|
+
readonly allowHosts: readonly HostRule[];
|
|
14
|
+
readonly resolve: HostResolve;
|
|
15
|
+
/** Through a proxy the EXIT resolves the name, so this box's answer says nothing — the HTTP leg's rule. */
|
|
16
|
+
readonly proxied: boolean;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** The schemes the browser opens a socket for by name. A WebSocket is a dial like any other. */
|
|
20
|
+
const DIALLED_SCHEMES = new Set(['http:', 'https:', 'ws:', 'wss:']);
|
|
21
|
+
|
|
22
|
+
const isWildcard = (rule: HostRule): boolean => {
|
|
23
|
+
const cleaned = rule.trim();
|
|
24
|
+
return cleaned === '*' || cleaned.startsWith('*.');
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* `undefined` — synchronously — when there is nothing to resolve: a hostless or non-http scheme,
|
|
29
|
+
* an address literal (core's floor already judged it), a host an EXACT rule names (the app said
|
|
30
|
+
* that host in a line a reviewer can see), or a proxied session. Otherwise a promise of whether
|
|
31
|
+
* the request must be REFUSED: one non-public address, or no answer at all, refuses — the browser
|
|
32
|
+
* would not have reached an unresolvable name either, and failing closed is the only safe read.
|
|
33
|
+
* Asked only of a URL `interceptVerdict` has already allowed.
|
|
34
|
+
*/
|
|
35
|
+
export function browserDialRefusal(
|
|
36
|
+
url: string,
|
|
37
|
+
screen: BrowserDialScreen,
|
|
38
|
+
): Promise<boolean> | undefined {
|
|
39
|
+
if (screen.proxied) return undefined;
|
|
40
|
+
let parsed: URL;
|
|
41
|
+
try {
|
|
42
|
+
parsed = new URL(url);
|
|
43
|
+
} catch {
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|
|
46
|
+
if (!DIALLED_SCHEMES.has(parsed.protocol)) return undefined;
|
|
47
|
+
const hostname = parsed.hostname.replace(/^\[|\]$/g, '');
|
|
48
|
+
if (hostname === '' || classifyAddress(hostname) !== undefined) return undefined;
|
|
49
|
+
const exact = screen.allowHosts.some((rule) => !isWildcard(rule) && hostMatches(hostname, rule));
|
|
50
|
+
if (exact) return undefined;
|
|
51
|
+
return screenDial(parsed, { resolve: screen.resolve }).then(
|
|
52
|
+
(verdict) => !('pin' in verdict),
|
|
53
|
+
() => true,
|
|
54
|
+
);
|
|
55
|
+
}
|
package/src/cdp-arm.ts
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
// Everything `cdpTarget()` arms on the page BEFORE its first navigation, plus the readers that
|
|
2
2
|
// turn the library's `unknown` event payloads into ring entries. Split out of `cdp-target.ts` on
|
|
3
3
|
// 2026-09-19 for one reason: that file stood at 478 lines against the 500-line ceiling, and the
|
|
4
|
-
// three verbs `press`/`focus`/`accessibility` did not fit under it.
|
|
5
|
-
//
|
|
4
|
+
// three verbs `press`/`focus`/`accessibility` did not fit under it. Since 31.0.0 an allowed request
|
|
5
|
+
// is released only after `browser-dial.ts` screens a wildcard-admitted name's addresses.
|
|
6
6
|
|
|
7
|
+
import type { SleepClock } from '@ultimat3/core';
|
|
8
|
+
import { resolveDialHost } from '@ultimat3/core/outbound';
|
|
9
|
+
import { type BrowserDialScreen, browserDialRefusal } from './browser-dial';
|
|
7
10
|
import type { CdpBrowserLike, CdpPageLike, CdpRequestLike } from './cdp-port';
|
|
8
|
-
import type { ScrapeClock } from './clock';
|
|
9
11
|
import type { InterceptRules } from './intercept';
|
|
10
12
|
import { interceptVerdict, refusalEntry } from './intercept';
|
|
13
|
+
import type { HostResolve } from './pinned-host';
|
|
11
14
|
import type {
|
|
12
15
|
ConsoleLine,
|
|
13
16
|
ConsoleRing,
|
|
@@ -18,13 +21,47 @@ import type {
|
|
|
18
21
|
} from './rings';
|
|
19
22
|
import { clamped, pageErrorEntry, RESOURCE_TYPES } from './rings';
|
|
20
23
|
|
|
24
|
+
/**
|
|
25
|
+
* What the DNS screen reads (`browser-dial.ts`). `resolve` defaults to `Bun.dns`, the one the HTTP
|
|
26
|
+
* leg uses; `proxied` is true when the session dials through an exit, which resolves names itself.
|
|
27
|
+
*/
|
|
28
|
+
export interface CdpDialInit {
|
|
29
|
+
readonly resolve?: HostResolve | undefined;
|
|
30
|
+
readonly proxied?: boolean | undefined;
|
|
31
|
+
}
|
|
32
|
+
|
|
21
33
|
/** What `arm()` reads: the page it subscribes to, the rules it judges by, the clock it stamps with. */
|
|
22
|
-
export interface CdpArmInit {
|
|
34
|
+
export interface CdpArmInit extends CdpDialInit {
|
|
23
35
|
readonly page: CdpPageLike;
|
|
24
36
|
readonly rules: InterceptRules;
|
|
25
|
-
readonly clock:
|
|
37
|
+
readonly clock: SleepClock;
|
|
26
38
|
}
|
|
27
39
|
|
|
40
|
+
const dialScreen = (init: CdpDialInit & { readonly rules: InterceptRules }): BrowserDialScreen => ({
|
|
41
|
+
allowHosts: init.rules.allowHosts,
|
|
42
|
+
resolve: init.resolve ?? resolveDialHost,
|
|
43
|
+
proxied: init.proxied === true,
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Release or refuse, after the DNS screen when there is one. Synchronous when nothing needs
|
|
48
|
+
* resolving — an exact rule's request leaves on the event's own tick, as before this screen.
|
|
49
|
+
* A refusal is a `host` refusal: the name was admitted, the address it answers is not.
|
|
50
|
+
*/
|
|
51
|
+
const screened = (
|
|
52
|
+
url: string,
|
|
53
|
+
screen: BrowserDialScreen,
|
|
54
|
+
release: () => void,
|
|
55
|
+
refuse: () => void,
|
|
56
|
+
): void => {
|
|
57
|
+
const pending = browserDialRefusal(url, screen);
|
|
58
|
+
if (pending === undefined) {
|
|
59
|
+
release();
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
pending.then((refused) => (refused ? refuse() : release()), refuse);
|
|
63
|
+
};
|
|
64
|
+
|
|
28
65
|
const asResourceType = (raw: string): ResourceType =>
|
|
29
66
|
(RESOURCE_TYPES as readonly string[]).includes(raw) ? (raw as ResourceType) : 'other';
|
|
30
67
|
|
|
@@ -121,6 +158,7 @@ export interface CdpSinks {
|
|
|
121
158
|
*/
|
|
122
159
|
export async function arm(init: CdpArmInit, sinks: CdpSinks): Promise<void> {
|
|
123
160
|
const { network, console: console_, pageErrors, crashed } = sinks;
|
|
161
|
+
const screen = dialScreen(init);
|
|
124
162
|
await init.page.setRequestInterception(true);
|
|
125
163
|
init.page.on('request', (payload) => {
|
|
126
164
|
const request = asRequest(payload);
|
|
@@ -133,16 +171,26 @@ export async function arm(init: CdpArmInit, sinks: CdpSinks): Promise<void> {
|
|
|
133
171
|
const method = readStringFrom(request, 'method') ?? 'GET';
|
|
134
172
|
const verdict = interceptVerdict(url, type, init.rules);
|
|
135
173
|
const at = init.clock.now().getTime();
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
request.
|
|
174
|
+
// Caught, never floated: both reject when the target closed mid-request (and on any other
|
|
175
|
+
// protocol error), and a floated rejection is one Bun ends the process on. The request is
|
|
176
|
+
// already recorded; there is nothing left to do for a target that is gone.
|
|
177
|
+
const refuse = (reason: 'host' | 'blocked'): void => {
|
|
178
|
+
network.push(refusalEntry(url, type, reason, at, method));
|
|
179
|
+
request.abort().catch(() => undefined);
|
|
180
|
+
};
|
|
181
|
+
if (verdict !== 'allow') {
|
|
182
|
+
refuse(verdict);
|
|
142
183
|
return;
|
|
143
184
|
}
|
|
144
|
-
|
|
145
|
-
|
|
185
|
+
screened(
|
|
186
|
+
url,
|
|
187
|
+
screen,
|
|
188
|
+
() => {
|
|
189
|
+
network.push({ method, url: clamped(url), resourceType: type, at });
|
|
190
|
+
request.continue().catch(() => undefined);
|
|
191
|
+
},
|
|
192
|
+
() => refuse('host'),
|
|
193
|
+
);
|
|
146
194
|
});
|
|
147
195
|
init.page.on('console', (payload) => {
|
|
148
196
|
console_.push({
|
|
@@ -212,13 +260,14 @@ const asPaused = (payload: unknown): PausedRequest | undefined => {
|
|
|
212
260
|
* `driver-cdp.ts`'s `opened()` closes it — a session whose rule would hold on one tab is refused.
|
|
213
261
|
*/
|
|
214
262
|
export async function armBrowser(
|
|
215
|
-
init: {
|
|
263
|
+
init: CdpDialInit & {
|
|
216
264
|
readonly browser: CdpBrowserLike;
|
|
217
265
|
readonly rules: InterceptRules;
|
|
218
|
-
readonly clock:
|
|
266
|
+
readonly clock: SleepClock;
|
|
219
267
|
},
|
|
220
268
|
network: NetworkRing,
|
|
221
269
|
): Promise<void> {
|
|
270
|
+
const screen = dialScreen(init);
|
|
222
271
|
const session = await init.browser.target().createCDPSession();
|
|
223
272
|
session.on('Fetch.requestPaused', (payload) => {
|
|
224
273
|
const paused = asPaused(payload);
|
|
@@ -226,16 +275,26 @@ export async function armBrowser(
|
|
|
226
275
|
const type = asResourceType(paused.resourceType.toLowerCase());
|
|
227
276
|
const verdict = interceptVerdict(paused.url, type, init.rules);
|
|
228
277
|
const requestId = { requestId: paused.requestId };
|
|
278
|
+
const at = init.clock.now().getTime();
|
|
229
279
|
// Caught, never floated, for `arm`'s reason: the target may be gone by the time this answers.
|
|
230
|
-
|
|
231
|
-
|
|
280
|
+
const refuse = (reason: 'host' | 'blocked'): void => {
|
|
281
|
+
network.push(refusalEntry(paused.url, type, reason, at, paused.method));
|
|
282
|
+
session
|
|
283
|
+
.send('Fetch.failRequest', { ...requestId, errorReason: 'BlockedByClient' })
|
|
284
|
+
.catch(() => undefined);
|
|
285
|
+
};
|
|
286
|
+
if (verdict !== 'allow') {
|
|
287
|
+
refuse(verdict);
|
|
232
288
|
return;
|
|
233
289
|
}
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
290
|
+
screened(
|
|
291
|
+
paused.url,
|
|
292
|
+
screen,
|
|
293
|
+
() => {
|
|
294
|
+
session.send('Fetch.continueRequest', requestId).catch(() => undefined);
|
|
295
|
+
},
|
|
296
|
+
() => refuse('host'),
|
|
297
|
+
);
|
|
239
298
|
});
|
|
240
299
|
await session.send('Fetch.enable', { patterns: [{ urlPattern: '*' }] });
|
|
241
300
|
}
|