@ultimat3/scraping 30.9.0 → 31.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +73 -8
  2. package/package.json +9 -6
  3. package/src/actionability.ts +2 -2
  4. package/src/adapter.ts +22 -0
  5. package/src/auth.ts +4 -5
  6. package/src/browser-dial.ts +55 -0
  7. package/src/cdp-arm.ts +81 -22
  8. package/src/cdp-connection.ts +290 -0
  9. package/src/cdp-error-codes.ts +28 -0
  10. package/src/cdp-errors.ts +179 -0
  11. package/src/cdp-launch-attempt.ts +234 -0
  12. package/src/cdp-launch-candidates.ts +62 -0
  13. package/src/cdp-launch-probe.ts +75 -0
  14. package/src/cdp-launch-profile.ts +106 -0
  15. package/src/cdp-launch-reap.ts +68 -0
  16. package/src/cdp-launch-wire.ts +171 -0
  17. package/src/cdp-launch.ts +216 -0
  18. package/src/cdp-pipe.ts +77 -0
  19. package/src/cdp-wire-watch.ts +113 -0
  20. package/src/cdp.ts +46 -0
  21. package/src/clock.ts +4 -111
  22. package/src/driver-cdp.ts +3 -0
  23. package/src/driver-fake.ts +4 -5
  24. package/src/driver.ts +16 -11
  25. package/src/error-throws-pdf.ts +45 -0
  26. package/src/error-throws-session.ts +1 -1
  27. package/src/error-throws.ts +4 -4
  28. package/src/errors.ts +17 -1
  29. package/src/event-prompt.ts +10 -4
  30. package/src/events.ts +2 -3
  31. package/src/html-target.ts +2 -2
  32. package/src/http-recorded.ts +2 -2
  33. package/src/http.ts +2 -2
  34. package/src/index.ts +21 -75
  35. package/src/launch-args.ts +10 -1
  36. package/src/page-over-target.ts +2 -3
  37. package/src/pdf-action.ts +116 -0
  38. package/src/pdf-cdp-params.ts +65 -0
  39. package/src/pdf-fence.ts +86 -0
  40. package/src/pdf-render.ts +241 -0
  41. package/src/pdf-slots.ts +68 -0
  42. package/src/rate.ts +2 -2
  43. package/src/scrape-run.ts +3 -4
  44. package/src/scrape.ts +5 -5
  45. package/src/target.ts +1 -1
  46. package/src/usage.ts +2 -2
  47. package/src/watchdog.ts +3 -3
@@ -0,0 +1,171 @@
1
+ // One responsibility: the WIRE a launched Chrome is driven over, per platform — its debugging pipe
2
+ // (fds 3 and 4) where the parent can hold those fds, else a loopback port the browser picks and
3
+ // announces in its profile's `DevToolsActivePort`. Spawning and reaping are `cdp-launch-attempt.ts`.
4
+
5
+ // why: Bun exposes no path-join primitive; the announcement file lives in the profile directory.
6
+ import { join } from 'node:path';
7
+ import { renderThrowable, stringField } from '@ultimat3/core';
8
+ import type { CdpConnection } from './cdp-connection';
9
+ import { cdpConnect, cdpConnectOver } from './cdp-connection';
10
+ import { CdpCallFailedError, CdpTimeoutError } from './cdp-errors';
11
+ import { pipeTransport } from './cdp-pipe';
12
+
13
+ /**
14
+ * `pipe` everywhere Bun hands the parent the child's extra fds as descriptors, which is POSIX: no
15
+ * port, so two suites on one machine never collide and nothing else can drive the browser.
16
+ * `port` on Windows, where those fds are not descriptors `Bun.file` can open: Chrome binds a
17
+ * loopback port of its own choosing (`=0`, so never a collision either) and writes it down.
18
+ */
19
+ export type CdpWire = 'pipe' | 'port';
20
+
21
+ export const defaultWire = (platform: string): CdpWire => (platform === 'win32' ? 'port' : 'pipe');
22
+
23
+ /** The one flag that differs between the two wires. */
24
+ export const wireFlag = (wire: CdpWire): string =>
25
+ wire === 'pipe' ? '--remote-debugging-pipe' : '--remote-debugging-port=0';
26
+
27
+ /** Chrome's fd 3 is where it READS commands and fd 4 where it WRITES replies — pipe wire only. */
28
+ const PIPE_STDIO = ['ignore', 'ignore', 'pipe', 'pipe', 'pipe'] as const;
29
+ const PORT_STDIO = ['ignore', 'ignore', 'pipe'] as const;
30
+ export const wireStdio = (wire: CdpWire): typeof PIPE_STDIO | typeof PORT_STDIO =>
31
+ wire === 'pipe' ? PIPE_STDIO : PORT_STDIO;
32
+
33
+ /** The pipe wire over a spawned child's fds 3 and 4. Answers nothing until the first `send`. */
34
+ export function pipeConnection(stdio: unknown, timeoutMs: number): CdpConnection {
35
+ const [, , , toBrowser, fromBrowser] = stdio as readonly number[];
36
+ const sink = Bun.file(toBrowser ?? -1).writer();
37
+ return cdpConnectOver(
38
+ pipeTransport({
39
+ write: (bytes) => {
40
+ sink.write(bytes);
41
+ void sink.flush();
42
+ },
43
+ read: Bun.file(fromBrowser ?? -1).stream(),
44
+ end: () => {
45
+ void Promise.resolve(sink.end()).catch(() => undefined);
46
+ },
47
+ }),
48
+ timeoutMs,
49
+ );
50
+ }
51
+
52
+ export const DEVTOOLS_ACTIVE_PORT = 'DevToolsActivePort';
53
+
54
+ /**
55
+ * `<port>\n/devtools/browser/<id>\n` → the browser's WebSocket url, or `undefined` while the file is
56
+ * still half-written. Loopback by address, never `localhost`, which may resolve to `::1` first
57
+ * while Chrome listens on IPv4.
58
+ */
59
+ export function devToolsEndpoint(text: string): string | undefined {
60
+ const [port, path] = text.split(/\r?\n/);
61
+ if (port === undefined || !/^\d{1,5}$/.test(port) || Number(port) < 1 || Number(port) > 65_535) {
62
+ return undefined;
63
+ }
64
+ if (path === undefined || !/^\/devtools\/browser\/[\w-]+$/.test(path)) return undefined;
65
+ return `ws://127.0.0.1:${port}${path}`;
66
+ }
67
+
68
+ const ANNOUNCE_POLL_MS = 25;
69
+
70
+ /**
71
+ * Dial, inside what is left of the LAUNCH deadline — `cdpConnect`'s own deadline is the per-call one
72
+ * the connection keeps afterwards, so it cannot also be the launch's. A dial that loses the race
73
+ * and connects later is closed, never left holding a socket nobody reads.
74
+ */
75
+ async function dialWithin(
76
+ endpoint: string,
77
+ leftMs: number,
78
+ timeoutMs: number,
79
+ ): Promise<CdpConnection> {
80
+ const dialing = cdpConnect({ endpoint, timeoutMs });
81
+ let timer: ReturnType<typeof setTimeout> | undefined;
82
+ const late = new Promise<never>((_, reject) => {
83
+ timer = setTimeout(
84
+ () => reject(new CdpTimeoutError({ method: 'connect', timeoutMs: leftMs })),
85
+ leftMs,
86
+ );
87
+ });
88
+ try {
89
+ return await Promise.race([dialing, late]);
90
+ } catch (error) {
91
+ void dialing.then((connection) => connection.close()).catch(() => undefined);
92
+ // A timeout stays one — the launcher reads it as the deadline. Any other refusal names the
93
+ // endpoint it was for: the browser announced it, so the dial is the step that failed.
94
+ if (error instanceof CdpTimeoutError) throw error;
95
+ throw new CdpCallFailedError({
96
+ method: `connect ${endpoint}`,
97
+ detail: renderThrowable(error),
98
+ });
99
+ } finally {
100
+ clearTimeout(timer);
101
+ }
102
+ }
103
+
104
+ export interface PortConnectionInput {
105
+ readonly profileDir: string;
106
+ /** True once the browser process has exited — it will never write the file then. */
107
+ readonly exited: () => boolean;
108
+ /** How long the browser has to announce itself AND accept the connection, from now. */
109
+ readonly deadlineMs: number;
110
+ /** Every call's deadline once connected. */
111
+ readonly timeoutMs: number;
112
+ /**
113
+ * How the announcement is read — `Bun.file(path).text()` unless a test hands a reader that
114
+ * refuses the way Windows does, which no Linux file system can be made to.
115
+ */
116
+ readonly readText?: ((path: string) => Promise<string>) | undefined;
117
+ }
118
+
119
+ /**
120
+ * The read errors that mean "not announced YET", never "the launch failed": `ENOENT`, the file
121
+ * gone between the existence check and the read, and `EBUSY`, which is how Bun reports a Windows
122
+ * sharing violation — Chrome writes `DevToolsActivePort` in place and holds it open without read
123
+ * sharing while it does. Seen on windows-latest (PR #668): "DevTools listening" on stderr, the file
124
+ * present, the read refused, and the launch reported `closed` with the browser alive.
125
+ */
126
+ const NOT_YET = new Set(['ENOENT', 'EBUSY']);
127
+
128
+ const readFile = (path: string): Promise<string> => Bun.file(path).text();
129
+
130
+ /** The announcement's text, or `undefined` while its writer still holds it. */
131
+ async function readAnnouncement(
132
+ path: string,
133
+ read: (path: string) => Promise<string>,
134
+ ): Promise<string | undefined> {
135
+ try {
136
+ return await read(path);
137
+ } catch (error) {
138
+ if (NOT_YET.has(stringField(error, 'code') ?? '')) return undefined;
139
+ throw error;
140
+ }
141
+ }
142
+
143
+ /**
144
+ * The port wire: wait for the profile's `DevToolsActivePort`, then dial it. A browser that exits
145
+ * first is a CLOSED launch and one still silent at the deadline a TIMED-OUT one — the same two
146
+ * answers the pipe gives, so the launcher's failure report reads alike on either wire.
147
+ */
148
+ export async function portConnection(input: PortConnectionInput): Promise<CdpConnection> {
149
+ const path = join(input.profileDir, DEVTOOLS_ACTIVE_PORT);
150
+ const began = performance.now();
151
+ for (;;) {
152
+ // A FRESH handle per look: a `BunFile` that once answered "absent" keeps answering it (measured
153
+ // on Bun 1.4.2 — `exists()` false and `text()` empty after the file was written).
154
+ const text = (await Bun.file(path).exists())
155
+ ? await readAnnouncement(path, input.readText ?? readFile)
156
+ : undefined;
157
+ const endpoint = text === undefined ? undefined : devToolsEndpoint(text);
158
+ const left = input.deadlineMs - (performance.now() - began);
159
+ if (endpoint !== undefined) return dialWithin(endpoint, Math.max(1, left), input.timeoutMs);
160
+ if (input.exited()) {
161
+ throw new CdpCallFailedError({
162
+ method: 'the launch',
163
+ detail: `the browser exited before it wrote ${DEVTOOLS_ACTIVE_PORT}`,
164
+ });
165
+ }
166
+ if (left <= 0) {
167
+ throw new CdpTimeoutError({ method: DEVTOOLS_ACTIVE_PORT, timeoutMs: input.deadlineMs });
168
+ }
169
+ await Bun.sleep(Math.min(ANNOUNCE_POLL_MS, left));
170
+ }
171
+ }
@@ -0,0 +1,216 @@
1
+ // One responsibility: WHICH Chrome, with which flags, and how many starts it gets — the candidate
2
+ // list, the launch deadline and the one relaunch. One start of the process is
3
+ // `cdp-launch-attempt.ts`; the connection is `cdp-connection.ts`. Its callers: `pdf()` here, and
4
+ // `@ultimat3/testing`'s e2e driver and `x shot` through `@ultimat3/scraping/cdp`.
5
+
6
+ import { finiteCount } from '@ultimat3/core';
7
+ import type { CdpLaunchAttempt } from './cdp-errors';
8
+ import { CdpBrowserMissingError, CdpLaunchFailedError } from './cdp-errors';
9
+ import type { LaunchedBrowser } from './cdp-launch-attempt';
10
+ import { LAUNCH_REAP_MS, launchAttempt } from './cdp-launch-attempt';
11
+ import { chromeCandidates } from './cdp-launch-candidates';
12
+ import { startsChrome } from './cdp-launch-probe';
13
+ import type { CdpWire } from './cdp-launch-wire';
14
+ import { defaultWire, wireFlag } from './cdp-launch-wire';
15
+
16
+ export type { LaunchedBrowser } from './cdp-launch-attempt';
17
+ export { CLOSE_GRACE_MS, LAUNCH_REAP_MS } from './cdp-launch-attempt';
18
+ export { chromeCandidates } from './cdp-launch-candidates';
19
+ export type { CdpWire } from './cdp-launch-wire';
20
+ export { defaultWire } from './cdp-launch-wire';
21
+
22
+ /**
23
+ * Where a Chrome is, in the order worth trying. `CHROME_PATH` first because it is the operator's
24
+ * answer and the only one that can be right on a machine none of the rest describes; after it, the
25
+ * install locations of THIS platform (`cdp-launch-candidates.ts`) — on Linux the `/usr/bin` names
26
+ * GitHub-hosted `ubuntu-latest` ships, which is what lets the browser-backed suite run in CI with
27
+ * **no download step and no new dependency**; on Windows Chrome and Edge; on macOS the app bundles.
28
+ */
29
+ export const CHROME_PATH_ENV = 'CHROME_PATH';
30
+ export const CHROME_CANDIDATES: readonly string[] = chromeCandidates(process.platform, process.env);
31
+
32
+ /**
33
+ * The first candidate that is a browser this machine can start, or `undefined`. An absent browser is
34
+ * a SKIP, never a failure — and so is one that is installed but cannot start (`startsChrome`,
35
+ * `cdp-launch-probe.ts`; on `win32` existence alone, since `chrome.exe --version` never exits).
36
+ * `platform` is a parameter so the Windows and macOS lists are asserted on a Linux runner too.
37
+ *
38
+ * `CHROME_PATH` is answered as given, and never probed: it is the operator's answer, the only one
39
+ * that can be right on a machine none of the candidates describes, and it is how a wrapper or a
40
+ * non-standard build is named at all.
41
+ */
42
+ export async function findChrome(
43
+ env: Readonly<Record<string, string | undefined>>,
44
+ platform: string = process.platform,
45
+ ): Promise<string | undefined> {
46
+ const declared = env[CHROME_PATH_ENV];
47
+ if (declared !== undefined && declared !== '') {
48
+ return (await Bun.file(declared).exists()) ? declared : undefined;
49
+ }
50
+ for (const candidate of chromeCandidates(platform, env)) {
51
+ if (!(await Bun.file(candidate).exists())) continue;
52
+ if (await startsChrome(candidate, platform)) return candidate;
53
+ }
54
+ return undefined;
55
+ }
56
+
57
+ /**
58
+ * The two flags a CONTAINER needs, regardless of which process launches Chrome: the sandbox needs
59
+ * privileges CI (and an Ubuntu 23.10+ host with AppArmor's unprivileged-user-namespace restriction
60
+ * — Chrome exits "No usable sandbox" there with neither) does not grant, and `/dev/shm` is 64 MB in
61
+ * a default container, which crashes the renderer on any real page.
62
+ *
63
+ * Exported, and in `chromeLaunchFlags` below, so every launcher carries the SAME two. `x shot` had
64
+ * neither while it launched through `puppeteer-core`, so a box where `x verify`'s e2e gate ran green
65
+ * could not run `x shot` at all; since 22.0.0 it launches through `launchChrome` here.
66
+ */
67
+ export const CONTAINER_CHROME_ARGS: readonly string[] = ['--no-sandbox', '--disable-dev-shm-usage'];
68
+
69
+ /**
70
+ * The flags, and every one of them earns its line.
71
+ *
72
+ * `--headless=new` is Chrome's own headless rather than the retired shim. The wire flag is
73
+ * `--remote-debugging-pipe` wherever the pipe can be held (`cdp-pipe.ts` says why it is not the
74
+ * WebSocket): no port, so two suites on one machine can never collide, and nothing but this process
75
+ * can drive the browser — and `--remote-debugging-port=0` on Windows (`cdp-launch-wire.ts`). A
76
+ * throwaway `--user-data-dir` because a run sharing a profile with a real browser inherits its
77
+ * cookies and locks its files.
78
+ */
79
+ export const chromeLaunchFlags = (
80
+ profileDir: string,
81
+ wire: CdpWire = defaultWire(process.platform),
82
+ sandbox = false,
83
+ ): readonly string[] => [
84
+ '--headless=new',
85
+ wireFlag(wire),
86
+ `--user-data-dir=${profileDir}`,
87
+ // `sandbox: true` drops `--no-sandbox` alone: `/dev/shm` is as small in a production container.
88
+ ...(sandbox
89
+ ? CONTAINER_CHROME_ARGS.filter((arg) => arg !== '--no-sandbox')
90
+ : CONTAINER_CHROME_ARGS),
91
+ '--disable-gpu',
92
+ // The cookie store's encryption key comes from the OS keyring, asked over D-Bus on the first
93
+ // cookie access — which is the first navigation. With no keyring answering, Chrome waits out the
94
+ // D-Bus timeout: measured 7-25 s on the first `Page.navigate` of every launch, against a 30 s CDP
95
+ // deadline, which is the intermittent `X_CDP_TIMEOUT` of a full e2e run. A throwaway profile has
96
+ // no secret worth a keyring. `x shot` launches through here too, so it has both.
97
+ '--password-store=basic',
98
+ '--use-mock-keychain',
99
+ // Nothing here should reach the network on its own account, and a first-run bubble or an update
100
+ // check is a page load the test did not ask for.
101
+ '--no-first-run',
102
+ '--no-default-browser-check',
103
+ '--disable-extensions',
104
+ // `--disable-extensions` leaves Google Chrome's COMPONENT extensions running, each with a page of
105
+ // its own: the hangouts one (nkeimhogjdpnpccoofpliimaahmaaome) loads `thunk.js` while the browser
106
+ // starts, and `x shot`'s browser-wide interception paused it and logged it as an off-list request
107
+ // of the page under test (CI, 2026-10-09). A test browser runs no page it was not handed.
108
+ '--disable-component-extensions-with-background-pages',
109
+ 'about:blank',
110
+ ];
111
+
112
+ /**
113
+ * How long a COLD start may take before the browser is given up on. Its own number, never the
114
+ * per-call deadline: measured 2026-10-02 over `scaffold-smoke` on free `ubuntu-latest` runners, the
115
+ * first launch of a job cost 5-19 s more than a warm one in 11 green jobs, and the one red job was
116
+ * still printing start-up lines when a 30 s deadline killed it. Twice that, and finite: a browser
117
+ * that exits says so at once (its pipe closes), so only one alive and silent waits this long.
118
+ */
119
+ export const LAUNCH_TIMEOUT_MS = 60_000;
120
+
121
+ /**
122
+ * Starts per launch: the first, and ONE more on a fresh profile after the first was reaped. A
123
+ * process start is the one step here whose failure can belong to the machine rather than the
124
+ * binary — and the second start of a cold binary is a warm one. Never a loop: two starts that both
125
+ * went unanswered are `X_CDP_LAUNCH_FAILED` carrying both.
126
+ */
127
+ export const LAUNCH_ATTEMPTS = 2;
128
+
129
+ /**
130
+ * The longest `launchChrome` is DESIGNED to take, for any `timeoutMs` up to `LAUNCH_TIMEOUT_MS`:
131
+ * every start given its whole deadline, each failed one its bounded reap. A hook that wraps a launch
132
+ * and gives it less is killed by Bun before the relaunch, and the `X_CDP_LAUNCH_FAILED` that would
133
+ * have named the slow step never surfaces — the shape of four CI flakes (plan 101, F1).
134
+ */
135
+ export const LAUNCH_BUDGET_MS = LAUNCH_ATTEMPTS * (LAUNCH_TIMEOUT_MS + LAUNCH_REAP_MS);
136
+
137
+ /**
138
+ * The longest a launched browser lives when its caller names no bound: one hour from its spawn, then
139
+ * `SIGKILL` from the OS. Never the way a browser ends — `close()` is — but the way one ends whose
140
+ * caller vanished. An hour, because an e2e lease holds ONE browser for a whole run; a caller with a
141
+ * shorter job (`pdf()`'s one render) passes its own `lifetimeMs`.
142
+ */
143
+ export const BROWSER_LIFETIME_MS = 60 * 60_000;
144
+
145
+ export interface LaunchOptions {
146
+ readonly executable: string;
147
+ /** Every CDP call's deadline once the browser is up. */
148
+ readonly timeoutMs: number;
149
+ /** The first answer's deadline, per start. Defaults to the larger of `timeoutMs` and `LAUNCH_TIMEOUT_MS`. */
150
+ readonly launchTimeoutMs?: number | undefined;
151
+ /** The wire. Defaults to this platform's (`defaultWire`): the pipe, and on Windows the port. */
152
+ readonly wire?: CdpWire | undefined;
153
+ /** From each start, the longest the process may live before the OS kills it. `BROWSER_LIFETIME_MS`. */
154
+ readonly lifetimeMs?: number | undefined;
155
+ /**
156
+ * Keep Chrome's sandbox — no `--no-sandbox`. Off by default, for the e2e driver and `x shot` on CI
157
+ * runners that cannot grant it; ON for `pdf()`, which renders user content in production
158
+ * (`pdfLaunchOptions`, `pdf-render.ts`, says why).
159
+ */
160
+ readonly sandbox?: boolean | undefined;
161
+ }
162
+
163
+ /** What `launchFoundChrome` passes through to `launchChrome`, beside the deadline. */
164
+ export type FoundChromeOptions = Pick<LaunchOptions, 'lifetimeMs' | 'sandbox'>;
165
+
166
+ /**
167
+ * Start Chrome on a throwaway profile and answer once it has answered one CDP call. A browser that
168
+ * dies or stays silent is started once more; twice is `X_CDP_LAUNCH_FAILED` with each start's own
169
+ * stderr and whether it exited or was killed at the deadline.
170
+ */
171
+ export async function launchChrome(options: LaunchOptions): Promise<LaunchedBrowser> {
172
+ // Screened: `setTimeout(fn, NaN)` fires in 1 ms, which reports a healthy browser as silent.
173
+ const launchTimeoutMs = finiteCount(
174
+ 'launchChrome',
175
+ 'launchTimeoutMs',
176
+ options.launchTimeoutMs ?? Math.max(options.timeoutMs, LAUNCH_TIMEOUT_MS),
177
+ 1,
178
+ );
179
+ const lifetimeMs = finiteCount(
180
+ 'launchChrome',
181
+ 'lifetimeMs',
182
+ options.lifetimeMs ?? BROWSER_LIFETIME_MS,
183
+ 1,
184
+ );
185
+ const wire = options.wire ?? defaultWire(process.platform);
186
+ const failures: CdpLaunchAttempt[] = [];
187
+ while (failures.length < LAUNCH_ATTEMPTS) {
188
+ const started = await launchAttempt({
189
+ executable: options.executable,
190
+ flags: (profileDir) => chromeLaunchFlags(profileDir, wire, options.sandbox === true),
191
+ wire,
192
+ timeoutMs: options.timeoutMs,
193
+ launchTimeoutMs,
194
+ lifetimeMs,
195
+ });
196
+ if (started.ok) return started.browser;
197
+ failures.push(started.failure);
198
+ // A process that outlived SIGKILL's grace still holds the machine; a second beside it is the
199
+ // load that made the first one late.
200
+ if (!started.reaped) break;
201
+ }
202
+ throw new CdpLaunchFailedError({ executable: options.executable, attempts: failures });
203
+ }
204
+
205
+ /** `findChrome` then `launchChrome`. Refuses by name when there is no browser to drive. */
206
+ export async function launchFoundChrome(
207
+ env: Readonly<Record<string, string | undefined>>,
208
+ timeoutMs: number,
209
+ options: FoundChromeOptions = {},
210
+ ): Promise<LaunchedBrowser> {
211
+ const executable = await findChrome(env);
212
+ if (executable === undefined) {
213
+ throw new CdpBrowserMissingError({ tried: chromeCandidates(process.platform, env) });
214
+ }
215
+ return launchChrome({ executable, timeoutMs, ...options });
216
+ }
@@ -0,0 +1,77 @@
1
+ // One responsibility: CDP over the pipe `--remote-debugging-pipe` opens — Chrome reads commands on
2
+ // its fd 3 and writes replies and events on its fd 4, each message one JSON text ended by a NUL
3
+ // byte. This is the e2e driver's wire; the WebSocket in `cdp-connection.ts` is for a remote browser.
4
+ //
5
+ // Why a pipe rather than the WebSocket Chrome also offers: Bun 1.4.0's WebSocket client handed
6
+ // `onmessage` text spliced from several frames under the dummy's `offline-feed` load — 64
7
+ // unparseable frames in one run, one of them a `Runtime.evaluate` reply that then waited out its
8
+ // 30 s deadline. A frame nobody can parse has no `id`, so no layer above can even tell which call
9
+ // it lost. A pipe is bytes and a delimiter, read here, and nothing in between.
10
+
11
+ import type { CdpTransport } from './cdp-connection';
12
+
13
+ /** The two ends a transport needs: a sink for whole messages, and the byte stream Chrome writes. */
14
+ export interface PipeEnds {
15
+ /** Write these bytes to Chrome's fd 3, in order. */
16
+ readonly write: (bytes: Uint8Array) => void;
17
+ /** Chrome's fd 4. */
18
+ readonly read: ReadableStream<Uint8Array>;
19
+ /** Release the write end — Chrome treats its fd 3 closing as the client going away. */
20
+ readonly end: () => void;
21
+ }
22
+
23
+ const NUL = 0;
24
+
25
+ /** Concatenate two byte arrays; the reader only ever holds the unterminated tail. */
26
+ const join = (a: Uint8Array, b: Uint8Array): Uint8Array => {
27
+ if (a.length === 0) return b;
28
+ const out = new Uint8Array(a.length + b.length);
29
+ out.set(a);
30
+ out.set(b, a.length);
31
+ return out;
32
+ };
33
+
34
+ export function pipeTransport(ends: PipeEnds): CdpTransport {
35
+ const encoder = new TextEncoder();
36
+ let closed = false;
37
+ const reader = ends.read.getReader();
38
+ return {
39
+ send(text: string): void {
40
+ if (closed) return;
41
+ const body = encoder.encode(text);
42
+ const framed = new Uint8Array(body.length + 1);
43
+ framed.set(body);
44
+ framed[body.length] = NUL;
45
+ ends.write(framed);
46
+ },
47
+ close(): void {
48
+ if (closed) return;
49
+ closed = true;
50
+ ends.end();
51
+ void reader.cancel().catch(() => undefined);
52
+ },
53
+ listen(handlers): void {
54
+ void (async () => {
55
+ // Split on BYTES, decoded per message: a multi-byte character may straddle two reads, and
56
+ // decoding each read on its own would corrupt it — the failure this file exists to end.
57
+ const decoder = new TextDecoder();
58
+ let tail: Uint8Array = new Uint8Array(0);
59
+ try {
60
+ for (;;) {
61
+ const { value, done } = await reader.read();
62
+ if (done) break;
63
+ tail = join(tail, value);
64
+ for (let at = tail.indexOf(NUL); at !== -1; at = tail.indexOf(NUL)) {
65
+ handlers.message(decoder.decode(tail.subarray(0, at)));
66
+ tail = tail.subarray(at + 1);
67
+ }
68
+ }
69
+ } catch {
70
+ // A read that fails is a pipe that is gone — reported as the close it is, below.
71
+ }
72
+ closed = true;
73
+ handlers.closed('the browser closed the CDP pipe');
74
+ })();
75
+ },
76
+ };
77
+ }
@@ -0,0 +1,113 @@
1
+ // One responsibility: remember what the wire did, so a call that is never answered can say WHY —
2
+ // its target went away, frames arrived that nothing could parse, or the target simply stayed
3
+ // silent. The connection (`cdp-connection.ts`) feeds every frame in; `CdpTimeoutError` reads it out.
4
+
5
+ import type { CdpTargetGone, CdpTimeoutObservation } from './cdp-errors';
6
+
7
+ /** How many of the latest frames a timeout quotes. Bounded: a busy page sends thousands. */
8
+ export const LAST_FRAMES = 8;
9
+
10
+ /** Where the wire stood when a call was sent — what "since the call" is measured from. */
11
+ export interface WireMark {
12
+ readonly arrived: number;
13
+ readonly dropped: number;
14
+ readonly navigations: number;
15
+ }
16
+
17
+ export interface WireWatch {
18
+ /** One frame that parsed: a reply or an event. */
19
+ frame(frame: Record<string, unknown>): void;
20
+ /** One frame nothing could parse. It has no `id`, so no call can be failed with it — only counted. */
21
+ drop(): void;
22
+ mark(sessionId: string | undefined): WireMark;
23
+ /** What happened between `mark` and now, for the call sent on `sessionId`. */
24
+ observe(mark: WireMark, sessionId: string | undefined, open: boolean): CdpTimeoutObservation;
25
+ }
26
+
27
+ const text = (from: unknown, key: string): string | undefined => {
28
+ const value =
29
+ typeof from === 'object' && from !== null ? (from as Record<string, unknown>)[key] : undefined;
30
+ return typeof value === 'string' ? value : undefined;
31
+ };
32
+
33
+ export function wireWatch(): WireWatch {
34
+ let arrived = 0;
35
+ let dropped = 0;
36
+ const last: string[] = [];
37
+ const targetOf = new Map<string, string>(); // sessionId → targetId
38
+ const gone = new Map<string, CdpTargetGone>(); // sessionId → how it went
39
+ const navigations = new Map<string, number>(); // sessionId → main-frame navigations
40
+
41
+ const goneTarget = (targetId: string | undefined, how: CdpTargetGone): void => {
42
+ for (const [session, target] of targetOf) if (target === targetId) gone.set(session, how);
43
+ };
44
+
45
+ /** The few events that say a session can no longer answer, or that its page was replaced. */
46
+ const track = (method: string, session: string | undefined, params: unknown): void => {
47
+ if (method === 'Target.attachedToTarget') {
48
+ const attached = text(params, 'sessionId');
49
+ const target = text(
50
+ (params as Record<string, unknown> | undefined)?.['targetInfo'],
51
+ 'targetId',
52
+ );
53
+ if (attached !== undefined && target !== undefined) targetOf.set(attached, target);
54
+ } else if (method === 'Target.detachedFromTarget') {
55
+ const detached = text(params, 'sessionId');
56
+ if (detached !== undefined) gone.set(detached, 'detached');
57
+ } else if (method === 'Target.targetDestroyed') {
58
+ goneTarget(text(params, 'targetId'), 'destroyed');
59
+ } else if (method === 'Target.targetCrashed') {
60
+ goneTarget(text(params, 'targetId'), 'crashed');
61
+ } else if (method === 'Inspector.targetCrashed' && session !== undefined) {
62
+ gone.set(session, 'crashed');
63
+ } else if (method === 'Page.frameNavigated' && session !== undefined) {
64
+ const frame = (params as Record<string, unknown> | undefined)?.['frame'];
65
+ // A frame with a parent is an iframe: the document the call was sent to is still there.
66
+ if (text(frame, 'parentId') === undefined) {
67
+ navigations.set(session, (navigations.get(session) ?? 0) + 1);
68
+ }
69
+ }
70
+ };
71
+
72
+ const note = (label: string): void => {
73
+ last.push(label);
74
+ if (last.length > LAST_FRAMES) last.shift();
75
+ };
76
+
77
+ return {
78
+ frame(frame): void {
79
+ arrived += 1;
80
+ const session = text(frame, 'sessionId');
81
+ const id = frame['id'];
82
+ const method = text(frame, 'method');
83
+ if (typeof id === 'number') note(`reply ${String(id)}`);
84
+ else if (method === undefined) note('a frame with neither an id nor a method');
85
+ else {
86
+ note(session === undefined ? method : `${method} @${session}`);
87
+ track(method, session, frame['params']);
88
+ }
89
+ },
90
+ drop(): void {
91
+ dropped += 1;
92
+ },
93
+ mark: (sessionId) => ({
94
+ arrived,
95
+ dropped,
96
+ navigations: sessionId === undefined ? 0 : (navigations.get(sessionId) ?? 0),
97
+ }),
98
+ observe(mark, sessionId, open): CdpTimeoutObservation {
99
+ const since = arrived - mark.arrived;
100
+ return {
101
+ sessionId: sessionId ?? null,
102
+ framesArrived: since,
103
+ // Only frames that arrived AFTER the call: an older one says nothing about its reply.
104
+ lastFrames: since === 0 ? [] : last.slice(-Math.min(since, LAST_FRAMES)),
105
+ framesDropped: dropped - mark.dropped,
106
+ transportOpen: open,
107
+ targetGone: sessionId === undefined ? null : (gone.get(sessionId) ?? null),
108
+ navigations:
109
+ sessionId === undefined ? 0 : (navigations.get(sessionId) ?? 0) - mark.navigations,
110
+ };
111
+ },
112
+ };
113
+ }
package/src/cdp.ts ADDED
@@ -0,0 +1,46 @@
1
+ // `@ultimat3/scraping/cdp` — the raw-CDP browser as a runtime: find a Chrome, launch it over its
2
+ // pipe, talk to it, and the four `X_CDP_*` refusals. Its own entry, off the `.` barrel: a caller
3
+ // that only launches (`@ultimat3/testing`'s e2e driver, `x shot`) loads no job, storage or scrape
4
+ // module for it. `pdf()` on the barrel prints through the same launcher.
5
+
6
+ export type {
7
+ CdpConnection,
8
+ CdpConnectionOptions,
9
+ CdpEventListener,
10
+ CdpResult,
11
+ CdpTransport,
12
+ } from './cdp-connection';
13
+ export { cdpConnect, cdpConnectOver } from './cdp-connection';
14
+ export type { CdpErrorCode } from './cdp-error-codes';
15
+ export { CDP_ERROR_CODES, CDP_ERROR_TITLES } from './cdp-error-codes';
16
+ export type {
17
+ CdpLaunchAttempt,
18
+ CdpTargetGone,
19
+ CdpTimeoutObservation,
20
+ CdpTimeoutReading,
21
+ } from './cdp-errors';
22
+ export {
23
+ CdpBrowserMissingError,
24
+ CdpCallFailedError,
25
+ CdpLaunchFailedError,
26
+ CdpTimeoutError,
27
+ } from './cdp-errors';
28
+ export type { CdpWire, FoundChromeOptions, LaunchedBrowser, LaunchOptions } from './cdp-launch';
29
+ export {
30
+ BROWSER_LIFETIME_MS,
31
+ CHROME_CANDIDATES,
32
+ CHROME_PATH_ENV,
33
+ CLOSE_GRACE_MS,
34
+ CONTAINER_CHROME_ARGS,
35
+ chromeCandidates,
36
+ chromeLaunchFlags,
37
+ defaultWire,
38
+ findChrome,
39
+ LAUNCH_ATTEMPTS,
40
+ LAUNCH_BUDGET_MS,
41
+ LAUNCH_REAP_MS,
42
+ LAUNCH_TIMEOUT_MS,
43
+ launchChrome,
44
+ launchFoundChrome,
45
+ } from './cdp-launch';
46
+ export { LAUNCHED_CLOSE_MS } from './cdp-launch-attempt';