@trawlme/cli 3.10.0 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/commands/scraps.js +250 -10
- package/dist/commands/spec.d.ts +32 -1
- package/dist/commands/spec.js +83 -11
- package/dist/index.d.ts +17 -0
- package/dist/index.js +39 -0
- package/dist/lib/api.d.ts +5 -0
- package/dist/lib/api.js +118 -0
- package/dist/lib/cdp-pipe.d.ts +103 -0
- package/dist/lib/cdp-pipe.js +221 -0
- package/dist/lib/chrome-discovery.d.ts +12 -0
- package/dist/lib/chrome-discovery.js +49 -0
- package/dist/lib/chrome-launch.d.ts +48 -0
- package/dist/lib/chrome-launch.js +122 -0
- package/dist/lib/docs.d.ts +140 -0
- package/dist/lib/docs.js +238 -0
- package/dist/lib/secure-transport.d.ts +8 -0
- package/dist/lib/secure-transport.js +39 -0
- package/dist/lib/session-capture-guard.d.ts +21 -0
- package/dist/lib/session-capture-guard.js +14 -0
- package/dist/lib/session-capture.d.ts +180 -0
- package/dist/lib/session-capture.js +600 -0
- package/dist/lib/storage-state.d.ts +167 -0
- package/dist/lib/storage-state.js +227 -0
- package/docs/agent-quickstart.md +9 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -20,6 +20,8 @@ import { initPostHog, captureCommand, shutdown, registerAllowedCommands } from '
|
|
|
20
20
|
import { classifyError, reportError, stripCommanderErrorPrefix, retryFieldsFor } from './lib/errors.js';
|
|
21
21
|
import { renderPinch, pinchEnabled } from './lib/pinch.js';
|
|
22
22
|
import { maybeNotifyUpdate, scheduleUpdateCheck } from './lib/updateNotifier.js';
|
|
23
|
+
import { getApiUrl } from './lib/config.js';
|
|
24
|
+
import { resolveDocsUrls, docsFooterLine } from './lib/docs.js';
|
|
23
25
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
24
26
|
const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf8'));
|
|
25
27
|
/**
|
|
@@ -142,8 +144,37 @@ export function createProgram() {
|
|
|
142
144
|
program.addCommand(logout);
|
|
143
145
|
program.addCommand(telemetry);
|
|
144
146
|
program.addCommand(upgrade);
|
|
147
|
+
// #185 — one dim line pointing at the docs, appended after EVERY `--help`
|
|
148
|
+
// in the tree (registering 'afterAll' on the root fires for a subcommand's
|
|
149
|
+
// own `--help` too — commander emits the afterAllHelp event on every
|
|
150
|
+
// ancestor of whichever command's outputHelp() ran, root included).
|
|
151
|
+
// Gated the way lib/tips.ts gates its own footer-shaped output: a pure,
|
|
152
|
+
// synchronous helper (docsFooterText, exported for unit testing) wrapped
|
|
153
|
+
// in try/catch so a broken footer can never turn a clean `--help` into a
|
|
154
|
+
// crash — but deliberately WITHOUT tips.ts's TTY/--json gate: `--help`
|
|
155
|
+
// itself never emits a --json payload (nothing here can pollute that
|
|
156
|
+
// channel), and a docs line inside `--help | less` is exactly the kind of
|
|
157
|
+
// thing worth keeping, unlike a promotional nudge.
|
|
158
|
+
program.addHelpText('afterAll', docsFooterText);
|
|
145
159
|
return program;
|
|
146
160
|
}
|
|
161
|
+
/**
|
|
162
|
+
* #185 — see the `addHelpText('afterAll', …)` call above for why this is
|
|
163
|
+
* unconditional (no TTY/--json gate) and why a thrown error must never
|
|
164
|
+
* escape: commander calls this synchronously while already writing help
|
|
165
|
+
* output, so a throw here would crash a `--help` invocation, the one code
|
|
166
|
+
* path this whole feature is not allowed to touch (see docs.ts's own
|
|
167
|
+
* `docsFooterLine` for the "never a guessed URL" contract this composes).
|
|
168
|
+
*/
|
|
169
|
+
export function docsFooterText() {
|
|
170
|
+
try {
|
|
171
|
+
const line = docsFooterLine(resolveDocsUrls({ apiBaseUrl: getApiUrl() }).docsUrl);
|
|
172
|
+
return line ? chalk.dim(line) : '';
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return '';
|
|
176
|
+
}
|
|
177
|
+
}
|
|
147
178
|
/**
|
|
148
179
|
* True when this module is the process entrypoint (not merely imported by a test).
|
|
149
180
|
*
|
|
@@ -228,6 +259,14 @@ export function isBareInvocation(argv) {
|
|
|
228
259
|
* overstates its own invariant is worse than no comment, because the next
|
|
229
260
|
* reader trusts it.
|
|
230
261
|
*
|
|
262
|
+
* #185 — `spec --json` (not plain `spec`) now DOES make its own bounded
|
|
263
|
+
* (2s), swallowed-on-failure GET for `externalDocs.url` (see spec.ts's
|
|
264
|
+
* `fetchExternalDocsUrl`) — this is not a regression of the "no mutation"
|
|
265
|
+
* invariant above (a GET mutates nothing, and any failure — including no
|
|
266
|
+
* egress at all — degrades to the sync host-derivation fallback, never a
|
|
267
|
+
* thrown error), just a second, narrower kind of side effect this guard was
|
|
268
|
+
* never meant to suppress in the first place.
|
|
269
|
+
*
|
|
231
270
|
* Before this, `trawl spec --json` silently re-synced
|
|
232
271
|
* `~/.claude/skills` (and `./.claude/skills`) and printed `trawl: re-synced
|
|
233
272
|
* skill …` lines to stderr BEFORE the JSON payload — an agent's very first
|
package/dist/lib/api.d.ts
CHANGED
|
@@ -95,6 +95,11 @@ export interface RequestOptions {
|
|
|
95
95
|
export declare const api: {
|
|
96
96
|
get: <T>(path: string, opts?: RequestOptions) => Promise<T>;
|
|
97
97
|
publicGet: <T>(path: string, opts?: RequestOptions) => Promise<T>;
|
|
98
|
+
/** See `probeJson`'s own doc comment — hard-bounded, probe-only, never for
|
|
99
|
+
* a command's real work. Today's one caller: spec.ts's fetchExternalDocsUrl. */
|
|
100
|
+
probeJson: <T>(path: string, opts: {
|
|
101
|
+
timeoutMs: number;
|
|
102
|
+
}) => Promise<T | null>;
|
|
98
103
|
getText: (path: string, opts?: RequestOptions) => Promise<string>;
|
|
99
104
|
post: <T>(path: string, body?: unknown, opts?: RequestOptions) => Promise<T>;
|
|
100
105
|
put: <T>(path: string, body?: unknown, opts?: RequestOptions) => Promise<T>;
|
package/dist/lib/api.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
import { fileURLToPath } from 'node:url';
|
|
3
3
|
import { dirname, resolve } from 'node:path';
|
|
4
|
+
import { request as httpRequest } from 'node:http';
|
|
5
|
+
import { request as httpsRequest } from 'node:https';
|
|
4
6
|
import { getApiUrl, getToken, getAuthMode, getLiveAuthEnvVar } from './config.js';
|
|
5
7
|
import { parseServerJson } from './json.js';
|
|
6
8
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
@@ -518,6 +520,119 @@ async function publicGet(path, reqOpts = {}) {
|
|
|
518
520
|
throw new Error('Invalid JSON in server response');
|
|
519
521
|
}
|
|
520
522
|
}
|
|
523
|
+
/**
|
|
524
|
+
* A hard-bounded, PROBE-ONLY GET — never use this for a command's real work
|
|
525
|
+
* (no auth headers, no retry, no NetworkError-with-cause classification).
|
|
526
|
+
*
|
|
527
|
+
* trawl_cli#185 defect: every OTHER fetch in this file rides Node's global
|
|
528
|
+
* `fetch()` + `AbortSignal.timeout()`, which bounds the PROMISE, not the
|
|
529
|
+
* process. Aborting a fetch mid-connect does not necessarily tear down an
|
|
530
|
+
* in-flight TCP/TLS handshake — a documented Node/undici characteristic —
|
|
531
|
+
* so against a routable-but-silently-dropping host (a dropped SYN, never a
|
|
532
|
+
* RST — a realistic firewalled/air-gapped shape) the dangling socket keeps
|
|
533
|
+
* the event loop alive until Node's OWN internal connect-timeout eventually
|
|
534
|
+
* fires. Measured on this host: a bare `fetch()` + `AbortSignal.timeout(2000)`
|
|
535
|
+
* against such a host rejects its promise at ~2005ms, but the PROCESS does
|
|
536
|
+
* not exit until ~10.5s later — not configurable from here, and not paid by
|
|
537
|
+
* `ECONNREFUSED` (~0.13s) or a failed DNS lookup (~0.29s), only by the
|
|
538
|
+
* silent-drop shape. Every other command in this file accepts that risk
|
|
539
|
+
* because the user explicitly asked for that network call (`trawl ping`
|
|
540
|
+
* hanging on an unreachable host is the command doing its job — nothing to
|
|
541
|
+
* fix there). This primitive exists for the one case where the network call
|
|
542
|
+
* ITSELF is optional: `spec --json`'s own doc-discovery probe (the one
|
|
543
|
+
* command an agent runs to orient itself) has no business costing 10.5s to
|
|
544
|
+
* learn a host is unreachable, so this manages the raw socket directly.
|
|
545
|
+
*
|
|
546
|
+
* The timeout here is an INDEPENDENT `setTimeout` that calls `req.destroy()`
|
|
547
|
+
* — deliberately NOT `http.request`'s own `timeout` option (idle-based via
|
|
548
|
+
* `socket.setTimeout`, so it only fires when zero bytes arrive for that
|
|
549
|
+
* long — a coincidentally-same condition against a silent black hole, but a
|
|
550
|
+
* dishonest bound in general: it would never fire against a host that
|
|
551
|
+
* connects instantly then trickles bytes just often enough to reset the
|
|
552
|
+
* idle clock). Destroying the request/socket on this timer is what actually
|
|
553
|
+
* frees the OS-level handle so the process can exit promptly instead of
|
|
554
|
+
* waiting out Node's own multi-second connect-timeout — verified empirically
|
|
555
|
+
* (same black-holed host): total process time dropped from ~10.5s to ~2.0s
|
|
556
|
+
* with this change.
|
|
557
|
+
*
|
|
558
|
+
* Resolves to `null` on ANY failure (timeout, refusal, non-2xx, malformed
|
|
559
|
+
* JSON, an unsupported protocol) — never throws — so the one caller
|
|
560
|
+
* (spec.ts's fetchExternalDocsUrl) needs no try/catch of its own. This is
|
|
561
|
+
* NOT automatic: `node:http`/`node:https`' own `request()` throws
|
|
562
|
+
* SYNCHRONOUSLY (`ERR_INVALID_PROTOCOL`) for a URL whose protocol doesn't
|
|
563
|
+
* match the module (e.g. a stray `ftp://` in a misconfigured `TRAWL_API_URL`)
|
|
564
|
+
* — inside a Promise executor a synchronous throw becomes a REJECTED
|
|
565
|
+
* promise, which would have propagated straight through `fetchExternalDocsUrl`
|
|
566
|
+
* and turned `spec --json` into an error envelope instead of the spec.
|
|
567
|
+
* Reproduced: `TRAWL_API_URL=ftp://x node dist/index.js spec --json` failed
|
|
568
|
+
* entirely before this guard was added. The explicit protocol check below
|
|
569
|
+
* closes that.
|
|
570
|
+
*
|
|
571
|
+
* Two more deliberate departures from every other call in this file:
|
|
572
|
+
* - `TRAWL_TIMEOUT` (getTimeoutMs's env override, #91) is NOT consulted
|
|
573
|
+
* here — `opts.timeoutMs` is absolute. An operator raising that ceiling
|
|
574
|
+
* for a genuinely long-running command (e.g. LONG_RUN_TIMEOUT_MS's 300s)
|
|
575
|
+
* must never make an OPTIONAL probe wait 300s too; the two knobs measure
|
|
576
|
+
* different things and must not share a dial.
|
|
577
|
+
* - Redirects are NOT followed (global `fetch()`, used everywhere else in
|
|
578
|
+
* this file, follows them by default; raw `http`/`https` `request()`
|
|
579
|
+
* does not). For this probe a 3xx is just another non-2xx -> `null` ->
|
|
580
|
+
* fall through to rung 2 — an acceptable degradation, not a bug.
|
|
581
|
+
*/
|
|
582
|
+
function probeJson(path, opts) {
|
|
583
|
+
return new Promise((settle) => {
|
|
584
|
+
let url;
|
|
585
|
+
try {
|
|
586
|
+
url = new URL(`${getApiUrl()}${path}`);
|
|
587
|
+
}
|
|
588
|
+
catch {
|
|
589
|
+
settle(null);
|
|
590
|
+
return;
|
|
591
|
+
}
|
|
592
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
593
|
+
settle(null);
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
let settled = false;
|
|
597
|
+
const finish = (value) => {
|
|
598
|
+
if (settled)
|
|
599
|
+
return;
|
|
600
|
+
settled = true;
|
|
601
|
+
clearTimeout(timer);
|
|
602
|
+
settle(value);
|
|
603
|
+
};
|
|
604
|
+
const transport = url.protocol === 'http:' ? httpRequest : httpsRequest;
|
|
605
|
+
const req = transport(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
|
|
606
|
+
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
|
|
607
|
+
res.resume(); // drain so the socket can be released
|
|
608
|
+
finish(null);
|
|
609
|
+
return;
|
|
610
|
+
}
|
|
611
|
+
let body = '';
|
|
612
|
+
res.setEncoding('utf8');
|
|
613
|
+
res.on('data', (chunk) => {
|
|
614
|
+
body += chunk;
|
|
615
|
+
});
|
|
616
|
+
res.on('end', () => {
|
|
617
|
+
try {
|
|
618
|
+
finish(JSON.parse(body));
|
|
619
|
+
}
|
|
620
|
+
catch {
|
|
621
|
+
finish(null);
|
|
622
|
+
}
|
|
623
|
+
});
|
|
624
|
+
res.on('error', () => finish(null));
|
|
625
|
+
});
|
|
626
|
+
// The independent, non-idle-based ceiling this function exists for —
|
|
627
|
+
// fires regardless of connect/response state and forcibly destroys the
|
|
628
|
+
// socket, unlike AbortSignal.timeout() everywhere else in this file.
|
|
629
|
+
const timer = setTimeout(() => {
|
|
630
|
+
req.destroy(new Error(`probe to ${url.href} timed out after ${opts.timeoutMs}ms`));
|
|
631
|
+
}, opts.timeoutMs);
|
|
632
|
+
req.on('error', () => finish(null));
|
|
633
|
+
req.end();
|
|
634
|
+
});
|
|
635
|
+
}
|
|
521
636
|
async function getText(path, reqOpts = {}) {
|
|
522
637
|
const token = getToken();
|
|
523
638
|
if (!token)
|
|
@@ -537,6 +652,9 @@ async function getText(path, reqOpts = {}) {
|
|
|
537
652
|
export const api = {
|
|
538
653
|
get: (path, opts) => request(path, {}, opts),
|
|
539
654
|
publicGet: (path, opts) => publicGet(path, opts),
|
|
655
|
+
/** See `probeJson`'s own doc comment — hard-bounded, probe-only, never for
|
|
656
|
+
* a command's real work. Today's one caller: spec.ts's fetchExternalDocsUrl. */
|
|
657
|
+
probeJson: (path, opts) => probeJson(path, opts),
|
|
540
658
|
getText: (path, opts) => getText(path, opts),
|
|
541
659
|
post: (path, body, opts) => request(path, {
|
|
542
660
|
method: 'POST',
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Raw Chrome DevTools Protocol client over Chrome's `--remote-debugging-pipe`
|
|
3
|
+
* transport (trawl_cli#183) — zero new dependencies. This is the SAME
|
|
4
|
+
* transport Puppeteer's own `pipe: true` mode uses: Chrome reads NUL
|
|
5
|
+
* (`\0`)-delimited JSON command messages on fd 3 and writes NUL-delimited
|
|
6
|
+
* JSON response/event messages on fd 4. No port, no `ws`, no HTTP polling
|
|
7
|
+
* of a `/json/version` endpoint, no parsing Chrome's stderr for a
|
|
8
|
+
* `DevTools listening on ws://…` line.
|
|
9
|
+
*
|
|
10
|
+
* Deliberately generic over any `Writable`/`Readable` pair (not tied to a
|
|
11
|
+
* real ChildProcess) so the framing/dispatch logic is fully testable with
|
|
12
|
+
* in-memory streams standing in for "Chrome" — see cdp-pipe.test.ts.
|
|
13
|
+
*/
|
|
14
|
+
import type { Readable, Writable } from 'node:stream';
|
|
15
|
+
type EventHandler = (params: unknown, sessionId?: string) => void;
|
|
16
|
+
/**
|
|
17
|
+
* A frame Chrome sent could not be parsed as JSON (trawl_cli#183 review
|
|
18
|
+
* finding 3). Distinguished from the generic "Chrome closed the CDP pipe"
|
|
19
|
+
* error so a caller can tell "the peer sent garbage and was cut off" apart
|
|
20
|
+
* from "the process actually exited" — `captureSession` reports the two
|
|
21
|
+
* with different, factual reasons rather than defaulting a corrupt-frame
|
|
22
|
+
* case to "Chrome exited".
|
|
23
|
+
*/
|
|
24
|
+
export declare class CdpProtocolError extends Error {
|
|
25
|
+
constructor(message: string);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* A `send()` command's own deadline elapsed with no response (trawl_cli#183
|
|
29
|
+
* review finding 3's fix; the gap it reopened is finding 4's bug class —
|
|
30
|
+
* see session-capture.ts's `readCaptureDefault`). Deliberately a SIBLING of
|
|
31
|
+
* `CdpProtocolError`, never a subclass: a protocol error means the pipe
|
|
32
|
+
* itself can no longer be trusted, so it is torn down (`isClosed` flips
|
|
33
|
+
* true, every OTHER pending command rejects too). A timeout means exactly
|
|
34
|
+
* ONE command never got an answer — Chrome is still running, the pipe is
|
|
35
|
+
* still open, every other in-flight/future command is unaffected. Making
|
|
36
|
+
* this a `CdpProtocolError` subclass would let a caller's
|
|
37
|
+
* `instanceof CdpProtocolError` (or a future one) treat a single slow
|
|
38
|
+
* command as proof the whole browser is gone, which is false. Carries only
|
|
39
|
+
* `method` and `timeoutMs` — never `params`, which can carry page-
|
|
40
|
+
* controlled data.
|
|
41
|
+
*/
|
|
42
|
+
export declare class CdpTimeoutError extends Error {
|
|
43
|
+
readonly method: string;
|
|
44
|
+
readonly timeoutMs: number;
|
|
45
|
+
constructor(method: string, timeoutMs: number);
|
|
46
|
+
}
|
|
47
|
+
export declare class CdpPipe {
|
|
48
|
+
private readonly writeStream;
|
|
49
|
+
private readonly commandTimeoutMs;
|
|
50
|
+
private nextId;
|
|
51
|
+
private buffer;
|
|
52
|
+
private readonly pending;
|
|
53
|
+
private readonly listeners;
|
|
54
|
+
private closed;
|
|
55
|
+
private protocolErr;
|
|
56
|
+
constructor(writeStream: Writable, readStream: Readable, commandTimeoutMs?: number);
|
|
57
|
+
private onClosed;
|
|
58
|
+
/**
|
|
59
|
+
* @desc A frame that fails to parse as JSON used to be dropped silently
|
|
60
|
+
* (`msg = undefined`, dispatch skipped) — any `send()` whose response was
|
|
61
|
+
* that exact frame then hung forever, since nothing ever rejected it.
|
|
62
|
+
* Now the whole pipe is torn down the same way a real Chrome exit is:
|
|
63
|
+
* every pending command rejects (with a `CdpProtocolError`, not the
|
|
64
|
+
* generic close message, so the cause is distinguishable), `isClosed`
|
|
65
|
+
* flips true, and `onClose` subscribers fire — the peer sent a frame
|
|
66
|
+
* that doesn't fit the protocol, so nothing it says next can be trusted
|
|
67
|
+
* either. The raw frame content is never included in the error message —
|
|
68
|
+
* it can carry page-controlled data (e.g. a `Runtime.evaluate` result).
|
|
69
|
+
*/
|
|
70
|
+
private onProtocolError;
|
|
71
|
+
private onData;
|
|
72
|
+
private dispatch;
|
|
73
|
+
/**
|
|
74
|
+
* Send a CDP command and resolve with its `result`. Rejects if the pipe
|
|
75
|
+
* closes (Chrome exited, or a corrupt frame tore it down — see
|
|
76
|
+
* `onProtocolError`) before a response arrives, or with a
|
|
77
|
+
* `CdpTimeoutError` if no response arrives within `timeoutMs` (defaults
|
|
78
|
+
* to the pipe's own `commandTimeoutMs`, 30s) — the timeout names the
|
|
79
|
+
* METHOD, never `params` (which can carry page-controlled data).
|
|
80
|
+
* @param timeoutMs — per-call override. Lets a caller pin its own named
|
|
81
|
+
* deadline as an assertable constant rather than relying on the pipe's
|
|
82
|
+
* default silently matching (see session-capture.ts's
|
|
83
|
+
* `LOCALSTORAGE_READ_TIMEOUT_MS`, which currently equals this pipe's own
|
|
84
|
+
* 30s default but is passed explicitly anyway — a post-cap review found
|
|
85
|
+
* that giving `Runtime.evaluate` a SHORTER override here, on the
|
|
86
|
+
* assumption a page-blocked renderer "won't unblock itself in 30s any
|
|
87
|
+
* more than in 5", cut off real-but-slow work that finished on its own;
|
|
88
|
+
* there is no override value proven safe, so this parameter exists for
|
|
89
|
+
* callers that want one, not as a recommendation to use a shorter one).
|
|
90
|
+
*/
|
|
91
|
+
send<T = unknown>(method: string, params?: unknown, sessionId?: string, timeoutMs?: number): Promise<T>;
|
|
92
|
+
/** Subscribe to a CDP event (`'Target.targetDestroyed'`, …). Returns an
|
|
93
|
+
* unsubscribe function. */
|
|
94
|
+
on(method: string, handler: EventHandler): () => void;
|
|
95
|
+
/** Subscribe to the pipe closing (Chrome process gone). */
|
|
96
|
+
onClose(handler: () => void): () => void;
|
|
97
|
+
get isClosed(): boolean;
|
|
98
|
+
/** Set only when the pipe was torn down because of a corrupt frame — as
|
|
99
|
+
* opposed to a real Chrome exit — so a caller can report the actual
|
|
100
|
+
* cause instead of defaulting every close to "Chrome exited". */
|
|
101
|
+
get protocolError(): CdpProtocolError | undefined;
|
|
102
|
+
}
|
|
103
|
+
export {};
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/** Reserved internal event name — never collides with a real CDP method
|
|
2
|
+
* (every real one is `Domain.method`, always containing a dot). Emitted
|
|
3
|
+
* once the read side of the pipe ends/errors, so any in-flight capture
|
|
4
|
+
* can tell "Chrome went away" apart from "the browser answered". */
|
|
5
|
+
const PIPE_CLOSED = '__pipe_closed__';
|
|
6
|
+
/** A per-command deadline that never fires under normal operation. */
|
|
7
|
+
const DEFAULT_COMMAND_TIMEOUT_MS = 30_000;
|
|
8
|
+
/**
|
|
9
|
+
* A frame Chrome sent could not be parsed as JSON (trawl_cli#183 review
|
|
10
|
+
* finding 3). Distinguished from the generic "Chrome closed the CDP pipe"
|
|
11
|
+
* error so a caller can tell "the peer sent garbage and was cut off" apart
|
|
12
|
+
* from "the process actually exited" — `captureSession` reports the two
|
|
13
|
+
* with different, factual reasons rather than defaulting a corrupt-frame
|
|
14
|
+
* case to "Chrome exited".
|
|
15
|
+
*/
|
|
16
|
+
export class CdpProtocolError extends Error {
|
|
17
|
+
constructor(message) {
|
|
18
|
+
super(message);
|
|
19
|
+
this.name = 'CdpProtocolError';
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* A `send()` command's own deadline elapsed with no response (trawl_cli#183
|
|
24
|
+
* review finding 3's fix; the gap it reopened is finding 4's bug class —
|
|
25
|
+
* see session-capture.ts's `readCaptureDefault`). Deliberately a SIBLING of
|
|
26
|
+
* `CdpProtocolError`, never a subclass: a protocol error means the pipe
|
|
27
|
+
* itself can no longer be trusted, so it is torn down (`isClosed` flips
|
|
28
|
+
* true, every OTHER pending command rejects too). A timeout means exactly
|
|
29
|
+
* ONE command never got an answer — Chrome is still running, the pipe is
|
|
30
|
+
* still open, every other in-flight/future command is unaffected. Making
|
|
31
|
+
* this a `CdpProtocolError` subclass would let a caller's
|
|
32
|
+
* `instanceof CdpProtocolError` (or a future one) treat a single slow
|
|
33
|
+
* command as proof the whole browser is gone, which is false. Carries only
|
|
34
|
+
* `method` and `timeoutMs` — never `params`, which can carry page-
|
|
35
|
+
* controlled data.
|
|
36
|
+
*/
|
|
37
|
+
export class CdpTimeoutError extends Error {
|
|
38
|
+
method;
|
|
39
|
+
timeoutMs;
|
|
40
|
+
constructor(method, timeoutMs) {
|
|
41
|
+
super(`CDP command '${method}' timed out after ${timeoutMs}ms without a response`);
|
|
42
|
+
this.method = method;
|
|
43
|
+
this.timeoutMs = timeoutMs;
|
|
44
|
+
this.name = 'CdpTimeoutError';
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
export class CdpPipe {
|
|
48
|
+
writeStream;
|
|
49
|
+
commandTimeoutMs;
|
|
50
|
+
nextId = 1;
|
|
51
|
+
buffer = '';
|
|
52
|
+
pending = new Map();
|
|
53
|
+
listeners = new Map();
|
|
54
|
+
closed = false;
|
|
55
|
+
protocolErr;
|
|
56
|
+
constructor(writeStream, readStream, commandTimeoutMs = DEFAULT_COMMAND_TIMEOUT_MS) {
|
|
57
|
+
this.writeStream = writeStream;
|
|
58
|
+
this.commandTimeoutMs = commandTimeoutMs;
|
|
59
|
+
readStream.setEncoding('utf8');
|
|
60
|
+
readStream.on('data', (chunk) => this.onData(chunk));
|
|
61
|
+
readStream.on('end', () => this.onClosed());
|
|
62
|
+
readStream.on('close', () => this.onClosed());
|
|
63
|
+
readStream.on('error', () => this.onClosed());
|
|
64
|
+
// fd 3 (write) and fd 4 (read) are separate sockets — Chrome dying can
|
|
65
|
+
// surface as an EPIPE on a write that lands before the read side's
|
|
66
|
+
// 'end'/'close' ever fires. An unhandled 'error' on a Writable is a
|
|
67
|
+
// thrown exception with no listener, which would kill the whole CLI
|
|
68
|
+
// process and skip captureSession's `finally` (so the temp profile is
|
|
69
|
+
// never cleaned up) — routing it through the same onClosed() path
|
|
70
|
+
// keeps "Chrome went away" a single, always-handled state.
|
|
71
|
+
writeStream.on('error', () => this.onClosed());
|
|
72
|
+
}
|
|
73
|
+
onClosed(err) {
|
|
74
|
+
if (this.closed)
|
|
75
|
+
return;
|
|
76
|
+
this.closed = true;
|
|
77
|
+
const closeErr = err ?? new Error('Chrome closed the CDP pipe');
|
|
78
|
+
for (const { reject } of this.pending.values())
|
|
79
|
+
reject(closeErr);
|
|
80
|
+
this.pending.clear();
|
|
81
|
+
const set = this.listeners.get(PIPE_CLOSED);
|
|
82
|
+
if (set)
|
|
83
|
+
for (const fn of set)
|
|
84
|
+
fn(undefined);
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* @desc A frame that fails to parse as JSON used to be dropped silently
|
|
88
|
+
* (`msg = undefined`, dispatch skipped) — any `send()` whose response was
|
|
89
|
+
* that exact frame then hung forever, since nothing ever rejected it.
|
|
90
|
+
* Now the whole pipe is torn down the same way a real Chrome exit is:
|
|
91
|
+
* every pending command rejects (with a `CdpProtocolError`, not the
|
|
92
|
+
* generic close message, so the cause is distinguishable), `isClosed`
|
|
93
|
+
* flips true, and `onClose` subscribers fire — the peer sent a frame
|
|
94
|
+
* that doesn't fit the protocol, so nothing it says next can be trusted
|
|
95
|
+
* either. The raw frame content is never included in the error message —
|
|
96
|
+
* it can carry page-controlled data (e.g. a `Runtime.evaluate` result).
|
|
97
|
+
*/
|
|
98
|
+
onProtocolError() {
|
|
99
|
+
if (this.closed)
|
|
100
|
+
return;
|
|
101
|
+
const err = new CdpProtocolError('Chrome sent a CDP frame that failed to parse as JSON — the pipe can no longer be trusted and has been torn down; any in-flight command was rejected.');
|
|
102
|
+
this.protocolErr = err;
|
|
103
|
+
this.onClosed(err);
|
|
104
|
+
}
|
|
105
|
+
onData(chunk) {
|
|
106
|
+
// Once torn down (a real close, or a corrupt frame — onProtocolError)
|
|
107
|
+
// the read stream can still be alive and keep delivering chunks from
|
|
108
|
+
// an untrusted peer; without this, a later VALID-looking event frame
|
|
109
|
+
// would still `dispatch()` to any listener still registered instead
|
|
110
|
+
// of being ignored, and `this.buffer` would keep growing forever.
|
|
111
|
+
if (this.closed)
|
|
112
|
+
return;
|
|
113
|
+
this.buffer += chunk;
|
|
114
|
+
let idx = this.buffer.indexOf('\0');
|
|
115
|
+
while (idx !== -1) {
|
|
116
|
+
const raw = this.buffer.slice(0, idx);
|
|
117
|
+
this.buffer = this.buffer.slice(idx + 1);
|
|
118
|
+
if (raw.length > 0) {
|
|
119
|
+
let msg;
|
|
120
|
+
try {
|
|
121
|
+
msg = JSON.parse(raw);
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
this.onProtocolError();
|
|
125
|
+
return; // torn down — whatever else is buffered is moot
|
|
126
|
+
}
|
|
127
|
+
this.dispatch(msg);
|
|
128
|
+
}
|
|
129
|
+
idx = this.buffer.indexOf('\0');
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
dispatch(msg) {
|
|
133
|
+
if (typeof msg.id === 'number' && this.pending.has(msg.id)) {
|
|
134
|
+
const entry = this.pending.get(msg.id);
|
|
135
|
+
this.pending.delete(msg.id);
|
|
136
|
+
if (msg.error)
|
|
137
|
+
entry.reject(new Error(`${msg.error.message} (code ${msg.error.code})`));
|
|
138
|
+
else
|
|
139
|
+
entry.resolve(msg.result);
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
if (typeof msg.method === 'string') {
|
|
143
|
+
const set = this.listeners.get(msg.method);
|
|
144
|
+
if (set)
|
|
145
|
+
for (const fn of set)
|
|
146
|
+
fn(msg.params, msg.sessionId);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Send a CDP command and resolve with its `result`. Rejects if the pipe
|
|
151
|
+
* closes (Chrome exited, or a corrupt frame tore it down — see
|
|
152
|
+
* `onProtocolError`) before a response arrives, or with a
|
|
153
|
+
* `CdpTimeoutError` if no response arrives within `timeoutMs` (defaults
|
|
154
|
+
* to the pipe's own `commandTimeoutMs`, 30s) — the timeout names the
|
|
155
|
+
* METHOD, never `params` (which can carry page-controlled data).
|
|
156
|
+
* @param timeoutMs — per-call override. Lets a caller pin its own named
|
|
157
|
+
* deadline as an assertable constant rather than relying on the pipe's
|
|
158
|
+
* default silently matching (see session-capture.ts's
|
|
159
|
+
* `LOCALSTORAGE_READ_TIMEOUT_MS`, which currently equals this pipe's own
|
|
160
|
+
* 30s default but is passed explicitly anyway — a post-cap review found
|
|
161
|
+
* that giving `Runtime.evaluate` a SHORTER override here, on the
|
|
162
|
+
* assumption a page-blocked renderer "won't unblock itself in 30s any
|
|
163
|
+
* more than in 5", cut off real-but-slow work that finished on its own;
|
|
164
|
+
* there is no override value proven safe, so this parameter exists for
|
|
165
|
+
* callers that want one, not as a recommendation to use a shorter one).
|
|
166
|
+
*/
|
|
167
|
+
send(method, params = {}, sessionId, timeoutMs = this.commandTimeoutMs) {
|
|
168
|
+
if (this.closed)
|
|
169
|
+
return Promise.reject(this.protocolErr ?? new Error('Chrome closed the CDP pipe'));
|
|
170
|
+
const id = this.nextId++;
|
|
171
|
+
const req = { id, method, params };
|
|
172
|
+
if (sessionId)
|
|
173
|
+
req.sessionId = sessionId;
|
|
174
|
+
return new Promise((resolve, reject) => {
|
|
175
|
+
// Every settle path (dispatch's resolve/reject, this timeout, a
|
|
176
|
+
// write-side EPIPE below, and onClosed's reject-all on a pipe
|
|
177
|
+
// teardown) goes through these two wrappers, so the timer is always
|
|
178
|
+
// cleared exactly once no matter which path wins the race.
|
|
179
|
+
const settleResolve = (v) => {
|
|
180
|
+
clearTimeout(timer);
|
|
181
|
+
resolve(v);
|
|
182
|
+
};
|
|
183
|
+
const settleReject = (e) => {
|
|
184
|
+
clearTimeout(timer);
|
|
185
|
+
reject(e);
|
|
186
|
+
};
|
|
187
|
+
const timer = setTimeout(() => {
|
|
188
|
+
this.pending.delete(id);
|
|
189
|
+
settleReject(new CdpTimeoutError(method, timeoutMs));
|
|
190
|
+
}, timeoutMs);
|
|
191
|
+
this.pending.set(id, { resolve: settleResolve, reject: settleReject });
|
|
192
|
+
this.writeStream.write(`${JSON.stringify(req)}\0`, (err) => {
|
|
193
|
+
if (err) {
|
|
194
|
+
this.pending.delete(id);
|
|
195
|
+
settleReject(err);
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
/** Subscribe to a CDP event (`'Target.targetDestroyed'`, …). Returns an
|
|
201
|
+
* unsubscribe function. */
|
|
202
|
+
on(method, handler) {
|
|
203
|
+
if (!this.listeners.has(method))
|
|
204
|
+
this.listeners.set(method, new Set());
|
|
205
|
+
this.listeners.get(method).add(handler);
|
|
206
|
+
return () => this.listeners.get(method)?.delete(handler);
|
|
207
|
+
}
|
|
208
|
+
/** Subscribe to the pipe closing (Chrome process gone). */
|
|
209
|
+
onClose(handler) {
|
|
210
|
+
return this.on(PIPE_CLOSED, () => handler());
|
|
211
|
+
}
|
|
212
|
+
get isClosed() {
|
|
213
|
+
return this.closed;
|
|
214
|
+
}
|
|
215
|
+
/** Set only when the pipe was torn down because of a corrupt frame — as
|
|
216
|
+
* opposed to a real Chrome exit — so a caller can report the actual
|
|
217
|
+
* cause instead of defaulting every close to "Chrome exited". */
|
|
218
|
+
get protocolError() {
|
|
219
|
+
return this.protocolErr;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @desc Resolve the Chrome executable to launch. Precedence: an explicit
|
|
3
|
+
* `--chrome <path>` flag (validated by the caller) > `TRAWL_CHROME_PATH` >
|
|
4
|
+
* `PUPPETEER_EXECUTABLE_PATH`/`CHROME_PATH` (common conventions other
|
|
5
|
+
* tools already set) > the well-known per-OS install paths. Returns null
|
|
6
|
+
* when nothing is found — the caller is responsible for a factual,
|
|
7
|
+
* non-guessing error message.
|
|
8
|
+
* @param {NodeJS.ProcessEnv} env
|
|
9
|
+
* @param {NodeJS.Platform} platform
|
|
10
|
+
* @param {(path: string) => boolean} exists — injectable for tests.
|
|
11
|
+
*/
|
|
12
|
+
export declare function findChrome(env?: NodeJS.ProcessEnv, platform?: NodeJS.Platform, exists?: (path: string) => boolean): string | null;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Locate a system Chrome/Chromium executable (trawl_cli#183). Same
|
|
3
|
+
* discovery approach as `@trawlme/skills`'s
|
|
4
|
+
* `trawl-scrap-local-test/scripts/run-local.mjs` `findChrome()` — a short
|
|
5
|
+
* list of well-known install paths per OS, first match wins — extended
|
|
6
|
+
* with an env override and a Windows list (the skill script only needed
|
|
7
|
+
* mac/Linux for a dev-machine local test; this ships to every user's OS).
|
|
8
|
+
*/
|
|
9
|
+
import { existsSync } from 'node:fs';
|
|
10
|
+
const CANDIDATES = {
|
|
11
|
+
darwin: [
|
|
12
|
+
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
13
|
+
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
|
14
|
+
'/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta',
|
|
15
|
+
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
|
|
16
|
+
],
|
|
17
|
+
linux: [
|
|
18
|
+
'/usr/bin/google-chrome',
|
|
19
|
+
'/usr/bin/google-chrome-stable',
|
|
20
|
+
'/usr/bin/chromium-browser',
|
|
21
|
+
'/usr/bin/chromium',
|
|
22
|
+
'/snap/bin/chromium',
|
|
23
|
+
],
|
|
24
|
+
win32: [
|
|
25
|
+
'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
|
|
26
|
+
'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
|
|
27
|
+
'C:\\Program Files\\Chromium\\Application\\chrome.exe',
|
|
28
|
+
],
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* @desc Resolve the Chrome executable to launch. Precedence: an explicit
|
|
32
|
+
* `--chrome <path>` flag (validated by the caller) > `TRAWL_CHROME_PATH` >
|
|
33
|
+
* `PUPPETEER_EXECUTABLE_PATH`/`CHROME_PATH` (common conventions other
|
|
34
|
+
* tools already set) > the well-known per-OS install paths. Returns null
|
|
35
|
+
* when nothing is found — the caller is responsible for a factual,
|
|
36
|
+
* non-guessing error message.
|
|
37
|
+
* @param {NodeJS.ProcessEnv} env
|
|
38
|
+
* @param {NodeJS.Platform} platform
|
|
39
|
+
* @param {(path: string) => boolean} exists — injectable for tests.
|
|
40
|
+
*/
|
|
41
|
+
export function findChrome(env = process.env, platform = process.platform, exists = existsSync) {
|
|
42
|
+
const envCandidates = [env.TRAWL_CHROME_PATH, env.PUPPETEER_EXECUTABLE_PATH, env.CHROME_PATH].filter((p) => typeof p === 'string' && p.length > 0);
|
|
43
|
+
for (const p of envCandidates) {
|
|
44
|
+
if (exists(p))
|
|
45
|
+
return p;
|
|
46
|
+
}
|
|
47
|
+
const candidates = CANDIDATES[platform] ?? [];
|
|
48
|
+
return candidates.find(exists) ?? null;
|
|
49
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spawn a headed, isolated Chrome instance wired for raw CDP over
|
|
3
|
+
* `--remote-debugging-pipe` (trawl_cli#183). Thin glue only — the actual
|
|
4
|
+
* protocol lives in cdp-pipe.ts, kept separate so this file's job (build
|
|
5
|
+
* the right flags, wire fd 3/4, fail loudly if the platform doesn't expose
|
|
6
|
+
* them) stays small enough to unit test with a mocked `spawn`.
|
|
7
|
+
*/
|
|
8
|
+
import { spawn, type ChildProcess } from 'node:child_process';
|
|
9
|
+
import { CdpPipe } from './cdp-pipe.js';
|
|
10
|
+
export interface LaunchedChrome {
|
|
11
|
+
proc: ChildProcess;
|
|
12
|
+
cdp: CdpPipe;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* @desc The exact CLI flags Chrome is launched with. Pure + exported so
|
|
16
|
+
* the flag set is testable without spawning a real process.
|
|
17
|
+
* - `--remote-debugging-pipe` — CDP over fd 3 (in) / fd 4 (out), the
|
|
18
|
+
* zero-dependency transport this command relies on.
|
|
19
|
+
* - `--user-data-dir` — an ephemeral, isolated profile (never the user's
|
|
20
|
+
* real Chrome profile/cookies).
|
|
21
|
+
* - `--no-first-run` / `--no-default-browser-check` — skip first-run UI
|
|
22
|
+
* that would otherwise sit in front of the target URL.
|
|
23
|
+
* - `--disable-sync` — never touches the user's real Google account sync.
|
|
24
|
+
* - `--password-store=basic` / `--use-mock-keychain` — a fresh profile
|
|
25
|
+
* would otherwise prompt for the macOS login keychain (Safe Storage) the
|
|
26
|
+
* first time Chrome touches its cookie/password store; these keep the
|
|
27
|
+
* whole flow non-blocking on an ephemeral profile that's deleted right
|
|
28
|
+
* after (mirrors Crawl4AI's `crwl profiles`, cited in the issue).
|
|
29
|
+
* - `--new-window` — always its own window, never a tab folded into an
|
|
30
|
+
* already-running Chrome instance.
|
|
31
|
+
* - No `--no-sandbox`: this runs headed on the user's own machine, not in
|
|
32
|
+
* a locked-down CI container — the sandbox stays on.
|
|
33
|
+
*/
|
|
34
|
+
export declare function buildChromeArgs(userDataDir: string, targetUrl: string): string[];
|
|
35
|
+
/**
|
|
36
|
+
* @desc Launch Chrome and wire a CdpPipe to its fd 3 (write) / fd 4 (read).
|
|
37
|
+
* Returns a Promise rather than the `ChildProcess` synchronously: `spawn()`
|
|
38
|
+
* can fail AFTER returning — EACCES on a non-executable path, an ENOENT
|
|
39
|
+
* race, EPERM, E2BIG — by emitting an `'error'` event rather than throwing,
|
|
40
|
+
* and that event is attached to BEFORE the fd 3/4 check below so a failure
|
|
41
|
+
* from either cause rejects this promise instead of crashing the process
|
|
42
|
+
* (an EventEmitter with zero `'error'` listeners throws the error itself).
|
|
43
|
+
* @rejects {Error} if `spawn()` itself failed (the original error, so
|
|
44
|
+
* `.code` like `EACCES` survives), or if the platform/Chrome build didn't
|
|
45
|
+
* expose fd 3/4 as pipes (e.g. `--remote-debugging-pipe` unsupported) — a
|
|
46
|
+
* factual message, never a silent hang.
|
|
47
|
+
*/
|
|
48
|
+
export declare function launchChrome(executablePath: string, targetUrl: string, userDataDir: string, spawnFn?: typeof spawn): Promise<LaunchedChrome>;
|