@trawlme/cli 3.11.0 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/commands/scraps.js +208 -8
- package/dist/lib/cdp-pipe.d.ts +103 -0
- package/dist/lib/cdp-pipe.js +221 -0
- package/dist/lib/chrome-discovery.d.ts +12 -0
- package/dist/lib/chrome-discovery.js +49 -0
- package/dist/lib/chrome-launch.d.ts +48 -0
- package/dist/lib/chrome-launch.js +122 -0
- package/dist/lib/secure-transport.d.ts +8 -0
- package/dist/lib/secure-transport.js +39 -0
- package/dist/lib/session-capture-guard.d.ts +21 -0
- package/dist/lib/session-capture-guard.js +14 -0
- package/dist/lib/session-capture.d.ts +180 -0
- package/dist/lib/session-capture.js +600 -0
- package/dist/lib/storage-state.d.ts +167 -0
- package/dist/lib/storage-state.js +227 -0
- package/package.json +1 -1
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Locate a system Chrome/Chromium executable (trawl_cli#183). Same
|
|
3
|
+
* discovery approach as `@trawlme/skills`'s
|
|
4
|
+
* `trawl-scrap-local-test/scripts/run-local.mjs` `findChrome()` — a short
|
|
5
|
+
* list of well-known install paths per OS, first match wins — extended
|
|
6
|
+
* with an env override and a Windows list (the skill script only needed
|
|
7
|
+
* mac/Linux for a dev-machine local test; this ships to every user's OS).
|
|
8
|
+
*/
|
|
9
|
+
import { existsSync } from 'node:fs';
|
|
10
|
+
const CANDIDATES = {
|
|
11
|
+
darwin: [
|
|
12
|
+
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
13
|
+
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
|
14
|
+
'/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta',
|
|
15
|
+
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
|
|
16
|
+
],
|
|
17
|
+
linux: [
|
|
18
|
+
'/usr/bin/google-chrome',
|
|
19
|
+
'/usr/bin/google-chrome-stable',
|
|
20
|
+
'/usr/bin/chromium-browser',
|
|
21
|
+
'/usr/bin/chromium',
|
|
22
|
+
'/snap/bin/chromium',
|
|
23
|
+
],
|
|
24
|
+
win32: [
|
|
25
|
+
'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
|
|
26
|
+
'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
|
|
27
|
+
'C:\\Program Files\\Chromium\\Application\\chrome.exe',
|
|
28
|
+
],
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* @desc Resolve the Chrome executable to launch. Precedence: an explicit
|
|
32
|
+
* `--chrome <path>` flag (validated by the caller) > `TRAWL_CHROME_PATH` >
|
|
33
|
+
* `PUPPETEER_EXECUTABLE_PATH`/`CHROME_PATH` (common conventions other
|
|
34
|
+
* tools already set) > the well-known per-OS install paths. Returns null
|
|
35
|
+
* when nothing is found — the caller is responsible for a factual,
|
|
36
|
+
* non-guessing error message.
|
|
37
|
+
* @param {NodeJS.ProcessEnv} env
|
|
38
|
+
* @param {NodeJS.Platform} platform
|
|
39
|
+
* @param {(path: string) => boolean} exists — injectable for tests.
|
|
40
|
+
*/
|
|
41
|
+
export function findChrome(env = process.env, platform = process.platform, exists = existsSync) {
|
|
42
|
+
const envCandidates = [env.TRAWL_CHROME_PATH, env.PUPPETEER_EXECUTABLE_PATH, env.CHROME_PATH].filter((p) => typeof p === 'string' && p.length > 0);
|
|
43
|
+
for (const p of envCandidates) {
|
|
44
|
+
if (exists(p))
|
|
45
|
+
return p;
|
|
46
|
+
}
|
|
47
|
+
const candidates = CANDIDATES[platform] ?? [];
|
|
48
|
+
return candidates.find(exists) ?? null;
|
|
49
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spawn a headed, isolated Chrome instance wired for raw CDP over
|
|
3
|
+
* `--remote-debugging-pipe` (trawl_cli#183). Thin glue only — the actual
|
|
4
|
+
* protocol lives in cdp-pipe.ts, kept separate so this file's job (build
|
|
5
|
+
* the right flags, wire fd 3/4, fail loudly if the platform doesn't expose
|
|
6
|
+
* them) stays small enough to unit test with a mocked `spawn`.
|
|
7
|
+
*/
|
|
8
|
+
import { spawn, type ChildProcess } from 'node:child_process';
|
|
9
|
+
import { CdpPipe } from './cdp-pipe.js';
|
|
10
|
+
export interface LaunchedChrome {
|
|
11
|
+
proc: ChildProcess;
|
|
12
|
+
cdp: CdpPipe;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* @desc The exact CLI flags Chrome is launched with. Pure + exported so
|
|
16
|
+
* the flag set is testable without spawning a real process.
|
|
17
|
+
* - `--remote-debugging-pipe` — CDP over fd 3 (in) / fd 4 (out), the
|
|
18
|
+
* zero-dependency transport this command relies on.
|
|
19
|
+
* - `--user-data-dir` — an ephemeral, isolated profile (never the user's
|
|
20
|
+
* real Chrome profile/cookies).
|
|
21
|
+
* - `--no-first-run` / `--no-default-browser-check` — skip first-run UI
|
|
22
|
+
* that would otherwise sit in front of the target URL.
|
|
23
|
+
* - `--disable-sync` — never touches the user's real Google account sync.
|
|
24
|
+
* - `--password-store=basic` / `--use-mock-keychain` — a fresh profile
|
|
25
|
+
* would otherwise prompt for the macOS login keychain (Safe Storage) the
|
|
26
|
+
* first time Chrome touches its cookie/password store; these keep the
|
|
27
|
+
* whole flow non-blocking on an ephemeral profile that's deleted right
|
|
28
|
+
* after (mirrors Crawl4AI's `crwl profiles`, cited in the issue).
|
|
29
|
+
* - `--new-window` — always its own window, never a tab folded into an
|
|
30
|
+
* already-running Chrome instance.
|
|
31
|
+
* - No `--no-sandbox`: this runs headed on the user's own machine, not in
|
|
32
|
+
* a locked-down CI container — the sandbox stays on.
|
|
33
|
+
*/
|
|
34
|
+
export declare function buildChromeArgs(userDataDir: string, targetUrl: string): string[];
|
|
35
|
+
/**
|
|
36
|
+
* @desc Launch Chrome and wire a CdpPipe to its fd 3 (write) / fd 4 (read).
|
|
37
|
+
* Returns a Promise rather than the `ChildProcess` synchronously: `spawn()`
|
|
38
|
+
* can fail AFTER returning — EACCES on a non-executable path, an ENOENT
|
|
39
|
+
* race, EPERM, E2BIG — by emitting an `'error'` event rather than throwing,
|
|
40
|
+
* and that event is attached to BEFORE the fd 3/4 check below so a failure
|
|
41
|
+
* from either cause rejects this promise instead of crashing the process
|
|
42
|
+
* (an EventEmitter with zero `'error'` listeners throws the error itself).
|
|
43
|
+
* @rejects {Error} if `spawn()` itself failed (the original error, so
|
|
44
|
+
* `.code` like `EACCES` survives), or if the platform/Chrome build didn't
|
|
45
|
+
* expose fd 3/4 as pipes (e.g. `--remote-debugging-pipe` unsupported) — a
|
|
46
|
+
* factual message, never a silent hang.
|
|
47
|
+
*/
|
|
48
|
+
export declare function launchChrome(executablePath: string, targetUrl: string, userDataDir: string, spawnFn?: typeof spawn): Promise<LaunchedChrome>;
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spawn a headed, isolated Chrome instance wired for raw CDP over
|
|
3
|
+
* `--remote-debugging-pipe` (trawl_cli#183). Thin glue only — the actual
|
|
4
|
+
* protocol lives in cdp-pipe.ts, kept separate so this file's job (build
|
|
5
|
+
* the right flags, wire fd 3/4, fail loudly if the platform doesn't expose
|
|
6
|
+
* them) stays small enough to unit test with a mocked `spawn`.
|
|
7
|
+
*/
|
|
8
|
+
import { spawn } from 'node:child_process';
|
|
9
|
+
import { CdpPipe } from './cdp-pipe.js';
|
|
10
|
+
/**
|
|
11
|
+
* @desc The exact CLI flags Chrome is launched with. Pure + exported so
|
|
12
|
+
* the flag set is testable without spawning a real process.
|
|
13
|
+
* - `--remote-debugging-pipe` — CDP over fd 3 (in) / fd 4 (out), the
|
|
14
|
+
* zero-dependency transport this command relies on.
|
|
15
|
+
* - `--user-data-dir` — an ephemeral, isolated profile (never the user's
|
|
16
|
+
* real Chrome profile/cookies).
|
|
17
|
+
* - `--no-first-run` / `--no-default-browser-check` — skip first-run UI
|
|
18
|
+
* that would otherwise sit in front of the target URL.
|
|
19
|
+
* - `--disable-sync` — never touches the user's real Google account sync.
|
|
20
|
+
* - `--password-store=basic` / `--use-mock-keychain` — a fresh profile
|
|
21
|
+
* would otherwise prompt for the macOS login keychain (Safe Storage) the
|
|
22
|
+
* first time Chrome touches its cookie/password store; these keep the
|
|
23
|
+
* whole flow non-blocking on an ephemeral profile that's deleted right
|
|
24
|
+
* after (mirrors Crawl4AI's `crwl profiles`, cited in the issue).
|
|
25
|
+
* - `--new-window` — always its own window, never a tab folded into an
|
|
26
|
+
* already-running Chrome instance.
|
|
27
|
+
* - No `--no-sandbox`: this runs headed on the user's own machine, not in
|
|
28
|
+
* a locked-down CI container — the sandbox stays on.
|
|
29
|
+
*/
|
|
30
|
+
export function buildChromeArgs(userDataDir, targetUrl) {
|
|
31
|
+
return [
|
|
32
|
+
'--remote-debugging-pipe',
|
|
33
|
+
`--user-data-dir=${userDataDir}`,
|
|
34
|
+
'--no-first-run',
|
|
35
|
+
'--no-default-browser-check',
|
|
36
|
+
'--disable-sync',
|
|
37
|
+
'--password-store=basic',
|
|
38
|
+
'--use-mock-keychain',
|
|
39
|
+
'--new-window',
|
|
40
|
+
targetUrl,
|
|
41
|
+
];
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* @desc Launch Chrome and wire a CdpPipe to its fd 3 (write) / fd 4 (read).
|
|
45
|
+
* Returns a Promise rather than the `ChildProcess` synchronously: `spawn()`
|
|
46
|
+
* can fail AFTER returning — EACCES on a non-executable path, an ENOENT
|
|
47
|
+
* race, EPERM, E2BIG — by emitting an `'error'` event rather than throwing,
|
|
48
|
+
* and that event is attached to BEFORE the fd 3/4 check below so a failure
|
|
49
|
+
* from either cause rejects this promise instead of crashing the process
|
|
50
|
+
* (an EventEmitter with zero `'error'` listeners throws the error itself).
|
|
51
|
+
* @rejects {Error} if `spawn()` itself failed (the original error, so
|
|
52
|
+
* `.code` like `EACCES` survives), or if the platform/Chrome build didn't
|
|
53
|
+
* expose fd 3/4 as pipes (e.g. `--remote-debugging-pipe` unsupported) — a
|
|
54
|
+
* factual message, never a silent hang.
|
|
55
|
+
*/
|
|
56
|
+
export function launchChrome(executablePath, targetUrl, userDataDir, spawnFn = spawn) {
|
|
57
|
+
return new Promise((resolve, reject) => {
|
|
58
|
+
// stderr is 'ignore', not 'pipe': nothing here ever reads it (unlike the
|
|
59
|
+
// rejected `ws`-over-port design, `--remote-debugging-pipe` needs none of
|
|
60
|
+
// Chrome's stderr output), and a 'pipe' nobody drains fills its OS pipe
|
|
61
|
+
// buffer — Chrome's own logging would then block on write mid-session,
|
|
62
|
+
// freezing the browser the human is mid-login in.
|
|
63
|
+
const proc = spawnFn(executablePath, buildChromeArgs(userDataDir, targetUrl), {
|
|
64
|
+
stdio: ['ignore', 'ignore', 'ignore', 'pipe', 'pipe'],
|
|
65
|
+
// POSIX only: makes Chrome the leader of its OWN process group, so its
|
|
66
|
+
// helper subprocesses (renderer/GPU/network service — Chrome forks
|
|
67
|
+
// several even for one window) live in that group too, distinct from
|
|
68
|
+
// ours. Verified against a real launch (trawl_cli#183's loopback E2E):
|
|
69
|
+
// without this, `proc.kill('SIGKILL')` only killed the main Chrome
|
|
70
|
+
// process — its helpers briefly kept the user-data-dir's files open,
|
|
71
|
+
// and `rmSync` silently lost that race, leaking the temp profile on
|
|
72
|
+
// every run. session-capture.ts's cleanup kills `-proc.pid` (the whole
|
|
73
|
+
// group) instead of `proc.pid` alone, which requires this.
|
|
74
|
+
detached: process.platform !== 'win32',
|
|
75
|
+
});
|
|
76
|
+
let settled = false;
|
|
77
|
+
// Attached as the very first thing after `spawnFn` returns — before the
|
|
78
|
+
// fd 3/4 check below, and before anything here ever awaits — so nothing
|
|
79
|
+
// can slip through unobserved. Stays attached FOREVER, including after
|
|
80
|
+
// this promise settles: a LATE error (spawn succeeded, then something
|
|
81
|
+
// else goes wrong long after this promise already resolved) must not
|
|
82
|
+
// crash the process either, and once `settled` is true this listener is
|
|
83
|
+
// a permanent no-op — whatever consumes `proc` from here on notices
|
|
84
|
+
// Chrome went away some other way (the CDP pipe closing).
|
|
85
|
+
proc.on('error', (err) => {
|
|
86
|
+
if (settled)
|
|
87
|
+
return;
|
|
88
|
+
settled = true;
|
|
89
|
+
try {
|
|
90
|
+
proc.kill('SIGKILL');
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
// already gone
|
|
94
|
+
}
|
|
95
|
+
reject(err);
|
|
96
|
+
});
|
|
97
|
+
const writeStream = proc.stdio[3];
|
|
98
|
+
const readStream = proc.stdio[4];
|
|
99
|
+
if (!writeStream || !readStream || typeof writeStream.write !== 'function') {
|
|
100
|
+
settled = true;
|
|
101
|
+
try {
|
|
102
|
+
proc.kill('SIGKILL');
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
// already gone
|
|
106
|
+
}
|
|
107
|
+
reject(new Error('Chrome did not expose the CDP pipe (fd 3/4) — this Chrome build or platform may not support --remote-debugging-pipe.'));
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
// Only declare success once the OS confirms the process actually
|
|
111
|
+
// started — the documented signal for that ('spawn', not merely
|
|
112
|
+
// "spawn() returned without throwing") — rather than assuming
|
|
113
|
+
// immediately: that's exactly the assumption the bug this fixes was
|
|
114
|
+
// built on.
|
|
115
|
+
proc.once('spawn', () => {
|
|
116
|
+
if (settled)
|
|
117
|
+
return;
|
|
118
|
+
settled = true;
|
|
119
|
+
resolve({ proc, cdp: new CdpPipe(writeStream, readStream) });
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @desc Returns a factual reason string when `apiUrl` is not an
|
|
3
|
+
* acceptable transport for a session upload, or null when it is
|
|
4
|
+
* (`https:`, or any scheme against a loopback host). An unparseable URL
|
|
5
|
+
* returns null — it fails with its own clear error at the actual fetch
|
|
6
|
+
* call, not here.
|
|
7
|
+
*/
|
|
8
|
+
export declare function assertSecureTransport(apiUrl: string): string | null;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Refuses a plaintext API base for the two commands that ship a browser
|
|
3
|
+
* session — a bearer-equivalent secret — in the request body
|
|
4
|
+
* (trawl_cli#183 review finding 5). Reproduced: `TRAWL_API_URL=http://…
|
|
5
|
+
* trawl scraps account session capture <id>` uploaded the full session in
|
|
6
|
+
* cleartext with no warning; this command is the first in the CLI to PUT
|
|
7
|
+
* that kind of secret, so the fix is scoped HERE rather than to the
|
|
8
|
+
* generic `request()` in api.ts — every other command's blast radius
|
|
9
|
+
* stays exactly what it was.
|
|
10
|
+
*
|
|
11
|
+
* Loopback hosts are exempt (by exact hostname only — a plaintext host
|
|
12
|
+
* that merely happens to RESOLVE to loopback is not detected here) so
|
|
13
|
+
* local/self-hosted dev and the test suite keep working over plain HTTP.
|
|
14
|
+
*/
|
|
15
|
+
// `new URL(...).hostname` returns the IPv6 form WITH its brackets
|
|
16
|
+
// (`'[::1]'`, never bare `'::1'`) — both are listed defensively so this
|
|
17
|
+
// never regresses silently if that ever changes.
|
|
18
|
+
const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', '[::1]', 'localhost']);
|
|
19
|
+
/**
|
|
20
|
+
* @desc Returns a factual reason string when `apiUrl` is not an
|
|
21
|
+
* acceptable transport for a session upload, or null when it is
|
|
22
|
+
* (`https:`, or any scheme against a loopback host). An unparseable URL
|
|
23
|
+
* returns null — it fails with its own clear error at the actual fetch
|
|
24
|
+
* call, not here.
|
|
25
|
+
*/
|
|
26
|
+
export function assertSecureTransport(apiUrl) {
|
|
27
|
+
let url;
|
|
28
|
+
try {
|
|
29
|
+
url = new URL(apiUrl);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
if (url.protocol === 'https:')
|
|
35
|
+
return null;
|
|
36
|
+
if (LOOPBACK_HOSTS.has(url.hostname.toLowerCase()))
|
|
37
|
+
return null;
|
|
38
|
+
return `The configured API URL (${url.origin}) is not https: — a captured/uploaded browser session is a bearer-equivalent secret and this command refuses to send it in cleartext. An https:// API URL is accepted (trawl login --url https://…); so is any URL on 127.0.0.1, ::1, or localhost, for local testing.`;
|
|
39
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The non-interactive guard for `scraps account session capture`
|
|
3
|
+
* (trawl_cli#183) — deliberately its OWN function, not a reuse of
|
|
4
|
+
* `confirm.ts`'s `isInteractive`. That one returns false under `--json`,
|
|
5
|
+
* which is wrong here: `--json` + a real interactive terminal is a valid
|
|
6
|
+
* way to run this command (a human still drives the browser; only the
|
|
7
|
+
* final result on stdout is machine-shaped). What actually makes this
|
|
8
|
+
* command impossible is no TTY on stdin (nobody can press Enter) or, on
|
|
9
|
+
* Linux, no display server to open a visible window on.
|
|
10
|
+
*/
|
|
11
|
+
export interface NonInteractiveEnv {
|
|
12
|
+
stdinIsTTY: boolean;
|
|
13
|
+
platform: NodeJS.Platform;
|
|
14
|
+
env: NodeJS.ProcessEnv;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* @desc Returns a factual reason string when this command cannot work in
|
|
18
|
+
* the current environment, or null when it can proceed. Never guesses —
|
|
19
|
+
* both checks are direct, observable facts (no TTY / no display var).
|
|
20
|
+
*/
|
|
21
|
+
export declare function detectNonInteractive({ stdinIsTTY, platform, env }: NonInteractiveEnv): string | null;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @desc Returns a factual reason string when this command cannot work in
|
|
3
|
+
* the current environment, or null when it can proceed. Never guesses —
|
|
4
|
+
* both checks are direct, observable facts (no TTY / no display var).
|
|
5
|
+
*/
|
|
6
|
+
export function detectNonInteractive({ stdinIsTTY, platform, env }) {
|
|
7
|
+
if (!stdinIsTTY) {
|
|
8
|
+
return 'No interactive terminal attached (stdin is not a TTY) — this command opens a visible Chrome window for a human to log into and needs a terminal to confirm completion. It does not work headless, in CI, or piped.';
|
|
9
|
+
}
|
|
10
|
+
if (platform === 'linux' && !env.DISPLAY && !env.WAYLAND_DISPLAY) {
|
|
11
|
+
return 'No display detected (DISPLAY/WAYLAND_DISPLAY are both unset) — this command opens a visible Chrome window and cannot run over a plain SSH session, in a container, or headless.';
|
|
12
|
+
}
|
|
13
|
+
return null;
|
|
14
|
+
}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import type { ChildProcess } from 'node:child_process';
|
|
2
|
+
import type { CdpPipe } from './cdp-pipe.js';
|
|
3
|
+
import { type RawCdpCookie, type RawOriginLocalStorage, type StorageState } from './storage-state.js';
|
|
4
|
+
export interface CaptureCounts {
|
|
5
|
+
cookiesCaptured: number;
|
|
6
|
+
cookiesDroppedOutOfScope: number;
|
|
7
|
+
cookiesDroppedInvalid: number;
|
|
8
|
+
originsCaptured: number;
|
|
9
|
+
originsDroppedOutOfScope: number;
|
|
10
|
+
/** An in-scope page's localStorage could not be captured, for one of
|
|
11
|
+
* four reasons — never a CLI bug, which propagates as a failed capture
|
|
12
|
+
* instead (trawl_cli#183 post-cap review finding B, sharpened by the R4
|
|
13
|
+
* fix below): a `readPageOverCdp` rejection — ANY CDP-level rejection
|
|
14
|
+
* scoped to THIS page's `attachToTarget`/`Runtime.evaluate` round-trip,
|
|
15
|
+
* whatever its error class (a timeout because the page's renderer was
|
|
16
|
+
* blocked — a native dialog, a synchronous script, a paused debugger; or
|
|
17
|
+
* a plain rejection because the target closed mid-attach/mid-evaluate,
|
|
18
|
+
* e.g. an OAuth popup the human closed at the wrong instant); the read
|
|
19
|
+
* threw INSIDE the page (e.g. a `SecurityError` on partitioned/sandboxed
|
|
20
|
+
* storage); or `Runtime.evaluate` succeeded with no exception but
|
|
21
|
+
* returned a value that doesn't match the `{name,value}[]` shape asked
|
|
22
|
+
* for (a page tampering with a global to hand back garbage without ever
|
|
23
|
+
* throwing). The discriminant is PROVENANCE, not error type: did the
|
|
24
|
+
* rejection come out of a CDP call scoped to this one page, or out of
|
|
25
|
+
* this file's OWN code running on data CDP already handed back? Only the
|
|
26
|
+
* latter is a CLI bug, and only the latter propagates. None of these
|
|
27
|
+
* four is the same fact as "0 keys": CDP itself was reachable (or the
|
|
28
|
+
* pipe wouldn't still be open), the origin just could not be read.
|
|
29
|
+
* Counted, never the exception text/URL/value itself (a page controls
|
|
30
|
+
* that content). */
|
|
31
|
+
originsUnreadable: number;
|
|
32
|
+
/** The tracked tab was closed before Enter — cookies were still read,
|
|
33
|
+
* localStorage was not (the completion path the issue calls out:
|
|
34
|
+
* "closing the window also completes"). */
|
|
35
|
+
closedEarly: boolean;
|
|
36
|
+
}
|
|
37
|
+
export type CaptureFailureReason = 'non_interactive' | 'no_chrome' | 'launch_failed' | 'capture_failed' | 'process_exited_before_capture' | 'cdp_protocol_error' | 'no_cookies_in_scope';
|
|
38
|
+
export type CaptureResult = {
|
|
39
|
+
ok: true;
|
|
40
|
+
storageState: StorageState;
|
|
41
|
+
counts: CaptureCounts;
|
|
42
|
+
targetDomain: string;
|
|
43
|
+
} | {
|
|
44
|
+
ok: false;
|
|
45
|
+
reason: CaptureFailureReason;
|
|
46
|
+
message: string;
|
|
47
|
+
};
|
|
48
|
+
interface RawCapture {
|
|
49
|
+
cookies: RawCdpCookie[];
|
|
50
|
+
origins: RawOriginLocalStorage[];
|
|
51
|
+
originsUnreadable: number;
|
|
52
|
+
}
|
|
53
|
+
/** Every effectful step captureSession needs — overridable for tests. */
|
|
54
|
+
export interface CaptureDeps {
|
|
55
|
+
detectNonInteractive(): string | null;
|
|
56
|
+
findChrome(): string | null;
|
|
57
|
+
mkdtemp(): string;
|
|
58
|
+
rmSync(dir: string): void;
|
|
59
|
+
launch(chromePath: string, targetUrl: string, userDataDir: string): Promise<{
|
|
60
|
+
proc: ChildProcess;
|
|
61
|
+
cdp: CdpPipe;
|
|
62
|
+
}>;
|
|
63
|
+
/** Resolve once Chrome has opened the target URL as a page target. */
|
|
64
|
+
findPageTarget(cdp: CdpPipe): Promise<string>;
|
|
65
|
+
/**
|
|
66
|
+
* Race the "done" signal: Enter in the terminal, or the tracked tab
|
|
67
|
+
* being closed. Resolves with whether the tab closed before Enter.
|
|
68
|
+
* `cleanup` MUST be wired to the readline Interface's own `'SIGINT'`
|
|
69
|
+
* event (not just `process.on('SIGINT', …)`) — a TTY readline prompt
|
|
70
|
+
* puts stdin in raw mode, which stops Ctrl-C from ever generating a real
|
|
71
|
+
* OS SIGINT in the first place; only the Interface itself observes the
|
|
72
|
+
* raw 0x03 byte and re-synthesizes it as its own `'SIGINT'` event.
|
|
73
|
+
*/
|
|
74
|
+
waitForDone(cdp: CdpPipe, targetId: string, cleanup: () => void): Promise<{
|
|
75
|
+
closedEarly: boolean;
|
|
76
|
+
}>;
|
|
77
|
+
/** Read the full (unscoped) cookie jar + open pages' localStorage. */
|
|
78
|
+
readCapture(cdp: CdpPipe, targetUrl: string, closedEarly: boolean): Promise<RawCapture>;
|
|
79
|
+
/**
|
|
80
|
+
* Register `cleanup` to run on every ORDINARY terminating signal this
|
|
81
|
+
* process can still run code for: SIGINT, SIGTERM, and SIGHUP
|
|
82
|
+
* (trawl_cli#183 review finding 1 — SIGTERM/SIGHUP were missing
|
|
83
|
+
* entirely, so `timeout` without `--signal`, a bare `kill <pid>`, most
|
|
84
|
+
* supervisors, and closing the terminal tab mid-login all skipped
|
|
85
|
+
* cleanup and left Chrome running with the profile on disk). SIGKILL is
|
|
86
|
+
* the only signal that cannot be handled at all, by any process, ever —
|
|
87
|
+
* see this function's own default implementation
|
|
88
|
+
* (`registerTerminationHandlers`) for that boundary spelled out.
|
|
89
|
+
* Returns an unregister function.
|
|
90
|
+
*/
|
|
91
|
+
onSigint(cleanup: () => void): () => void;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* The three real (non-fake) CDP orchestration steps, exported for direct
|
|
95
|
+
* testing against a minimal fake CdpPipe — session-capture.test.ts covers
|
|
96
|
+
* the OUTER guard/race/cleanup logic with these swapped out entirely;
|
|
97
|
+
* session-capture-defaults.test.ts covers these directly instead, since
|
|
98
|
+
* "test the wiring" and "test the wired thing" are two different jobs.
|
|
99
|
+
*/
|
|
100
|
+
export declare function findPageTargetDefault(cdp: CdpPipe): Promise<string>;
|
|
101
|
+
export declare function waitForDoneDefault(cdp: CdpPipe, targetId: string, cleanup: () => void): Promise<{
|
|
102
|
+
closedEarly: boolean;
|
|
103
|
+
}>;
|
|
104
|
+
/**
|
|
105
|
+
* @desc The deadline for the ONE CDP command in this file that runs inside
|
|
106
|
+
* a page's own renderer rather than Chrome's browser process:
|
|
107
|
+
* `Runtime.evaluate` reading `window.localStorage`. Deliberately the SAME
|
|
108
|
+
* 30s as `CdpPipe`'s own browser-process default, not a shorter override —
|
|
109
|
+
* a post-cap review (trawl_cli#183) found that an earlier 5s value here cut
|
|
110
|
+
* off a real, terminating synchronous computation (the shape of an
|
|
111
|
+
* anti-bot challenge solve) at ~5s: `originsUnreadable:1` for a page that
|
|
112
|
+
* would have captured cleanly at 7s. There is no value that is provably
|
|
113
|
+
* "long enough" — a renderer truly blocked forever (a native dialog, a
|
|
114
|
+
* paused debugger) costs the same whether the ceiling is 5s or 30s, while
|
|
115
|
+
* a renderer doing bounded work of unknown-but-finite length keeps a
|
|
116
|
+
* chance of finishing for as long as this stays generous. 30s is a
|
|
117
|
+
* JUDGEMENT call on that trade-off, not a provably-correct number — kept
|
|
118
|
+
* equal to the pipe's own default so this file doesn't invent a second
|
|
119
|
+
* number to defend. Passed explicitly to `send()` anyway (rather than
|
|
120
|
+
* relying on the pipe's own default silently matching) so the deadline
|
|
121
|
+
* stays a named, assertable constant here regardless of what the pipe
|
|
122
|
+
* default happens to be. Whatever the outcome, it is always COUNTED and
|
|
123
|
+
* SURFACED (`originsUnreadable`, plus the human-facing ⚠ line in
|
|
124
|
+
* scraps.ts) — never silently swallowed; see `readCaptureDefault` below
|
|
125
|
+
* for the stderr progress line that covers the wait itself.
|
|
126
|
+
*/
|
|
127
|
+
export declare const LOCALSTORAGE_READ_TIMEOUT_MS = 30000;
|
|
128
|
+
export declare function readCaptureDefault(cdp: CdpPipe, targetUrl: string, closedEarly: boolean): Promise<RawCapture>;
|
|
129
|
+
/**
|
|
130
|
+
* @desc `rmSync` with a short, bounded, SYNCHRONOUS retry (max ~200ms
|
|
131
|
+
* total). Verified against a real Chrome launch (trawl_cli#183's loopback
|
|
132
|
+
* E2E): even after killing Chrome's whole process group (see
|
|
133
|
+
* chrome-launch.ts's `detached`), a helper subprocess can hold a file
|
|
134
|
+
* under the profile open for a few milliseconds after the kill signal is
|
|
135
|
+
* delivered — a bare `rmSync` right after `kill()` lost that race and
|
|
136
|
+
* silently leaked the temp profile every time. Stays synchronous
|
|
137
|
+
* (`Atomics.wait`, no `await`) because cleanup must be callable from a
|
|
138
|
+
* signal handler right before `process.exit()`.
|
|
139
|
+
*/
|
|
140
|
+
export declare function rmSyncWithRetry(dir: string): void;
|
|
141
|
+
/**
|
|
142
|
+
* @desc The default `onSigint` implementation, exported for direct testing
|
|
143
|
+
* (same pattern as `findPageTargetDefault`/`waitForDoneDefault`/
|
|
144
|
+
* `readCaptureDefault` above — this file's own convention keeps "test the
|
|
145
|
+
* wiring" and "test the wired thing" separate). Registers `cleanup` on
|
|
146
|
+
* SIGINT, SIGTERM, AND SIGHUP (trawl_cli#183 review finding 1 — only
|
|
147
|
+
* SIGINT was registered before this; `timeout` without `--signal`, a bare
|
|
148
|
+
* `kill <pid>`, most supervisors, and closing the terminal tab mid-login
|
|
149
|
+
* all default-terminate a process via SIGTERM/SIGHUP, and Node runs no
|
|
150
|
+
* `finally` for a default-terminated process). SIGKILL is the one signal
|
|
151
|
+
* this — or any handler, in any process — cannot observe: the OS tears
|
|
152
|
+
* the process down directly, no userspace code runs at all, so Chrome and
|
|
153
|
+
* the temp profile are left behind whenever that specific signal is what
|
|
154
|
+
* ends this process. That gap is inherent, not something a different
|
|
155
|
+
* signal list here could close.
|
|
156
|
+
*/
|
|
157
|
+
export declare function registerTerminationHandlers(cleanup: () => void): () => void;
|
|
158
|
+
/**
|
|
159
|
+
* @desc The same guard `captureSession` runs internally as its very first
|
|
160
|
+
* step, exposed so the command layer can call it BEFORE printing anything
|
|
161
|
+
* about a Chrome window that may never open. Without this, `scraps.ts`
|
|
162
|
+
* printed its "a Chrome window will open" banner unconditionally, ahead of
|
|
163
|
+
* this exact check inside `captureSession` — so a non-interactive run saw
|
|
164
|
+
* that banner immediately followed by "this cannot work headless"
|
|
165
|
+
* (trawl_cli#183 gap). Reads live process state once; pure
|
|
166
|
+
* `detectNonInteractive` stays the single source of truth for both call
|
|
167
|
+
* sites.
|
|
168
|
+
* @returns {string | null} a factual reason when this command cannot
|
|
169
|
+
* proceed in the current environment, or null when it can.
|
|
170
|
+
*/
|
|
171
|
+
export declare function checkInteractiveEnvironment(): string | null;
|
|
172
|
+
/**
|
|
173
|
+
* @desc Run the full capture flow for one scrap's target URL.
|
|
174
|
+
* @param {string} targetUrl
|
|
175
|
+
* @param {Partial<CaptureDeps>} depsOverride — for tests; real deps fill in
|
|
176
|
+
* the rest.
|
|
177
|
+
* @returns {Promise<CaptureResult>}
|
|
178
|
+
*/
|
|
179
|
+
export declare function captureSession(targetUrl: string, depsOverride?: Partial<CaptureDeps>): Promise<CaptureResult>;
|
|
180
|
+
export {};
|