@trawlme/cli 3.12.0 → 3.12.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +2 -2
  2. package/dist/commands/create.d.ts +0 -28
  3. package/dist/commands/create.js +0 -89
  4. package/dist/commands/doctor.d.ts +0 -79
  5. package/dist/commands/doctor.js +1 -187
  6. package/dist/commands/login.js +0 -67
  7. package/dist/commands/ping.d.ts +0 -15
  8. package/dist/commands/ping.js +0 -15
  9. package/dist/commands/scraps.d.ts +0 -120
  10. package/dist/commands/scraps.js +10 -724
  11. package/dist/commands/skills.js +0 -22
  12. package/dist/commands/spec.d.ts +0 -85
  13. package/dist/commands/spec.js +0 -67
  14. package/dist/commands/telemetry.js +0 -4
  15. package/dist/commands/token.js +0 -28
  16. package/dist/commands/upgrade.js +0 -22
  17. package/dist/commands/whoami.d.ts +0 -12
  18. package/dist/commands/whoami.js +0 -6
  19. package/dist/index.d.ts +0 -188
  20. package/dist/index.js +0 -349
  21. package/dist/lib/api.d.ts +0 -78
  22. package/dist/lib/api.js +1 -320
  23. package/dist/lib/cdp-pipe.d.ts +0 -72
  24. package/dist/lib/cdp-pipe.js +1 -81
  25. package/dist/lib/chrome-discovery.d.ts +0 -11
  26. package/dist/lib/chrome-discovery.js +0 -19
  27. package/dist/lib/chrome-launch.d.ts +0 -40
  28. package/dist/lib/chrome-launch.js +0 -69
  29. package/dist/lib/config.d.ts +0 -53
  30. package/dist/lib/config.js +0 -55
  31. package/dist/lib/confirm.d.ts +0 -55
  32. package/dist/lib/confirm.js +0 -47
  33. package/dist/lib/docs.d.ts +0 -123
  34. package/dist/lib/docs.js +0 -169
  35. package/dist/lib/errors.d.ts +0 -134
  36. package/dist/lib/errors.js +0 -151
  37. package/dist/lib/format.d.ts +0 -6
  38. package/dist/lib/format.js +0 -6
  39. package/dist/lib/json.d.ts +0 -35
  40. package/dist/lib/json.js +0 -48
  41. package/dist/lib/jwt.d.ts +0 -7
  42. package/dist/lib/jwt.js +0 -7
  43. package/dist/lib/pinch.d.ts +0 -53
  44. package/dist/lib/pinch.js +6 -112
  45. package/dist/lib/pinchAnimation.d.ts +0 -16
  46. package/dist/lib/pinchAnimation.js +8 -29
  47. package/dist/lib/posthog.d.ts +0 -9
  48. package/dist/lib/posthog.js +0 -23
  49. package/dist/lib/prompt.js +1 -20
  50. package/dist/lib/secure-transport.d.ts +0 -7
  51. package/dist/lib/secure-transport.js +0 -24
  52. package/dist/lib/session-capture-guard.d.ts +0 -15
  53. package/dist/lib/session-capture-guard.js +0 -5
  54. package/dist/lib/session-capture.d.ts +0 -125
  55. package/dist/lib/session-capture.js +0 -281
  56. package/dist/lib/skills.d.ts +0 -175
  57. package/dist/lib/skills.js +1 -216
  58. package/dist/lib/skillsNudge.d.ts +0 -17
  59. package/dist/lib/skillsNudge.js +0 -83
  60. package/dist/lib/spinner.d.ts +0 -39
  61. package/dist/lib/spinner.js +0 -40
  62. package/dist/lib/storage-state.d.ts +0 -112
  63. package/dist/lib/storage-state.js +0 -131
  64. package/dist/lib/tips.d.ts +0 -38
  65. package/dist/lib/tips.js +0 -77
  66. package/dist/lib/updateCheckWorker.js +0 -14
  67. package/dist/lib/updateNotifier.d.ts +0 -17
  68. package/dist/lib/updateNotifier.js +0 -53
  69. package/dist/lib/validate.d.ts +0 -8
  70. package/dist/lib/validate.js +0 -8
  71. package/dist/lib/version.d.ts +0 -12
  72. package/dist/lib/version.js +1 -13
  73. package/docs/agent-quickstart.md +2 -2
  74. package/package.json +2 -2
@@ -7,31 +7,7 @@ export interface CaptureCounts {
7
7
  cookiesDroppedInvalid: number;
8
8
  originsCaptured: number;
9
9
  originsDroppedOutOfScope: number;
10
- /** An in-scope page's localStorage could not be captured, for one of
11
- * four reasons — never a CLI bug, which propagates as a failed capture
12
- * instead (trawl_cli#183 post-cap review finding B, sharpened by the R4
13
- * fix below): a `readPageOverCdp` rejection — ANY CDP-level rejection
14
- * scoped to THIS page's `attachToTarget`/`Runtime.evaluate` round-trip,
15
- * whatever its error class (a timeout because the page's renderer was
16
- * blocked — a native dialog, a synchronous script, a paused debugger; or
17
- * a plain rejection because the target closed mid-attach/mid-evaluate,
18
- * e.g. an OAuth popup the human closed at the wrong instant); the read
19
- * threw INSIDE the page (e.g. a `SecurityError` on partitioned/sandboxed
20
- * storage); or `Runtime.evaluate` succeeded with no exception but
21
- * returned a value that doesn't match the `{name,value}[]` shape asked
22
- * for (a page tampering with a global to hand back garbage without ever
23
- * throwing). The discriminant is PROVENANCE, not error type: did the
24
- * rejection come out of a CDP call scoped to this one page, or out of
25
- * this file's OWN code running on data CDP already handed back? Only the
26
- * latter is a CLI bug, and only the latter propagates. None of these
27
- * four is the same fact as "0 keys": CDP itself was reachable (or the
28
- * pipe wouldn't still be open), the origin just could not be read.
29
- * Counted, never the exception text/URL/value itself (a page controls
30
- * that content). */
31
10
  originsUnreadable: number;
32
- /** The tracked tab was closed before Enter — cookies were still read,
33
- * localStorage was not (the completion path the issue calls out:
34
- * "closing the window also completes"). */
35
11
  closedEarly: boolean;
36
12
  }
37
13
  export type CaptureFailureReason = 'non_interactive' | 'no_chrome' | 'launch_failed' | 'capture_failed' | 'process_exited_before_capture' | 'cdp_protocol_error' | 'no_cookies_in_scope';
@@ -50,7 +26,6 @@ interface RawCapture {
50
26
  origins: RawOriginLocalStorage[];
51
27
  originsUnreadable: number;
52
28
  }
53
- /** Every effectful step captureSession needs — overridable for tests. */
54
29
  export interface CaptureDeps {
55
30
  detectNonInteractive(): string | null;
56
31
  findChrome(): string | null;
@@ -60,121 +35,21 @@ export interface CaptureDeps {
60
35
  proc: ChildProcess;
61
36
  cdp: CdpPipe;
62
37
  }>;
63
- /** Resolve once Chrome has opened the target URL as a page target. */
64
38
  findPageTarget(cdp: CdpPipe): Promise<string>;
65
- /**
66
- * Race the "done" signal: Enter in the terminal, or the tracked tab
67
- * being closed. Resolves with whether the tab closed before Enter.
68
- * `cleanup` MUST be wired to the readline Interface's own `'SIGINT'`
69
- * event (not just `process.on('SIGINT', …)`) — a TTY readline prompt
70
- * puts stdin in raw mode, which stops Ctrl-C from ever generating a real
71
- * OS SIGINT in the first place; only the Interface itself observes the
72
- * raw 0x03 byte and re-synthesizes it as its own `'SIGINT'` event.
73
- */
74
39
  waitForDone(cdp: CdpPipe, targetId: string, cleanup: () => void): Promise<{
75
40
  closedEarly: boolean;
76
41
  }>;
77
- /** Read the full (unscoped) cookie jar + open pages' localStorage. */
78
42
  readCapture(cdp: CdpPipe, targetUrl: string, closedEarly: boolean): Promise<RawCapture>;
79
- /**
80
- * Register `cleanup` to run on every ORDINARY terminating signal this
81
- * process can still run code for: SIGINT, SIGTERM, and SIGHUP
82
- * (trawl_cli#183 review finding 1 — SIGTERM/SIGHUP were missing
83
- * entirely, so `timeout` without `--signal`, a bare `kill <pid>`, most
84
- * supervisors, and closing the terminal tab mid-login all skipped
85
- * cleanup and left Chrome running with the profile on disk). SIGKILL is
86
- * the only signal that cannot be handled at all, by any process, ever —
87
- * see this function's own default implementation
88
- * (`registerTerminationHandlers`) for that boundary spelled out.
89
- * Returns an unregister function.
90
- */
91
43
  onSigint(cleanup: () => void): () => void;
92
44
  }
93
- /**
94
- * The three real (non-fake) CDP orchestration steps, exported for direct
95
- * testing against a minimal fake CdpPipe — session-capture.test.ts covers
96
- * the OUTER guard/race/cleanup logic with these swapped out entirely;
97
- * session-capture-defaults.test.ts covers these directly instead, since
98
- * "test the wiring" and "test the wired thing" are two different jobs.
99
- */
100
45
  export declare function findPageTargetDefault(cdp: CdpPipe): Promise<string>;
101
46
  export declare function waitForDoneDefault(cdp: CdpPipe, targetId: string, cleanup: () => void): Promise<{
102
47
  closedEarly: boolean;
103
48
  }>;
104
- /**
105
- * @desc The deadline for the ONE CDP command in this file that runs inside
106
- * a page's own renderer rather than Chrome's browser process:
107
- * `Runtime.evaluate` reading `window.localStorage`. Deliberately the SAME
108
- * 30s as `CdpPipe`'s own browser-process default, not a shorter override —
109
- * a post-cap review (trawl_cli#183) found that an earlier 5s value here cut
110
- * off a real, terminating synchronous computation (the shape of an
111
- * anti-bot challenge solve) at ~5s: `originsUnreadable:1` for a page that
112
- * would have captured cleanly at 7s. There is no value that is provably
113
- * "long enough" — a renderer truly blocked forever (a native dialog, a
114
- * paused debugger) costs the same whether the ceiling is 5s or 30s, while
115
- * a renderer doing bounded work of unknown-but-finite length keeps a
116
- * chance of finishing for as long as this stays generous. 30s is a
117
- * JUDGEMENT call on that trade-off, not a provably-correct number — kept
118
- * equal to the pipe's own default so this file doesn't invent a second
119
- * number to defend. Passed explicitly to `send()` anyway (rather than
120
- * relying on the pipe's own default silently matching) so the deadline
121
- * stays a named, assertable constant here regardless of what the pipe
122
- * default happens to be. Whatever the outcome, it is always COUNTED and
123
- * SURFACED (`originsUnreadable`, plus the human-facing ⚠ line in
124
- * scraps.ts) — never silently swallowed; see `readCaptureDefault` below
125
- * for the stderr progress line that covers the wait itself.
126
- */
127
49
  export declare const LOCALSTORAGE_READ_TIMEOUT_MS = 30000;
128
50
  export declare function readCaptureDefault(cdp: CdpPipe, targetUrl: string, closedEarly: boolean): Promise<RawCapture>;
129
- /**
130
- * @desc `rmSync` with a short, bounded, SYNCHRONOUS retry (max ~200ms
131
- * total). Verified against a real Chrome launch (trawl_cli#183's loopback
132
- * E2E): even after killing Chrome's whole process group (see
133
- * chrome-launch.ts's `detached`), a helper subprocess can hold a file
134
- * under the profile open for a few milliseconds after the kill signal is
135
- * delivered — a bare `rmSync` right after `kill()` lost that race and
136
- * silently leaked the temp profile every time. Stays synchronous
137
- * (`Atomics.wait`, no `await`) because cleanup must be callable from a
138
- * signal handler right before `process.exit()`.
139
- */
140
51
  export declare function rmSyncWithRetry(dir: string): void;
141
- /**
142
- * @desc The default `onSigint` implementation, exported for direct testing
143
- * (same pattern as `findPageTargetDefault`/`waitForDoneDefault`/
144
- * `readCaptureDefault` above — this file's own convention keeps "test the
145
- * wiring" and "test the wired thing" separate). Registers `cleanup` on
146
- * SIGINT, SIGTERM, AND SIGHUP (trawl_cli#183 review finding 1 — only
147
- * SIGINT was registered before this; `timeout` without `--signal`, a bare
148
- * `kill <pid>`, most supervisors, and closing the terminal tab mid-login
149
- * all default-terminate a process via SIGTERM/SIGHUP, and Node runs no
150
- * `finally` for a default-terminated process). SIGKILL is the one signal
151
- * this — or any handler, in any process — cannot observe: the OS tears
152
- * the process down directly, no userspace code runs at all, so Chrome and
153
- * the temp profile are left behind whenever that specific signal is what
154
- * ends this process. That gap is inherent, not something a different
155
- * signal list here could close.
156
- */
157
52
  export declare function registerTerminationHandlers(cleanup: () => void): () => void;
158
- /**
159
- * @desc The same guard `captureSession` runs internally as its very first
160
- * step, exposed so the command layer can call it BEFORE printing anything
161
- * about a Chrome window that may never open. Without this, `scraps.ts`
162
- * printed its "a Chrome window will open" banner unconditionally, ahead of
163
- * this exact check inside `captureSession` — so a non-interactive run saw
164
- * that banner immediately followed by "this cannot work headless"
165
- * (trawl_cli#183 gap). Reads live process state once; pure
166
- * `detectNonInteractive` stays the single source of truth for both call
167
- * sites.
168
- * @returns {string | null} a factual reason when this command cannot
169
- * proceed in the current environment, or null when it can.
170
- */
171
53
  export declare function checkInteractiveEnvironment(): string | null;
172
- /**
173
- * @desc Run the full capture flow for one scrap's target URL.
174
- * @param {string} targetUrl
175
- * @param {Partial<CaptureDeps>} depsOverride — for tests; real deps fill in
176
- * the rest.
177
- * @returns {Promise<CaptureResult>}
178
- */
179
54
  export declare function captureSession(targetUrl: string, depsOverride?: Partial<CaptureDeps>): Promise<CaptureResult>;
180
55
  export {};
@@ -1,24 +1,3 @@
1
- /**
2
- * Orchestrates `trawl scraps account session capture <id>` (trawl_cli#183):
3
- * launch a headed, isolated Chrome; let a human log in (2FA included —
4
- * Trawl/the agent never sees credentials); read the resulting session over
5
- * CDP; scope it to the target domain; hand back a storageState-shaped
6
- * result. Does NOT talk to the Trawl API — the command layer
7
- * (commands/scraps.ts) owns the `PUT /api/scraps/:id/account/session`
8
- * call, same as every other command in this CLI owns its own `api.*`
9
- * calls.
10
- *
11
- * Every I/O boundary (Chrome discovery/launch, the CDP calls themselves,
12
- * the temp profile, the "done" prompt) is behind the `Deps` interface so
13
- * the orchestration logic — the guard, the race, cleanup-always, count
14
- * assembly — is fully testable without a real browser. `captureSession`
15
- * itself never throws for an EXPECTED failure (no chrome, non-interactive,
16
- * zero in-scope cookies) — those come back as `{ ok: false, reason,
17
- * message }` for the command layer to map to the right exit code. An
18
- * UNEXPECTED failure (Chrome crashed mid-flight, a CDP call errored) is
19
- * caught and returned the same way rather than propagated, so cleanup
20
- * always still runs via the single `finally`.
21
- */
22
1
  import { mkdtempSync, rmSync } from 'node:fs';
23
2
  import { tmpdir } from 'node:os';
24
3
  import { join } from 'node:path';
@@ -27,23 +6,10 @@ import { findChrome } from './chrome-discovery.js';
27
6
  import { launchChrome } from './chrome-launch.js';
28
7
  import { detectNonInteractive } from './session-capture-guard.js';
29
8
  import { mapCookies, mapOrigins, getTargetHost, isOriginHostInScope, } from './storage-state.js';
30
- /** Shared between the two call sites that can observe Chrome having
31
- * exited before a capture completed (mid-wait, and mid-readCapture). */
32
9
  const PROCESS_EXITED_MESSAGE = 'Chrome exited before the session was read — nothing uploaded. The capture completes on Enter in the terminal; Chrome exiting first ends the run without a session.';
33
- /** Shared between the two call sites that can observe a corrupt CDP frame
34
- * tearing the pipe down (mid-wait, and mid-readCapture) — trawl_cli#183
35
- * review finding 3. `protocolError.message` is internal, factual text
36
- * with no page-controlled content (see cdp-pipe.ts's onProtocolError). */
37
10
  function cdpProtocolErrorMessage(protocolError) {
38
11
  return `${protocolError.message} Nothing was uploaded.`;
39
12
  }
40
- /**
41
- * The three real (non-fake) CDP orchestration steps, exported for direct
42
- * testing against a minimal fake CdpPipe — session-capture.test.ts covers
43
- * the OUTER guard/race/cleanup logic with these swapped out entirely;
44
- * session-capture-defaults.test.ts covers these directly instead, since
45
- * "test the wiring" and "test the wired thing" are two different jobs.
46
- */
47
13
  export async function findPageTargetDefault(cdp) {
48
14
  const deadline = Date.now() + 10_000;
49
15
  for (;;) {
@@ -70,13 +36,6 @@ export async function waitForDoneDefault(cdp, targetId, cleanup) {
70
36
  resolve({ closedEarly: true });
71
37
  }
72
38
  });
73
- // If the whole Chrome process dies mid-wait (crash, or a platform where
74
- // closing the last window quits it) no `targetDestroyed` event can ever
75
- // arrive on a dead pipe — without this, the prompt sits there forever
76
- // giving the human no feedback. `captureSession` checks its own
77
- // `pipeClosed` flag right after this resolves and turns it into the
78
- // factual `process_exited_before_capture` result immediately, rather
79
- // than waiting on a keypress that was never going to fix anything.
80
39
  const offClose = cdp.onClose(() => {
81
40
  if (!settled) {
82
41
  settled = true;
@@ -86,9 +45,6 @@ export async function waitForDoneDefault(cdp, targetId, cleanup) {
86
45
  }
87
46
  });
88
47
  const rl = createInterface({ input: process.stdin, output: process.stderr, terminal: true });
89
- // Ctrl-C during this prompt never reaches `process` as a real SIGINT —
90
- // raw-mode TTY input disables signal generation for it — so the
91
- // Interface's own synthesized 'SIGINT' is the only place to catch it.
92
48
  rl.on('SIGINT', () => {
93
49
  settled = true;
94
50
  offDestroyed();
@@ -108,73 +64,12 @@ export async function waitForDoneDefault(cdp, targetId, cleanup) {
108
64
  });
109
65
  });
110
66
  }
111
- /**
112
- * @desc The deadline for the ONE CDP command in this file that runs inside
113
- * a page's own renderer rather than Chrome's browser process:
114
- * `Runtime.evaluate` reading `window.localStorage`. Deliberately the SAME
115
- * 30s as `CdpPipe`'s own browser-process default, not a shorter override —
116
- * a post-cap review (trawl_cli#183) found that an earlier 5s value here cut
117
- * off a real, terminating synchronous computation (the shape of an
118
- * anti-bot challenge solve) at ~5s: `originsUnreadable:1` for a page that
119
- * would have captured cleanly at 7s. There is no value that is provably
120
- * "long enough" — a renderer truly blocked forever (a native dialog, a
121
- * paused debugger) costs the same whether the ceiling is 5s or 30s, while
122
- * a renderer doing bounded work of unknown-but-finite length keeps a
123
- * chance of finishing for as long as this stays generous. 30s is a
124
- * JUDGEMENT call on that trade-off, not a provably-correct number — kept
125
- * equal to the pipe's own default so this file doesn't invent a second
126
- * number to defend. Passed explicitly to `send()` anyway (rather than
127
- * relying on the pipe's own default silently matching) so the deadline
128
- * stays a named, assertable constant here regardless of what the pipe
129
- * default happens to be. Whatever the outcome, it is always COUNTED and
130
- * SURFACED (`originsUnreadable`, plus the human-facing ⚠ line in
131
- * scraps.ts) — never silently swallowed; see `readCaptureDefault` below
132
- * for the stderr progress line that covers the wait itself.
133
- */
134
67
  export const LOCALSTORAGE_READ_TIMEOUT_MS = 30_000;
135
- /** How long to wait, mid-read, before printing a factual "still waiting"
136
- * progress line to stderr — a human who just pressed Enter and is now
137
- * staring at silence for up to `LOCALSTORAGE_READ_TIMEOUT_MS` has no way
138
- * to tell "still working" from "hung". Well under the read deadline so it
139
- * fires long before the read could time out; never printed for a read
140
- * that resolves before this fires (cleared in a `finally`). */
141
68
  const LOCALSTORAGE_READ_PROGRESS_MS = 5_000;
142
- /**
143
- * @desc Structural guard for `Runtime.evaluate`'s returned value (see
144
- * `readCaptureDefault`'s call site) — trawl_cli#183 post-cap review finding
145
- * B. `returnByValue: true` means CDP itself, not page script, produced this
146
- * value, but a page can still make the EXPRESSION return arbitrary garbage
147
- * without throwing (tampering with `Object.entries`, `Array.prototype.map`,
148
- * etc.) — this is the backstop for that, not a claim that the expression
149
- * itself is un-tamperable. Anything that fails this check is treated as
150
- * page-controlled misbehavior, counted the same as a thrown read.
151
- */
152
69
  function isLocalStorageEntriesShape(value) {
153
70
  return (Array.isArray(value) &&
154
71
  value.every((e) => typeof e === 'object' && e !== null && typeof e.name === 'string' && typeof e.value === 'string'));
155
72
  }
156
- /**
157
- * @desc Phase 1 of the per-page read (trawl_cli#183 R4 fix): every CDP
158
- * round-trip SCOPED TO ONE PAGE — attach, the localStorage evaluate,
159
- * detach. Every round this catch block went through classified errors by
160
- * TYPE (a `CdpTimeoutError` counts, "anything else" rethrows) — the wrong
161
- * criterion, because a plain `Error` is exactly what BOTH a page-caused CDP
162
- * rejection and a CLI bug look like; no error-class taxonomy tells them
163
- * apart. The right discriminant is PROVENANCE: did the rejection come out
164
- * of a CDP call for this page, or out of this file's own code running on
165
- * data CDP already handed back? This function's OWN BOUNDARY draws that
166
- * line structurally instead — `readCaptureDefault` treats ANY rejection
167
- * out of here as page-caused, whatever its error class: a `CdpTimeoutError`
168
- * (the renderer never answered — a native dialog, a synchronous script, a
169
- * paused debugger), or a plain rejection because the target closed
170
- * mid-attach/mid-evaluate (reproduced against real Chrome: a second
171
- * in-scope page — an OAuth popup, say — closing at the moment the loop
172
- * reaches it rejects `Target.attachToTarget` with a plain
173
- * `Error("No target with given id found (code -32602)")`, pipe still open,
174
- * not a timeout). The one exception `readCaptureDefault` still special-
175
- * cases on the way out is `cdp.isClosed` — that means the PIPE itself
176
- * died, not this page, and belongs to every other page too.
177
- */
178
73
  async function readPageOverCdp(cdp, targetId) {
179
74
  let sessionId;
180
75
  try {
@@ -183,28 +78,11 @@ async function readPageOverCdp(cdp, targetId) {
183
78
  flatten: true,
184
79
  });
185
80
  sessionId = attached.sessionId;
186
- // A human who just pressed Enter has no signal at all while this one
187
- // call is in flight — print a factual "still waiting" line to stderr
188
- // if it runs past LOCALSTORAGE_READ_PROGRESS_MS, so a bounded-but-slow
189
- // page (or a genuinely blocked one) doesn't read as a hung CLI.
190
- // Cleared unconditionally in the inner `finally` so it never fires
191
- // for a read that already resolved. Never the page's URL/title —
192
- // only the fact that a read is in progress.
193
81
  const progressTimer = setTimeout(() => {
194
82
  process.stderr.write(` still waiting for a page to finish its scripts (up to ${LOCALSTORAGE_READ_TIMEOUT_MS / 1000}s)…\n`);
195
83
  }, LOCALSTORAGE_READ_PROGRESS_MS);
196
84
  try {
197
85
  return await cdp.send('Runtime.evaluate', {
198
- // Builds the {name,value}[] array directly and returns it via
199
- // CDP's OWN serialisation (`returnByValue: true`) rather than
200
- // calling the page's `JSON.stringify` and parsing the result
201
- // ourselves — trawl_cli#183 post-cap review finding B: a page
202
- // that overrides `window.JSON.stringify` to return `"null"`
203
- // made our own later `JSON.parse` + `.length` throw a TypeError
204
- // that used to be laundered into `originsUnreadable`,
205
- // indistinguishable from an ordinary blocked-tab capture.
206
- // Chrome's inspector serialiser cannot be tampered with from
207
- // page script the way `JSON.stringify` can.
208
86
  expression: 'Object.entries(window.localStorage).map(([name, value]) => ({ name, value }))',
209
87
  returnByValue: true,
210
88
  }, sessionId, LOCALSTORAGE_READ_TIMEOUT_MS);
@@ -219,8 +97,6 @@ async function readPageOverCdp(cdp, targetId) {
219
97
  await cdp.send('Target.detachFromTarget', { sessionId });
220
98
  }
221
99
  catch {
222
- // already detached / target gone — non-fatal either way: this
223
- // page's origin is already about to be counted or captured
224
100
  }
225
101
  }
226
102
  }
@@ -245,23 +121,6 @@ export async function readCaptureDefault(cdp, targetUrl, closedEarly) {
245
121
  catch {
246
122
  continue;
247
123
  }
248
- // Phase 1 — readPageOverCdp above; see its own doc comment. Any
249
- // rejection out of it is page-caused BY PROVENANCE, not by error class
250
- // (trawl_cli#183 R4) — count it and move on to the next page. The one
251
- // exception is `cdp.isClosed`: if the WHOLE pipe was torn down (Chrome
252
- // exited, or a corrupt frame — trawl_cli#183 review finding 3), every
253
- // remaining `cdp.send` in this loop would reject the same way;
254
- // swallowing that here would finish the loop and report an apparently-
255
- // successful capture with fewer origins than real. Rethrown so
256
- // captureSession's own pipeClosed/protocolError handling reports the
257
- // actual cause. Checked unconditionally, before looking at the
258
- // rejection at all — a dead pipe is true regardless of which error
259
- // under it happens to be surfacing, including a `CdpTimeoutError` that
260
- // raced a teardown on the same tick (a `CdpProtocolError` teardown
261
- // flips `isClosed` before it rejects any pending command — see
262
- // cdp-pipe.ts's `onClosed` — so this one check already covers both
263
- // causes; there is no separate `instanceof CdpProtocolError` branch to
264
- // maintain).
265
124
  let evalResult;
266
125
  try {
267
126
  evalResult = await readPageOverCdp(cdp, page.targetId);
@@ -273,42 +132,12 @@ export async function readCaptureDefault(cdp, targetUrl, closedEarly) {
273
132
  process.stderr.write(' one in-scope page could not be read over CDP — counted, continuing…\n');
274
133
  continue;
275
134
  }
276
- // Phase 2 — post-processing: a decision THIS FILE's OWN code makes on
277
- // data CDP already handed back, never another CDP call. Deliberately
278
- // OUTSIDE the try/catch above (trawl_cli#183 R4 fix): a bug in this
279
- // code must propagate as a genuinely FAILED capture, never get caught
280
- // by the page-provenance catch above and laundered into one more
281
- // `originsUnreadable` count indistinguishable from an ordinary blocked
282
- // tab. That laundering is exactly the bug class R3 fixed once already
283
- // (trawl_cli#183 post-cap review finding B) — classifying by error TYPE
284
- // inside a single catch cannot keep it fixed, because a plain `Error`
285
- // is exactly what BOTH a page-caused CDP rejection (R4) and a CLI bug
286
- // (R3) look like; a FUNCTION BOUNDARY can, because Phase 2 never runs
287
- // inside Phase 1's `try`. Page garbage is still handled here, by the
288
- // structural `isLocalStorageEntriesShape` guard below — but that is a
289
- // decision this code makes on data, not an exception this code merely
290
- // swallows.
291
135
  if (evalResult.exceptionDetails) {
292
- // Runtime.evaluate SUCCEEDED at the CDP layer but the expression
293
- // threw INSIDE the page (e.g. a SecurityError on partitioned/
294
- // sandboxed storage) — result.value is then undefined, which used
295
- // to be silently read as "0 keys" and the origin dropped, making a
296
- // blocked read indistinguishable from genuinely empty localStorage
297
- // (trawl_cli#183 review finding 4). Counted separately; never the
298
- // exception text itself — a page controls that string.
299
136
  originsUnreadable++;
300
137
  continue;
301
138
  }
302
139
  const value = evalResult.result?.value;
303
140
  if (!isLocalStorageEntriesShape(value)) {
304
- // Runtime.evaluate raised no page-side exception and CDP's own
305
- // serialiser did its job, but the returned value is not the
306
- // `{name,value}[]` shape the expression asked for — a page can
307
- // still tamper with a global to hand back arbitrary garbage WITHOUT
308
- // ever throwing (trawl_cli#183 post-cap review finding B). That is
309
- // page-controlled misbehavior, exactly the same actionable fact as a
310
- // thrown read — count it, never trust the shape blindly, and never
311
- // surface its content (a page controls it).
312
141
  originsUnreadable++;
313
142
  continue;
314
143
  }
@@ -318,17 +147,6 @@ export async function readCaptureDefault(cdp, targetUrl, closedEarly) {
318
147
  }
319
148
  return { cookies, origins, originsUnreadable };
320
149
  }
321
- /**
322
- * @desc `rmSync` with a short, bounded, SYNCHRONOUS retry (max ~200ms
323
- * total). Verified against a real Chrome launch (trawl_cli#183's loopback
324
- * E2E): even after killing Chrome's whole process group (see
325
- * chrome-launch.ts's `detached`), a helper subprocess can hold a file
326
- * under the profile open for a few milliseconds after the kill signal is
327
- * delivered — a bare `rmSync` right after `kill()` lost that race and
328
- * silently leaked the temp profile every time. Stays synchronous
329
- * (`Atomics.wait`, no `await`) because cleanup must be callable from a
330
- * signal handler right before `process.exit()`.
331
- */
332
150
  export function rmSyncWithRetry(dir) {
333
151
  const MAX_ATTEMPTS = 5;
334
152
  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
@@ -343,30 +161,11 @@ export function rmSyncWithRetry(dir) {
343
161
  }
344
162
  }
345
163
  }
346
- /** The conventional shell exit code for each terminating signal this
347
- * process still handles (128 + signal number) — reused here so a killed
348
- * capture reports the same code a bare, unhandled kill would have. */
349
164
  const TERMINATION_EXIT_CODES = {
350
165
  SIGHUP: 129,
351
166
  SIGINT: 130,
352
167
  SIGTERM: 143,
353
168
  };
354
- /**
355
- * @desc The default `onSigint` implementation, exported for direct testing
356
- * (same pattern as `findPageTargetDefault`/`waitForDoneDefault`/
357
- * `readCaptureDefault` above — this file's own convention keeps "test the
358
- * wiring" and "test the wired thing" separate). Registers `cleanup` on
359
- * SIGINT, SIGTERM, AND SIGHUP (trawl_cli#183 review finding 1 — only
360
- * SIGINT was registered before this; `timeout` without `--signal`, a bare
361
- * `kill <pid>`, most supervisors, and closing the terminal tab mid-login
362
- * all default-terminate a process via SIGTERM/SIGHUP, and Node runs no
363
- * `finally` for a default-terminated process). SIGKILL is the one signal
364
- * this — or any handler, in any process — cannot observe: the OS tears
365
- * the process down directly, no userspace code runs at all, so Chrome and
366
- * the temp profile are left behind whenever that specific signal is what
367
- * ends this process. That gap is inherent, not something a different
368
- * signal list here could close.
369
- */
370
169
  export function registerTerminationHandlers(cleanup) {
371
170
  const signals = Object.keys(TERMINATION_EXIT_CODES);
372
171
  const handlers = signals.map((signal) => {
@@ -382,19 +181,6 @@ export function registerTerminationHandlers(cleanup) {
382
181
  process.off(signal, handler);
383
182
  };
384
183
  }
385
- /**
386
- * @desc The same guard `captureSession` runs internally as its very first
387
- * step, exposed so the command layer can call it BEFORE printing anything
388
- * about a Chrome window that may never open. Without this, `scraps.ts`
389
- * printed its "a Chrome window will open" banner unconditionally, ahead of
390
- * this exact check inside `captureSession` — so a non-interactive run saw
391
- * that banner immediately followed by "this cannot work headless"
392
- * (trawl_cli#183 gap). Reads live process state once; pure
393
- * `detectNonInteractive` stays the single source of truth for both call
394
- * sites.
395
- * @returns {string | null} a factual reason when this command cannot
396
- * proceed in the current environment, or null when it can.
397
- */
398
184
  export function checkInteractiveEnvironment() {
399
185
  return detectNonInteractive({ stdinIsTTY: Boolean(process.stdin.isTTY), platform: process.platform, env: process.env });
400
186
  }
@@ -411,13 +197,6 @@ function defaultDeps() {
411
197
  onSigint: registerTerminationHandlers,
412
198
  };
413
199
  }
414
- /**
415
- * @desc Run the full capture flow for one scrap's target URL.
416
- * @param {string} targetUrl
417
- * @param {Partial<CaptureDeps>} depsOverride — for tests; real deps fill in
418
- * the rest.
419
- * @returns {Promise<CaptureResult>}
420
- */
421
200
  export async function captureSession(targetUrl, depsOverride = {}) {
422
201
  const deps = { ...defaultDeps(), ...depsOverride };
423
202
  const guardMessage = deps.detectNonInteractive();
@@ -434,31 +213,12 @@ export async function captureSession(targetUrl, depsOverride = {}) {
434
213
  const userDataDir = deps.mkdtemp();
435
214
  let proc;
436
215
  let cleaned = false;
437
- // Covers every path this process can observe: return, throw, and SIGINT,
438
- // SIGTERM, and SIGHUP (via onSigint/registerTerminationHandlers above —
439
- // trawl_cli#183 review finding 1: SIGTERM/SIGHUP were missing before
440
- // this, so `timeout` without `--signal`, a bare `kill <pid>`, most
441
- // supervisors, and closing the terminal tab mid-login all skipped
442
- // cleanup). A SIGKILL of this process itself (OOM, a CI timeout,
443
- // `kill -9`) is NOT observable by any handler here — Chrome exits on its
444
- // own when the CDP pipe hits EOF, but this cleanup, and so the temp
445
- // profile removal below, never runs. That gap is inherent to any
446
- // local-Chrome-launch design (not specific to the pipe transport) and is
447
- // not closed here.
448
216
  const cleanup = () => {
449
217
  if (cleaned)
450
218
  return;
451
219
  cleaned = true;
452
220
  try {
453
221
  if (proc?.pid) {
454
- // Kill Chrome's WHOLE process group (it and its own renderer/GPU/
455
- // network-service helpers — chrome-launch.ts launches it
456
- // `detached` on POSIX for exactly this), never just the single
457
- // main-process pid: a bare `proc.kill()` left helper processes
458
- // holding the profile open, which is what made rmSync lose the
459
- // race below (see rmSyncWithRetry's own comment). Never `-pid` on
460
- // win32 — there are no POSIX process groups there, and it isn't
461
- // detached in the first place (chrome-launch.ts).
462
222
  if (process.platform === 'win32')
463
223
  proc.kill('SIGKILL');
464
224
  else
@@ -466,19 +226,15 @@ export async function captureSession(targetUrl, depsOverride = {}) {
466
226
  }
467
227
  }
468
228
  catch {
469
- // already gone
470
229
  }
471
230
  try {
472
231
  deps.rmSync(userDataDir);
473
232
  }
474
233
  catch {
475
- // best-effort — never let cleanup itself throw out of a signal handler
476
234
  }
477
235
  };
478
236
  const unregisterSigint = deps.onSigint(cleanup);
479
237
  let launched;
480
- // See CaptureStage's own doc comment. Read only in the catch block below
481
- // to decide launch_failed vs capture_failed — never assigned 'upload'.
482
238
  let stage = 'launch';
483
239
  try {
484
240
  launched = await deps.launch(chromePath, targetUrl, userDataDir);
@@ -491,10 +247,6 @@ export async function captureSession(targetUrl, depsOverride = {}) {
491
247
  const targetId = await deps.findPageTarget(launched.cdp);
492
248
  const { closedEarly } = await deps.waitForDone(launched.cdp, targetId, cleanup);
493
249
  if (pipeClosed) {
494
- // A corrupt EVENT frame during the wait (e.g. a garbled
495
- // targetDestroyed) tears the pipe down the same way a real Chrome
496
- // exit does — `protocolError` distinguishes the two so this doesn't
497
- // default to "Chrome exited" for a cause that wasn't that.
498
250
  if (launched.cdp.protocolError) {
499
251
  return {
500
252
  ok: false,
@@ -513,9 +265,6 @@ export async function captureSession(targetUrl, depsOverride = {}) {
513
265
  offClose();
514
266
  const cookieResult = mapCookies(raw.cookies, targetUrl);
515
267
  const originResult = mapOrigins(raw.origins, targetUrl);
516
- // The exact host cookies/origins were scoped against above — not a
517
- // peeled "registrable domain" (trawl_cli#183 review finding 2; see
518
- // storage-state.ts's module doc comment for why peeling was the bug).
519
268
  const targetDomain = getTargetHost(targetUrl);
520
269
  if (cookieResult.cookies.length === 0) {
521
270
  return {
@@ -540,14 +289,6 @@ export async function captureSession(targetUrl, depsOverride = {}) {
540
289
  };
541
290
  }
542
291
  catch (e) {
543
- // A CDP call (readCapture, most likely) can reject mid-flight because
544
- // Chrome died, or a corrupt frame tore the pipe down, AFTER waitForDone
545
- // already resolved normally — the `pipeClosed` check above only covers
546
- // the window up to that point. `protocolError` is checked FIRST (it
547
- // implies `isClosed` too) so a corrupt-frame failure reports its own
548
- // factual reason instead of the generic "Chrome exited"; `isClosed`
549
- // alone still distinguishes a real exit from every other unexpected
550
- // failure here (a malformed response, a genuinely broken launch).
551
292
  if (launched?.cdp.protocolError) {
552
293
  return {
553
294
  ok: false,
@@ -562,29 +303,7 @@ export async function captureSession(targetUrl, depsOverride = {}) {
562
303
  message: PROCESS_EXITED_MESSAGE,
563
304
  };
564
305
  }
565
- // trawl_cli#183 R4: everything above already special-cases a dead pipe
566
- // (real exit or a corrupt frame). What's left here — the pipe still
567
- // open, no protocol error — used to fall into one undifferentiated
568
- // `launch_failed` regardless of WHEN it happened. That mislabels a
569
- // `readCapture` failure that occurs after launch + handshake already
570
- // succeeded (Chrome reachable over CDP, a page already open, the human
571
- // already done logging in): every LEGITIMATE per-page CDP rejection at
572
- // that point is already counted as `originsUnreadable` by
573
- // `readCaptureDefault`'s own provenance boundary (see
574
- // `readPageOverCdp`'s doc comment) — so anything that still reaches
575
- // here from the `capture` stage is a genuine bug in THIS CLI (or an
576
- // unanticipated non-page-scoped failure, e.g. `Storage.getCookies`
577
- // itself), never a "Chrome could not be launched/driven" problem.
578
- // `launch`/`handshake` stay `launch_failed`, message unchanged.
579
306
  if (stage === 'capture') {
580
- // Never `(e as Error).message` here: the whole point of the R3/R4
581
- // history (trawl_cli#183 post-cap review finding B, then R4) is that
582
- // a bug in THIS FILE's own code can be TRIGGERED by page-influenced
583
- // data (a tampered global, a crafted return value) — nothing
584
- // guarantees the resulting exception's own message is free of it, so
585
- // the stage name is the only safe, fixed vocabulary to report here.
586
- // No counts either: `readCapture` rejected before returning any —
587
- // there is no partial `raw` to report counts from.
588
307
  return {
589
308
  ok: false,
590
309
  reason: 'capture_failed',