@trawlme/cli 3.12.0 → 3.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/commands/create.d.ts +0 -28
- package/dist/commands/create.js +0 -89
- package/dist/commands/doctor.d.ts +0 -79
- package/dist/commands/doctor.js +1 -187
- package/dist/commands/login.js +0 -67
- package/dist/commands/ping.d.ts +0 -15
- package/dist/commands/ping.js +0 -15
- package/dist/commands/scraps.d.ts +0 -120
- package/dist/commands/scraps.js +10 -724
- package/dist/commands/skills.js +0 -22
- package/dist/commands/spec.d.ts +0 -85
- package/dist/commands/spec.js +0 -67
- package/dist/commands/telemetry.js +0 -4
- package/dist/commands/token.js +0 -28
- package/dist/commands/upgrade.js +0 -22
- package/dist/commands/whoami.d.ts +0 -12
- package/dist/commands/whoami.js +0 -6
- package/dist/index.d.ts +0 -188
- package/dist/index.js +0 -349
- package/dist/lib/api.d.ts +0 -78
- package/dist/lib/api.js +1 -320
- package/dist/lib/cdp-pipe.d.ts +0 -72
- package/dist/lib/cdp-pipe.js +1 -81
- package/dist/lib/chrome-discovery.d.ts +0 -11
- package/dist/lib/chrome-discovery.js +0 -19
- package/dist/lib/chrome-launch.d.ts +0 -40
- package/dist/lib/chrome-launch.js +0 -69
- package/dist/lib/config.d.ts +0 -53
- package/dist/lib/config.js +0 -55
- package/dist/lib/confirm.d.ts +0 -55
- package/dist/lib/confirm.js +0 -47
- package/dist/lib/docs.d.ts +0 -123
- package/dist/lib/docs.js +0 -169
- package/dist/lib/errors.d.ts +0 -134
- package/dist/lib/errors.js +0 -151
- package/dist/lib/format.d.ts +0 -6
- package/dist/lib/format.js +0 -6
- package/dist/lib/json.d.ts +0 -35
- package/dist/lib/json.js +0 -48
- package/dist/lib/jwt.d.ts +0 -7
- package/dist/lib/jwt.js +0 -7
- package/dist/lib/pinch.d.ts +0 -53
- package/dist/lib/pinch.js +6 -112
- package/dist/lib/pinchAnimation.d.ts +0 -16
- package/dist/lib/pinchAnimation.js +8 -29
- package/dist/lib/posthog.d.ts +0 -9
- package/dist/lib/posthog.js +0 -23
- package/dist/lib/prompt.js +1 -20
- package/dist/lib/secure-transport.d.ts +0 -7
- package/dist/lib/secure-transport.js +0 -24
- package/dist/lib/session-capture-guard.d.ts +0 -15
- package/dist/lib/session-capture-guard.js +0 -5
- package/dist/lib/session-capture.d.ts +0 -125
- package/dist/lib/session-capture.js +0 -281
- package/dist/lib/skills.d.ts +0 -175
- package/dist/lib/skills.js +1 -216
- package/dist/lib/skillsNudge.d.ts +0 -17
- package/dist/lib/skillsNudge.js +0 -83
- package/dist/lib/spinner.d.ts +0 -39
- package/dist/lib/spinner.js +0 -40
- package/dist/lib/storage-state.d.ts +0 -112
- package/dist/lib/storage-state.js +0 -131
- package/dist/lib/tips.d.ts +0 -38
- package/dist/lib/tips.js +0 -77
- package/dist/lib/updateCheckWorker.js +0 -14
- package/dist/lib/updateNotifier.d.ts +0 -17
- package/dist/lib/updateNotifier.js +0 -53
- package/dist/lib/validate.d.ts +0 -8
- package/dist/lib/validate.js +0 -8
- package/dist/lib/version.d.ts +0 -12
- package/dist/lib/version.js +1 -13
- package/package.json +2 -2
package/dist/lib/api.js
CHANGED
|
@@ -11,15 +11,6 @@ const USER_AGENT = `@trawlme/cli/${pkg.version}`;
|
|
|
11
11
|
export class ApiError extends Error {
|
|
12
12
|
status;
|
|
13
13
|
next;
|
|
14
|
-
/**
|
|
15
|
-
* `next` is an OPTIONAL per-instance override of the envelope's default
|
|
16
|
-
* `next` steps (errors.ts's frozen `RETRY_POLICY`) — set only at the two
|
|
17
|
-
* apiKey-mode 401 call sites below (via authNextSteps()), where the
|
|
18
|
-
* generic "trawl login --token <jwt>" default is inert until a live
|
|
19
|
-
* TRAWL_API_KEY/TRAWL_TOKEN is unset first (#169 review). Absent for every
|
|
20
|
-
* other error, so classifyError falls back to the frozen default exactly
|
|
21
|
-
* as before.
|
|
22
|
-
*/
|
|
23
14
|
constructor(status, message, next) {
|
|
24
15
|
super(message);
|
|
25
16
|
this.status = status;
|
|
@@ -27,164 +18,35 @@ export class ApiError extends Error {
|
|
|
27
18
|
this.name = 'ApiError';
|
|
28
19
|
}
|
|
29
20
|
}
|
|
30
|
-
/**
|
|
31
|
-
* A fetch-level failure — the request never got a response at all (DNS,
|
|
32
|
-
* connection refused, timeout, TLS, …). Distinguished from ApiError (which
|
|
33
|
-
* always carries a real HTTP status) so the top-level handler can map it to
|
|
34
|
-
* its own exit code instead of the generic uniform 1. (#71 findings 4/58)
|
|
35
|
-
*/
|
|
36
21
|
export class NetworkError extends Error {
|
|
37
22
|
constructor(message) {
|
|
38
23
|
super(message);
|
|
39
24
|
this.name = 'NetworkError';
|
|
40
25
|
}
|
|
41
26
|
}
|
|
42
|
-
/**
|
|
43
|
-
* A LOCAL auth failure — no token available, or a locally-decoded token
|
|
44
|
-
* that's provably expired, discovered entirely client-side before any HTTP
|
|
45
|
-
* call was ever made. Distinguished from ApiError(401) (a real server-issued
|
|
46
|
-
* 401 response) so the --json envelope never claims `status:401` for
|
|
47
|
-
* something the server never said — that would be a fabricated fact,
|
|
48
|
-
* indistinguishable from an actual server round-trip to a machine consumer.
|
|
49
|
-
* Both classify to the same exit code (3) / kind "auth" in classifyError
|
|
50
|
-
* (errors.ts); only the envelope's `status` field differs (present for
|
|
51
|
-
* ApiError, absent here). (#88 item 4)
|
|
52
|
-
*/
|
|
53
27
|
export class AuthError extends Error {
|
|
54
28
|
next;
|
|
55
|
-
/** See ApiError's `next` for what this overrides and why. */
|
|
56
29
|
constructor(message, next) {
|
|
57
30
|
super(message);
|
|
58
31
|
this.next = next;
|
|
59
32
|
this.name = 'AuthError';
|
|
60
33
|
}
|
|
61
34
|
}
|
|
62
|
-
/**
|
|
63
|
-
* The single "no token available" error — every call site in this file that
|
|
64
|
-
* needs a token (request/upload/getText/stream) used to throw its own copy
|
|
65
|
-
* of `new Error('Not logged in. Run: trawl login')`, which fell through
|
|
66
|
-
* classifyError's generic branch (exit 1, kind:"unknown") — indistinguishable
|
|
67
|
-
* from an arbitrary bug. Auth-classifying it puts it on the exact same
|
|
68
|
-
* exit-3 / kind:"auth" path a real 401 response already takes — but as an
|
|
69
|
-
* AuthError (no HTTP call happened here), never a fabricated ApiError(401).
|
|
70
|
-
* `trawl token` (src/commands/token.ts) reuses this too, so "no token" means
|
|
71
|
-
* the same thing everywhere it can be observed. (#86 findings 1/2, #88 item 4)
|
|
72
|
-
*/
|
|
73
35
|
export function notLoggedInError() {
|
|
74
36
|
return new AuthError('Not logged in. Run: trawl login');
|
|
75
37
|
}
|
|
76
|
-
/**
|
|
77
|
-
* The actionable "how to recover" tail for an apiKey-mode auth failure —
|
|
78
|
-
* shared by sessionAuthMessage's apiKey branch and apiKeyUnsupportedError so
|
|
79
|
-
* neither one contradicts the other. `Run: trawl login` alone is INERT
|
|
80
|
-
* whenever TRAWL_API_KEY/TRAWL_TOKEN is live: getToken()'s own precedence
|
|
81
|
-
* (config.ts) keeps resolving the env credential over whatever `trawl login`
|
|
82
|
-
* just stored, so the login it just told the operator to run changes
|
|
83
|
-
* nothing on the very next request — reproduced end to end (`login --token
|
|
84
|
-
* <jwt>` reports success; the next call 401s again with the same rejected
|
|
85
|
-
* key). Leads with the unset step whenever there's a live var to unset;
|
|
86
|
-
* falls back to the plain original wording when there isn't (the stored
|
|
87
|
-
* config token case — `trawl login` alone already fixes that one). (#169
|
|
88
|
-
* review round 2 — finding 2)
|
|
89
|
-
*/
|
|
90
38
|
function loginRemedyText() {
|
|
91
39
|
const liveVar = getLiveAuthEnvVar();
|
|
92
40
|
return liveVar ? `unset ${liveVar}, then run: trawl login` : 'Run: trawl login';
|
|
93
41
|
}
|
|
94
|
-
/**
|
|
95
|
-
* The machine-readable counterpart to loginRemedyText() — feeds `ApiError`/
|
|
96
|
-
* `AuthError`'s `next` override (see ApiError's doc comment) so an agent
|
|
97
|
-
* reading `--json` on stdout gets the SAME "unset first" step the prose
|
|
98
|
-
* message carries, never a `next` that quietly disagrees with the message
|
|
99
|
-
* next to it. Only ever called from a branch that already knows the auth
|
|
100
|
-
* mode is 'apiKey' — see both call sites. (#169 review round 2 — finding 2)
|
|
101
|
-
*/
|
|
102
42
|
function authNextSteps() {
|
|
103
43
|
const liveVar = getLiveAuthEnvVar();
|
|
104
44
|
const login = 'trawl login --token <jwt>';
|
|
105
45
|
return liveVar ? [`unset ${liveVar}`, login] : [login];
|
|
106
46
|
}
|
|
107
|
-
/**
|
|
108
|
-
* The "this route is JWT-only" error — thrown client-side, before any HTTP
|
|
109
|
-
* call, by a command that only works with a session JWT (today: `whoami`,
|
|
110
|
-
* see its own doc comment for why `GET /api/users/me` was descoped rather
|
|
111
|
-
* than relocated) when the resolved credential is a scoped API key instead.
|
|
112
|
-
* Distinct from notLoggedInError(): there IS a credential here, it's just
|
|
113
|
-
* the wrong shape for this one route. Left unhandled, that route would 401
|
|
114
|
-
* and surface the generic `notLoggedInError`-adjacent message ("Session
|
|
115
|
-
* expired or invalid. Run: trawl login") — wrong twice over under a key: the
|
|
116
|
-
* route never accepts keys at all (no session ever "expired"), and
|
|
117
|
-
* re-running `trawl login` genuinely IS the fix here, just not because
|
|
118
|
-
* anything expired. Classifies to the SAME exit 3 / kind:"auth" as every
|
|
119
|
-
* other auth failure (#169) — reusing the existing `auth` kind rather than
|
|
120
|
-
* minting a new one, since the taxonomy question is still "not
|
|
121
|
-
* authenticated the way this route needs," never a new category.
|
|
122
|
-
*
|
|
123
|
-
* Always called already knowing authMode is 'apiKey' (see whoami.ts) — the
|
|
124
|
-
* message and `next` both route through loginRemedyText()/authNextSteps()
|
|
125
|
-
* so this never emits the same inert "Run: trawl login" that finding 2
|
|
126
|
-
* corrects everywhere else. (#169 review round 2 — finding 2)
|
|
127
|
-
*/
|
|
128
47
|
export function apiKeyUnsupportedError(command) {
|
|
129
48
|
return new AuthError(`${command} requires a session (JWT) — this credential is a scoped API key, which this route does not accept. ${loginRemedyText()}`, authNextSteps());
|
|
130
49
|
}
|
|
131
|
-
/**
|
|
132
|
-
* The ONE place that decides what a real server-issued 401 means, for
|
|
133
|
-
* whichever credential shape produced it (#169 review finding 1). Before this, a
|
|
134
|
-
* genuine 401 always got the same "Session expired or invalid" text — true
|
|
135
|
-
* for a JWT, but wrong twice over for a scoped API key: an API key doesn't
|
|
136
|
-
* have a "session" to expire, and re-running `trawl login` fixes nothing
|
|
137
|
-
* when the ROUTE itself is JWT-only (`scraps update`/`delete`, `scraps
|
|
138
|
-
* account *`/`scraps session *`, `scraps banner`, the SSE `scraps watch` —
|
|
139
|
-
* `whoami` already catches this client-side, see apiKeyUnsupportedError
|
|
140
|
-
* above, but those five/six never had an equivalent guard).
|
|
141
|
-
*
|
|
142
|
-
* #169 review round 2 — finding 1: the FIRST fix here asserted two things
|
|
143
|
-
* never established — that the route is JWT-only, AND that "the API key in
|
|
144
|
-
* use is valid." Both are false whenever the 401 is the KEY's own fault
|
|
145
|
-
* (revoked/rotated/malformed) rather than the route's — reproduced live
|
|
146
|
-
* against `GET /api/scraps` (dual-auth, works with a key) using a revoked
|
|
147
|
-
* key: the old text confidently certified the key as valid and blamed the
|
|
148
|
-
* route, which sends the operator away from the actual problem. trawl_node's
|
|
149
|
-
* authenticateApiKey.js (modules/developers/middlewares/authenticateApiKey.js)
|
|
150
|
-
* already tells the two cases apart in the response body — `responses.error`
|
|
151
|
-
* puts the real reason in `description` ("Invalid or expired API key" /
|
|
152
|
-
* "Invalid API key format" / "Missing or invalid Authorization header") for
|
|
153
|
-
* a REJECTED key, while a JWT-only route's `passport.authenticate('jwt')`
|
|
154
|
-
* (cookie-only extractor — a Bearer header never even reaches the verify
|
|
155
|
-
* step) sends passport's own bare `Unauthorized` body with nothing to
|
|
156
|
-
* disambiguate. `extractErrorMessage` already unwraps that envelope and
|
|
157
|
-
* prefers `description` over a `message` that only echoes the reason phrase
|
|
158
|
-
* — reused here instead of re-parsing, and its plain-text fallback (`return
|
|
159
|
-
* raw`) is exactly the bare `Unauthorized` case. So: a body that resolves to
|
|
160
|
-
* anything other than that bare word IS the server naming the real cause —
|
|
161
|
-
* trust it over any guess. Anything else (unrecoverable, or genuinely just
|
|
162
|
-
* "Unauthorized") gets an honest hedge between the two live possibilities,
|
|
163
|
-
* never a pick.
|
|
164
|
-
*
|
|
165
|
-
* Deliberately driven off the REAL response (called only where a 401 just
|
|
166
|
-
* came back from the server), not a client-side allow-list of "routes a key
|
|
167
|
-
* can reach" duplicated into every command handler — the server is the only
|
|
168
|
-
* side that actually knows which routes accept a key, and a second,
|
|
169
|
-
* hand-maintained list here would silently drift the moment trawl_node opens
|
|
170
|
-
* (or closes) a route to keys. That's also why the message can't name the
|
|
171
|
-
* specific CLI command the way apiKeyUnsupportedError does: this function
|
|
172
|
-
* only ever sees the HTTP response, never which `trawl` verb issued the
|
|
173
|
-
* request.
|
|
174
|
-
*
|
|
175
|
-
* `res` is optional and read defensively (`typeof res.text === 'function'`,
|
|
176
|
-
* `.catch(() => '')`) — the SSE call site (`stream()`) passes a real
|
|
177
|
-
* `Response` in production, but its own unit tests mock a bare `{ok,
|
|
178
|
-
* status}` with no `text()` at all; a missing/failed body read just falls
|
|
179
|
-
* through to the honest hedge rather than throwing a SECOND error while
|
|
180
|
-
* already handling the first.
|
|
181
|
-
*
|
|
182
|
-
* Still an ApiError(401), not an AuthError — a real HTTP round-trip
|
|
183
|
-
* happened, so the envelope's `status:401` field must stay honest about
|
|
184
|
-
* that (see AuthError's own doc comment on why a LOCAL failure never
|
|
185
|
-
* fabricates one). Same exit 3 / kind:"auth" either way — only the message
|
|
186
|
-
* text differs. (#169 review)
|
|
187
|
-
*/
|
|
188
50
|
async function sessionAuthMessage(authMode, res) {
|
|
189
51
|
if (authMode === 'jwt')
|
|
190
52
|
return 'Session expired or invalid. Run: trawl login';
|
|
@@ -195,52 +57,11 @@ async function sessionAuthMessage(authMode, res) {
|
|
|
195
57
|
}
|
|
196
58
|
return `The API key was rejected (401) — either this route requires a session (JWT), or the key is invalid or revoked. ${loginRemedyText()}.`;
|
|
197
59
|
}
|
|
198
|
-
/**
|
|
199
|
-
* One shared header set for whichever credential `getToken()` resolved: a
|
|
200
|
-
* scoped API key (`trawl_*`) goes in `Authorization: Bearer`; a session JWT
|
|
201
|
-
* stays in the `TOKEN` cookie — trawl_node's passport JWT strategy is
|
|
202
|
-
* cookie-only on the REST surface, so a Bearer header there is silently
|
|
203
|
-
* ignored, which is why this is a SWITCH and not an addition. Never sends
|
|
204
|
-
* both — two credentials on one request is an ambiguity the server does not
|
|
205
|
-
* have to resolve in our favour. Takes the token the caller already
|
|
206
|
-
* resolved (never re-calls getToken() itself) so a request's headers and its
|
|
207
|
-
* own classification of that same token can't drift mid-flight. Used at all
|
|
208
|
-
* 5 call sites that attach a credential: request/upload/publicGet/getText/
|
|
209
|
-
* stream. (#169)
|
|
210
|
-
*/
|
|
211
60
|
function authHeaders(token) {
|
|
212
61
|
return getAuthMode(token) === 'apiKey' ? { Authorization: `Bearer ${token}` } : { Cookie: `TOKEN=${token}` };
|
|
213
62
|
}
|
|
214
63
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
215
|
-
/**
|
|
216
|
-
* #91 P0 — some endpoints legitimately run 30–250s server-side: a scrap
|
|
217
|
-
* execute (worker-puppeteer navigation + antibot tier escalation + AI-fix
|
|
218
|
-
* dry-run retries). The generic 30s default was aborting those mid-flight
|
|
219
|
-
* and surfacing a fabricated `NetworkError timed out` (exit 5) for a request
|
|
220
|
-
* that was always going to succeed given enough time. Passed as the per-call
|
|
221
|
-
* `{timeoutMs}` override at exactly the 4 call sites that hit those
|
|
222
|
-
* endpoints: `scraps run` / `data --fresh` (GET /api/scraps/load/:id) and
|
|
223
|
-
* `scraps trigger --wait` (POST /api/scraps/worker/:id, synchronous branch
|
|
224
|
-
* only — the default async `?wait=false` POST returns almost immediately and
|
|
225
|
-
* keeps the 30s default) — all three in src/commands/scraps.ts — plus (#114)
|
|
226
|
-
* `create` (POST /api/ai/wizard, src/commands/create.ts), whose AI-generation
|
|
227
|
-
* + scrap creation + first run + autofix pipeline runs the same 30–250s+
|
|
228
|
-
* server-side. 300s leaves margin over the ~250s worst case without being
|
|
229
|
-
* unboundedly long.
|
|
230
|
-
*/
|
|
231
64
|
export const LONG_RUN_TIMEOUT_MS = 300_000;
|
|
232
|
-
/**
|
|
233
|
-
* Effective fetch timeout for one request.
|
|
234
|
-
*
|
|
235
|
-
* Precedence (#91): TRAWL_TIMEOUT (env) ALWAYS wins when set to a valid
|
|
236
|
-
* positive number — an operator/CI override must be able to force a shorter
|
|
237
|
-
* or longer ceiling globally (e.g. a slow CI network, or a deliberately tight
|
|
238
|
-
* smoke-test budget) without editing call sites. Only when the env var is
|
|
239
|
-
* unset/invalid does the caller-supplied per-call `overrideMs` apply (e.g.
|
|
240
|
-
* LONG_RUN_TIMEOUT_MS for the 3 long-run call sites); otherwise
|
|
241
|
-
* DEFAULT_TIMEOUT_MS. This preserves #71's original "env always overrides"
|
|
242
|
-
* contract while layering the new per-call default beneath it, never above.
|
|
243
|
-
*/
|
|
244
65
|
function getTimeoutMs(overrideMs) {
|
|
245
66
|
const raw = process.env['TRAWL_TIMEOUT']?.trim();
|
|
246
67
|
if (raw) {
|
|
@@ -250,19 +71,6 @@ function getTimeoutMs(overrideMs) {
|
|
|
250
71
|
}
|
|
251
72
|
return overrideMs !== undefined && overrideMs > 0 ? overrideMs : DEFAULT_TIMEOUT_MS;
|
|
252
73
|
}
|
|
253
|
-
/**
|
|
254
|
-
* Wrap a `fetch()` call so connection-level failures (ECONNREFUSED, DNS,
|
|
255
|
-
* timeout, …) surface as a NetworkError carrying the effective URL + the
|
|
256
|
-
* unwrapped `err.cause` detail, instead of a bare "fetch failed" with no
|
|
257
|
-
* actionable information. (#71 findings 4/58)
|
|
258
|
-
*
|
|
259
|
-
* `effectiveTimeoutMs` is the ACTUAL timeout that was armed for this specific
|
|
260
|
-
* call (already resolved via getTimeoutMs by the caller) — used only to
|
|
261
|
-
* report an honest number in the timeout message; #91 — before this it
|
|
262
|
-
* always re-derived a fresh `getTimeoutMs()` with no override, so a request
|
|
263
|
-
* armed with LONG_RUN_TIMEOUT_MS that timed out would have lied and reported
|
|
264
|
-
* "timed out after 30000ms".
|
|
265
|
-
*/
|
|
266
74
|
async function safeFetch(url, options, effectiveTimeoutMs) {
|
|
267
75
|
try {
|
|
268
76
|
return await fetch(url, options);
|
|
@@ -277,23 +85,10 @@ async function safeFetch(url, options, effectiveTimeoutMs) {
|
|
|
277
85
|
throw new NetworkError(`Network error reaching ${url}${causeDetail}: ${e?.message ?? String(err)}`);
|
|
278
86
|
}
|
|
279
87
|
}
|
|
280
|
-
/**
|
|
281
|
-
* Extract the honest client-facing error string from a raw response body.
|
|
282
|
-
* The server envelope (lib/helpers/responses.js) shape is
|
|
283
|
-
* `{ type, message, code, status, errorCode, description, error? }` — where
|
|
284
|
-
* `message` is sometimes a bare HTTP reason phrase (e.g. "Payment Required")
|
|
285
|
-
* duplicating `res.statusText`, producing a tautology like
|
|
286
|
-
* "402 Payment Required: Payment Required". When that happens, prefer the
|
|
287
|
-
* richer `description` field instead. (#71 finding 76 — error-copy part only)
|
|
288
|
-
*/
|
|
289
88
|
function extractErrorMessage(raw, statusText) {
|
|
290
89
|
if (!raw)
|
|
291
90
|
return '';
|
|
292
91
|
try {
|
|
293
|
-
// #159 — an error envelope's message/description/nested `error` string can
|
|
294
|
-
// carry the same raw-control-char server quirk as a success payload;
|
|
295
|
-
// recover it here too instead of falling all the way back to the raw,
|
|
296
|
-
// unparsed blob as the "error message".
|
|
297
92
|
const parsed = parseServerJson(raw);
|
|
298
93
|
if (parsed && typeof parsed === 'object') {
|
|
299
94
|
const env = parsed;
|
|
@@ -323,18 +118,9 @@ function extractErrorMessage(raw, statusText) {
|
|
|
323
118
|
}
|
|
324
119
|
}
|
|
325
120
|
catch {
|
|
326
|
-
// not JSON — fall through
|
|
327
121
|
}
|
|
328
122
|
return raw;
|
|
329
123
|
}
|
|
330
|
-
/**
|
|
331
|
-
* Best-effort extraction of an upgrade URL from a 402 response body. In
|
|
332
|
-
* production the envelope rarely carries it directly (billing.quota.service
|
|
333
|
-
* nests `upgradeUrl` inside AppError.details, which `responses.error` only
|
|
334
|
-
* serializes to the dev-only `error` string) — so this checks the top-level
|
|
335
|
-
* field, `details.upgradeUrl`, and the dev-only nested `error` JSON string,
|
|
336
|
-
* and returns null (never fabricates) when none are present. (#71 finding 76)
|
|
337
|
-
*/
|
|
338
124
|
function extractUpgradeUrl(raw) {
|
|
339
125
|
try {
|
|
340
126
|
const parsed = parseServerJson(raw);
|
|
@@ -357,23 +143,15 @@ function extractUpgradeUrl(raw) {
|
|
|
357
143
|
}
|
|
358
144
|
}
|
|
359
145
|
catch {
|
|
360
|
-
// dev-only nested string wasn't JSON — nothing to extract
|
|
361
146
|
}
|
|
362
147
|
}
|
|
363
148
|
}
|
|
364
149
|
catch {
|
|
365
|
-
// not JSON — nothing to extract
|
|
366
150
|
}
|
|
367
151
|
return null;
|
|
368
152
|
}
|
|
369
153
|
async function throwIfError(res, isPublic = false, authMode = 'jwt') {
|
|
370
154
|
if (res.status === 401 && !isPublic) {
|
|
371
|
-
// #169 review round 2 — finding 1: sessionAuthMessage now reads the body
|
|
372
|
-
// itself (the bug this whole branch used to have was discarding it
|
|
373
|
-
// without ever calling res.text()), so it's awaited here instead of
|
|
374
|
-
// called synchronously. `next` is only overridden in apiKey mode — see
|
|
375
|
-
// authNextSteps()'s own doc comment for why jwt mode keeps the frozen
|
|
376
|
-
// default (finding 2).
|
|
377
155
|
throw new ApiError(401, await sessionAuthMessage(authMode, res), authMode === 'apiKey' ? authNextSteps() : undefined);
|
|
378
156
|
}
|
|
379
157
|
if (!res.ok) {
|
|
@@ -418,7 +196,6 @@ async function request(path, options = {}, reqOpts = {}) {
|
|
|
418
196
|
if (!text)
|
|
419
197
|
return {};
|
|
420
198
|
const parsed = parseServerJson(text);
|
|
421
|
-
// Unwrap API envelope { type, message, data: T }
|
|
422
199
|
if (parsed !== null && typeof parsed === 'object' && 'data' in parsed) {
|
|
423
200
|
return parsed.data;
|
|
424
201
|
}
|
|
@@ -434,7 +211,6 @@ async function upload(path, formData, reqOpts = {}) {
|
|
|
434
211
|
throw notLoggedInError();
|
|
435
212
|
const url = `${getApiUrl()}${path}`;
|
|
436
213
|
const timeoutMs = getTimeoutMs(reqOpts.timeoutMs);
|
|
437
|
-
// Do NOT set Content-Type — fetch sets it automatically with the correct multipart boundary
|
|
438
214
|
const res = await safeFetch(url, {
|
|
439
215
|
method: 'POST',
|
|
440
216
|
body: formData,
|
|
@@ -450,7 +226,6 @@ async function upload(path, formData, reqOpts = {}) {
|
|
|
450
226
|
if (!text)
|
|
451
227
|
return {};
|
|
452
228
|
const parsed = parseServerJson(text);
|
|
453
|
-
// Unwrap API envelope { type, message, data: T }
|
|
454
229
|
if (parsed !== null && typeof parsed === 'object' && 'data' in parsed) {
|
|
455
230
|
return parsed.data;
|
|
456
231
|
}
|
|
@@ -479,20 +254,6 @@ async function publicPost(path, body, baseUrlOverride, reqOpts = {}) {
|
|
|
479
254
|
throw new Error('Invalid JSON in server response');
|
|
480
255
|
}
|
|
481
256
|
}
|
|
482
|
-
/**
|
|
483
|
-
* Unauthenticated GET — for endpoints the server itself treats as public/
|
|
484
|
-
* optional-auth (e.g. `GET /api/health`, home.route.js's `optionalAuth`
|
|
485
|
-
* middleware: enriches the response for an admin JWT, passes through
|
|
486
|
-
* otherwise). Unlike `request()`, this NEVER throws `notLoggedInError()` for
|
|
487
|
-
* a missing token — that guard exists for endpoints that genuinely require
|
|
488
|
-
* auth, and applying it here defeated the whole point of a pre-login
|
|
489
|
-
* sanity-check (#148). The token is still attached as a best-effort Cookie
|
|
490
|
-
* when one happens to be stored/env-set, so an already-authenticated caller
|
|
491
|
-
* still gets the admin-enriched payload — it's just never REQUIRED.
|
|
492
|
-
* `throwIfError(res, true)` mirrors `publicPost`'s "public endpoint" 401
|
|
493
|
-
* wording, even though this route's optionalAuth middleware never actually
|
|
494
|
-
* rejects a request for lacking/invalid credentials.
|
|
495
|
-
*/
|
|
496
257
|
async function publicGet(path, reqOpts = {}) {
|
|
497
258
|
const token = getToken();
|
|
498
259
|
const url = `${getApiUrl()}${path}`;
|
|
@@ -510,7 +271,6 @@ async function publicGet(path, reqOpts = {}) {
|
|
|
510
271
|
if (!text)
|
|
511
272
|
return {};
|
|
512
273
|
const parsed = parseServerJson(text);
|
|
513
|
-
// Unwrap API envelope { type, message, data: T }
|
|
514
274
|
if (parsed !== null && typeof parsed === 'object' && 'data' in parsed) {
|
|
515
275
|
return parsed.data;
|
|
516
276
|
}
|
|
@@ -520,65 +280,6 @@ async function publicGet(path, reqOpts = {}) {
|
|
|
520
280
|
throw new Error('Invalid JSON in server response');
|
|
521
281
|
}
|
|
522
282
|
}
|
|
523
|
-
/**
|
|
524
|
-
* A hard-bounded, PROBE-ONLY GET — never use this for a command's real work
|
|
525
|
-
* (no auth headers, no retry, no NetworkError-with-cause classification).
|
|
526
|
-
*
|
|
527
|
-
* trawl_cli#185 defect: every OTHER fetch in this file rides Node's global
|
|
528
|
-
* `fetch()` + `AbortSignal.timeout()`, which bounds the PROMISE, not the
|
|
529
|
-
* process. Aborting a fetch mid-connect does not necessarily tear down an
|
|
530
|
-
* in-flight TCP/TLS handshake — a documented Node/undici characteristic —
|
|
531
|
-
* so against a routable-but-silently-dropping host (a dropped SYN, never a
|
|
532
|
-
* RST — a realistic firewalled/air-gapped shape) the dangling socket keeps
|
|
533
|
-
* the event loop alive until Node's OWN internal connect-timeout eventually
|
|
534
|
-
* fires. Measured on this host: a bare `fetch()` + `AbortSignal.timeout(2000)`
|
|
535
|
-
* against such a host rejects its promise at ~2005ms, but the PROCESS does
|
|
536
|
-
* not exit until ~10.5s later — not configurable from here, and not paid by
|
|
537
|
-
* `ECONNREFUSED` (~0.13s) or a failed DNS lookup (~0.29s), only by the
|
|
538
|
-
* silent-drop shape. Every other command in this file accepts that risk
|
|
539
|
-
* because the user explicitly asked for that network call (`trawl ping`
|
|
540
|
-
* hanging on an unreachable host is the command doing its job — nothing to
|
|
541
|
-
* fix there). This primitive exists for the one case where the network call
|
|
542
|
-
* ITSELF is optional: `spec --json`'s own doc-discovery probe (the one
|
|
543
|
-
* command an agent runs to orient itself) has no business costing 10.5s to
|
|
544
|
-
* learn a host is unreachable, so this manages the raw socket directly.
|
|
545
|
-
*
|
|
546
|
-
* The timeout here is an INDEPENDENT `setTimeout` that calls `req.destroy()`
|
|
547
|
-
* — deliberately NOT `http.request`'s own `timeout` option (idle-based via
|
|
548
|
-
* `socket.setTimeout`, so it only fires when zero bytes arrive for that
|
|
549
|
-
* long — a coincidentally-same condition against a silent black hole, but a
|
|
550
|
-
* dishonest bound in general: it would never fire against a host that
|
|
551
|
-
* connects instantly then trickles bytes just often enough to reset the
|
|
552
|
-
* idle clock). Destroying the request/socket on this timer is what actually
|
|
553
|
-
* frees the OS-level handle so the process can exit promptly instead of
|
|
554
|
-
* waiting out Node's own multi-second connect-timeout — verified empirically
|
|
555
|
-
* (same black-holed host): total process time dropped from ~10.5s to ~2.0s
|
|
556
|
-
* with this change.
|
|
557
|
-
*
|
|
558
|
-
* Resolves to `null` on ANY failure (timeout, refusal, non-2xx, malformed
|
|
559
|
-
* JSON, an unsupported protocol) — never throws — so the one caller
|
|
560
|
-
* (spec.ts's fetchExternalDocsUrl) needs no try/catch of its own. This is
|
|
561
|
-
* NOT automatic: `node:http`/`node:https`' own `request()` throws
|
|
562
|
-
* SYNCHRONOUSLY (`ERR_INVALID_PROTOCOL`) for a URL whose protocol doesn't
|
|
563
|
-
* match the module (e.g. a stray `ftp://` in a misconfigured `TRAWL_API_URL`)
|
|
564
|
-
* — inside a Promise executor a synchronous throw becomes a REJECTED
|
|
565
|
-
* promise, which would have propagated straight through `fetchExternalDocsUrl`
|
|
566
|
-
* and turned `spec --json` into an error envelope instead of the spec.
|
|
567
|
-
* Reproduced: `TRAWL_API_URL=ftp://x node dist/index.js spec --json` failed
|
|
568
|
-
* entirely before this guard was added. The explicit protocol check below
|
|
569
|
-
* closes that.
|
|
570
|
-
*
|
|
571
|
-
* Two more deliberate departures from every other call in this file:
|
|
572
|
-
* - `TRAWL_TIMEOUT` (getTimeoutMs's env override, #91) is NOT consulted
|
|
573
|
-
* here — `opts.timeoutMs` is absolute. An operator raising that ceiling
|
|
574
|
-
* for a genuinely long-running command (e.g. LONG_RUN_TIMEOUT_MS's 300s)
|
|
575
|
-
* must never make an OPTIONAL probe wait 300s too; the two knobs measure
|
|
576
|
-
* different things and must not share a dial.
|
|
577
|
-
* - Redirects are NOT followed (global `fetch()`, used everywhere else in
|
|
578
|
-
* this file, follows them by default; raw `http`/`https` `request()`
|
|
579
|
-
* does not). For this probe a 3xx is just another non-2xx -> `null` ->
|
|
580
|
-
* fall through to rung 2 — an acceptable degradation, not a bug.
|
|
581
|
-
*/
|
|
582
283
|
function probeJson(path, opts) {
|
|
583
284
|
return new Promise((settle) => {
|
|
584
285
|
let url;
|
|
@@ -604,7 +305,7 @@ function probeJson(path, opts) {
|
|
|
604
305
|
const transport = url.protocol === 'http:' ? httpRequest : httpsRequest;
|
|
605
306
|
const req = transport(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
|
|
606
307
|
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
|
|
607
|
-
res.resume();
|
|
308
|
+
res.resume();
|
|
608
309
|
finish(null);
|
|
609
310
|
return;
|
|
610
311
|
}
|
|
@@ -623,9 +324,6 @@ function probeJson(path, opts) {
|
|
|
623
324
|
});
|
|
624
325
|
res.on('error', () => finish(null));
|
|
625
326
|
});
|
|
626
|
-
// The independent, non-idle-based ceiling this function exists for —
|
|
627
|
-
// fires regardless of connect/response state and forcibly destroys the
|
|
628
|
-
// socket, unlike AbortSignal.timeout() everywhere else in this file.
|
|
629
327
|
const timer = setTimeout(() => {
|
|
630
328
|
req.destroy(new Error(`probe to ${url.href} timed out after ${opts.timeoutMs}ms`));
|
|
631
329
|
}, opts.timeoutMs);
|
|
@@ -652,8 +350,6 @@ async function getText(path, reqOpts = {}) {
|
|
|
652
350
|
export const api = {
|
|
653
351
|
get: (path, opts) => request(path, {}, opts),
|
|
654
352
|
publicGet: (path, opts) => publicGet(path, opts),
|
|
655
|
-
/** See `probeJson`'s own doc comment — hard-bounded, probe-only, never for
|
|
656
|
-
* a command's real work. Today's one caller: spec.ts's fetchExternalDocsUrl. */
|
|
657
353
|
probeJson: (path, opts) => probeJson(path, opts),
|
|
658
354
|
getText: (path, opts) => getText(path, opts),
|
|
659
355
|
post: (path, body, opts) => request(path, {
|
|
@@ -672,11 +368,6 @@ export const api = {
|
|
|
672
368
|
if (!token)
|
|
673
369
|
throw notLoggedInError();
|
|
674
370
|
const url = `${getApiUrl()}${path}`;
|
|
675
|
-
// No AbortSignal.timeout here — a long-running `watch`/`--watch` stream is
|
|
676
|
-
// expected to sit open indefinitely; only connection-level failures
|
|
677
|
-
// (never a timeout) should surface via safeFetch's cause-unwrapping. (#71)
|
|
678
|
-
// effectiveTimeoutMs passed to safeFetch here is only ever used to format
|
|
679
|
-
// a timeout message that can't actually fire (no signal attached).
|
|
680
371
|
const res = await safeFetch(url, {
|
|
681
372
|
headers: {
|
|
682
373
|
Accept: 'text/event-stream',
|
|
@@ -684,15 +375,6 @@ export const api = {
|
|
|
684
375
|
...authHeaders(token),
|
|
685
376
|
},
|
|
686
377
|
}, getTimeoutMs());
|
|
687
|
-
// `scraps watch` is one of the JWT-only routes (#169 review finding 1) —
|
|
688
|
-
// singled out here (not routed through throwIfError, since this call
|
|
689
|
-
// never sets `isPublic` and the SSE fetch has no timeout signal to
|
|
690
|
-
// thread through) so a real 401 gets the same honest, auth-mode-aware
|
|
691
|
-
// message every other JWT-only route does, instead of the opaque `SSE
|
|
692
|
-
// failed: 401`. sessionAuthMessage reads `res` defensively — a real SSE
|
|
693
|
-
// Response supports `.text()` in production; only this file's own mocks
|
|
694
|
-
// sometimes don't, and that falls through to the honest hedge rather
|
|
695
|
-
// than throwing a second error (see sessionAuthMessage's doc comment).
|
|
696
378
|
if (res.status === 401) {
|
|
697
379
|
const mode = getAuthMode(token);
|
|
698
380
|
throw new ApiError(401, await sessionAuthMessage(mode, res), mode === 'apiKey' ? authNextSteps() : undefined);
|
|
@@ -716,7 +398,6 @@ export const api = {
|
|
|
716
398
|
}
|
|
717
399
|
}
|
|
718
400
|
}
|
|
719
|
-
// Flush any remaining data in buffer (stream ended without trailing newline)
|
|
720
401
|
if (buffer.startsWith('data: ')) {
|
|
721
402
|
yield buffer.slice(6);
|
|
722
403
|
}
|
package/dist/lib/cdp-pipe.d.ts
CHANGED
|
@@ -1,44 +1,8 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Raw Chrome DevTools Protocol client over Chrome's `--remote-debugging-pipe`
|
|
3
|
-
* transport (trawl_cli#183) — zero new dependencies. This is the SAME
|
|
4
|
-
* transport Puppeteer's own `pipe: true` mode uses: Chrome reads NUL
|
|
5
|
-
* (`\0`)-delimited JSON command messages on fd 3 and writes NUL-delimited
|
|
6
|
-
* JSON response/event messages on fd 4. No port, no `ws`, no HTTP polling
|
|
7
|
-
* of a `/json/version` endpoint, no parsing Chrome's stderr for a
|
|
8
|
-
* `DevTools listening on ws://…` line.
|
|
9
|
-
*
|
|
10
|
-
* Deliberately generic over any `Writable`/`Readable` pair (not tied to a
|
|
11
|
-
* real ChildProcess) so the framing/dispatch logic is fully testable with
|
|
12
|
-
* in-memory streams standing in for "Chrome" — see cdp-pipe.test.ts.
|
|
13
|
-
*/
|
|
14
1
|
import type { Readable, Writable } from 'node:stream';
|
|
15
2
|
type EventHandler = (params: unknown, sessionId?: string) => void;
|
|
16
|
-
/**
|
|
17
|
-
* A frame Chrome sent could not be parsed as JSON (trawl_cli#183 review
|
|
18
|
-
* finding 3). Distinguished from the generic "Chrome closed the CDP pipe"
|
|
19
|
-
* error so a caller can tell "the peer sent garbage and was cut off" apart
|
|
20
|
-
* from "the process actually exited" — `captureSession` reports the two
|
|
21
|
-
* with different, factual reasons rather than defaulting a corrupt-frame
|
|
22
|
-
* case to "Chrome exited".
|
|
23
|
-
*/
|
|
24
3
|
export declare class CdpProtocolError extends Error {
|
|
25
4
|
constructor(message: string);
|
|
26
5
|
}
|
|
27
|
-
/**
|
|
28
|
-
* A `send()` command's own deadline elapsed with no response (trawl_cli#183
|
|
29
|
-
* review finding 3's fix; the gap it reopened is finding 4's bug class —
|
|
30
|
-
* see session-capture.ts's `readCaptureDefault`). Deliberately a SIBLING of
|
|
31
|
-
* `CdpProtocolError`, never a subclass: a protocol error means the pipe
|
|
32
|
-
* itself can no longer be trusted, so it is torn down (`isClosed` flips
|
|
33
|
-
* true, every OTHER pending command rejects too). A timeout means exactly
|
|
34
|
-
* ONE command never got an answer — Chrome is still running, the pipe is
|
|
35
|
-
* still open, every other in-flight/future command is unaffected. Making
|
|
36
|
-
* this a `CdpProtocolError` subclass would let a caller's
|
|
37
|
-
* `instanceof CdpProtocolError` (or a future one) treat a single slow
|
|
38
|
-
* command as proof the whole browser is gone, which is false. Carries only
|
|
39
|
-
* `method` and `timeoutMs` — never `params`, which can carry page-
|
|
40
|
-
* controlled data.
|
|
41
|
-
*/
|
|
42
6
|
export declare class CdpTimeoutError extends Error {
|
|
43
7
|
readonly method: string;
|
|
44
8
|
readonly timeoutMs: number;
|
|
@@ -55,49 +19,13 @@ export declare class CdpPipe {
|
|
|
55
19
|
private protocolErr;
|
|
56
20
|
constructor(writeStream: Writable, readStream: Readable, commandTimeoutMs?: number);
|
|
57
21
|
private onClosed;
|
|
58
|
-
/**
|
|
59
|
-
* @desc A frame that fails to parse as JSON used to be dropped silently
|
|
60
|
-
* (`msg = undefined`, dispatch skipped) — any `send()` whose response was
|
|
61
|
-
* that exact frame then hung forever, since nothing ever rejected it.
|
|
62
|
-
* Now the whole pipe is torn down the same way a real Chrome exit is:
|
|
63
|
-
* every pending command rejects (with a `CdpProtocolError`, not the
|
|
64
|
-
* generic close message, so the cause is distinguishable), `isClosed`
|
|
65
|
-
* flips true, and `onClose` subscribers fire — the peer sent a frame
|
|
66
|
-
* that doesn't fit the protocol, so nothing it says next can be trusted
|
|
67
|
-
* either. The raw frame content is never included in the error message —
|
|
68
|
-
* it can carry page-controlled data (e.g. a `Runtime.evaluate` result).
|
|
69
|
-
*/
|
|
70
22
|
private onProtocolError;
|
|
71
23
|
private onData;
|
|
72
24
|
private dispatch;
|
|
73
|
-
/**
|
|
74
|
-
* Send a CDP command and resolve with its `result`. Rejects if the pipe
|
|
75
|
-
* closes (Chrome exited, or a corrupt frame tore it down — see
|
|
76
|
-
* `onProtocolError`) before a response arrives, or with a
|
|
77
|
-
* `CdpTimeoutError` if no response arrives within `timeoutMs` (defaults
|
|
78
|
-
* to the pipe's own `commandTimeoutMs`, 30s) — the timeout names the
|
|
79
|
-
* METHOD, never `params` (which can carry page-controlled data).
|
|
80
|
-
* @param timeoutMs — per-call override. Lets a caller pin its own named
|
|
81
|
-
* deadline as an assertable constant rather than relying on the pipe's
|
|
82
|
-
* default silently matching (see session-capture.ts's
|
|
83
|
-
* `LOCALSTORAGE_READ_TIMEOUT_MS`, which currently equals this pipe's own
|
|
84
|
-
* 30s default but is passed explicitly anyway — a post-cap review found
|
|
85
|
-
* that giving `Runtime.evaluate` a SHORTER override here, on the
|
|
86
|
-
* assumption a page-blocked renderer "won't unblock itself in 30s any
|
|
87
|
-
* more than in 5", cut off real-but-slow work that finished on its own;
|
|
88
|
-
* there is no override value proven safe, so this parameter exists for
|
|
89
|
-
* callers that want one, not as a recommendation to use a shorter one).
|
|
90
|
-
*/
|
|
91
25
|
send<T = unknown>(method: string, params?: unknown, sessionId?: string, timeoutMs?: number): Promise<T>;
|
|
92
|
-
/** Subscribe to a CDP event (`'Target.targetDestroyed'`, …). Returns an
|
|
93
|
-
* unsubscribe function. */
|
|
94
26
|
on(method: string, handler: EventHandler): () => void;
|
|
95
|
-
/** Subscribe to the pipe closing (Chrome process gone). */
|
|
96
27
|
onClose(handler: () => void): () => void;
|
|
97
28
|
get isClosed(): boolean;
|
|
98
|
-
/** Set only when the pipe was torn down because of a corrupt frame — as
|
|
99
|
-
* opposed to a real Chrome exit — so a caller can report the actual
|
|
100
|
-
* cause instead of defaulting every close to "Chrome exited". */
|
|
101
29
|
get protocolError(): CdpProtocolError | undefined;
|
|
102
30
|
}
|
|
103
31
|
export {};
|