@trawlme/cli 3.12.0 → 3.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/commands/create.d.ts +0 -28
- package/dist/commands/create.js +0 -89
- package/dist/commands/doctor.d.ts +0 -79
- package/dist/commands/doctor.js +1 -187
- package/dist/commands/login.js +0 -67
- package/dist/commands/ping.d.ts +0 -15
- package/dist/commands/ping.js +0 -15
- package/dist/commands/scraps.d.ts +0 -120
- package/dist/commands/scraps.js +10 -724
- package/dist/commands/skills.js +0 -22
- package/dist/commands/spec.d.ts +0 -85
- package/dist/commands/spec.js +0 -67
- package/dist/commands/telemetry.js +0 -4
- package/dist/commands/token.js +0 -28
- package/dist/commands/upgrade.js +0 -22
- package/dist/commands/whoami.d.ts +0 -12
- package/dist/commands/whoami.js +0 -6
- package/dist/index.d.ts +0 -188
- package/dist/index.js +0 -349
- package/dist/lib/api.d.ts +0 -78
- package/dist/lib/api.js +1 -320
- package/dist/lib/cdp-pipe.d.ts +0 -72
- package/dist/lib/cdp-pipe.js +1 -81
- package/dist/lib/chrome-discovery.d.ts +0 -11
- package/dist/lib/chrome-discovery.js +0 -19
- package/dist/lib/chrome-launch.d.ts +0 -40
- package/dist/lib/chrome-launch.js +0 -69
- package/dist/lib/config.d.ts +0 -53
- package/dist/lib/config.js +0 -55
- package/dist/lib/confirm.d.ts +0 -55
- package/dist/lib/confirm.js +0 -47
- package/dist/lib/docs.d.ts +0 -123
- package/dist/lib/docs.js +0 -169
- package/dist/lib/errors.d.ts +0 -134
- package/dist/lib/errors.js +0 -151
- package/dist/lib/format.d.ts +0 -6
- package/dist/lib/format.js +0 -6
- package/dist/lib/json.d.ts +0 -35
- package/dist/lib/json.js +0 -48
- package/dist/lib/jwt.d.ts +0 -7
- package/dist/lib/jwt.js +0 -7
- package/dist/lib/pinch.d.ts +0 -53
- package/dist/lib/pinch.js +6 -112
- package/dist/lib/pinchAnimation.d.ts +0 -16
- package/dist/lib/pinchAnimation.js +8 -29
- package/dist/lib/posthog.d.ts +0 -9
- package/dist/lib/posthog.js +0 -23
- package/dist/lib/prompt.js +1 -20
- package/dist/lib/secure-transport.d.ts +0 -7
- package/dist/lib/secure-transport.js +0 -24
- package/dist/lib/session-capture-guard.d.ts +0 -15
- package/dist/lib/session-capture-guard.js +0 -5
- package/dist/lib/session-capture.d.ts +0 -125
- package/dist/lib/session-capture.js +0 -281
- package/dist/lib/skills.d.ts +0 -175
- package/dist/lib/skills.js +1 -216
- package/dist/lib/skillsNudge.d.ts +0 -17
- package/dist/lib/skillsNudge.js +0 -83
- package/dist/lib/spinner.d.ts +0 -39
- package/dist/lib/spinner.js +0 -40
- package/dist/lib/storage-state.d.ts +0 -112
- package/dist/lib/storage-state.js +0 -131
- package/dist/lib/tips.d.ts +0 -38
- package/dist/lib/tips.js +0 -77
- package/dist/lib/updateCheckWorker.js +0 -14
- package/dist/lib/updateNotifier.d.ts +0 -17
- package/dist/lib/updateNotifier.js +0 -53
- package/dist/lib/validate.d.ts +0 -8
- package/dist/lib/validate.js +0 -8
- package/dist/lib/version.d.ts +0 -12
- package/dist/lib/version.js +1 -13
- package/docs/agent-quickstart.md +2 -2
- package/package.json +2 -2
package/dist/commands/scraps.js
CHANGED
|
@@ -16,56 +16,16 @@ import { captureSession, checkInteractiveEnvironment, LOCALSTORAGE_READ_TIMEOUT_
|
|
|
16
16
|
import { assertSecureTransport } from '../lib/secure-transport.js';
|
|
17
17
|
import { getApiUrl } from '../lib/config.js';
|
|
18
18
|
import { resolveDocsUrls, resolveFailureKindDocsUrl } from '../lib/docs.js';
|
|
19
|
-
/**
|
|
20
|
-
* #185 — `{ docs }` (spreadable, empty object when nothing to add) for a
|
|
21
|
-
* run's JSON payload (`doctor`/`data --errors`/`run-info`), keyed on
|
|
22
|
-
* trawl_node's `failureKind` (NOT errors.ts's unrelated `ErrorEnvelope.kind`
|
|
23
|
-
* — see docs.ts's `FAILURE_KIND_DOC_PATHS` doc comment for why those two
|
|
24
|
-
* `'auth'`s must never be conflated). Reuses `isAuthWall` — the SAME
|
|
25
|
-
* staleness guard `formatDoctor`'s human hint and the #184 skills nudge
|
|
26
|
-
* already share (a `failureKind:'auth'` stamp survives a later
|
|
27
|
-
* success/regression patch, see `isAuthWall`'s own doc comment) — so a run
|
|
28
|
-
* that ultimately succeeded or degraded never carries a stale `docs` link
|
|
29
|
-
* either, exactly the invariant `detectWallVendor`/`isAuthWall` already
|
|
30
|
-
* enforce for the human-facing surfaces. One call site, one rule, reused by
|
|
31
|
-
* all three JSON surfaces below instead of three copies that could drift.
|
|
32
|
-
*/
|
|
33
19
|
function runDocsField(run) {
|
|
34
20
|
if (!isAuthWall(run))
|
|
35
21
|
return {};
|
|
36
|
-
// Whole resolved object (never just `.docsUrl`) so resolveFailureKindDocsUrl
|
|
37
|
-
// can gate on rung provenance — see docs.ts's DocsUrls.docsUrlIsDerived.
|
|
38
|
-
// This call never supplies `externalDocsUrl`, so `docsUrl` here can only
|
|
39
|
-
// ever be rung-2-derived or absent — never a rung-1 server-declared root —
|
|
40
|
-
// but the gate stays explicit rather than relying on that as an invariant.
|
|
41
22
|
const resolved = resolveDocsUrls({ apiBaseUrl: getApiUrl() });
|
|
42
23
|
const docs = resolveFailureKindDocsUrl(run.failureKind, resolved);
|
|
43
24
|
return docs ? { docs } : {};
|
|
44
25
|
}
|
|
45
|
-
/**
|
|
46
|
-
* Print a usage/validation error consistently: human text to stderr, or a
|
|
47
|
-
* machine envelope on stdout under --json (never both — reportError is the
|
|
48
|
-
* single formatting path shared with index.ts's central catch, #86 finding
|
|
49
|
-
* 9). Sets exit code 2 (usage) — distinct from a business-logic refusal
|
|
50
|
-
* (which stays 1) or an unmapped ApiError/NetworkError (handled centrally in
|
|
51
|
-
* index.ts). (#71)
|
|
52
|
-
*/
|
|
53
26
|
function usageError(message, opts = {}) {
|
|
54
27
|
process.exitCode = reportError(new UsageError(message), { json: opts.json });
|
|
55
28
|
}
|
|
56
|
-
/**
|
|
57
|
-
* #163 — client-side URL validation shared by `create`/`update`, mirroring
|
|
58
|
-
* the server's now-required-and-parseable `url` field (trawl_node#1823,
|
|
59
|
-
* PR #1828: `z.string().trim().default('')` -> `z.string().trim().url()`).
|
|
60
|
-
* Gives a readable message BEFORE the request goes out instead of a raw
|
|
61
|
-
* 400/422 from the server. Reuses the exact same `requireUrl` helper
|
|
62
|
-
* `trawl create`/`trawl login --url` already validate against (lib/validate.js),
|
|
63
|
-
* just routed through this file's own return-style `usageError()` convention
|
|
64
|
-
* (never throw — every other validation in this file, tier/limit/page/params/
|
|
65
|
-
* status, follows the same pattern) instead of the throw-style those two
|
|
66
|
-
* commands use. Returns `undefined` (never throws) on a bad value — the
|
|
67
|
-
* caller must check for that and `return` without calling the API.
|
|
68
|
-
*/
|
|
69
29
|
function tryValidateUrl(value, opts) {
|
|
70
30
|
try {
|
|
71
31
|
return requireUrl(value, '--url');
|
|
@@ -78,28 +38,13 @@ function tryValidateUrl(value, opts) {
|
|
|
78
38
|
throw err;
|
|
79
39
|
}
|
|
80
40
|
}
|
|
81
|
-
/**
|
|
82
|
-
* One-line rendering of an emptyContext blob for the run-info table. Two
|
|
83
|
-
* buckets are actionable and must stay distinct: selectors that matched zero
|
|
84
|
-
* nodes (the page changed) and selectors the browser rejected outright
|
|
85
|
-
* (`-1`, a bug in the scrap script). Folding the second into "all matched"
|
|
86
|
-
* would report a broken selector as healthy. `--json` still emits the full
|
|
87
|
-
* object, `page.topClasses` / `topAnchorPaths` included.
|
|
88
|
-
*/
|
|
89
41
|
function summarizeEmptyContext(ctx) {
|
|
90
42
|
if (!ctx || typeof ctx !== 'object')
|
|
91
43
|
return null;
|
|
92
44
|
const parts = [];
|
|
93
45
|
const selectors = ctx.selectors && typeof ctx.selectors === 'object' ? ctx.selectors : {};
|
|
94
46
|
const names = Object.keys(selectors);
|
|
95
|
-
// Selectors may themselves contain a comma (`div, span`), so quote each one
|
|
96
|
-
// — a raw join renders one grouping selector as if it were two.
|
|
97
47
|
const list = (ns) => `${ns.slice(0, 3).map((s) => `\`${s}\``).join(', ')}${ns.length > 3 ? ` (+${ns.length - 3})` : ''}`;
|
|
98
|
-
// Four disjoint, exhaustive buckets — every selector lands in exactly one, so
|
|
99
|
-
// no count is ever dropped and none is summarised as something it is not.
|
|
100
|
-
// `selectors` comes from a Mongoose Mixed field, so a count may be `null` or
|
|
101
|
-
// a string: that is `unreadable`, which is neither healthy nor zero-match and
|
|
102
|
-
// must be named as its own thing next to the others, not collapsed into them.
|
|
103
48
|
const isCount = (v) => typeof v === 'number' && Number.isFinite(v);
|
|
104
49
|
const zero = names.filter((s) => selectors[s] === 0);
|
|
105
50
|
const invalid = names.filter((s) => selectors[s] === -1);
|
|
@@ -113,15 +58,8 @@ function summarizeEmptyContext(ctx) {
|
|
|
113
58
|
if (unreadable.length)
|
|
114
59
|
parts.push(`unreadable: ${list(unreadable)}`);
|
|
115
60
|
if (matched.length) {
|
|
116
|
-
// Named even in a mixed set: "3 of the selectors still work" is the
|
|
117
|
-
// difference between a page that moved and a page that vanished.
|
|
118
61
|
parts.push(mixed ? `${matched.length} matched` : `${matched.length} selector(s), all matched`);
|
|
119
62
|
}
|
|
120
|
-
// Same Mixed-field caution on the title: coerce before calling a string
|
|
121
|
-
// method (a numeric title would otherwise crash `run-info` on the very run
|
|
122
|
-
// it exists to explain), and test against null/undefined rather than
|
|
123
|
-
// truthiness so a falsy-but-real title like `0` still shows. Inner double
|
|
124
|
-
// quotes would close the wrapper early and corrupt the line.
|
|
125
63
|
const title = ctx.page?.title;
|
|
126
64
|
if (title !== undefined && title !== null && String(title) !== '') {
|
|
127
65
|
parts.push(`title="${String(title).replace(/"/g, "'")}"`);
|
|
@@ -134,25 +72,16 @@ function lastStatus(scrap) {
|
|
|
134
72
|
const last = scrap.history?.[0];
|
|
135
73
|
if (!last)
|
|
136
74
|
return 'never';
|
|
137
|
-
// status:null means a run is IN FLIGHT (node persists {status:null,
|
|
138
|
-
// statusDetail:null, inFlight:true} the moment a run starts) — that is NOT
|
|
139
|
-
// the same thing as "this scrap has never run" (no history row at all).
|
|
140
|
-
// (#88 item 1)
|
|
141
75
|
if (last.status === null)
|
|
142
76
|
return 'running';
|
|
143
77
|
if (last.status === undefined)
|
|
144
78
|
return 'never';
|
|
145
|
-
// #91 LOW — same regression-as-failure bucketing bug as doctor's badge
|
|
146
|
-
// (formatDoctor): a regression row's write actually succeeded (item count
|
|
147
|
-
// just dropped vs baseline, flagged by an async patch afterward — #88 item
|
|
148
|
-
// 2), so it must not collapse into the same 'failure' bucket a genuine
|
|
149
|
-
// failed run gets.
|
|
150
79
|
if (last.status === false && last.statusDetail === 'regression')
|
|
151
80
|
return 'regression';
|
|
152
81
|
return last.status === true ? 'success' : 'failure';
|
|
153
82
|
}
|
|
154
83
|
function lastRun(scrap) {
|
|
155
|
-
return formatDate(scrap.history?.[0]?.createdAt);
|
|
84
|
+
return formatDate(scrap.history?.[0]?.createdAt);
|
|
156
85
|
}
|
|
157
86
|
function statusIcon(status) {
|
|
158
87
|
if (status === 'success')
|
|
@@ -161,28 +90,10 @@ function statusIcon(status) {
|
|
|
161
90
|
return chalk.red('✗');
|
|
162
91
|
if (status === 'running')
|
|
163
92
|
return chalk.cyan('↻');
|
|
164
|
-
// #91 LOW — distinct amber icon, never the red ✗ a genuine failure gets
|
|
165
|
-
// (mirrors doctor's "● regression" badge — see formatDoctor).
|
|
166
93
|
if (status === 'regression')
|
|
167
94
|
return chalk.hex('#FFA500')('▼');
|
|
168
95
|
return chalk.dim('—');
|
|
169
96
|
}
|
|
170
|
-
/**
|
|
171
|
-
* #179 — health badge, next to the existing status icon. Two distinct
|
|
172
|
-
* sources, deliberately never blended into one derived verdict:
|
|
173
|
-
* 1. `consecutiveFailedRuns`/`unhealthySince` — present ONLY when `--unhealthy`
|
|
174
|
-
* asked the API to attach them (see attachListCommand). This is the
|
|
175
|
-
* live, exact server verdict — shown whenever available, since it is
|
|
176
|
-
* strictly more informative than the breadcrumb below.
|
|
177
|
-
* 2. `lastCronOutcome === 'skipped_unhealthy'` — always present (a normal
|
|
178
|
-
* scrap field, no extra query cost), but a STALE breadcrumb of the last
|
|
179
|
-
* scheduled tick, not a live read: never set on a no-cron scrap, lags
|
|
180
|
-
* by up to one tick, and flips back to 'ok'/'error' once the recovery
|
|
181
|
-
* probe fires. Labeled as what it is ("cron paused"), never upgraded to
|
|
182
|
-
* "unhealthy" — that word is reserved for the live verdict above.
|
|
183
|
-
* Never re-derives either signal from `history[]` — both come straight off
|
|
184
|
-
* the server (#1952 in trawl_node).
|
|
185
|
-
*/
|
|
186
97
|
function healthBadge(scrap) {
|
|
187
98
|
if (typeof scrap.consecutiveFailedRuns === 'number' && scrap.consecutiveFailedRuns > 0) {
|
|
188
99
|
const n = scrap.consecutiveFailedRuns;
|
|
@@ -193,19 +104,6 @@ function healthBadge(scrap) {
|
|
|
193
104
|
}
|
|
194
105
|
return chalk.dim('—');
|
|
195
106
|
}
|
|
196
|
-
/**
|
|
197
|
-
* #179 — old-server fallback for `list --unhealthy`, same shape as
|
|
198
|
-
* `warnIfUnconfirmedTier` above (#86 finding 4b): an older trawl_node that
|
|
199
|
-
* predates #1952 ignores the unknown `minConsecutiveFailures` query param
|
|
200
|
-
* entirely and returns the FULL unfiltered list — silently presenting every
|
|
201
|
-
* scrap as "unhealthy" is exactly the kind of unconfirmed-outcome lie that
|
|
202
|
-
* precedent exists to prevent. A confirming server attaches
|
|
203
|
-
* `consecutiveFailedRuns` to EVERY item once the param is set (the
|
|
204
|
-
* controller writes `s.consecutiveFailedRuns || 0` unconditionally in that
|
|
205
|
-
* branch — see trawl_node scraps.controller.js), so its total absence across
|
|
206
|
-
* a non-empty response is the honest tell. Stderr only (safe under --json —
|
|
207
|
-
* stdout purity is untouched), never blocks the (unreliable) results below.
|
|
208
|
-
*/
|
|
209
107
|
function healthFilterUnconfirmed(data, unhealthyWasRequested) {
|
|
210
108
|
if (!unhealthyWasRequested || data.length === 0)
|
|
211
109
|
return false;
|
|
@@ -214,35 +112,19 @@ function healthFilterUnconfirmed(data, unhealthyWasRequested) {
|
|
|
214
112
|
function warnIfUnconfirmedHealthFilter(data, unhealthyWasRequested) {
|
|
215
113
|
if (!healthFilterUnconfirmed(data, unhealthyWasRequested))
|
|
216
114
|
return;
|
|
217
|
-
console.error(chalk.yellow('⚠ Server did not confirm the --unhealthy filter (older server, predates
|
|
115
|
+
console.error(chalk.yellow('⚠ Server did not confirm the --unhealthy filter (older server version, predates this filter) — results below are UNFILTERED, not just the unhealthy scraps.'));
|
|
218
116
|
}
|
|
219
|
-
/**
|
|
220
|
-
* #179 — the machine-readable counterpart to the stderr warning above, in the
|
|
221
|
-
* exact shape `withTierUnconfirmed` already uses for the tier case. A --json
|
|
222
|
-
* caller has no reliable reason to read stderr, so without this a script sees
|
|
223
|
-
* a full unfiltered list and cannot tell it is not the unhealthy set.
|
|
224
|
-
* Emitted ONLY when the filter went unconfirmed — never fabricated otherwise.
|
|
225
|
-
*/
|
|
226
117
|
function withHealthFilterUnconfirmed(rows, data, unhealthyWasRequested) {
|
|
227
118
|
if (!healthFilterUnconfirmed(data, unhealthyWasRequested))
|
|
228
119
|
return rows;
|
|
229
120
|
return { scraps: rows, _healthFilterUnconfirmed: true };
|
|
230
121
|
}
|
|
231
122
|
export const scraps = new Command('scraps').description('Manage scraps');
|
|
232
|
-
// shared SSE streaming helper. `asJson` (#107) emits one raw JSON object per
|
|
233
|
-
// line (NDJSON) on stdout instead of the human-formatted timestamped text —
|
|
234
|
-
// a streaming command still needs a pure-stdout machine mode, just line-
|
|
235
|
-
// delimited instead of a single blob (there's no single "final" payload to
|
|
236
|
-
// wait for).
|
|
237
123
|
async function watchActivities(id, asJson) {
|
|
238
124
|
if (!asJson)
|
|
239
125
|
console.log(chalk.dim('Streaming activities (Ctrl+C to stop)…\n'));
|
|
240
126
|
for await (const event of api.stream(`/api/scraps/${id}/activities/stream`)) {
|
|
241
127
|
try {
|
|
242
|
-
// #159 — same tolerant parse as api.ts's response-parsing seam: an SSE
|
|
243
|
-
// activity line can carry the same server-side raw-control-character
|
|
244
|
-
// quirk as any other API payload, and this feeds `--json` NDJSON
|
|
245
|
-
// output too (not just create).
|
|
246
128
|
const activity = parseServerJson(event);
|
|
247
129
|
if (asJson) {
|
|
248
130
|
console.log(JSON.stringify(activity));
|
|
@@ -252,34 +134,12 @@ async function watchActivities(id, asJson) {
|
|
|
252
134
|
console.log(`${chalk.dim(`[${time}]`)} ${activity.message}`);
|
|
253
135
|
}
|
|
254
136
|
catch {
|
|
255
|
-
// non-JSON lines are heartbeats or SSE comments — skip silently
|
|
256
137
|
if (process.env['DEBUG'] && event.trim()) {
|
|
257
138
|
console.error(chalk.dim(`[SSE] skipping non-JSON: ${event}`));
|
|
258
139
|
}
|
|
259
140
|
}
|
|
260
141
|
}
|
|
261
142
|
}
|
|
262
|
-
/**
|
|
263
|
-
* #91 P1 / #93 item 1 — snapshot the current top history entry BEFORE
|
|
264
|
-
* triggering a run, so pollRunProgress can later tell "the run we just
|
|
265
|
-
* launched" apart from whatever the last run happened to be. Node persists a
|
|
266
|
-
* fresh {status:null, inFlight:true} row the moment a run starts (see
|
|
267
|
-
* HistorysService.create in trawl_node), so a changed history[0]._id is
|
|
268
|
-
* normally an honest "the new run has begun" signal — EXCEPT `trigger`
|
|
269
|
-
* (method:'worker') dedups onto an already-pending/running worker job
|
|
270
|
-
* (ScrapJobsService, LIVE_STATUSES) instead of creating a new row: the top
|
|
271
|
-
* row's `_id` never changes even though this launch IS that run. Recording
|
|
272
|
-
* `alreadyInFlight` (status===null at capture time) lets pollRunProgress
|
|
273
|
-
* recognize that dedup case too, instead of waiting forever for an `_id`
|
|
274
|
-
* that will never arrive (#93 item 1).
|
|
275
|
-
*
|
|
276
|
-
* #97 — a failed lookup used to fall back to `{alreadyInFlight:false}` with
|
|
277
|
-
* no `id`, which LOOKED identical to "scrap has never run" — but is not:
|
|
278
|
-
* the scrap may well have prior (possibly terminal) history this GET simply
|
|
279
|
-
* never observed. `captured:false` flags that distinction explicitly so
|
|
280
|
-
* pollRunProgress can defer trusting a baseline instead of risking a stale
|
|
281
|
-
* pre-existing row being reported as the run just launched.
|
|
282
|
-
*/
|
|
283
143
|
async function captureBeforeRunState(id) {
|
|
284
144
|
try {
|
|
285
145
|
const scrap = await api.get(`/api/scraps/${id}`);
|
|
@@ -294,115 +154,14 @@ function sleep(ms) {
|
|
|
294
154
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
295
155
|
}
|
|
296
156
|
const POLL_INTERVAL_MS = 2000;
|
|
297
|
-
// Mirrors LONG_RUN_TIMEOUT_MS (#91 item 1) — the server-side worst case this
|
|
298
|
-
// polls for is the same one that timeout was sized for.
|
|
299
157
|
const POLL_TIMEOUT_MS = LONG_RUN_TIMEOUT_MS;
|
|
300
|
-
// #107 review F1 — a genuinely unreachable API must not run the watch
|
|
301
|
-
// silently for the full 300s timeout with dead air; after this many
|
|
302
|
-
// CONSECUTIVE (not cumulative — any successful poll resets the counter)
|
|
303
|
-
// failed polls, treat it as a persistent error and surface it immediately
|
|
304
|
-
// instead of waiting out the clock. At the default 2s interval this gives up
|
|
305
|
-
// after ~10s of continuous failures — comfortably longer than any single
|
|
306
|
-
// transient blip, nowhere near the 300s ceiling.
|
|
307
158
|
const MAX_CONSECUTIVE_POLL_ERRORS = 5;
|
|
308
|
-
/**
|
|
309
|
-
* #91 P1 — replaces "await the run to completion, THEN open the activities
|
|
310
|
-
* SSE stream" (which showed NOTHING: the activities SSE
|
|
311
|
-
* (GET /api/scraps/:id/activities/stream) is backed by an in-process
|
|
312
|
-
* EventEmitter with no backlog — trawl_node
|
|
313
|
-
* modules/activities/services/activities.service.js — so by the time a
|
|
314
|
-
* synchronous run has already finished there is nothing left to emit; and
|
|
315
|
-
* the async `trigger` default enqueues the run onto a durable job queue a
|
|
316
|
-
* SEPARATE cron-consumer pod drains, whose in-process emitter never reaches
|
|
317
|
-
* the API pod holding the SSE connection at all).
|
|
318
|
-
*
|
|
319
|
-
* Instead this polls two REST reads that are BOTH Mongo-backed (not
|
|
320
|
-
* in-process), so they work no matter which pod actually executed the run:
|
|
321
|
-
* - GET /api/scraps/:id/activities?history=<hid>&limit=20 — the SAME
|
|
322
|
-
* activities-list endpoint `doctor`'s fetchRunAndFix already calls
|
|
323
|
-
* (src/commands/doctor.ts) — prints each new activity line once the new
|
|
324
|
-
* run's history id is known.
|
|
325
|
-
* - GET /api/scraps/:id — history[0].status/statusDetail, the SAME
|
|
326
|
-
* terminal-status signal `lastStatus()` above already trusts (status:null
|
|
327
|
-
* === in flight, #88 item 1) to know when the run is done.
|
|
328
|
-
*
|
|
329
|
-
* #93 item 1 — dedup-race fix. "Is this the run we're watching?" used to be
|
|
330
|
-
* a single check: `history[0]._id !== beforeHistoryId`. That's wrong when
|
|
331
|
-
* `before.alreadyInFlight` is true (a `trigger` call deduped onto a worker
|
|
332
|
-
* job that was ALREADY pending/running at capture time): the top row IS the
|
|
333
|
-
* run we're watching, but its `_id` never changes, so the old guard never
|
|
334
|
-
* released and the poll ran the full timeout to a false "Timed out". The run
|
|
335
|
-
* we're watching is now EITHER a brand-new id (fresh trigger, the common
|
|
336
|
-
* case) OR the same id that was already in-flight (status:null) at capture
|
|
337
|
-
* (the dedup case) — a same-id row that was already TERMINAL at capture is
|
|
338
|
-
* neither, and must not be latched onto as "done" (it's just the previous
|
|
339
|
-
* run, still sitting there until a genuinely new run supersedes it).
|
|
340
|
-
*
|
|
341
|
-
* #97 — capture-failed fallback. The dedup-race fix above assumes `before`
|
|
342
|
-
* is trustworthy. When `captureBeforeRunState`'s own GET threw,
|
|
343
|
-
* `before.id` is `undefined` — and that is INDISTINGUISHABLE from "the
|
|
344
|
-
* scrap has genuinely never run" (also `id: undefined`), which is exactly
|
|
345
|
-
* the case `isNewRun` above is designed to match on the very first row that
|
|
346
|
-
* ever appears. So on the very first poll, ANY pre-existing history row —
|
|
347
|
-
* even the STALE PREVIOUS run, already terminal — satisfied
|
|
348
|
-
* `last._id !== undefined` and got reported as "the run we just launched"
|
|
349
|
-
* finishing, when it was really just whatever ran before.
|
|
350
|
-
*
|
|
351
|
-
* Fix: `before.captured === false` defers trusting a baseline at all.
|
|
352
|
-
* Instead of comparing against the (unknown) `beforeId` from the start, the
|
|
353
|
-
* FIRST successful poll read is treated as the deferred capture itself —
|
|
354
|
-
* exactly what `captureBeforeRunState` would have returned had its GET
|
|
355
|
-
* succeeded — and only READS from that point on are compared against it,
|
|
356
|
-
* via the exact same `isNewRun` / `isDedupOntoInFlight` logic above. A
|
|
357
|
-
* pre-existing terminal row observed on that first read becomes `beforeId`
|
|
358
|
-
* (not a match for itself), so it correctly falls into "still the stale
|
|
359
|
-
* previous run" below and the poll keeps waiting; a row still in flight
|
|
360
|
-
* becomes the `alreadyInFlight` baseline, exactly like a successful capture
|
|
361
|
-
* would have recorded. Either way this costs at most one extra poll
|
|
362
|
-
* interval, bounded by the same deadline as everything else. The one
|
|
363
|
-
* remaining edge case — capture failed AND the scrap never ran before AND
|
|
364
|
-
* the triggered run already finished by the very first poll — is genuinely
|
|
365
|
-
* undecidable from "stale pre-existing row" with no more information than
|
|
366
|
-
* this function has, so it resolves to the same honest timeout rather than
|
|
367
|
-
* risk reporting a possibly-wrong outcome (never a lie, at worst a timeout
|
|
368
|
-
* telling the caller to check `doctor`).
|
|
369
|
-
*
|
|
370
|
-
* #107 review F1 — before this fix, `run|trigger --json --watch` was
|
|
371
|
-
* outcome-blind: `quiet` suppressed ALL output (including "Run finished:
|
|
372
|
-
* failure" and the timeout notice), a transient poll error was caught and
|
|
373
|
-
* silently retried FOREVER within the deadline, and the process always
|
|
374
|
-
* exited 0 after the poll loop regardless of what the watched run actually
|
|
375
|
-
* did — dead air, then a clean exit code, even for a failed or timed-out
|
|
376
|
-
* run. An agent scripting this CLI had no way to tell success from failure
|
|
377
|
-
* from "we gave up". Fixed by:
|
|
378
|
-
* - emitting exactly ONE final NDJSON line on stdout under `--json` once
|
|
379
|
-
* the watch reaches ANY of its three exits (terminal status, timeout, or
|
|
380
|
-
* a persistent poll error) — `{runId,status}` (+`error` for a poll
|
|
381
|
-
* error) — while every intermediate progress line stays suppressed
|
|
382
|
-
* (unchanged from before);
|
|
383
|
-
* - setting `process.exitCode` non-zero on a genuine run failure, a
|
|
384
|
-
* timeout, or a persistent poll error, and `0` on a real success — in
|
|
385
|
-
* BOTH `--json` and human `--watch` modes (human mode used to exit 0
|
|
386
|
-
* unconditionally, the same bug, just silent instead of dishonest);
|
|
387
|
-
* - giving up after `MAX_CONSECUTIVE_POLL_ERRORS` consecutive failed reads
|
|
388
|
-
* instead of retrying the same dead endpoint for the full 300s.
|
|
389
|
-
*/
|
|
390
159
|
export async function pollRunProgress(id, before, opts = {}) {
|
|
391
160
|
const intervalMs = opts.intervalMs ?? POLL_INTERVAL_MS;
|
|
392
161
|
const timeoutMs = opts.timeoutMs ?? POLL_TIMEOUT_MS;
|
|
393
|
-
// #107 — `run --json --watch` / `trigger --json --watch` must keep stdout
|
|
394
|
-
// pure JSON: none of this function's intermediate progress text is safe to
|
|
395
|
-
// print once a caller asked for --json. `asJson` suppresses every
|
|
396
|
-
// intermediate console.log below (including the pinch celebration frame)
|
|
397
|
-
// while the polling/wait logic itself runs unchanged; the ONE exception is
|
|
398
|
-
// the single final outcome line emitted right before each return below.
|
|
399
162
|
const asJson = opts.json ?? false;
|
|
400
163
|
let beforeId = before?.id;
|
|
401
164
|
let beforeAlreadyInFlight = before?.alreadyInFlight ?? false;
|
|
402
|
-
// #97 — only an EXPLICIT captured:false (capture's GET actually threw)
|
|
403
|
-
// defers the baseline. `before` itself being undefined (callers that skip
|
|
404
|
-
// capture entirely) or `captured` being true/absent both mean "trust
|
|
405
|
-
// beforeId as given", preserving every existing call site's behavior.
|
|
406
165
|
let baselineEstablished = before?.captured ?? true;
|
|
407
166
|
if (!asJson) {
|
|
408
167
|
console.log(chalk.dim('Live activity streaming has no signal for this run (async/cross-pod) — polling for progress instead…\n'));
|
|
@@ -422,11 +181,6 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
422
181
|
}
|
|
423
182
|
catch (err) {
|
|
424
183
|
consecutivePollErrors++;
|
|
425
|
-
// #107 review F1 — a single failed read is still a TRANSIENT blip,
|
|
426
|
-
// safe to retry within the deadline (unchanged). Only once reads fail
|
|
427
|
-
// this many times IN A ROW is the API treated as genuinely
|
|
428
|
-
// unreachable — surface that honestly instead of quietly burning the
|
|
429
|
-
// full 300s timeout on a dead endpoint.
|
|
430
184
|
if (consecutivePollErrors >= MAX_CONSECUTIVE_POLL_ERRORS) {
|
|
431
185
|
const message = err instanceof Error ? err.message : String(err);
|
|
432
186
|
if (asJson) {
|
|
@@ -443,13 +197,8 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
443
197
|
}
|
|
444
198
|
const last = scrap.history?.[0];
|
|
445
199
|
if (!last?._id)
|
|
446
|
-
continue;
|
|
200
|
+
continue;
|
|
447
201
|
if (!baselineEstablished) {
|
|
448
|
-
// #97 — deferred capture: whatever we see on this first successful
|
|
449
|
-
// read becomes the reference point, BEFORE the isNewRun check below
|
|
450
|
-
// ever runs against it. This must happen before that check, not
|
|
451
|
-
// after, so a pre-existing terminal row is recognized as stale on
|
|
452
|
-
// this very same iteration rather than one poll late.
|
|
453
202
|
beforeId = last._id;
|
|
454
203
|
beforeAlreadyInFlight = last.status === null;
|
|
455
204
|
baselineEstablished = true;
|
|
@@ -457,10 +206,9 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
457
206
|
const isNewRun = last._id !== beforeId;
|
|
458
207
|
const isDedupOntoInFlight = last._id === beforeId && beforeAlreadyInFlight;
|
|
459
208
|
if (!isNewRun && !isDedupOntoInFlight)
|
|
460
|
-
continue;
|
|
209
|
+
continue;
|
|
461
210
|
try {
|
|
462
211
|
const activities = await api.get(`/api/scraps/${id}/activities?history=${last._id}&limit=20`);
|
|
463
|
-
// Server returns newest-first — print unseen ones oldest-first.
|
|
464
212
|
for (const a of [...(activities ?? [])].reverse()) {
|
|
465
213
|
const key = a._id ?? `${a.createdAt}:${a.message}`;
|
|
466
214
|
if (seen.has(key))
|
|
@@ -473,16 +221,9 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
473
221
|
}
|
|
474
222
|
}
|
|
475
223
|
catch {
|
|
476
|
-
// best-effort — terminal detection below still works without activity lines
|
|
477
224
|
}
|
|
478
225
|
if (last.status !== null) {
|
|
479
226
|
const outcome = last.statusDetail ?? (last.status ? 'success' : 'failure');
|
|
480
|
-
// #107 review F1 — the exit code, in BOTH modes: a regression row's
|
|
481
|
-
// WRITE actually succeeded (see lastStatus()'s own #91 comment above,
|
|
482
|
-
// and `scraps data`'s isRegression branch) and an 'empty' row is a
|
|
483
|
-
// genuine zero-item success — neither is a real failure. Only
|
|
484
|
-
// status:false with neither of those details (e.g. 'error', or no
|
|
485
|
-
// detail at all) is a genuine failed run.
|
|
486
227
|
const isGenuineFailure = last.status === false && outcome !== 'empty' && outcome !== 'regression';
|
|
487
228
|
if (asJson) {
|
|
488
229
|
const payload = { runId: last._id, status: outcome };
|
|
@@ -490,10 +231,6 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
490
231
|
}
|
|
491
232
|
else {
|
|
492
233
|
console.log(chalk.dim(`Run finished: ${outcome}`));
|
|
493
|
-
// Pinch celebrates a clean run finish (#94) — mirrors doctor.ts's own
|
|
494
|
-
// `status === true` success definition (regardless of statusDetail),
|
|
495
|
-
// never for a failed/regression run. Guarded by pinchEnabled()
|
|
496
|
-
// (NO_COLOR/non-TTY) and never under --json (stdout must stay pure).
|
|
497
234
|
if (last.status === true && pinchEnabled()) {
|
|
498
235
|
console.log(renderPinch('celebrating'));
|
|
499
236
|
}
|
|
@@ -502,9 +239,6 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
502
239
|
return;
|
|
503
240
|
}
|
|
504
241
|
}
|
|
505
|
-
// #107 review F1 — timeout: honest non-zero exit in both modes, plus the
|
|
506
|
-
// machine-readable final line under --json (never silently exit 0 after a
|
|
507
|
-
// watch that never actually confirmed what happened).
|
|
508
242
|
if (asJson) {
|
|
509
243
|
const outcome = { runId: beforeId, status: 'timeout' };
|
|
510
244
|
console.log(JSON.stringify(outcome));
|
|
@@ -514,20 +248,8 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
514
248
|
}
|
|
515
249
|
process.exitCode = 1;
|
|
516
250
|
}
|
|
517
|
-
// #149 item 3 — same shared-enum shape as VALID_TIERS below (~line 603):
|
|
518
|
-
// keeps `--status`'s allowed values and its error message in one place,
|
|
519
|
-
// mirrored from lastStatus()'s own return type so the two can never drift.
|
|
520
251
|
const VALID_LAST_STATUSES = ['success', 'failure', 'never', 'running', 'regression'];
|
|
521
|
-
// #179 — mirrors `UNHEALTHY_STREAK_LENGTH` in trawl_node
|
|
522
|
-
// modules/scraps/helpers/scrapCronHealth.js. `--unhealthy` is a boolean
|
|
523
|
-
// convenience flag for that ONE product-defined threshold (also what the
|
|
524
|
-
// server's own cron-skip gate and email use), not a general numeric filter —
|
|
525
|
-
// the API's `minConsecutiveFailures` param accepts any threshold, but this
|
|
526
|
-
// CLI only ever asks for the one value the rest of the product means by
|
|
527
|
-
// "unhealthy". If trawl_node's threshold ever changes, update this constant
|
|
528
|
-
// to match.
|
|
529
252
|
const UNHEALTHY_STREAK_LENGTH = 3;
|
|
530
|
-
// list — promoted to a top-level verb (#108, see the AttachOptions comment above)
|
|
531
253
|
export function attachListCommand(parent, attachOpts = {}) {
|
|
532
254
|
return parent
|
|
533
255
|
.command('list', attachOpts)
|
|
@@ -535,33 +257,14 @@ export function attachListCommand(parent, attachOpts = {}) {
|
|
|
535
257
|
.description('List all scraps')
|
|
536
258
|
.option('--json', 'Output as JSON')
|
|
537
259
|
.option('--status <status>', 'Filter by last run status (success|failure|never|running|regression)')
|
|
538
|
-
// #88 item 8 — no custom parser here (unlike the old `(v) => parseInt(v,
|
|
539
|
-
// 10)`): a bad value like "abc" used to silently become NaN, which then
|
|
540
|
-
// sailed straight through `Number.isInteger`-less checks and into
|
|
541
|
-
// `.slice(0, NaN)` (silently truncates to 0 rows) or `?page=NaN` (silently
|
|
542
|
-
// sent to the server) — never a usage error. Keeping the raw string here
|
|
543
|
-
// lets the validation below mirror `history`'s own --limit check exactly
|
|
544
|
-
// (~line 660) and report the actual bad input in the error message.
|
|
545
260
|
.option('--limit <n>', 'Show only the first N results')
|
|
546
261
|
.option('--page <n>', 'Fetch a specific page only (50 per page, no auto-pagination)')
|
|
547
|
-
// #179 — backed by the server's `minConsecutiveFailures` filter
|
|
548
|
-
// (trawl_node#1952): restricts the result set server-side to scraps
|
|
549
|
-
// whose CURRENT consecutive-failure streak is >= UNHEALTHY_STREAK_LENGTH,
|
|
550
|
-
// and annotates each returned scrap with the exact `consecutiveFailedRuns`/
|
|
551
|
-
// `unhealthySince` the aggregation computed. Deliberately never derived
|
|
552
|
-
// client-side from `history[]` — that would drift from what the web
|
|
553
|
-
// dashboard shows (comes-io/trawl_vue#1299 reads the identical filter).
|
|
554
262
|
.option('--unhealthy', `Only show scraps failing their last ${UNHEALTHY_STREAK_LENGTH}+ runs (server-computed)`)
|
|
555
263
|
.action(async (opts, cmd) => {
|
|
556
|
-
// #149 item 3 — validate --status against its enum the same way
|
|
557
|
-
// --tier already validates (usageError + return, checked first, before
|
|
558
|
-
// any other guard/async work — mirrors `create`/`update`'s --tier
|
|
559
|
-
// check below).
|
|
560
264
|
if (opts.status !== undefined && !VALID_LAST_STATUSES.includes(opts.status)) {
|
|
561
265
|
usageError(`Invalid --status "${opts.status}" (allowed: ${VALID_LAST_STATUSES.join(', ')})`, { json: opts.json });
|
|
562
266
|
return;
|
|
563
267
|
}
|
|
564
|
-
// Guard: --limit and --page are mutually exclusive
|
|
565
268
|
if (opts.limit !== undefined && opts.page !== undefined) {
|
|
566
269
|
usageError('--limit and --page are mutually exclusive. Use one or the other.', { json: opts.json });
|
|
567
270
|
return;
|
|
@@ -582,18 +285,13 @@ export function attachListCommand(parent, attachOpts = {}) {
|
|
|
582
285
|
return;
|
|
583
286
|
}
|
|
584
287
|
}
|
|
585
|
-
// #179 — same param on every request this action can issue (both
|
|
586
|
-
// branches below), so a filtered result stays filtered across the
|
|
587
|
-
// fetch-all pagination loop, not just its first page.
|
|
588
288
|
const healthFilter = opts.unhealthy ? `&minConsecutiveFailures=${UNHEALTHY_STREAK_LENGTH}` : '';
|
|
589
289
|
let data;
|
|
590
290
|
try {
|
|
591
291
|
data = await spin(async () => {
|
|
592
292
|
if (page !== undefined) {
|
|
593
|
-
// Single-page mode: explicit page requested, no loop
|
|
594
293
|
return api.get(`/api/scraps?perPage=50&page=${page}${healthFilter}`);
|
|
595
294
|
}
|
|
596
|
-
// Fetch-all mode: paginate until a page returns < 200 items
|
|
597
295
|
const perPage = 200;
|
|
598
296
|
let result = [];
|
|
599
297
|
let pageNum = 1;
|
|
@@ -608,16 +306,6 @@ export function attachListCommand(parent, attachOpts = {}) {
|
|
|
608
306
|
}, 'Fetching scraps…');
|
|
609
307
|
}
|
|
610
308
|
catch (err) {
|
|
611
|
-
// Spinner already failed by oraPromise — report with the scrap-specific
|
|
612
|
-
// prefix kept, but route through the shared classifier so exit code +
|
|
613
|
-
// --json envelope stay consistent with every other command. (#71)
|
|
614
|
-
//
|
|
615
|
-
// This bespoke catch (kept for the "Failed to fetch scraps:" prefix,
|
|
616
|
-
// which the shared reportError() can't add) used to silently swallow
|
|
617
|
-
// --debug: unlike the central index.ts catch, it never printed the raw
|
|
618
|
-
// stack trace. optsWithGlobals() reads --debug off the ROOT command
|
|
619
|
-
// (this leaf has no --debug of its own) so it can honor the flag
|
|
620
|
-
// locally instead. (#86 finding 9)
|
|
621
309
|
const isDebug = Boolean(cmd.optsWithGlobals().debug || process.env['DEBUG']);
|
|
622
310
|
const { exitCode, envelope } = classifyError(err);
|
|
623
311
|
const message = `Failed to fetch scraps: ${envelope.message}`;
|
|
@@ -632,9 +320,6 @@ export function attachListCommand(parent, attachOpts = {}) {
|
|
|
632
320
|
process.exitCode = exitCode;
|
|
633
321
|
return;
|
|
634
322
|
}
|
|
635
|
-
// #179 — check the RAW response, before --status narrows it further,
|
|
636
|
-
// so a genuinely empty (all-healthy) result is never mistaken for an
|
|
637
|
-
// unconfirmed filter.
|
|
638
323
|
warnIfUnconfirmedHealthFilter(data, opts.unhealthy);
|
|
639
324
|
if (opts.status)
|
|
640
325
|
data = data.filter((s) => lastStatus(s) === opts.status);
|
|
@@ -647,21 +332,17 @@ export function attachListCommand(parent, attachOpts = {}) {
|
|
|
647
332
|
title: s.title || '(untitled)',
|
|
648
333
|
cron: s.cron || '—',
|
|
649
334
|
status: statusIcon(lastStatus(s)),
|
|
650
|
-
// #179 — next to the existing status badge, per the issue's own
|
|
651
|
-
// wording (see healthBadge() above for the two source signals).
|
|
652
335
|
health: healthBadge(s),
|
|
653
336
|
'last run': lastRun(s),
|
|
654
337
|
updated: formatDate(s.updatedAt),
|
|
655
338
|
}));
|
|
656
339
|
table(tableRows, ['id', 'title', 'cron', 'status', 'health', 'last run', 'updated']);
|
|
657
|
-
// Print footer when --limit truncates
|
|
658
340
|
if (limit !== undefined && rows.length < totalMatched) {
|
|
659
341
|
console.log(chalk.dim(`Showing ${rows.length} of ${totalMatched} — omit --limit to see all`));
|
|
660
342
|
}
|
|
661
343
|
});
|
|
662
344
|
}
|
|
663
345
|
attachListCommand(scraps, { hidden: true });
|
|
664
|
-
// get — promoted to a top-level verb (#108)
|
|
665
346
|
export function attachGetCommand(parent, attachOpts = {}) {
|
|
666
347
|
return parent
|
|
667
348
|
.command('get <id>', attachOpts)
|
|
@@ -676,9 +357,6 @@ export function attachGetCommand(parent, attachOpts = {}) {
|
|
|
676
357
|
console.log(chalk.dim(` ID: `) + data._id);
|
|
677
358
|
console.log(chalk.dim(` Cron: `) + (data.cron || '—'));
|
|
678
359
|
console.log(chalk.dim(` Status: `) + statusIcon(lastStatus(data)));
|
|
679
|
-
// #179 — same badge `list` renders (see healthBadge() above); `get`
|
|
680
|
-
// never sends `minConsecutiveFailures`, so this always reads the
|
|
681
|
-
// free `lastCronOutcome` breadcrumb, never the live per-run count.
|
|
682
360
|
console.log(chalk.dim(` Health: `) + healthBadge(data));
|
|
683
361
|
console.log(chalk.dim(` Last run: `) + lastRun(data));
|
|
684
362
|
console.log(chalk.dim(` Updated: `) + formatDate(data.updatedAt));
|
|
@@ -686,14 +364,6 @@ export function attachGetCommand(parent, attachOpts = {}) {
|
|
|
686
364
|
}
|
|
687
365
|
attachGetCommand(scraps, { hidden: true });
|
|
688
366
|
const VALID_TIERS = ['tier0', 'tier1', 'tier2', 'tier3', 'tier4'];
|
|
689
|
-
/**
|
|
690
|
-
* #86 findings 4/5 — shared honest-tier renderer for `create --tier` and
|
|
691
|
-
* `update --tier/--force-tier`. Reads the typed `_tierOverride` echoed back
|
|
692
|
-
* by the server (#1559) so both commands show the SAME truth: a refusal, an
|
|
693
|
-
* allowed ceiling raise (+ spend warning), or a silently clamped proxyTier.
|
|
694
|
-
* Human-mode output only — a --json caller gets the same truth for free from
|
|
695
|
-
* the full scrap object (which already includes `_tierOverride`).
|
|
696
|
-
*/
|
|
697
367
|
function renderTierOverrideHuman(data) {
|
|
698
368
|
const ov = data._tierOverride;
|
|
699
369
|
if (!ov)
|
|
@@ -718,64 +388,25 @@ function renderTierOverrideHuman(data) {
|
|
|
718
388
|
}
|
|
719
389
|
}
|
|
720
390
|
}
|
|
721
|
-
/**
|
|
722
|
-
* #86 finding 4b — old-server fallback. When a tier was requested but the
|
|
723
|
-
* response carries no `_tierOverride` at all, the server is too old to
|
|
724
|
-
* confirm what actually got applied. Echoing the REQUESTED value as if it
|
|
725
|
-
* were the outcome is exactly the silent-clamp lie #1559 fixed — warn
|
|
726
|
-
* instead, on stderr (safe under --json too; stdout purity is untouched).
|
|
727
|
-
*/
|
|
728
391
|
function warnIfUnconfirmedTier(data, tierWasRequested, id) {
|
|
729
392
|
if (data._tierOverride || !tierWasRequested)
|
|
730
393
|
return;
|
|
731
394
|
console.error(chalk.yellow(` ⚠ Server did not confirm the tier change (older server) — verify with: trawl get ${id}`));
|
|
732
395
|
}
|
|
733
|
-
/**
|
|
734
|
-
* #88 item 3 — the --json machine-readable counterpart to
|
|
735
|
-
* warnIfUnconfirmedTier's stderr warning above. A --json caller (an agent
|
|
736
|
-
* scripting this CLI) has no reliable reason to read stderr — that channel
|
|
737
|
-
* is advisory-only everywhere else in this CLI, and stdout must stay the
|
|
738
|
-
* sole payload. Without this, the ONLY signal that the server never
|
|
739
|
-
* confirmed the tier change was a string on stderr, invisible to any --json
|
|
740
|
-
* consumer parsing stdout alone. Adds `_tierUnconfirmed: true` to the
|
|
741
|
-
* emitted object under EXACTLY the same condition warnIfUnconfirmedTier
|
|
742
|
-
* warns on (tier requested, response carries no `_tierOverride`) — never
|
|
743
|
-
* fabricated, never present otherwise.
|
|
744
|
-
*/
|
|
745
396
|
function withTierUnconfirmed(data, tierWasRequested) {
|
|
746
397
|
if (data._tierOverride || !tierWasRequested)
|
|
747
398
|
return data;
|
|
748
399
|
return { ...data, _tierUnconfirmed: true };
|
|
749
400
|
}
|
|
750
|
-
/** #86 finding 5 — the standard error envelope for a refused tier override,
|
|
751
|
-
* routed through the same reportError() central formatting path used
|
|
752
|
-
* everywhere else (exit 1: a business-logic refusal, not a usage error).
|
|
753
|
-
*
|
|
754
|
-
* #107 review F3 — uses `RefusalError` (kind:"refused"), not a bare `Error`
|
|
755
|
-
* (which fell through classifyError's default `kind:"unknown"` bucket,
|
|
756
|
-
* indistinguishable from a generic crash even though the README sells `kind`
|
|
757
|
-
* as the machine discriminant an agent branches on).
|
|
758
|
-
*/
|
|
759
401
|
function reportTierRefusal(data, wantsJson) {
|
|
760
402
|
const ov = data._tierOverride;
|
|
761
403
|
const message = `Tier ceiling override refused: ${ov?.reason ?? 'unknown'} (requested ${ov?.requestedMaxTier ?? '—'}; kept the registry cap)`;
|
|
762
404
|
return reportError(new RefusalError(message), { json: wantsJson });
|
|
763
405
|
}
|
|
764
|
-
// create
|
|
765
406
|
scraps
|
|
766
407
|
.command('create')
|
|
767
408
|
.description('Create a new scrap')
|
|
768
409
|
.requiredOption('-t, --title <title>', 'Scrap title')
|
|
769
|
-
// #170 review F6 — promoted from a plain `.option()` to `.requiredOption()`:
|
|
770
|
-
// this is unconditionally required (no positional alternative exists on
|
|
771
|
-
// THIS command, unlike the top-level `trawl create <url>`, which accepts
|
|
772
|
-
// `[url]` OR `--url` — deliberately left untouched, that either/or needs a
|
|
773
|
-
// human product decision, tracked separately). Before this, the flag's
|
|
774
|
-
// description carried the only "Required." signal, in English prose — the
|
|
775
|
-
// exact thing docs/agent-quickstart.md tells an agent not to parse.
|
|
776
|
-
// Commander itself now enforces it, so `trawl spec --json`'s published
|
|
777
|
-
// `mandatory:true` is honest instead of a second, driftable copy of this
|
|
778
|
-
// same fact.
|
|
779
410
|
.requiredOption('-u, --url <url>', 'Target URL — the site the scrap targets, not necessarily the exact URL fetched at run time (a request script can compute that, e.g. a templated search query).')
|
|
780
411
|
.option('-r, --request <request>', 'Request/query')
|
|
781
412
|
.option('-d, --description <text>', 'Scrap description')
|
|
@@ -786,10 +417,6 @@ scraps
|
|
|
786
417
|
usageError(`Invalid --tier "${opts.tier}" (allowed: ${VALID_TIERS.join(', ')})`, { json: opts.json });
|
|
787
418
|
return;
|
|
788
419
|
}
|
|
789
|
-
// #163 — the API now requires a parseable `url` on create (trawl_node
|
|
790
|
-
// #1823/#1828); validate client-side BEFORE the request goes out so a
|
|
791
|
-
// missing/unparseable value surfaces as a readable usage error instead
|
|
792
|
-
// of a raw server 400/422.
|
|
793
420
|
const url = tryValidateUrl(opts.url, opts);
|
|
794
421
|
if (url === undefined)
|
|
795
422
|
return;
|
|
@@ -800,9 +427,6 @@ scraps
|
|
|
800
427
|
...(opts.description !== undefined && { description: opts.description }),
|
|
801
428
|
...(opts.tier !== undefined && { proxyTier: opts.tier }),
|
|
802
429
|
}), { text: 'Creating scrap…', successText: (d) => `Scrap created: ${chalk.bold(d._id)}` });
|
|
803
|
-
// #86 finding 4a — read _tierOverride back from the POST response and
|
|
804
|
-
// render it exactly like `update` does; never echo the requested tier as
|
|
805
|
-
// if it were applied when an older server doesn't confirm it.
|
|
806
430
|
const tierWasRequested = opts.tier !== undefined;
|
|
807
431
|
warnIfUnconfirmedTier(data, tierWasRequested, data._id);
|
|
808
432
|
const refused = Boolean(data._tierOverride?.refused);
|
|
@@ -819,7 +443,6 @@ scraps
|
|
|
819
443
|
if (refused)
|
|
820
444
|
process.exitCode = 1;
|
|
821
445
|
});
|
|
822
|
-
// update
|
|
823
446
|
scraps
|
|
824
447
|
.command('update <id>')
|
|
825
448
|
.description('Update an existing scrap')
|
|
@@ -851,9 +474,6 @@ scraps
|
|
|
851
474
|
const body = {};
|
|
852
475
|
if (opts.title !== undefined)
|
|
853
476
|
body.title = opts.title;
|
|
854
|
-
// #163 — omitting --url stays a no-op (the API keeps that allowed on
|
|
855
|
-
// update), but a value that IS supplied must still be a parseable URL —
|
|
856
|
-
// validate it client-side before the request goes out, same as create.
|
|
857
477
|
if (opts.url !== undefined) {
|
|
858
478
|
const url = tryValidateUrl(opts.url, opts);
|
|
859
479
|
if (url === undefined)
|
|
@@ -902,7 +522,6 @@ scraps
|
|
|
902
522
|
if (opts.tier !== undefined)
|
|
903
523
|
body.proxyTier = opts.tier;
|
|
904
524
|
if (opts.forceTier !== undefined) {
|
|
905
|
-
// Raise the ceiling; also start the run at that tier unless --tier says otherwise.
|
|
906
525
|
body.proxyMaxTier = opts.forceTier;
|
|
907
526
|
if (opts.tier === undefined)
|
|
908
527
|
body.proxyTier = opts.forceTier;
|
|
@@ -919,10 +538,6 @@ scraps
|
|
|
919
538
|
text: 'Updating scrap…',
|
|
920
539
|
successText: (d) => `Scrap updated: ${chalk.bold(d._id)}`,
|
|
921
540
|
});
|
|
922
|
-
// #1559 / #86 findings 4b/5 — surface the effective tier + clamp/refuse
|
|
923
|
-
// reason (fixes the silent-clamp: the server may persist a lower tier
|
|
924
|
-
// than requested), and NEVER echo the requested value as applied when
|
|
925
|
-
// the server doesn't confirm it (old-server fallback below).
|
|
926
541
|
const tierWasRequested = opts.tier !== undefined || opts.forceTier !== undefined;
|
|
927
542
|
warnIfUnconfirmedTier(data, tierWasRequested, id);
|
|
928
543
|
const refused = Boolean(data._tierOverride?.refused);
|
|
@@ -936,10 +551,6 @@ scraps
|
|
|
936
551
|
}
|
|
937
552
|
const shown = data;
|
|
938
553
|
for (const key of Object.keys(body)) {
|
|
939
|
-
// Tier keys are rendered exclusively by renderTierOverrideHuman /
|
|
940
|
-
// warnIfUnconfirmedTier above — never echo them here, whether or not
|
|
941
|
-
// _tierOverride came back (an old-server echo of the REQUESTED value
|
|
942
|
-
// is exactly the silent-clamp lie #1559 fixed).
|
|
943
554
|
if (key === 'proxyTier' || key === 'proxyMaxTier')
|
|
944
555
|
continue;
|
|
945
556
|
const src = key in shown ? shown[key] : body[key];
|
|
@@ -949,7 +560,6 @@ scraps
|
|
|
949
560
|
if (refused)
|
|
950
561
|
process.exitCode = 1;
|
|
951
562
|
});
|
|
952
|
-
// run — promoted to a top-level verb (#108)
|
|
953
563
|
export function attachRunCommand(parent, attachOpts = {}) {
|
|
954
564
|
return parent
|
|
955
565
|
.command('run <id>', attachOpts)
|
|
@@ -958,66 +568,25 @@ export function attachRunCommand(parent, attachOpts = {}) {
|
|
|
958
568
|
.option('--json', 'Output the raw launch payload as JSON')
|
|
959
569
|
.action(async (id, opts) => {
|
|
960
570
|
validateObjectId(id);
|
|
961
|
-
// #91 P1 / #93 item 1 — captured BEFORE launching so pollRunProgress can
|
|
962
|
-
// tell "the run that's about to finish" apart from whatever the last run
|
|
963
|
-
// happened to be (including a dedup onto an already-in-flight run).
|
|
964
571
|
const beforeRun = opts.watch ? await captureBeforeRunState(id) : undefined;
|
|
965
|
-
// #91 P0 — GET /api/scraps/load/:id runs the scrap synchronously
|
|
966
|
-
// server-side (30-250s); the 30s default was aborting it mid-flight.
|
|
967
572
|
const call = () => api.get(`/api/scraps/load/${id}`, { timeoutMs: LONG_RUN_TIMEOUT_MS });
|
|
968
|
-
// #107 — under --json the stdout path stays pure: no spinner channel at
|
|
969
|
-
// all, mirroring `trawl create`'s own --json handling.
|
|
970
573
|
const data = opts.json
|
|
971
574
|
? await call()
|
|
972
575
|
: await spin(call, { text: 'Launching scrap…', successText: 'Scrap launched' });
|
|
973
576
|
if (opts.json)
|
|
974
577
|
json(data);
|
|
975
578
|
else if (!opts.watch) {
|
|
976
|
-
/* #166 — see confirmNonTTY. Two deliberate choices here:
|
|
977
|
-
*
|
|
978
|
-
* "run complete", not "launched": `run` is the SYNCHRONOUS verb —
|
|
979
|
-
* GET /api/scraps/load/:id executes the scrap server-side (30-250s)
|
|
980
|
-
* and returns the finished result, so by the time this prints the run
|
|
981
|
-
* is over. The spinner's successText says "Scrap launched", but that
|
|
982
|
-
* is stderr chrome for a human watching it happen; THIS line is the
|
|
983
|
-
* machine surface, where a script reading "launched" would poll for a
|
|
984
|
-
* completion that already happened. `trigger` already draws the same
|
|
985
|
-
* distinction ("Worker triggered" async vs "Worker run complete" under
|
|
986
|
-
* --wait) — this matches that vocabulary.
|
|
987
|
-
*
|
|
988
|
-
* `--watch` is excluded because pollRunProgress below does its own
|
|
989
|
-
* reporting; without that guard a watched run would print this line
|
|
990
|
-
* and then narrate the same run. */
|
|
991
579
|
confirmNonTTY(`Scrap ${id} run complete`);
|
|
992
580
|
}
|
|
993
581
|
if (opts.watch) {
|
|
994
|
-
// #107 — under --json, pollRunProgress suppresses its own intermediate
|
|
995
|
-
// console.log calls and instead emits exactly ONE final NDJSON outcome
|
|
996
|
-
// line (+ sets process.exitCode honestly) once the watch reaches a
|
|
997
|
-
// terminal status, a timeout, or a persistent poll error (review F1) —
|
|
998
|
-
// `run --json --watch` never again exits 0 after dead air regardless
|
|
999
|
-
// of what the watched run actually did.
|
|
1000
582
|
await pollRunProgress(id, beforeRun, { json: opts.json });
|
|
1001
583
|
}
|
|
1002
|
-
// #151 — throttled post-run referral tip. Reached only when the
|
|
1003
|
-
// launch call above didn't throw (a hard failure rejects it, unwinding
|
|
1004
|
-
// straight to index.ts's central catch before this line ever runs)
|
|
1005
|
-
// — and, when --watch polled to a terminal state, only when
|
|
1006
|
-
// pollRunProgress didn't flag a genuine run failure/timeout/poll-error
|
|
1007
|
-
// via a non-zero process.exitCode (#107 review F1). See lib/tips.ts
|
|
1008
|
-
// for the throttle/TTY/json/opt-out/server-flag gating itself. #153 —
|
|
1009
|
-
// now async (a due tip fetches the server's userFacing flag before
|
|
1010
|
-
// printing), so this must be awaited: otherwise commander's
|
|
1011
|
-
// parseAsync-driven action would resolve before the tip's own
|
|
1012
|
-
// await settles, racing the process's natural exit.
|
|
1013
584
|
const runFailed = typeof process.exitCode === 'number' && process.exitCode !== 0;
|
|
1014
585
|
if (!runFailed)
|
|
1015
586
|
await maybeShowReferralTip({ json: opts.json });
|
|
1016
587
|
});
|
|
1017
588
|
}
|
|
1018
589
|
attachRunCommand(scraps, { hidden: true });
|
|
1019
|
-
// #70 — render an items array either as a table summary or --json. Shared by
|
|
1020
|
-
// both the default (persisted read) and --fresh (live execute) paths of `data`.
|
|
1021
590
|
function renderScrapItems(items, asJson) {
|
|
1022
591
|
if (asJson) {
|
|
1023
592
|
json(items);
|
|
@@ -1030,43 +599,15 @@ function renderScrapItems(items, asJson) {
|
|
|
1030
599
|
}
|
|
1031
600
|
console.log(chalk.dim(' Use --json for full output.'));
|
|
1032
601
|
}
|
|
1033
|
-
/**
|
|
1034
|
-
* #86 finding 6 — `data`'s honest empty-vs-error distinction, for both --json
|
|
1035
|
-
* and human prose. Before this, EVERY non-array outcome (never run, last run
|
|
1036
|
-
* failed, or the payload aged out of retention) collapsed to the SAME `[]` /
|
|
1037
|
-
* "No data yet." — indistinguishable from a genuine zero-item successful run.
|
|
1038
|
-
* That's a lie under --json: an agent can't tell "nothing to show" from "go
|
|
1039
|
-
* look at what actually happened". Mirrors reportError's dual json/human
|
|
1040
|
-
* shape (human line to stderr always; --json ALSO gets a machine envelope on
|
|
1041
|
-
* stdout) with a caller-chosen exit code + kind, since these states are not
|
|
1042
|
-
* all "usage" (2) — never-run / aged-out-of-retention are not_found (4), a
|
|
1043
|
-
* failed last run is a business-logic failure (1).
|
|
1044
|
-
*/
|
|
1045
|
-
/**
|
|
1046
|
-
* #170 — `retryable`/`next` follow the SAME frozen `RETRY_POLICY` map
|
|
1047
|
-
* `classifyError` reads (errors.ts), keyed by `kind` — this path never
|
|
1048
|
-
* builds its own copy. `retryOverride` is for the one case here where the
|
|
1049
|
-
* kind alone is genuinely wrong: `in_progress`'s caller passes
|
|
1050
|
-
* `{retryable:true, next:['trawl data <id>']}` since retrying `data` (not
|
|
1051
|
-
* `--fresh`, which would 429 against the lock already held) IS worth doing
|
|
1052
|
-
* once the run finishes. Every other kind here (`not_found`, `run_failed`)
|
|
1053
|
-
* isn't in `RETRY_POLICY` at all — `retryFieldsFor` is total, so those fall
|
|
1054
|
-
* back to its conservative "not retryable, nothing to suggest" default.
|
|
1055
|
-
*/
|
|
1056
602
|
function reportDataState(message, exitCode, kind, wantsJson, retryOverride) {
|
|
1057
603
|
console.error(chalk.red(`✗ ${message}`));
|
|
1058
604
|
if (wantsJson) {
|
|
1059
605
|
const { retryable, next } = retryOverride ?? retryFieldsFor(kind);
|
|
1060
|
-
// #170 review F3 — typed as ErrorEnvelope so the compiler enforces
|
|
1061
|
-
// `retryable` here too (this was one of two emitters a mutation test
|
|
1062
|
-
// proved `tsc --noEmit` never actually guarded before this annotation —
|
|
1063
|
-
// the untyped literal let `retryable` be omitted silently).
|
|
1064
606
|
const envelope = { message, kind, retryable, ...(next ? { next } : {}) };
|
|
1065
607
|
console.log(JSON.stringify({ error: envelope }));
|
|
1066
608
|
}
|
|
1067
609
|
process.exitCode = exitCode;
|
|
1068
610
|
}
|
|
1069
|
-
// data — promoted to a top-level verb (#108)
|
|
1070
611
|
export function attachDataCommand(parent, attachOpts = {}) {
|
|
1071
612
|
return parent
|
|
1072
613
|
.command('data <id>', attachOpts)
|
|
@@ -1076,15 +617,9 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1076
617
|
.option('--fresh', 'Launch a fresh run instead of reading the last persisted payload (consumes execute quota, same as `run`)')
|
|
1077
618
|
.action(async (id, opts) => {
|
|
1078
619
|
validateObjectId(id);
|
|
1079
|
-
// --errors: fetch full run + fix detail via fetchRunAndFix (DRY with doctor).
|
|
1080
|
-
// Read-only: GET /api/scraps/:id + GET /api/historys/:hid, no quota, no run lock.
|
|
1081
620
|
if (opts.errors) {
|
|
1082
621
|
const result = await fetchRunAndFix(id);
|
|
1083
622
|
if (!result) {
|
|
1084
|
-
// #88 item 7 — unified no-runs shape with `doctor --json`: a bare
|
|
1085
|
-
// `null` was indistinguishable from any other absent-payload state
|
|
1086
|
-
// (a scrap CAN legitimately have a null-ish result elsewhere); an
|
|
1087
|
-
// explicit `{status:"no_runs"}` object is unambiguous everywhere.
|
|
1088
623
|
if (opts.json) {
|
|
1089
624
|
json({ status: 'no_runs' });
|
|
1090
625
|
return;
|
|
@@ -1092,13 +627,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1092
627
|
console.log(chalk.dim('No runs yet.'));
|
|
1093
628
|
return;
|
|
1094
629
|
}
|
|
1095
|
-
// --json is honored for BOTH outcomes (success or failure) — an agent
|
|
1096
|
-
// parsing `data --errors --json` must always get the flat run object,
|
|
1097
|
-
// never prose gated behind a status check. (#71 finding 13)
|
|
1098
|
-
// #185 — `docs` rides alongside as a sibling field on the same flat
|
|
1099
|
-
// object (never nested under `run`, since there's no wrapper shape
|
|
1100
|
-
// here), present only for a live mapped failureKind (see
|
|
1101
|
-
// runDocsField's doc comment).
|
|
1102
630
|
if (opts.json)
|
|
1103
631
|
return json({ ...pickRun(result.run), ...runDocsField(result.run) });
|
|
1104
632
|
if (result.run.status === true) {
|
|
@@ -1106,25 +634,18 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1106
634
|
return;
|
|
1107
635
|
}
|
|
1108
636
|
console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
|
|
637
|
+
if (isAuthWall(result.run)) {
|
|
638
|
+
maybeSuggestSkillsForAuthWall();
|
|
639
|
+
}
|
|
1109
640
|
return;
|
|
1110
641
|
}
|
|
1111
|
-
// #70 — --fresh is the explicit opt-in for a LIVE run. This is what the
|
|
1112
|
-
// default path used to do silently: GET /api/scraps/load/:id executes a
|
|
1113
|
-
// fresh scrap run server-side (requireQuota('scraps','execute') + a
|
|
1114
|
-
// distributed run lock — trawl_node modules/scraps/routes/scraps.routes.js),
|
|
1115
|
-
// burning execute quota and 429ing if a run is already in flight. A user
|
|
1116
|
-
// or agent "just reading data" must never trigger that by accident.
|
|
1117
642
|
if (opts.fresh) {
|
|
1118
|
-
// #91 P0 — same long-run endpoint as `run` (30-250s server-side);
|
|
1119
|
-
// the 30s default was aborting it mid-flight.
|
|
1120
643
|
const loaded = await spin(() => api.get(`/api/scraps/load/${id}`, { timeoutMs: LONG_RUN_TIMEOUT_MS }), {
|
|
1121
644
|
text: 'Launching a fresh scrap run (consumes execute quota)…',
|
|
1122
645
|
successText: 'Fresh run complete',
|
|
1123
646
|
});
|
|
1124
647
|
const items = loaded?.result?.data;
|
|
1125
648
|
if (!Array.isArray(items)) {
|
|
1126
|
-
// --json always returns an array from `data` — [] is the honest
|
|
1127
|
-
// "no items" signal instead of prose breaking JSON parsing. (#71)
|
|
1128
649
|
if (opts.json) {
|
|
1129
650
|
json([]);
|
|
1130
651
|
return;
|
|
@@ -1132,18 +653,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1132
653
|
console.log(chalk.dim('No data yet. Run the scrap first.'));
|
|
1133
654
|
return;
|
|
1134
655
|
}
|
|
1135
|
-
// #93 item 2 — --fresh used to render items without ever checking for a
|
|
1136
|
-
// regression, unlike the persisted `data` path below (#88 item 2). The
|
|
1137
|
-
// load() response's OWN embedded `scrap.history[0]` can't be trusted for
|
|
1138
|
-
// this (see the ScrapLoadResult comment above): it may be the previous
|
|
1139
|
-
// run's row, and even the right row never carries the regression flip.
|
|
1140
|
-
// `--fresh` runs synchronously to completion server-side though — by the
|
|
1141
|
-
// time this call returns, node has already awaited the regression patch
|
|
1142
|
-
// — so a fresh GET /api/scraps/:id (the SAME read the persisted path
|
|
1143
|
-
// below already trusts) reliably observes the finalized DB state.
|
|
1144
|
-
// Best-effort: never fail --fresh's real output over this side check,
|
|
1145
|
-
// and never gate on it — the items returned ARE this run's real,
|
|
1146
|
-
// synchronously-computed data regardless of what this check finds.
|
|
1147
656
|
try {
|
|
1148
657
|
const fresh = await api.get(`/api/scraps/${id}`);
|
|
1149
658
|
if (fresh.history?.[0]?.statusDetail === 'regression') {
|
|
@@ -1151,56 +660,20 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1151
660
|
}
|
|
1152
661
|
}
|
|
1153
662
|
catch {
|
|
1154
|
-
// best-effort — the fresh run's items are still valid without this check
|
|
1155
663
|
}
|
|
1156
664
|
renderScrapItems(items, opts.json);
|
|
1157
665
|
return;
|
|
1158
666
|
}
|
|
1159
|
-
// Default: read the last PERSISTED run payload — no quota, no run lock.
|
|
1160
|
-
// History.data (trawl_node modules/historys/services/historys.service.js
|
|
1161
|
-
// `update()`) is a JSON-stringified clone of the worker result, so it
|
|
1162
|
-
// carries the same `.data` items array the live load endpoint returns —
|
|
1163
|
-
// it's just pulled from the most recent history row instead of a fresh
|
|
1164
|
-
// run. Retention keeps this only for the newest row per (scrap, status)
|
|
1165
|
-
// bucket (config.trawl.keepData, default 1); older rows null it out.
|
|
1166
|
-
//
|
|
1167
|
-
// #86 finding 6 — [] is reserved for a GENUINE zero-item successful run.
|
|
1168
|
-
// Every other outcome below is an honest error envelope instead: never
|
|
1169
|
-
// run (not_found/4), last run failed (1), or the payload aged out of
|
|
1170
|
-
// retention (not_found/4) all used to collapse into the same silent [].
|
|
1171
667
|
const scrap = await api.get(`/api/scraps/${id}`);
|
|
1172
668
|
const last = scrap.history?.[0];
|
|
1173
669
|
if (!last?._id) {
|
|
1174
670
|
reportDataState(`Scrap ${id} has never run. Run it first (trawl run ${id}) or pass --fresh to launch one now.`, 4, 'not_found', opts.json);
|
|
1175
671
|
return;
|
|
1176
672
|
}
|
|
1177
|
-
// #88 item 1 — status:null is an IN-FLIGHT run (node persists
|
|
1178
|
-
// {status:null, statusDetail:null, inFlight:true} the moment a run
|
|
1179
|
-
// starts, and only flips status/statusDetail once it finishes). That is
|
|
1180
|
-
// neither "never run" nor "the last run failed" — a caller reading data
|
|
1181
|
-
// mid-run needs an honest "wait" signal. Never suggest --fresh here: a
|
|
1182
|
-
// run already holds the server-side distributed lock, so --fresh would
|
|
1183
|
-
// just 429 against it.
|
|
1184
673
|
if (last.status === null) {
|
|
1185
|
-
// #170 — retrying `trawl data <id>` (unchanged) once the run
|
|
1186
|
-
// finishes IS worth it; `--fresh` never is (it would 429 against the
|
|
1187
|
-
// lock this very run already holds) — never suggested here.
|
|
1188
674
|
reportDataState(`Run in progress for ${id} — retry shortly.`, 1, 'in_progress', opts.json, { retryable: true, next: [`trawl data ${id}`] });
|
|
1189
675
|
return;
|
|
1190
676
|
}
|
|
1191
|
-
// #86 review — node persists status=false for a GENUINE zero-item run
|
|
1192
|
-
// too (historys schema: status boolean|null + statusDetail
|
|
1193
|
-
// success/error/empty/regression; a zero-item run is status=false +
|
|
1194
|
-
// statusDetail='empty', and the embedded history rows from GET
|
|
1195
|
-
// /api/scraps/:id include statusDetail via the repository populate
|
|
1196
|
-
// select). An 'empty' run is the one case [] is FOR — only a real
|
|
1197
|
-
// failure (error/unknown detail) gets the run_failed envelope.
|
|
1198
|
-
//
|
|
1199
|
-
// #88 item 2 — statusDetail='regression' is ALSO status=false (an async
|
|
1200
|
-
// patch flips it after item count dropped vs baseline), but the row's
|
|
1201
|
-
// `data` still holds REAL, non-empty items — the write that persisted
|
|
1202
|
-
// them succeeded before the regression was even detected. Treating it as
|
|
1203
|
-
// run_failed would hide genuine data behind a false negative.
|
|
1204
677
|
const isEmptyRun = last.status === false && last.statusDetail === 'empty';
|
|
1205
678
|
const isRegression = last.status === false && last.statusDetail === 'regression';
|
|
1206
679
|
if (last.status === false && !isEmptyRun && !isRegression) {
|
|
@@ -1211,11 +684,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1211
684
|
let items;
|
|
1212
685
|
if (typeof detail?.data === 'string' && detail.data) {
|
|
1213
686
|
try {
|
|
1214
|
-
// #159 — `detail.data` is the same JSON-string-within-JSON shape
|
|
1215
|
-
// (`history.data`) the create --json bug lived in: a raw control
|
|
1216
|
-
// character here would otherwise throw and silently report "aged
|
|
1217
|
-
// out of retention" below for data that is actually present and
|
|
1218
|
-
// recoverable, burning a --fresh re-run + quota for nothing.
|
|
1219
687
|
items = parseServerJson(detail.data)?.data;
|
|
1220
688
|
}
|
|
1221
689
|
catch {
|
|
@@ -1224,23 +692,12 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1224
692
|
}
|
|
1225
693
|
if (!Array.isArray(items)) {
|
|
1226
694
|
if (isEmptyRun) {
|
|
1227
|
-
// A genuine zero-item run whose payload is '[]' or absent — both are
|
|
1228
|
-
// the SAME honest answer: no items, exit 0. Never the retention
|
|
1229
|
-
// message (nothing aged out; there was nothing to persist).
|
|
1230
695
|
renderScrapItems([], opts.json);
|
|
1231
696
|
return;
|
|
1232
697
|
}
|
|
1233
|
-
// #88 item 2 — a regression row whose payload aged out of retention has
|
|
1234
|
-
// nothing left to show either; fall through to the SAME honest
|
|
1235
|
-
// aged-out envelope a normal successful row would get (never fabricate
|
|
1236
|
-
// items, never silently succeed).
|
|
1237
698
|
reportDataState(`No persisted data for the last run of ${id} — it aged out of retention. Pass --fresh to launch a new run.`, 4, 'not_found', opts.json);
|
|
1238
699
|
return;
|
|
1239
700
|
}
|
|
1240
|
-
// #88 item 2 — a regression row's items are REAL (the write succeeded
|
|
1241
|
-
// before the async patch flagged the drop) — return them on stdout
|
|
1242
|
-
// (exit 0, both modes) with an honest stderr warning pointing at the
|
|
1243
|
-
// diagnostic command, instead of hiding genuine data behind run_failed.
|
|
1244
701
|
if (isRegression) {
|
|
1245
702
|
console.error(chalk.yellow(`⚠ Item count regressed vs baseline for the last run of ${id} — see: trawl scraps doctor ${id}`));
|
|
1246
703
|
}
|
|
@@ -1248,7 +705,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1248
705
|
});
|
|
1249
706
|
}
|
|
1250
707
|
attachDataCommand(scraps, { hidden: true });
|
|
1251
|
-
// history — list past runs for a scrap — promoted to a top-level verb (#108)
|
|
1252
708
|
export function attachHistoryCommand(parent, attachOpts = {}) {
|
|
1253
709
|
return parent
|
|
1254
710
|
.command('history <id>', attachOpts)
|
|
@@ -1269,8 +725,6 @@ export function attachHistoryCommand(parent, attachOpts = {}) {
|
|
|
1269
725
|
time: h.time ?? null,
|
|
1270
726
|
tier: h.proxyTier ?? null,
|
|
1271
727
|
failureKind: h.failureKind ?? null,
|
|
1272
|
-
// trawl_node#1950 renamed blockType to block.kind; fall back to the
|
|
1273
|
-
// deprecated flat field for a prod backend not yet on that tag.
|
|
1274
728
|
blockType: h.block?.kind ?? h.blockType ?? null,
|
|
1275
729
|
createdAt: h.createdAt ?? null,
|
|
1276
730
|
}));
|
|
@@ -1282,7 +736,6 @@ export function attachHistoryCommand(parent, attachOpts = {}) {
|
|
|
1282
736
|
});
|
|
1283
737
|
}
|
|
1284
738
|
attachHistoryCommand(scraps, { hidden: true });
|
|
1285
|
-
// run-info — single-run detail by history id — promoted to a top-level verb (#108)
|
|
1286
739
|
export function attachRunInfoCommand(parent, attachOpts = {}) {
|
|
1287
740
|
return parent
|
|
1288
741
|
.command('run-info <hid>', attachOpts)
|
|
@@ -1297,36 +750,24 @@ export function attachRunInfoCommand(parent, attachOpts = {}) {
|
|
|
1297
750
|
time: h.time ?? null,
|
|
1298
751
|
tier: h.proxyTier ?? null,
|
|
1299
752
|
failureKind: h.failureKind ?? null,
|
|
1300
|
-
// trawl_node#1950 renamed blockType to block.kind; fall back to the
|
|
1301
|
-
// deprecated flat field for a prod backend not yet on that tag.
|
|
1302
753
|
blockType: h.block?.kind ?? h.blockType ?? null,
|
|
1303
754
|
errorMessage: h.errorSnapshot?.errorMessage ?? null,
|
|
1304
755
|
selector: h.errorSnapshot?.selector ?? null,
|
|
1305
756
|
emptyContext: h.emptyContext ?? null,
|
|
1306
757
|
createdAt: h.createdAt ?? null,
|
|
1307
|
-
// #185 — present only for a live mapped failureKind (see
|
|
1308
|
-
// runDocsField's doc comment); omitted, never `docs: undefined`, for
|
|
1309
|
-
// every other run so this object's own JSON.stringify never grows an
|
|
1310
|
-
// unexpected key on existing consumers.
|
|
1311
758
|
...runDocsField(h),
|
|
1312
759
|
};
|
|
1313
760
|
if (opts.json) {
|
|
1314
761
|
json(info);
|
|
1315
762
|
return;
|
|
1316
763
|
}
|
|
1317
|
-
// The blob is an object; the table needs one line. --json keeps it whole.
|
|
1318
764
|
table([{ ...info, emptyContext: summarizeEmptyContext(info.emptyContext) }], ['hid', 'status', 'time', 'tier', 'failureKind', 'blockType', 'errorMessage', 'selector', 'emptyContext', 'createdAt']);
|
|
1319
|
-
// #184 safety-net — the run just shown carries a live login-wall
|
|
1320
|
-
// verdict; if Claude's skills aren't installed either, point at the
|
|
1321
|
-
// command that installs them too (never under --json, see the early
|
|
1322
|
-
// return above).
|
|
1323
765
|
if (isAuthWall(h)) {
|
|
1324
766
|
maybeSuggestSkillsForAuthWall();
|
|
1325
767
|
}
|
|
1326
768
|
});
|
|
1327
769
|
}
|
|
1328
770
|
attachRunInfoCommand(scraps, { hidden: true });
|
|
1329
|
-
// delete
|
|
1330
771
|
scraps
|
|
1331
772
|
.command('delete <id>')
|
|
1332
773
|
.alias('rm')
|
|
@@ -1335,16 +776,6 @@ scraps
|
|
|
1335
776
|
.option('--json', 'Output as JSON')
|
|
1336
777
|
.action(async (id, opts) => {
|
|
1337
778
|
validateObjectId(id);
|
|
1338
|
-
// #107 — never blocks on a y/N under --json or a non-TTY invocation
|
|
1339
|
-
// (agent/CI subprocess); -f/--force always pre-confirms.
|
|
1340
|
-
//
|
|
1341
|
-
// #107 review F2 — `message` (the plain-id form) feeds the refusal
|
|
1342
|
-
// UsageError, which can land verbatim in the `--json` error envelope;
|
|
1343
|
-
// `chalk.bold(id)` is passed ONLY as `promptMessage`, shown solely on
|
|
1344
|
-
// the interactive TTY `[y/N]` prompt. Before this split, the styled
|
|
1345
|
-
// string was the ONLY message confirmDestructive had, so a non-TTY/
|
|
1346
|
-
// --json refusal on `scraps delete X --json` emitted raw ANSI escape
|
|
1347
|
-
// bytes inside the JSON string.
|
|
1348
779
|
const { proceed, blocked } = await confirmDestructive(`Delete scrap ${id}?`, {
|
|
1349
780
|
force: opts.force,
|
|
1350
781
|
json: opts.json,
|
|
@@ -1363,9 +794,8 @@ scraps
|
|
|
1363
794
|
return;
|
|
1364
795
|
}
|
|
1365
796
|
await spin(call, { text: 'Deleting…', successText: 'Scrap deleted' });
|
|
1366
|
-
confirmNonTTY(`Scrap ${id} deleted`);
|
|
797
|
+
confirmNonTTY(`Scrap ${id} deleted`);
|
|
1367
798
|
});
|
|
1368
|
-
// banner
|
|
1369
799
|
scraps
|
|
1370
800
|
.command('banner <id>')
|
|
1371
801
|
.description('Upload a banner image for a scrap')
|
|
@@ -1387,9 +817,6 @@ scraps
|
|
|
1387
817
|
jpeg: 'image/jpeg',
|
|
1388
818
|
webp: 'image/webp',
|
|
1389
819
|
};
|
|
1390
|
-
// #91 P2 — an unsupported/unknown extension silently became image/png
|
|
1391
|
-
// (uploading the raw bytes of, say, a .gif or .pdf under an image/png
|
|
1392
|
-
// Content-Type — a lie about the actual file's format). Refuse instead.
|
|
1393
820
|
const mimeType = mimeMap[ext];
|
|
1394
821
|
if (!mimeType) {
|
|
1395
822
|
usageError(`Unsupported image type "${ext ? `.${ext}` : filename}" — use png, jpg, or webp.`, { json: opts.json });
|
|
@@ -1409,10 +836,8 @@ scraps
|
|
|
1409
836
|
text: 'Uploading banner…',
|
|
1410
837
|
successText: `Banner uploaded for scrap ${chalk.bold(id)}`,
|
|
1411
838
|
});
|
|
1412
|
-
// #166 — plain, unstyled: chalk.bold would emit escape codes into a pipe.
|
|
1413
839
|
confirmNonTTY(`Banner uploaded for scrap ${id}`);
|
|
1414
840
|
});
|
|
1415
|
-
// watch (stream activities)
|
|
1416
841
|
scraps
|
|
1417
842
|
.command('watch <id>')
|
|
1418
843
|
.description('Stream scrap activities in real-time')
|
|
@@ -1421,7 +846,6 @@ scraps
|
|
|
1421
846
|
validateObjectId(id);
|
|
1422
847
|
await watchActivities(id, opts.json);
|
|
1423
848
|
});
|
|
1424
|
-
// trigger — promoted to a top-level verb (#108)
|
|
1425
849
|
export function attachTriggerCommand(parent, attachOpts = {}) {
|
|
1426
850
|
return parent
|
|
1427
851
|
.command('trigger <id>', attachOpts)
|
|
@@ -1431,23 +855,9 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
|
|
|
1431
855
|
.option('--json', 'Output the raw trigger payload as JSON')
|
|
1432
856
|
.action(async (id, opts) => {
|
|
1433
857
|
validateObjectId(id);
|
|
1434
|
-
// #91 P1 / #93 item 1 — captured BEFORE triggering so pollRunProgress can
|
|
1435
|
-
// tell "the run we just triggered" apart from whatever the last run
|
|
1436
|
-
// happened to be. This is the dedup-prone path: `trigger`'s method:'worker'
|
|
1437
|
-
// collapses onto an already-pending/running worker job for the same scrap
|
|
1438
|
-
// (ScrapJobsService, LIVE_STATUSES) instead of creating a new history row —
|
|
1439
|
-
// captureBeforeRunState records that so pollRunProgress can still track it.
|
|
1440
858
|
const beforeRun = opts.watch ? await captureBeforeRunState(id) : undefined;
|
|
1441
|
-
// #50 — default async: the backend (#1313) kicks off the run and returns a
|
|
1442
|
-
// 'queued' envelope immediately instead of holding the connection for the
|
|
1443
|
-
// whole run. --wait restores the old synchronous round-trip.
|
|
1444
859
|
const path = opts.wait ? `/api/scraps/worker/${id}` : `/api/scraps/worker/${id}?wait=false`;
|
|
1445
|
-
// #91 P0 — the synchronous --wait branch runs the scrap server-side
|
|
1446
|
-
// (30-250s), same as `run`; the 30s default was aborting it
|
|
1447
|
-
// mid-flight. The async (default) POST returns almost immediately, so it
|
|
1448
|
-
// keeps the 30s default.
|
|
1449
860
|
const call = () => (opts.wait ? api.post(path, undefined, { timeoutMs: LONG_RUN_TIMEOUT_MS }) : api.post(path));
|
|
1450
|
-
// #107 — under --json the stdout path stays pure: no spinner channel.
|
|
1451
861
|
const data = opts.json
|
|
1452
862
|
? await call()
|
|
1453
863
|
: await spin(call, {
|
|
@@ -1458,26 +868,16 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
|
|
|
1458
868
|
json(data);
|
|
1459
869
|
}
|
|
1460
870
|
else {
|
|
1461
|
-
/* #160, folded onto the shared helper by #166 — the rationale and the
|
|
1462
|
-
* known residual now live once, on `confirmNonTTY` itself, instead of
|
|
1463
|
-
* in the one call site that happened to be fixed first. Wording is
|
|
1464
|
-
* unchanged on purpose: the trawl-internal QA runbook asserts on the
|
|
1465
|
-
* exact "Worker triggered" / "Worker run complete" substrings. */
|
|
1466
871
|
confirmNonTTY(opts.wait ? `Worker run complete for scrap ${id}` : `Worker triggered for scrap ${id}`);
|
|
1467
872
|
}
|
|
1468
|
-
// #107 — see the matching comment on `run`'s --watch call above (review
|
|
1469
|
-
// F1): honest final NDJSON line + exit code under --json, human mode
|
|
1470
|
-
// gets the same honest exit code too.
|
|
1471
873
|
if (opts.watch)
|
|
1472
874
|
await pollRunProgress(id, beforeRun, { json: opts.json });
|
|
1473
875
|
});
|
|
1474
876
|
}
|
|
1475
877
|
attachTriggerCommand(scraps, { hidden: true });
|
|
1476
|
-
// account subcommand group
|
|
1477
878
|
const account = scraps
|
|
1478
879
|
.command('account')
|
|
1479
880
|
.description('Manage scrap account credentials and session');
|
|
1480
|
-
// helper: prompt for a value with readline (visible input)
|
|
1481
881
|
async function promptLine(prompt) {
|
|
1482
882
|
const { createInterface } = await import('readline');
|
|
1483
883
|
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
@@ -1490,7 +890,6 @@ async function promptLine(prompt) {
|
|
|
1490
890
|
rl.close();
|
|
1491
891
|
}
|
|
1492
892
|
}
|
|
1493
|
-
// account set
|
|
1494
893
|
account
|
|
1495
894
|
.command('set <id>')
|
|
1496
895
|
.description('Set credentials for a scrap account')
|
|
@@ -1502,13 +901,8 @@ account
|
|
|
1502
901
|
let username = opts.username || '';
|
|
1503
902
|
let password = opts.password || '';
|
|
1504
903
|
if (opts.password) {
|
|
1505
|
-
// #107 — this advisory is stderr-only: stdout must stay pure under
|
|
1506
|
-
// --json, and every other advisory in this CLI already follows that
|
|
1507
|
-
// rule (warnIfUnconfirmedTier, token.ts's expiry hints, …).
|
|
1508
904
|
console.error(chalk.yellow('⚠ Passing --password on the command line is insecure and may be stored in shell history.'));
|
|
1509
905
|
}
|
|
1510
|
-
// #107 — never blocks on a readline prompt under --json or a non-TTY
|
|
1511
|
-
// invocation (agent/CI subprocess); pass -u/-p instead.
|
|
1512
906
|
const interactive = isInteractive({ json: opts.json });
|
|
1513
907
|
if (!username) {
|
|
1514
908
|
if (!interactive) {
|
|
@@ -1543,7 +937,6 @@ account
|
|
|
1543
937
|
console.log(chalk.dim(' Credentials: ') + (acc.hasCredentials ? chalk.green('✓ configured') : chalk.dim('not set')));
|
|
1544
938
|
console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('active') : chalk.dim('none')));
|
|
1545
939
|
});
|
|
1546
|
-
// account delete
|
|
1547
940
|
account
|
|
1548
941
|
.command('delete <id>')
|
|
1549
942
|
.description('Delete account credentials for a scrap')
|
|
@@ -1551,10 +944,6 @@ account
|
|
|
1551
944
|
.option('--json', 'Output as JSON')
|
|
1552
945
|
.action(async (id, opts) => {
|
|
1553
946
|
validateObjectId(id);
|
|
1554
|
-
// #107 — never blocks on a y/N under --json or a non-TTY invocation.
|
|
1555
|
-
// #107 review F2 — plain-id `message` for the refusal/JSON envelope,
|
|
1556
|
-
// styled `promptMessage` for the interactive TTY prompt only (see the
|
|
1557
|
-
// matching comment on `scraps delete` above).
|
|
1558
947
|
const { proceed, blocked } = await confirmDestructive(`Delete account credentials for scrap ${id}?`, { force: opts.force, json: opts.json, promptMessage: `Delete account credentials for scrap ${chalk.bold(id)}?` });
|
|
1559
948
|
if (blocked)
|
|
1560
949
|
return;
|
|
@@ -1572,9 +961,8 @@ account
|
|
|
1572
961
|
text: 'Deleting credentials…',
|
|
1573
962
|
successText: 'Account credentials deleted',
|
|
1574
963
|
});
|
|
1575
|
-
confirmNonTTY(`Account credentials deleted for scrap ${id}`);
|
|
964
|
+
confirmNonTTY(`Account credentials deleted for scrap ${id}`);
|
|
1576
965
|
});
|
|
1577
|
-
// account clear-session
|
|
1578
966
|
account
|
|
1579
967
|
.command('clear-session <id>')
|
|
1580
968
|
.description('Clear the saved session for a scrap account')
|
|
@@ -1591,17 +979,11 @@ account
|
|
|
1591
979
|
text: 'Clearing session…',
|
|
1592
980
|
successText: 'Session cleared',
|
|
1593
981
|
});
|
|
1594
|
-
confirmNonTTY(`Session cleared for scrap ${id}`);
|
|
982
|
+
confirmNonTTY(`Session cleared for scrap ${id}`);
|
|
1595
983
|
});
|
|
1596
|
-
// account session subcommand group
|
|
1597
984
|
const accountSession = account
|
|
1598
985
|
.command('session')
|
|
1599
986
|
.description('Manage scrap account session cookies (flavour B BYO-cookies)');
|
|
1600
|
-
/** Structural check only (bare origin format, {name,value} shape) — the
|
|
1601
|
-
* server's sessionShape.js does the authoritative validation. No domain
|
|
1602
|
-
* scoping here: unlike `capture`'s CDP path, a hand-authored/exported file
|
|
1603
|
-
* has no "target URL" to scope against, and the human supplying it already
|
|
1604
|
-
* chose what to include. */
|
|
1605
987
|
function isValidOriginsShape(value) {
|
|
1606
988
|
return (Array.isArray(value) &&
|
|
1607
989
|
value.every((o) => o &&
|
|
@@ -1610,7 +992,6 @@ function isValidOriginsShape(value) {
|
|
|
1610
992
|
Array.isArray(o.localStorage) &&
|
|
1611
993
|
o.localStorage.every((kv) => kv && typeof kv === 'object' && typeof kv.name === 'string' && typeof kv.value === 'string')));
|
|
1612
994
|
}
|
|
1613
|
-
// account session set
|
|
1614
995
|
accountSession
|
|
1615
996
|
.command('set <id>')
|
|
1616
997
|
.description('Upload a browser session for a scrap — a cookie JSON array, or a { cookies, origins } storageState file (see `session capture`)')
|
|
@@ -1618,9 +999,6 @@ accountSession
|
|
|
1618
999
|
.option('--json', 'Output as JSON')
|
|
1619
1000
|
.action(async (id, opts) => {
|
|
1620
1001
|
validateObjectId(id);
|
|
1621
|
-
// A session is a bearer-equivalent secret — refuse to PUT it over
|
|
1622
|
-
// plain HTTP (trawl_cli#183 review finding 5). Checked before anything
|
|
1623
|
-
// else in this action, including the file read below.
|
|
1624
1002
|
const secureTransportError = assertSecureTransport(getApiUrl());
|
|
1625
1003
|
if (secureTransportError) {
|
|
1626
1004
|
usageError(secureTransportError, { json: opts.json });
|
|
@@ -1640,7 +1018,6 @@ accountSession
|
|
|
1640
1018
|
cookies = parsed;
|
|
1641
1019
|
}
|
|
1642
1020
|
else if (parsed && typeof parsed === 'object' && Array.isArray(parsed.cookies)) {
|
|
1643
|
-
// storageState-shaped file — trawl_cli#183's `session capture` output shape.
|
|
1644
1021
|
const shaped = parsed;
|
|
1645
1022
|
cookies = shaped.cookies;
|
|
1646
1023
|
if (shaped.origins !== undefined) {
|
|
@@ -1684,32 +1061,6 @@ accountSession
|
|
|
1684
1061
|
const acc = data.account;
|
|
1685
1062
|
console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('✓ active') : chalk.dim('none')));
|
|
1686
1063
|
});
|
|
1687
|
-
/**
|
|
1688
|
-
* @desc Map a failed `captureSession()` result onto this CLI's existing
|
|
1689
|
-
* exit-code taxonomy (#71/#88/#107). `no_cookies_in_scope` is the one
|
|
1690
|
-
* outcome where everything ran correctly but nothing was found to upload —
|
|
1691
|
-
* a business-level refusal (exit 1, kind:"refused"), not a usage mistake.
|
|
1692
|
-
*
|
|
1693
|
-
* `capture_failed` (trawl_cli#183 R4) is a SECOND exception, for the
|
|
1694
|
-
* opposite reason: launch + handshake already succeeded — Chrome was
|
|
1695
|
-
* reachable over CDP, a page was already open — and the failure is a
|
|
1696
|
-
* genuine bug in this CLI (or an unanticipated non-page-scoped failure),
|
|
1697
|
-
* never a "cannot run in this environment" problem. Routing it through
|
|
1698
|
-
* `usageError`'s exit 2 / "cannot complete in the current environment"
|
|
1699
|
-
* framing would misclassify it exactly the way a bare `launch_failed` used
|
|
1700
|
-
* to before the R4 fix. Reported instead through `reportError` with a bare
|
|
1701
|
-
* `Error` — `classifyError` has no dedicated `kind` for it, so it falls
|
|
1702
|
-
* into the generic `kind:"unknown"`/exit 1 bucket, the SAME code an
|
|
1703
|
-
* unhandled bug reaching index.ts's own top-level catch would already get.
|
|
1704
|
-
* Never `RefusalError` (this is not a business-logic refusal, everything
|
|
1705
|
-
* up to this point may have gone fine) and never `UsageError` (nothing
|
|
1706
|
-
* about the invocation or the environment was wrong).
|
|
1707
|
-
*
|
|
1708
|
-
* Every remaining reason (non_interactive, no_chrome, launch_failed, the CDP
|
|
1709
|
-
* pipe closing before capture, a corrupt CDP frame tearing it down) means
|
|
1710
|
-
* THIS invocation cannot complete in the current environment — exit 2, the
|
|
1711
|
-
* same bucket `confirmDestructive`'s own non-interactive refusal uses.
|
|
1712
|
-
*/
|
|
1713
1064
|
function reportCaptureFailure(result, opts) {
|
|
1714
1065
|
if (result.reason === 'no_cookies_in_scope') {
|
|
1715
1066
|
process.exitCode = reportError(new RefusalError(result.message), { json: opts.json });
|
|
@@ -1721,7 +1072,6 @@ function reportCaptureFailure(result, opts) {
|
|
|
1721
1072
|
}
|
|
1722
1073
|
usageError(result.message, { json: opts.json });
|
|
1723
1074
|
}
|
|
1724
|
-
// account session capture — trawl_cli#183
|
|
1725
1075
|
accountSession
|
|
1726
1076
|
.command('capture <id>')
|
|
1727
1077
|
.description('Open a headed Chrome to the scrap\'s target URL, wait for you to log in (2FA included), and capture the session over CDP — cookies AND localStorage. Requires a local interactive terminal with a display; does not work headless, in CI, or over a plain SSH session. You stay authenticated as yourself — Trawl never sees your credentials. Responsibility for lawful use of the captured session stays with you.')
|
|
@@ -1729,10 +1079,6 @@ accountSession
|
|
|
1729
1079
|
.option('--json', 'Output as JSON (counts only — a session is a bearer secret and is never printed, in any mode)')
|
|
1730
1080
|
.action(async (id, opts) => {
|
|
1731
1081
|
validateObjectId(id);
|
|
1732
|
-
// A captured session is a bearer-equivalent secret — refuse to PUT it
|
|
1733
|
-
// over plain HTTP (trawl_cli#183 review finding 5). Checked before
|
|
1734
|
-
// anything else, including the scrap lookup below (which itself
|
|
1735
|
-
// already sends the auth token over whatever transport is configured).
|
|
1736
1082
|
const secureTransportError = assertSecureTransport(getApiUrl());
|
|
1737
1083
|
if (secureTransportError) {
|
|
1738
1084
|
usageError(secureTransportError, { json: opts.json });
|
|
@@ -1743,14 +1089,6 @@ accountSession
|
|
|
1743
1089
|
usageError(`Scrap ${id} has no target URL configured — nothing to open a browser to. A URL can be set with: trawl scraps update ${id} -u <url>`, { json: opts.json });
|
|
1744
1090
|
return;
|
|
1745
1091
|
}
|
|
1746
|
-
// Validated here, before captureSession ever creates a temp profile:
|
|
1747
|
-
// `existsSync` alone doesn't mean `spawn()` can run the file — a
|
|
1748
|
-
// non-executable path fails asynchronously deep inside chrome-launch.ts
|
|
1749
|
-
// instead. That failure is still caught there (an 'error' listener is
|
|
1750
|
-
// the safety net for anything not caught by this pre-flight — a race,
|
|
1751
|
-
// a permission change after this check, PUPPETEER_EXECUTABLE_PATH/
|
|
1752
|
-
// CHROME_PATH), but a pre-flight check gives a faster, more specific
|
|
1753
|
-
// message for the common case: an explicit path the user pointed at.
|
|
1754
1092
|
if (opts.chrome) {
|
|
1755
1093
|
const { existsSync, accessSync, constants } = await import('fs');
|
|
1756
1094
|
if (!existsSync(opts.chrome)) {
|
|
@@ -1766,10 +1104,6 @@ accountSession
|
|
|
1766
1104
|
}
|
|
1767
1105
|
}
|
|
1768
1106
|
else if (process.env.TRAWL_CHROME_PATH) {
|
|
1769
|
-
// Same check, same reasoning, for the env-var form of an explicit
|
|
1770
|
-
// path — findChrome() gives TRAWL_CHROME_PATH top precedence and
|
|
1771
|
-
// would otherwise hand this same non-executable path straight to
|
|
1772
|
-
// captureSession.
|
|
1773
1107
|
const trawlChromePath = process.env.TRAWL_CHROME_PATH;
|
|
1774
1108
|
const { existsSync, accessSync, constants } = await import('fs');
|
|
1775
1109
|
if (existsSync(trawlChromePath)) {
|
|
@@ -1782,18 +1116,11 @@ accountSession
|
|
|
1782
1116
|
}
|
|
1783
1117
|
}
|
|
1784
1118
|
}
|
|
1785
|
-
// Check the same guard captureSession runs internally BEFORE printing
|
|
1786
|
-
// anything about a Chrome window that may never open — otherwise a
|
|
1787
|
-
// non-interactive run saw "a window will open" immediately followed by
|
|
1788
|
-
// "this cannot work headless" (trawl_cli#183 gap).
|
|
1789
1119
|
const nonInteractiveReason = checkInteractiveEnvironment();
|
|
1790
1120
|
if (nonInteractiveReason) {
|
|
1791
1121
|
reportCaptureFailure({ ok: false, reason: 'non_interactive', message: nonInteractiveReason }, opts);
|
|
1792
1122
|
return;
|
|
1793
1123
|
}
|
|
1794
|
-
// Progress/prompts are stderr-only, in every mode — stdout under
|
|
1795
|
-
// --json must stay a single parseable document (#88/#107's rule,
|
|
1796
|
-
// restated for this command by trawl_cli#183's own hard rules).
|
|
1797
1124
|
console.error(chalk.dim(`Chrome will open at ${scrap.url} for you to log in there (2FA included); capture completes when you press Enter back in this terminal.`));
|
|
1798
1125
|
const result = await captureSession(scrap.url, opts.chrome ? { findChrome: () => opts.chrome ?? null } : {});
|
|
1799
1126
|
if (!result.ok) {
|
|
@@ -1806,9 +1133,6 @@ accountSession
|
|
|
1806
1133
|
});
|
|
1807
1134
|
if (opts.json) {
|
|
1808
1135
|
const data = await call();
|
|
1809
|
-
// Never the session itself — only what the server echoes back, plus
|
|
1810
|
-
// counts (#183 hard rule: no cookie/localStorage VALUE, ever, under
|
|
1811
|
-
// --json or otherwise).
|
|
1812
1136
|
json({ account: data.account, targetDomain: result.targetDomain, capture: result.counts });
|
|
1813
1137
|
return;
|
|
1814
1138
|
}
|
|
@@ -1825,24 +1149,9 @@ accountSession
|
|
|
1825
1149
|
console.log(chalk.dim(' The browser window was closed before Enter — localStorage was not captured (cookies only).'));
|
|
1826
1150
|
}
|
|
1827
1151
|
if (result.counts.originsUnreadable > 0) {
|
|
1828
|
-
// Impossible to overlook (chalk.yellow + ⚠, not the routine chalk.dim
|
|
1829
|
-
// scope-drop note above) — this is a DEGRADED capture: cookies still
|
|
1830
|
-
// uploaded, but at least one in-scope origin's localStorage did not.
|
|
1831
|
-
// Two distinct causes share this one count: the page threw reading
|
|
1832
|
-
// `window.localStorage` (e.g. a SecurityError on partitioned
|
|
1833
|
-
// storage), or the read never got a response within
|
|
1834
|
-
// LOCALSTORAGE_READ_TIMEOUT_MS (the page's renderer was blocked — a
|
|
1835
|
-
// native dialog, a synchronous script, a paused debugger;
|
|
1836
|
-
// trawl_cli#183 review finding 3's own pipe-level timeout fix
|
|
1837
|
-
// reopened finding 4's exact bug class on this one path, since a
|
|
1838
|
-
// timeout does not close the pipe the way a corrupt frame does). The
|
|
1839
|
-
// two are not split apart here: both mean the same actionable fact
|
|
1840
|
-
// ("re-run once nothing is blocking that tab") and neither one names
|
|
1841
|
-
// the page URL/title/exception text — a page controls that content.
|
|
1842
1152
|
console.log(chalk.yellow(` ⚠ ${result.counts.originsUnreadable} origin(s) could not be captured — threw reading localStorage, or gave no response within ${LOCALSTORAGE_READ_TIMEOUT_MS / 1000}s (a blocked tab). Not the same as being empty; cookies were still captured.`));
|
|
1843
1153
|
}
|
|
1844
1154
|
});
|
|
1845
|
-
// account status
|
|
1846
1155
|
account
|
|
1847
1156
|
.command('status <id>')
|
|
1848
1157
|
.description('Show account credentials and session status for a scrap')
|
|
@@ -1852,8 +1161,6 @@ account
|
|
|
1852
1161
|
const data = await spin(() => api.get(`/api/scraps/${id}`), 'Fetching scrap…');
|
|
1853
1162
|
const acc = data.account;
|
|
1854
1163
|
if (opts.json) {
|
|
1855
|
-
// #86 finding 12 — `json` is already statically imported at the top of
|
|
1856
|
-
// this file; the dynamic import here was pure dead weight.
|
|
1857
1164
|
return json(acc ?? null);
|
|
1858
1165
|
}
|
|
1859
1166
|
if (!acc) {
|
|
@@ -1882,7 +1189,6 @@ account
|
|
|
1882
1189
|
}
|
|
1883
1190
|
console.log(`${credLine} | ${sessionLine}`);
|
|
1884
1191
|
});
|
|
1885
|
-
// doctor — diagnose last run of a scrap (error, failed selector, block status, page state, autofix)
|
|
1886
1192
|
scraps
|
|
1887
1193
|
.command('doctor <id>')
|
|
1888
1194
|
.description('Diagnose the last run (error, failed selector, block status, page state, autofix outcome)')
|
|
@@ -1899,24 +1205,16 @@ scraps
|
|
|
1899
1205
|
console.log(chalk.dim('No runs yet.'));
|
|
1900
1206
|
return;
|
|
1901
1207
|
}
|
|
1902
|
-
// #185 — `docs` sits alongside `run`/`fix` (present only for a live
|
|
1903
|
-
// mapped failureKind — see runDocsField's doc comment), never nested
|
|
1904
|
-
// inside the allowlisted `run` object.
|
|
1905
1208
|
if (opts.json)
|
|
1906
1209
|
return json({ run: pickRun(result.run), fix: pickFix(result.fix), ...runDocsField(result.run) });
|
|
1907
1210
|
console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
|
|
1908
1211
|
if (opts.autofix && result.fix) {
|
|
1909
1212
|
console.log('\n' + formatAutofix(result.fix));
|
|
1910
1213
|
}
|
|
1911
|
-
// #184 safety-net — formatDoctor already prints the login-wall hint
|
|
1912
|
-
// (#182) above when this is a live auth wall; if Claude's skills aren't
|
|
1913
|
-
// installed either, point at the command that installs them too (never
|
|
1914
|
-
// under --json, see the early return above).
|
|
1915
1214
|
if (isAuthWall(result.run)) {
|
|
1916
1215
|
maybeSuggestSkillsForAuthWall();
|
|
1917
1216
|
}
|
|
1918
1217
|
});
|
|
1919
|
-
// autofix — show full auto-fix attempt detail (diff, dry-run, knowledge)
|
|
1920
1218
|
scraps
|
|
1921
1219
|
.command('autofix <id>')
|
|
1922
1220
|
.description('Show the last auto-fix attempt for a scrap (decision, diff, dry-run, knowledge)')
|
|
@@ -1925,9 +1223,6 @@ scraps
|
|
|
1925
1223
|
validateObjectId(id);
|
|
1926
1224
|
const result = await fetchRunAndFix(id);
|
|
1927
1225
|
if (!result) {
|
|
1928
|
-
// #88 item 7 — unified no-runs shape with `doctor --json` / `data
|
|
1929
|
-
// --errors --json`: a never-run scrap is a distinct, nameable state,
|
|
1930
|
-
// not the same bare `null` a run-with-no-fix-attempt returns below.
|
|
1931
1226
|
if (opts.json) {
|
|
1932
1227
|
json({ status: 'no_runs' });
|
|
1933
1228
|
return;
|
|
@@ -1936,8 +1231,6 @@ scraps
|
|
|
1936
1231
|
return;
|
|
1937
1232
|
}
|
|
1938
1233
|
if (!result.fix) {
|
|
1939
|
-
// A run DID happen, it just had no autofix attempt — genuinely "no
|
|
1940
|
-
// data", unlike the never-run case above.
|
|
1941
1234
|
if (opts.json) {
|
|
1942
1235
|
json(null);
|
|
1943
1236
|
return;
|
|
@@ -1949,7 +1242,6 @@ scraps
|
|
|
1949
1242
|
return json(result.fix);
|
|
1950
1243
|
console.log(formatAutofix(result.fix));
|
|
1951
1244
|
});
|
|
1952
|
-
// snapshot — download the captured page HTML (or error-path HTML) for the last run
|
|
1953
1245
|
scraps
|
|
1954
1246
|
.command('snapshot <id>')
|
|
1955
1247
|
.description('Download captured page HTML for the last run of a scrap')
|
|
@@ -1961,16 +1253,10 @@ scraps
|
|
|
1961
1253
|
const scrap = await api.get(`/api/scraps/${id}`);
|
|
1962
1254
|
const hid = scrap.history?.[0]?._id;
|
|
1963
1255
|
if (!hid) {
|
|
1964
|
-
// #91 P2 — same no-runs contract as `doctor`/`autofix`/`data --errors`
|
|
1965
|
-
// (`{status:"no_runs"}`, exit 0) under --json.
|
|
1966
1256
|
if (opts.json) {
|
|
1967
1257
|
json({ status: 'no_runs' });
|
|
1968
1258
|
return;
|
|
1969
1259
|
}
|
|
1970
|
-
// `-o` explicitly asked for a file to be written. Silently exiting 0
|
|
1971
|
-
// with nothing written (and only a dim console line) is indistinguishable
|
|
1972
|
-
// from success to a script checking the exit code alone — give it the
|
|
1973
|
-
// SAME not_found envelope `scraps data`'s own never-run case uses.
|
|
1974
1260
|
if (opts.out) {
|
|
1975
1261
|
reportDataState(`Scrap ${id} has never run — nothing to write to ${opts.out}.`, 4, 'not_found', false);
|
|
1976
1262
|
return;
|