@trawlme/cli 3.11.0 → 3.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/commands/create.d.ts +0 -28
- package/dist/commands/create.js +0 -89
- package/dist/commands/doctor.d.ts +0 -79
- package/dist/commands/doctor.js +1 -187
- package/dist/commands/login.js +0 -67
- package/dist/commands/ping.d.ts +0 -15
- package/dist/commands/ping.js +0 -15
- package/dist/commands/scraps.d.ts +0 -120
- package/dist/commands/scraps.js +142 -656
- package/dist/commands/skills.js +0 -22
- package/dist/commands/spec.d.ts +0 -85
- package/dist/commands/spec.js +0 -67
- package/dist/commands/telemetry.js +0 -4
- package/dist/commands/token.js +0 -28
- package/dist/commands/upgrade.js +0 -22
- package/dist/commands/whoami.d.ts +0 -12
- package/dist/commands/whoami.js +0 -6
- package/dist/index.d.ts +0 -188
- package/dist/index.js +0 -349
- package/dist/lib/api.d.ts +0 -78
- package/dist/lib/api.js +1 -320
- package/dist/lib/cdp-pipe.d.ts +31 -0
- package/dist/lib/cdp-pipe.js +141 -0
- package/dist/lib/chrome-discovery.d.ts +1 -0
- package/dist/lib/chrome-discovery.js +30 -0
- package/dist/lib/chrome-launch.d.ts +8 -0
- package/dist/lib/chrome-launch.js +53 -0
- package/dist/lib/config.d.ts +0 -53
- package/dist/lib/config.js +0 -55
- package/dist/lib/confirm.d.ts +0 -55
- package/dist/lib/confirm.js +0 -47
- package/dist/lib/docs.d.ts +0 -123
- package/dist/lib/docs.js +0 -169
- package/dist/lib/errors.d.ts +0 -134
- package/dist/lib/errors.js +0 -151
- package/dist/lib/format.d.ts +0 -6
- package/dist/lib/format.js +0 -6
- package/dist/lib/json.d.ts +0 -35
- package/dist/lib/json.js +0 -48
- package/dist/lib/jwt.d.ts +0 -7
- package/dist/lib/jwt.js +0 -7
- package/dist/lib/pinch.d.ts +0 -53
- package/dist/lib/pinch.js +6 -112
- package/dist/lib/pinchAnimation.d.ts +0 -16
- package/dist/lib/pinchAnimation.js +8 -29
- package/dist/lib/posthog.d.ts +0 -9
- package/dist/lib/posthog.js +0 -23
- package/dist/lib/prompt.js +1 -20
- package/dist/lib/secure-transport.d.ts +1 -0
- package/dist/lib/secure-transport.js +15 -0
- package/dist/lib/session-capture-guard.d.ts +6 -0
- package/dist/lib/session-capture-guard.js +9 -0
- package/dist/lib/session-capture.d.ts +55 -0
- package/dist/lib/session-capture.js +319 -0
- package/dist/lib/skills.d.ts +0 -175
- package/dist/lib/skills.js +1 -216
- package/dist/lib/skillsNudge.d.ts +0 -17
- package/dist/lib/skillsNudge.js +0 -83
- package/dist/lib/spinner.d.ts +0 -39
- package/dist/lib/spinner.js +0 -40
- package/dist/lib/storage-state.d.ts +55 -0
- package/dist/lib/storage-state.js +96 -0
- package/dist/lib/tips.d.ts +0 -38
- package/dist/lib/tips.js +0 -77
- package/dist/lib/updateCheckWorker.js +0 -14
- package/dist/lib/updateNotifier.d.ts +0 -17
- package/dist/lib/updateNotifier.js +0 -53
- package/dist/lib/validate.d.ts +0 -8
- package/dist/lib/validate.js +0 -8
- package/dist/lib/version.d.ts +0 -12
- package/dist/lib/version.js +1 -13
- package/package.json +2 -2
package/dist/commands/scraps.js
CHANGED
|
@@ -12,58 +12,20 @@ import { formatDoctor, formatAutofix, fetchRunAndFix, pickRun, pickFix, isAuthWa
|
|
|
12
12
|
import { renderPinch, pinchEnabled } from '../lib/pinch.js';
|
|
13
13
|
import { maybeShowReferralTip } from '../lib/tips.js';
|
|
14
14
|
import { maybeSuggestSkillsForAuthWall } from '../lib/skillsNudge.js';
|
|
15
|
+
import { captureSession, checkInteractiveEnvironment, LOCALSTORAGE_READ_TIMEOUT_MS, } from '../lib/session-capture.js';
|
|
16
|
+
import { assertSecureTransport } from '../lib/secure-transport.js';
|
|
15
17
|
import { getApiUrl } from '../lib/config.js';
|
|
16
18
|
import { resolveDocsUrls, resolveFailureKindDocsUrl } from '../lib/docs.js';
|
|
17
|
-
/**
|
|
18
|
-
* #185 — `{ docs }` (spreadable, empty object when nothing to add) for a
|
|
19
|
-
* run's JSON payload (`doctor`/`data --errors`/`run-info`), keyed on
|
|
20
|
-
* trawl_node's `failureKind` (NOT errors.ts's unrelated `ErrorEnvelope.kind`
|
|
21
|
-
* — see docs.ts's `FAILURE_KIND_DOC_PATHS` doc comment for why those two
|
|
22
|
-
* `'auth'`s must never be conflated). Reuses `isAuthWall` — the SAME
|
|
23
|
-
* staleness guard `formatDoctor`'s human hint and the #184 skills nudge
|
|
24
|
-
* already share (a `failureKind:'auth'` stamp survives a later
|
|
25
|
-
* success/regression patch, see `isAuthWall`'s own doc comment) — so a run
|
|
26
|
-
* that ultimately succeeded or degraded never carries a stale `docs` link
|
|
27
|
-
* either, exactly the invariant `detectWallVendor`/`isAuthWall` already
|
|
28
|
-
* enforce for the human-facing surfaces. One call site, one rule, reused by
|
|
29
|
-
* all three JSON surfaces below instead of three copies that could drift.
|
|
30
|
-
*/
|
|
31
19
|
function runDocsField(run) {
|
|
32
20
|
if (!isAuthWall(run))
|
|
33
21
|
return {};
|
|
34
|
-
// Whole resolved object (never just `.docsUrl`) so resolveFailureKindDocsUrl
|
|
35
|
-
// can gate on rung provenance — see docs.ts's DocsUrls.docsUrlIsDerived.
|
|
36
|
-
// This call never supplies `externalDocsUrl`, so `docsUrl` here can only
|
|
37
|
-
// ever be rung-2-derived or absent — never a rung-1 server-declared root —
|
|
38
|
-
// but the gate stays explicit rather than relying on that as an invariant.
|
|
39
22
|
const resolved = resolveDocsUrls({ apiBaseUrl: getApiUrl() });
|
|
40
23
|
const docs = resolveFailureKindDocsUrl(run.failureKind, resolved);
|
|
41
24
|
return docs ? { docs } : {};
|
|
42
25
|
}
|
|
43
|
-
/**
|
|
44
|
-
* Print a usage/validation error consistently: human text to stderr, or a
|
|
45
|
-
* machine envelope on stdout under --json (never both — reportError is the
|
|
46
|
-
* single formatting path shared with index.ts's central catch, #86 finding
|
|
47
|
-
* 9). Sets exit code 2 (usage) — distinct from a business-logic refusal
|
|
48
|
-
* (which stays 1) or an unmapped ApiError/NetworkError (handled centrally in
|
|
49
|
-
* index.ts). (#71)
|
|
50
|
-
*/
|
|
51
26
|
function usageError(message, opts = {}) {
|
|
52
27
|
process.exitCode = reportError(new UsageError(message), { json: opts.json });
|
|
53
28
|
}
|
|
54
|
-
/**
|
|
55
|
-
* #163 — client-side URL validation shared by `create`/`update`, mirroring
|
|
56
|
-
* the server's now-required-and-parseable `url` field (trawl_node#1823,
|
|
57
|
-
* PR #1828: `z.string().trim().default('')` -> `z.string().trim().url()`).
|
|
58
|
-
* Gives a readable message BEFORE the request goes out instead of a raw
|
|
59
|
-
* 400/422 from the server. Reuses the exact same `requireUrl` helper
|
|
60
|
-
* `trawl create`/`trawl login --url` already validate against (lib/validate.js),
|
|
61
|
-
* just routed through this file's own return-style `usageError()` convention
|
|
62
|
-
* (never throw — every other validation in this file, tier/limit/page/params/
|
|
63
|
-
* status, follows the same pattern) instead of the throw-style those two
|
|
64
|
-
* commands use. Returns `undefined` (never throws) on a bad value — the
|
|
65
|
-
* caller must check for that and `return` without calling the API.
|
|
66
|
-
*/
|
|
67
29
|
function tryValidateUrl(value, opts) {
|
|
68
30
|
try {
|
|
69
31
|
return requireUrl(value, '--url');
|
|
@@ -76,28 +38,13 @@ function tryValidateUrl(value, opts) {
|
|
|
76
38
|
throw err;
|
|
77
39
|
}
|
|
78
40
|
}
|
|
79
|
-
/**
|
|
80
|
-
* One-line rendering of an emptyContext blob for the run-info table. Two
|
|
81
|
-
* buckets are actionable and must stay distinct: selectors that matched zero
|
|
82
|
-
* nodes (the page changed) and selectors the browser rejected outright
|
|
83
|
-
* (`-1`, a bug in the scrap script). Folding the second into "all matched"
|
|
84
|
-
* would report a broken selector as healthy. `--json` still emits the full
|
|
85
|
-
* object, `page.topClasses` / `topAnchorPaths` included.
|
|
86
|
-
*/
|
|
87
41
|
function summarizeEmptyContext(ctx) {
|
|
88
42
|
if (!ctx || typeof ctx !== 'object')
|
|
89
43
|
return null;
|
|
90
44
|
const parts = [];
|
|
91
45
|
const selectors = ctx.selectors && typeof ctx.selectors === 'object' ? ctx.selectors : {};
|
|
92
46
|
const names = Object.keys(selectors);
|
|
93
|
-
// Selectors may themselves contain a comma (`div, span`), so quote each one
|
|
94
|
-
// — a raw join renders one grouping selector as if it were two.
|
|
95
47
|
const list = (ns) => `${ns.slice(0, 3).map((s) => `\`${s}\``).join(', ')}${ns.length > 3 ? ` (+${ns.length - 3})` : ''}`;
|
|
96
|
-
// Four disjoint, exhaustive buckets — every selector lands in exactly one, so
|
|
97
|
-
// no count is ever dropped and none is summarised as something it is not.
|
|
98
|
-
// `selectors` comes from a Mongoose Mixed field, so a count may be `null` or
|
|
99
|
-
// a string: that is `unreadable`, which is neither healthy nor zero-match and
|
|
100
|
-
// must be named as its own thing next to the others, not collapsed into them.
|
|
101
48
|
const isCount = (v) => typeof v === 'number' && Number.isFinite(v);
|
|
102
49
|
const zero = names.filter((s) => selectors[s] === 0);
|
|
103
50
|
const invalid = names.filter((s) => selectors[s] === -1);
|
|
@@ -111,15 +58,8 @@ function summarizeEmptyContext(ctx) {
|
|
|
111
58
|
if (unreadable.length)
|
|
112
59
|
parts.push(`unreadable: ${list(unreadable)}`);
|
|
113
60
|
if (matched.length) {
|
|
114
|
-
// Named even in a mixed set: "3 of the selectors still work" is the
|
|
115
|
-
// difference between a page that moved and a page that vanished.
|
|
116
61
|
parts.push(mixed ? `${matched.length} matched` : `${matched.length} selector(s), all matched`);
|
|
117
62
|
}
|
|
118
|
-
// Same Mixed-field caution on the title: coerce before calling a string
|
|
119
|
-
// method (a numeric title would otherwise crash `run-info` on the very run
|
|
120
|
-
// it exists to explain), and test against null/undefined rather than
|
|
121
|
-
// truthiness so a falsy-but-real title like `0` still shows. Inner double
|
|
122
|
-
// quotes would close the wrapper early and corrupt the line.
|
|
123
63
|
const title = ctx.page?.title;
|
|
124
64
|
if (title !== undefined && title !== null && String(title) !== '') {
|
|
125
65
|
parts.push(`title="${String(title).replace(/"/g, "'")}"`);
|
|
@@ -132,25 +72,16 @@ function lastStatus(scrap) {
|
|
|
132
72
|
const last = scrap.history?.[0];
|
|
133
73
|
if (!last)
|
|
134
74
|
return 'never';
|
|
135
|
-
// status:null means a run is IN FLIGHT (node persists {status:null,
|
|
136
|
-
// statusDetail:null, inFlight:true} the moment a run starts) — that is NOT
|
|
137
|
-
// the same thing as "this scrap has never run" (no history row at all).
|
|
138
|
-
// (#88 item 1)
|
|
139
75
|
if (last.status === null)
|
|
140
76
|
return 'running';
|
|
141
77
|
if (last.status === undefined)
|
|
142
78
|
return 'never';
|
|
143
|
-
// #91 LOW — same regression-as-failure bucketing bug as doctor's badge
|
|
144
|
-
// (formatDoctor): a regression row's write actually succeeded (item count
|
|
145
|
-
// just dropped vs baseline, flagged by an async patch afterward — #88 item
|
|
146
|
-
// 2), so it must not collapse into the same 'failure' bucket a genuine
|
|
147
|
-
// failed run gets.
|
|
148
79
|
if (last.status === false && last.statusDetail === 'regression')
|
|
149
80
|
return 'regression';
|
|
150
81
|
return last.status === true ? 'success' : 'failure';
|
|
151
82
|
}
|
|
152
83
|
function lastRun(scrap) {
|
|
153
|
-
return formatDate(scrap.history?.[0]?.createdAt);
|
|
84
|
+
return formatDate(scrap.history?.[0]?.createdAt);
|
|
154
85
|
}
|
|
155
86
|
function statusIcon(status) {
|
|
156
87
|
if (status === 'success')
|
|
@@ -159,28 +90,10 @@ function statusIcon(status) {
|
|
|
159
90
|
return chalk.red('✗');
|
|
160
91
|
if (status === 'running')
|
|
161
92
|
return chalk.cyan('↻');
|
|
162
|
-
// #91 LOW — distinct amber icon, never the red ✗ a genuine failure gets
|
|
163
|
-
// (mirrors doctor's "● regression" badge — see formatDoctor).
|
|
164
93
|
if (status === 'regression')
|
|
165
94
|
return chalk.hex('#FFA500')('▼');
|
|
166
95
|
return chalk.dim('—');
|
|
167
96
|
}
|
|
168
|
-
/**
|
|
169
|
-
* #179 — health badge, next to the existing status icon. Two distinct
|
|
170
|
-
* sources, deliberately never blended into one derived verdict:
|
|
171
|
-
* 1. `consecutiveFailedRuns`/`unhealthySince` — present ONLY when `--unhealthy`
|
|
172
|
-
* asked the API to attach them (see attachListCommand). This is the
|
|
173
|
-
* live, exact server verdict — shown whenever available, since it is
|
|
174
|
-
* strictly more informative than the breadcrumb below.
|
|
175
|
-
* 2. `lastCronOutcome === 'skipped_unhealthy'` — always present (a normal
|
|
176
|
-
* scrap field, no extra query cost), but a STALE breadcrumb of the last
|
|
177
|
-
* scheduled tick, not a live read: never set on a no-cron scrap, lags
|
|
178
|
-
* by up to one tick, and flips back to 'ok'/'error' once the recovery
|
|
179
|
-
* probe fires. Labeled as what it is ("cron paused"), never upgraded to
|
|
180
|
-
* "unhealthy" — that word is reserved for the live verdict above.
|
|
181
|
-
* Never re-derives either signal from `history[]` — both come straight off
|
|
182
|
-
* the server (#1952 in trawl_node).
|
|
183
|
-
*/
|
|
184
97
|
function healthBadge(scrap) {
|
|
185
98
|
if (typeof scrap.consecutiveFailedRuns === 'number' && scrap.consecutiveFailedRuns > 0) {
|
|
186
99
|
const n = scrap.consecutiveFailedRuns;
|
|
@@ -191,19 +104,6 @@ function healthBadge(scrap) {
|
|
|
191
104
|
}
|
|
192
105
|
return chalk.dim('—');
|
|
193
106
|
}
|
|
194
|
-
/**
|
|
195
|
-
* #179 — old-server fallback for `list --unhealthy`, same shape as
|
|
196
|
-
* `warnIfUnconfirmedTier` above (#86 finding 4b): an older trawl_node that
|
|
197
|
-
* predates #1952 ignores the unknown `minConsecutiveFailures` query param
|
|
198
|
-
* entirely and returns the FULL unfiltered list — silently presenting every
|
|
199
|
-
* scrap as "unhealthy" is exactly the kind of unconfirmed-outcome lie that
|
|
200
|
-
* precedent exists to prevent. A confirming server attaches
|
|
201
|
-
* `consecutiveFailedRuns` to EVERY item once the param is set (the
|
|
202
|
-
* controller writes `s.consecutiveFailedRuns || 0` unconditionally in that
|
|
203
|
-
* branch — see trawl_node scraps.controller.js), so its total absence across
|
|
204
|
-
* a non-empty response is the honest tell. Stderr only (safe under --json —
|
|
205
|
-
* stdout purity is untouched), never blocks the (unreliable) results below.
|
|
206
|
-
*/
|
|
207
107
|
function healthFilterUnconfirmed(data, unhealthyWasRequested) {
|
|
208
108
|
if (!unhealthyWasRequested || data.length === 0)
|
|
209
109
|
return false;
|
|
@@ -212,35 +112,19 @@ function healthFilterUnconfirmed(data, unhealthyWasRequested) {
|
|
|
212
112
|
function warnIfUnconfirmedHealthFilter(data, unhealthyWasRequested) {
|
|
213
113
|
if (!healthFilterUnconfirmed(data, unhealthyWasRequested))
|
|
214
114
|
return;
|
|
215
|
-
console.error(chalk.yellow('⚠ Server did not confirm the --unhealthy filter (older server, predates
|
|
115
|
+
console.error(chalk.yellow('⚠ Server did not confirm the --unhealthy filter (older server version, predates this filter) — results below are UNFILTERED, not just the unhealthy scraps.'));
|
|
216
116
|
}
|
|
217
|
-
/**
|
|
218
|
-
* #179 — the machine-readable counterpart to the stderr warning above, in the
|
|
219
|
-
* exact shape `withTierUnconfirmed` already uses for the tier case. A --json
|
|
220
|
-
* caller has no reliable reason to read stderr, so without this a script sees
|
|
221
|
-
* a full unfiltered list and cannot tell it is not the unhealthy set.
|
|
222
|
-
* Emitted ONLY when the filter went unconfirmed — never fabricated otherwise.
|
|
223
|
-
*/
|
|
224
117
|
function withHealthFilterUnconfirmed(rows, data, unhealthyWasRequested) {
|
|
225
118
|
if (!healthFilterUnconfirmed(data, unhealthyWasRequested))
|
|
226
119
|
return rows;
|
|
227
120
|
return { scraps: rows, _healthFilterUnconfirmed: true };
|
|
228
121
|
}
|
|
229
122
|
export const scraps = new Command('scraps').description('Manage scraps');
|
|
230
|
-
// shared SSE streaming helper. `asJson` (#107) emits one raw JSON object per
|
|
231
|
-
// line (NDJSON) on stdout instead of the human-formatted timestamped text —
|
|
232
|
-
// a streaming command still needs a pure-stdout machine mode, just line-
|
|
233
|
-
// delimited instead of a single blob (there's no single "final" payload to
|
|
234
|
-
// wait for).
|
|
235
123
|
async function watchActivities(id, asJson) {
|
|
236
124
|
if (!asJson)
|
|
237
125
|
console.log(chalk.dim('Streaming activities (Ctrl+C to stop)…\n'));
|
|
238
126
|
for await (const event of api.stream(`/api/scraps/${id}/activities/stream`)) {
|
|
239
127
|
try {
|
|
240
|
-
// #159 — same tolerant parse as api.ts's response-parsing seam: an SSE
|
|
241
|
-
// activity line can carry the same server-side raw-control-character
|
|
242
|
-
// quirk as any other API payload, and this feeds `--json` NDJSON
|
|
243
|
-
// output too (not just create).
|
|
244
128
|
const activity = parseServerJson(event);
|
|
245
129
|
if (asJson) {
|
|
246
130
|
console.log(JSON.stringify(activity));
|
|
@@ -250,34 +134,12 @@ async function watchActivities(id, asJson) {
|
|
|
250
134
|
console.log(`${chalk.dim(`[${time}]`)} ${activity.message}`);
|
|
251
135
|
}
|
|
252
136
|
catch {
|
|
253
|
-
// non-JSON lines are heartbeats or SSE comments — skip silently
|
|
254
137
|
if (process.env['DEBUG'] && event.trim()) {
|
|
255
138
|
console.error(chalk.dim(`[SSE] skipping non-JSON: ${event}`));
|
|
256
139
|
}
|
|
257
140
|
}
|
|
258
141
|
}
|
|
259
142
|
}
|
|
260
|
-
/**
|
|
261
|
-
* #91 P1 / #93 item 1 — snapshot the current top history entry BEFORE
|
|
262
|
-
* triggering a run, so pollRunProgress can later tell "the run we just
|
|
263
|
-
* launched" apart from whatever the last run happened to be. Node persists a
|
|
264
|
-
* fresh {status:null, inFlight:true} row the moment a run starts (see
|
|
265
|
-
* HistorysService.create in trawl_node), so a changed history[0]._id is
|
|
266
|
-
* normally an honest "the new run has begun" signal — EXCEPT `trigger`
|
|
267
|
-
* (method:'worker') dedups onto an already-pending/running worker job
|
|
268
|
-
* (ScrapJobsService, LIVE_STATUSES) instead of creating a new row: the top
|
|
269
|
-
* row's `_id` never changes even though this launch IS that run. Recording
|
|
270
|
-
* `alreadyInFlight` (status===null at capture time) lets pollRunProgress
|
|
271
|
-
* recognize that dedup case too, instead of waiting forever for an `_id`
|
|
272
|
-
* that will never arrive (#93 item 1).
|
|
273
|
-
*
|
|
274
|
-
* #97 — a failed lookup used to fall back to `{alreadyInFlight:false}` with
|
|
275
|
-
* no `id`, which LOOKED identical to "scrap has never run" — but is not:
|
|
276
|
-
* the scrap may well have prior (possibly terminal) history this GET simply
|
|
277
|
-
* never observed. `captured:false` flags that distinction explicitly so
|
|
278
|
-
* pollRunProgress can defer trusting a baseline instead of risking a stale
|
|
279
|
-
* pre-existing row being reported as the run just launched.
|
|
280
|
-
*/
|
|
281
143
|
async function captureBeforeRunState(id) {
|
|
282
144
|
try {
|
|
283
145
|
const scrap = await api.get(`/api/scraps/${id}`);
|
|
@@ -292,115 +154,14 @@ function sleep(ms) {
|
|
|
292
154
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
293
155
|
}
|
|
294
156
|
const POLL_INTERVAL_MS = 2000;
|
|
295
|
-
// Mirrors LONG_RUN_TIMEOUT_MS (#91 item 1) — the server-side worst case this
|
|
296
|
-
// polls for is the same one that timeout was sized for.
|
|
297
157
|
const POLL_TIMEOUT_MS = LONG_RUN_TIMEOUT_MS;
|
|
298
|
-
// #107 review F1 — a genuinely unreachable API must not run the watch
|
|
299
|
-
// silently for the full 300s timeout with dead air; after this many
|
|
300
|
-
// CONSECUTIVE (not cumulative — any successful poll resets the counter)
|
|
301
|
-
// failed polls, treat it as a persistent error and surface it immediately
|
|
302
|
-
// instead of waiting out the clock. At the default 2s interval this gives up
|
|
303
|
-
// after ~10s of continuous failures — comfortably longer than any single
|
|
304
|
-
// transient blip, nowhere near the 300s ceiling.
|
|
305
158
|
const MAX_CONSECUTIVE_POLL_ERRORS = 5;
|
|
306
|
-
/**
|
|
307
|
-
* #91 P1 — replaces "await the run to completion, THEN open the activities
|
|
308
|
-
* SSE stream" (which showed NOTHING: the activities SSE
|
|
309
|
-
* (GET /api/scraps/:id/activities/stream) is backed by an in-process
|
|
310
|
-
* EventEmitter with no backlog — trawl_node
|
|
311
|
-
* modules/activities/services/activities.service.js — so by the time a
|
|
312
|
-
* synchronous run has already finished there is nothing left to emit; and
|
|
313
|
-
* the async `trigger` default enqueues the run onto a durable job queue a
|
|
314
|
-
* SEPARATE cron-consumer pod drains, whose in-process emitter never reaches
|
|
315
|
-
* the API pod holding the SSE connection at all).
|
|
316
|
-
*
|
|
317
|
-
* Instead this polls two REST reads that are BOTH Mongo-backed (not
|
|
318
|
-
* in-process), so they work no matter which pod actually executed the run:
|
|
319
|
-
* - GET /api/scraps/:id/activities?history=<hid>&limit=20 — the SAME
|
|
320
|
-
* activities-list endpoint `doctor`'s fetchRunAndFix already calls
|
|
321
|
-
* (src/commands/doctor.ts) — prints each new activity line once the new
|
|
322
|
-
* run's history id is known.
|
|
323
|
-
* - GET /api/scraps/:id — history[0].status/statusDetail, the SAME
|
|
324
|
-
* terminal-status signal `lastStatus()` above already trusts (status:null
|
|
325
|
-
* === in flight, #88 item 1) to know when the run is done.
|
|
326
|
-
*
|
|
327
|
-
* #93 item 1 — dedup-race fix. "Is this the run we're watching?" used to be
|
|
328
|
-
* a single check: `history[0]._id !== beforeHistoryId`. That's wrong when
|
|
329
|
-
* `before.alreadyInFlight` is true (a `trigger` call deduped onto a worker
|
|
330
|
-
* job that was ALREADY pending/running at capture time): the top row IS the
|
|
331
|
-
* run we're watching, but its `_id` never changes, so the old guard never
|
|
332
|
-
* released and the poll ran the full timeout to a false "Timed out". The run
|
|
333
|
-
* we're watching is now EITHER a brand-new id (fresh trigger, the common
|
|
334
|
-
* case) OR the same id that was already in-flight (status:null) at capture
|
|
335
|
-
* (the dedup case) — a same-id row that was already TERMINAL at capture is
|
|
336
|
-
* neither, and must not be latched onto as "done" (it's just the previous
|
|
337
|
-
* run, still sitting there until a genuinely new run supersedes it).
|
|
338
|
-
*
|
|
339
|
-
* #97 — capture-failed fallback. The dedup-race fix above assumes `before`
|
|
340
|
-
* is trustworthy. When `captureBeforeRunState`'s own GET threw,
|
|
341
|
-
* `before.id` is `undefined` — and that is INDISTINGUISHABLE from "the
|
|
342
|
-
* scrap has genuinely never run" (also `id: undefined`), which is exactly
|
|
343
|
-
* the case `isNewRun` above is designed to match on the very first row that
|
|
344
|
-
* ever appears. So on the very first poll, ANY pre-existing history row —
|
|
345
|
-
* even the STALE PREVIOUS run, already terminal — satisfied
|
|
346
|
-
* `last._id !== undefined` and got reported as "the run we just launched"
|
|
347
|
-
* finishing, when it was really just whatever ran before.
|
|
348
|
-
*
|
|
349
|
-
* Fix: `before.captured === false` defers trusting a baseline at all.
|
|
350
|
-
* Instead of comparing against the (unknown) `beforeId` from the start, the
|
|
351
|
-
* FIRST successful poll read is treated as the deferred capture itself —
|
|
352
|
-
* exactly what `captureBeforeRunState` would have returned had its GET
|
|
353
|
-
* succeeded — and only READS from that point on are compared against it,
|
|
354
|
-
* via the exact same `isNewRun` / `isDedupOntoInFlight` logic above. A
|
|
355
|
-
* pre-existing terminal row observed on that first read becomes `beforeId`
|
|
356
|
-
* (not a match for itself), so it correctly falls into "still the stale
|
|
357
|
-
* previous run" below and the poll keeps waiting; a row still in flight
|
|
358
|
-
* becomes the `alreadyInFlight` baseline, exactly like a successful capture
|
|
359
|
-
* would have recorded. Either way this costs at most one extra poll
|
|
360
|
-
* interval, bounded by the same deadline as everything else. The one
|
|
361
|
-
* remaining edge case — capture failed AND the scrap never ran before AND
|
|
362
|
-
* the triggered run already finished by the very first poll — is genuinely
|
|
363
|
-
* undecidable from "stale pre-existing row" with no more information than
|
|
364
|
-
* this function has, so it resolves to the same honest timeout rather than
|
|
365
|
-
* risk reporting a possibly-wrong outcome (never a lie, at worst a timeout
|
|
366
|
-
* telling the caller to check `doctor`).
|
|
367
|
-
*
|
|
368
|
-
* #107 review F1 — before this fix, `run|trigger --json --watch` was
|
|
369
|
-
* outcome-blind: `quiet` suppressed ALL output (including "Run finished:
|
|
370
|
-
* failure" and the timeout notice), a transient poll error was caught and
|
|
371
|
-
* silently retried FOREVER within the deadline, and the process always
|
|
372
|
-
* exited 0 after the poll loop regardless of what the watched run actually
|
|
373
|
-
* did — dead air, then a clean exit code, even for a failed or timed-out
|
|
374
|
-
* run. An agent scripting this CLI had no way to tell success from failure
|
|
375
|
-
* from "we gave up". Fixed by:
|
|
376
|
-
* - emitting exactly ONE final NDJSON line on stdout under `--json` once
|
|
377
|
-
* the watch reaches ANY of its three exits (terminal status, timeout, or
|
|
378
|
-
* a persistent poll error) — `{runId,status}` (+`error` for a poll
|
|
379
|
-
* error) — while every intermediate progress line stays suppressed
|
|
380
|
-
* (unchanged from before);
|
|
381
|
-
* - setting `process.exitCode` non-zero on a genuine run failure, a
|
|
382
|
-
* timeout, or a persistent poll error, and `0` on a real success — in
|
|
383
|
-
* BOTH `--json` and human `--watch` modes (human mode used to exit 0
|
|
384
|
-
* unconditionally, the same bug, just silent instead of dishonest);
|
|
385
|
-
* - giving up after `MAX_CONSECUTIVE_POLL_ERRORS` consecutive failed reads
|
|
386
|
-
* instead of retrying the same dead endpoint for the full 300s.
|
|
387
|
-
*/
|
|
388
159
|
export async function pollRunProgress(id, before, opts = {}) {
|
|
389
160
|
const intervalMs = opts.intervalMs ?? POLL_INTERVAL_MS;
|
|
390
161
|
const timeoutMs = opts.timeoutMs ?? POLL_TIMEOUT_MS;
|
|
391
|
-
// #107 — `run --json --watch` / `trigger --json --watch` must keep stdout
|
|
392
|
-
// pure JSON: none of this function's intermediate progress text is safe to
|
|
393
|
-
// print once a caller asked for --json. `asJson` suppresses every
|
|
394
|
-
// intermediate console.log below (including the pinch celebration frame)
|
|
395
|
-
// while the polling/wait logic itself runs unchanged; the ONE exception is
|
|
396
|
-
// the single final outcome line emitted right before each return below.
|
|
397
162
|
const asJson = opts.json ?? false;
|
|
398
163
|
let beforeId = before?.id;
|
|
399
164
|
let beforeAlreadyInFlight = before?.alreadyInFlight ?? false;
|
|
400
|
-
// #97 — only an EXPLICIT captured:false (capture's GET actually threw)
|
|
401
|
-
// defers the baseline. `before` itself being undefined (callers that skip
|
|
402
|
-
// capture entirely) or `captured` being true/absent both mean "trust
|
|
403
|
-
// beforeId as given", preserving every existing call site's behavior.
|
|
404
165
|
let baselineEstablished = before?.captured ?? true;
|
|
405
166
|
if (!asJson) {
|
|
406
167
|
console.log(chalk.dim('Live activity streaming has no signal for this run (async/cross-pod) — polling for progress instead…\n'));
|
|
@@ -420,11 +181,6 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
420
181
|
}
|
|
421
182
|
catch (err) {
|
|
422
183
|
consecutivePollErrors++;
|
|
423
|
-
// #107 review F1 — a single failed read is still a TRANSIENT blip,
|
|
424
|
-
// safe to retry within the deadline (unchanged). Only once reads fail
|
|
425
|
-
// this many times IN A ROW is the API treated as genuinely
|
|
426
|
-
// unreachable — surface that honestly instead of quietly burning the
|
|
427
|
-
// full 300s timeout on a dead endpoint.
|
|
428
184
|
if (consecutivePollErrors >= MAX_CONSECUTIVE_POLL_ERRORS) {
|
|
429
185
|
const message = err instanceof Error ? err.message : String(err);
|
|
430
186
|
if (asJson) {
|
|
@@ -441,13 +197,8 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
441
197
|
}
|
|
442
198
|
const last = scrap.history?.[0];
|
|
443
199
|
if (!last?._id)
|
|
444
|
-
continue;
|
|
200
|
+
continue;
|
|
445
201
|
if (!baselineEstablished) {
|
|
446
|
-
// #97 — deferred capture: whatever we see on this first successful
|
|
447
|
-
// read becomes the reference point, BEFORE the isNewRun check below
|
|
448
|
-
// ever runs against it. This must happen before that check, not
|
|
449
|
-
// after, so a pre-existing terminal row is recognized as stale on
|
|
450
|
-
// this very same iteration rather than one poll late.
|
|
451
202
|
beforeId = last._id;
|
|
452
203
|
beforeAlreadyInFlight = last.status === null;
|
|
453
204
|
baselineEstablished = true;
|
|
@@ -455,10 +206,9 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
455
206
|
const isNewRun = last._id !== beforeId;
|
|
456
207
|
const isDedupOntoInFlight = last._id === beforeId && beforeAlreadyInFlight;
|
|
457
208
|
if (!isNewRun && !isDedupOntoInFlight)
|
|
458
|
-
continue;
|
|
209
|
+
continue;
|
|
459
210
|
try {
|
|
460
211
|
const activities = await api.get(`/api/scraps/${id}/activities?history=${last._id}&limit=20`);
|
|
461
|
-
// Server returns newest-first — print unseen ones oldest-first.
|
|
462
212
|
for (const a of [...(activities ?? [])].reverse()) {
|
|
463
213
|
const key = a._id ?? `${a.createdAt}:${a.message}`;
|
|
464
214
|
if (seen.has(key))
|
|
@@ -471,16 +221,9 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
471
221
|
}
|
|
472
222
|
}
|
|
473
223
|
catch {
|
|
474
|
-
// best-effort — terminal detection below still works without activity lines
|
|
475
224
|
}
|
|
476
225
|
if (last.status !== null) {
|
|
477
226
|
const outcome = last.statusDetail ?? (last.status ? 'success' : 'failure');
|
|
478
|
-
// #107 review F1 — the exit code, in BOTH modes: a regression row's
|
|
479
|
-
// WRITE actually succeeded (see lastStatus()'s own #91 comment above,
|
|
480
|
-
// and `scraps data`'s isRegression branch) and an 'empty' row is a
|
|
481
|
-
// genuine zero-item success — neither is a real failure. Only
|
|
482
|
-
// status:false with neither of those details (e.g. 'error', or no
|
|
483
|
-
// detail at all) is a genuine failed run.
|
|
484
227
|
const isGenuineFailure = last.status === false && outcome !== 'empty' && outcome !== 'regression';
|
|
485
228
|
if (asJson) {
|
|
486
229
|
const payload = { runId: last._id, status: outcome };
|
|
@@ -488,10 +231,6 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
488
231
|
}
|
|
489
232
|
else {
|
|
490
233
|
console.log(chalk.dim(`Run finished: ${outcome}`));
|
|
491
|
-
// Pinch celebrates a clean run finish (#94) — mirrors doctor.ts's own
|
|
492
|
-
// `status === true` success definition (regardless of statusDetail),
|
|
493
|
-
// never for a failed/regression run. Guarded by pinchEnabled()
|
|
494
|
-
// (NO_COLOR/non-TTY) and never under --json (stdout must stay pure).
|
|
495
234
|
if (last.status === true && pinchEnabled()) {
|
|
496
235
|
console.log(renderPinch('celebrating'));
|
|
497
236
|
}
|
|
@@ -500,9 +239,6 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
500
239
|
return;
|
|
501
240
|
}
|
|
502
241
|
}
|
|
503
|
-
// #107 review F1 — timeout: honest non-zero exit in both modes, plus the
|
|
504
|
-
// machine-readable final line under --json (never silently exit 0 after a
|
|
505
|
-
// watch that never actually confirmed what happened).
|
|
506
242
|
if (asJson) {
|
|
507
243
|
const outcome = { runId: beforeId, status: 'timeout' };
|
|
508
244
|
console.log(JSON.stringify(outcome));
|
|
@@ -512,20 +248,8 @@ export async function pollRunProgress(id, before, opts = {}) {
|
|
|
512
248
|
}
|
|
513
249
|
process.exitCode = 1;
|
|
514
250
|
}
|
|
515
|
-
// #149 item 3 — same shared-enum shape as VALID_TIERS below (~line 603):
|
|
516
|
-
// keeps `--status`'s allowed values and its error message in one place,
|
|
517
|
-
// mirrored from lastStatus()'s own return type so the two can never drift.
|
|
518
251
|
const VALID_LAST_STATUSES = ['success', 'failure', 'never', 'running', 'regression'];
|
|
519
|
-
// #179 — mirrors `UNHEALTHY_STREAK_LENGTH` in trawl_node
|
|
520
|
-
// modules/scraps/helpers/scrapCronHealth.js. `--unhealthy` is a boolean
|
|
521
|
-
// convenience flag for that ONE product-defined threshold (also what the
|
|
522
|
-
// server's own cron-skip gate and email use), not a general numeric filter —
|
|
523
|
-
// the API's `minConsecutiveFailures` param accepts any threshold, but this
|
|
524
|
-
// CLI only ever asks for the one value the rest of the product means by
|
|
525
|
-
// "unhealthy". If trawl_node's threshold ever changes, update this constant
|
|
526
|
-
// to match.
|
|
527
252
|
const UNHEALTHY_STREAK_LENGTH = 3;
|
|
528
|
-
// list — promoted to a top-level verb (#108, see the AttachOptions comment above)
|
|
529
253
|
export function attachListCommand(parent, attachOpts = {}) {
|
|
530
254
|
return parent
|
|
531
255
|
.command('list', attachOpts)
|
|
@@ -533,33 +257,14 @@ export function attachListCommand(parent, attachOpts = {}) {
|
|
|
533
257
|
.description('List all scraps')
|
|
534
258
|
.option('--json', 'Output as JSON')
|
|
535
259
|
.option('--status <status>', 'Filter by last run status (success|failure|never|running|regression)')
|
|
536
|
-
// #88 item 8 — no custom parser here (unlike the old `(v) => parseInt(v,
|
|
537
|
-
// 10)`): a bad value like "abc" used to silently become NaN, which then
|
|
538
|
-
// sailed straight through `Number.isInteger`-less checks and into
|
|
539
|
-
// `.slice(0, NaN)` (silently truncates to 0 rows) or `?page=NaN` (silently
|
|
540
|
-
// sent to the server) — never a usage error. Keeping the raw string here
|
|
541
|
-
// lets the validation below mirror `history`'s own --limit check exactly
|
|
542
|
-
// (~line 660) and report the actual bad input in the error message.
|
|
543
260
|
.option('--limit <n>', 'Show only the first N results')
|
|
544
261
|
.option('--page <n>', 'Fetch a specific page only (50 per page, no auto-pagination)')
|
|
545
|
-
// #179 — backed by the server's `minConsecutiveFailures` filter
|
|
546
|
-
// (trawl_node#1952): restricts the result set server-side to scraps
|
|
547
|
-
// whose CURRENT consecutive-failure streak is >= UNHEALTHY_STREAK_LENGTH,
|
|
548
|
-
// and annotates each returned scrap with the exact `consecutiveFailedRuns`/
|
|
549
|
-
// `unhealthySince` the aggregation computed. Deliberately never derived
|
|
550
|
-
// client-side from `history[]` — that would drift from what the web
|
|
551
|
-
// dashboard shows (comes-io/trawl_vue#1299 reads the identical filter).
|
|
552
262
|
.option('--unhealthy', `Only show scraps failing their last ${UNHEALTHY_STREAK_LENGTH}+ runs (server-computed)`)
|
|
553
263
|
.action(async (opts, cmd) => {
|
|
554
|
-
// #149 item 3 — validate --status against its enum the same way
|
|
555
|
-
// --tier already validates (usageError + return, checked first, before
|
|
556
|
-
// any other guard/async work — mirrors `create`/`update`'s --tier
|
|
557
|
-
// check below).
|
|
558
264
|
if (opts.status !== undefined && !VALID_LAST_STATUSES.includes(opts.status)) {
|
|
559
265
|
usageError(`Invalid --status "${opts.status}" (allowed: ${VALID_LAST_STATUSES.join(', ')})`, { json: opts.json });
|
|
560
266
|
return;
|
|
561
267
|
}
|
|
562
|
-
// Guard: --limit and --page are mutually exclusive
|
|
563
268
|
if (opts.limit !== undefined && opts.page !== undefined) {
|
|
564
269
|
usageError('--limit and --page are mutually exclusive. Use one or the other.', { json: opts.json });
|
|
565
270
|
return;
|
|
@@ -580,18 +285,13 @@ export function attachListCommand(parent, attachOpts = {}) {
|
|
|
580
285
|
return;
|
|
581
286
|
}
|
|
582
287
|
}
|
|
583
|
-
// #179 — same param on every request this action can issue (both
|
|
584
|
-
// branches below), so a filtered result stays filtered across the
|
|
585
|
-
// fetch-all pagination loop, not just its first page.
|
|
586
288
|
const healthFilter = opts.unhealthy ? `&minConsecutiveFailures=${UNHEALTHY_STREAK_LENGTH}` : '';
|
|
587
289
|
let data;
|
|
588
290
|
try {
|
|
589
291
|
data = await spin(async () => {
|
|
590
292
|
if (page !== undefined) {
|
|
591
|
-
// Single-page mode: explicit page requested, no loop
|
|
592
293
|
return api.get(`/api/scraps?perPage=50&page=${page}${healthFilter}`);
|
|
593
294
|
}
|
|
594
|
-
// Fetch-all mode: paginate until a page returns < 200 items
|
|
595
295
|
const perPage = 200;
|
|
596
296
|
let result = [];
|
|
597
297
|
let pageNum = 1;
|
|
@@ -606,16 +306,6 @@ export function attachListCommand(parent, attachOpts = {}) {
|
|
|
606
306
|
}, 'Fetching scraps…');
|
|
607
307
|
}
|
|
608
308
|
catch (err) {
|
|
609
|
-
// Spinner already failed by oraPromise — report with the scrap-specific
|
|
610
|
-
// prefix kept, but route through the shared classifier so exit code +
|
|
611
|
-
// --json envelope stay consistent with every other command. (#71)
|
|
612
|
-
//
|
|
613
|
-
// This bespoke catch (kept for the "Failed to fetch scraps:" prefix,
|
|
614
|
-
// which the shared reportError() can't add) used to silently swallow
|
|
615
|
-
// --debug: unlike the central index.ts catch, it never printed the raw
|
|
616
|
-
// stack trace. optsWithGlobals() reads --debug off the ROOT command
|
|
617
|
-
// (this leaf has no --debug of its own) so it can honor the flag
|
|
618
|
-
// locally instead. (#86 finding 9)
|
|
619
309
|
const isDebug = Boolean(cmd.optsWithGlobals().debug || process.env['DEBUG']);
|
|
620
310
|
const { exitCode, envelope } = classifyError(err);
|
|
621
311
|
const message = `Failed to fetch scraps: ${envelope.message}`;
|
|
@@ -630,9 +320,6 @@ export function attachListCommand(parent, attachOpts = {}) {
|
|
|
630
320
|
process.exitCode = exitCode;
|
|
631
321
|
return;
|
|
632
322
|
}
|
|
633
|
-
// #179 — check the RAW response, before --status narrows it further,
|
|
634
|
-
// so a genuinely empty (all-healthy) result is never mistaken for an
|
|
635
|
-
// unconfirmed filter.
|
|
636
323
|
warnIfUnconfirmedHealthFilter(data, opts.unhealthy);
|
|
637
324
|
if (opts.status)
|
|
638
325
|
data = data.filter((s) => lastStatus(s) === opts.status);
|
|
@@ -645,21 +332,17 @@ export function attachListCommand(parent, attachOpts = {}) {
|
|
|
645
332
|
title: s.title || '(untitled)',
|
|
646
333
|
cron: s.cron || '—',
|
|
647
334
|
status: statusIcon(lastStatus(s)),
|
|
648
|
-
// #179 — next to the existing status badge, per the issue's own
|
|
649
|
-
// wording (see healthBadge() above for the two source signals).
|
|
650
335
|
health: healthBadge(s),
|
|
651
336
|
'last run': lastRun(s),
|
|
652
337
|
updated: formatDate(s.updatedAt),
|
|
653
338
|
}));
|
|
654
339
|
table(tableRows, ['id', 'title', 'cron', 'status', 'health', 'last run', 'updated']);
|
|
655
|
-
// Print footer when --limit truncates
|
|
656
340
|
if (limit !== undefined && rows.length < totalMatched) {
|
|
657
341
|
console.log(chalk.dim(`Showing ${rows.length} of ${totalMatched} — omit --limit to see all`));
|
|
658
342
|
}
|
|
659
343
|
});
|
|
660
344
|
}
|
|
661
345
|
attachListCommand(scraps, { hidden: true });
|
|
662
|
-
// get — promoted to a top-level verb (#108)
|
|
663
346
|
export function attachGetCommand(parent, attachOpts = {}) {
|
|
664
347
|
return parent
|
|
665
348
|
.command('get <id>', attachOpts)
|
|
@@ -674,9 +357,6 @@ export function attachGetCommand(parent, attachOpts = {}) {
|
|
|
674
357
|
console.log(chalk.dim(` ID: `) + data._id);
|
|
675
358
|
console.log(chalk.dim(` Cron: `) + (data.cron || '—'));
|
|
676
359
|
console.log(chalk.dim(` Status: `) + statusIcon(lastStatus(data)));
|
|
677
|
-
// #179 — same badge `list` renders (see healthBadge() above); `get`
|
|
678
|
-
// never sends `minConsecutiveFailures`, so this always reads the
|
|
679
|
-
// free `lastCronOutcome` breadcrumb, never the live per-run count.
|
|
680
360
|
console.log(chalk.dim(` Health: `) + healthBadge(data));
|
|
681
361
|
console.log(chalk.dim(` Last run: `) + lastRun(data));
|
|
682
362
|
console.log(chalk.dim(` Updated: `) + formatDate(data.updatedAt));
|
|
@@ -684,14 +364,6 @@ export function attachGetCommand(parent, attachOpts = {}) {
|
|
|
684
364
|
}
|
|
685
365
|
attachGetCommand(scraps, { hidden: true });
|
|
686
366
|
const VALID_TIERS = ['tier0', 'tier1', 'tier2', 'tier3', 'tier4'];
|
|
687
|
-
/**
|
|
688
|
-
* #86 findings 4/5 — shared honest-tier renderer for `create --tier` and
|
|
689
|
-
* `update --tier/--force-tier`. Reads the typed `_tierOverride` echoed back
|
|
690
|
-
* by the server (#1559) so both commands show the SAME truth: a refusal, an
|
|
691
|
-
* allowed ceiling raise (+ spend warning), or a silently clamped proxyTier.
|
|
692
|
-
* Human-mode output only — a --json caller gets the same truth for free from
|
|
693
|
-
* the full scrap object (which already includes `_tierOverride`).
|
|
694
|
-
*/
|
|
695
367
|
function renderTierOverrideHuman(data) {
|
|
696
368
|
const ov = data._tierOverride;
|
|
697
369
|
if (!ov)
|
|
@@ -716,64 +388,25 @@ function renderTierOverrideHuman(data) {
|
|
|
716
388
|
}
|
|
717
389
|
}
|
|
718
390
|
}
|
|
719
|
-
/**
|
|
720
|
-
* #86 finding 4b — old-server fallback. When a tier was requested but the
|
|
721
|
-
* response carries no `_tierOverride` at all, the server is too old to
|
|
722
|
-
* confirm what actually got applied. Echoing the REQUESTED value as if it
|
|
723
|
-
* were the outcome is exactly the silent-clamp lie #1559 fixed — warn
|
|
724
|
-
* instead, on stderr (safe under --json too; stdout purity is untouched).
|
|
725
|
-
*/
|
|
726
391
|
function warnIfUnconfirmedTier(data, tierWasRequested, id) {
|
|
727
392
|
if (data._tierOverride || !tierWasRequested)
|
|
728
393
|
return;
|
|
729
394
|
console.error(chalk.yellow(` ⚠ Server did not confirm the tier change (older server) — verify with: trawl get ${id}`));
|
|
730
395
|
}
|
|
731
|
-
/**
|
|
732
|
-
* #88 item 3 — the --json machine-readable counterpart to
|
|
733
|
-
* warnIfUnconfirmedTier's stderr warning above. A --json caller (an agent
|
|
734
|
-
* scripting this CLI) has no reliable reason to read stderr — that channel
|
|
735
|
-
* is advisory-only everywhere else in this CLI, and stdout must stay the
|
|
736
|
-
* sole payload. Without this, the ONLY signal that the server never
|
|
737
|
-
* confirmed the tier change was a string on stderr, invisible to any --json
|
|
738
|
-
* consumer parsing stdout alone. Adds `_tierUnconfirmed: true` to the
|
|
739
|
-
* emitted object under EXACTLY the same condition warnIfUnconfirmedTier
|
|
740
|
-
* warns on (tier requested, response carries no `_tierOverride`) — never
|
|
741
|
-
* fabricated, never present otherwise.
|
|
742
|
-
*/
|
|
743
396
|
function withTierUnconfirmed(data, tierWasRequested) {
|
|
744
397
|
if (data._tierOverride || !tierWasRequested)
|
|
745
398
|
return data;
|
|
746
399
|
return { ...data, _tierUnconfirmed: true };
|
|
747
400
|
}
|
|
748
|
-
/** #86 finding 5 — the standard error envelope for a refused tier override,
|
|
749
|
-
* routed through the same reportError() central formatting path used
|
|
750
|
-
* everywhere else (exit 1: a business-logic refusal, not a usage error).
|
|
751
|
-
*
|
|
752
|
-
* #107 review F3 — uses `RefusalError` (kind:"refused"), not a bare `Error`
|
|
753
|
-
* (which fell through classifyError's default `kind:"unknown"` bucket,
|
|
754
|
-
* indistinguishable from a generic crash even though the README sells `kind`
|
|
755
|
-
* as the machine discriminant an agent branches on).
|
|
756
|
-
*/
|
|
757
401
|
function reportTierRefusal(data, wantsJson) {
|
|
758
402
|
const ov = data._tierOverride;
|
|
759
403
|
const message = `Tier ceiling override refused: ${ov?.reason ?? 'unknown'} (requested ${ov?.requestedMaxTier ?? '—'}; kept the registry cap)`;
|
|
760
404
|
return reportError(new RefusalError(message), { json: wantsJson });
|
|
761
405
|
}
|
|
762
|
-
// create
|
|
763
406
|
scraps
|
|
764
407
|
.command('create')
|
|
765
408
|
.description('Create a new scrap')
|
|
766
409
|
.requiredOption('-t, --title <title>', 'Scrap title')
|
|
767
|
-
// #170 review F6 — promoted from a plain `.option()` to `.requiredOption()`:
|
|
768
|
-
// this is unconditionally required (no positional alternative exists on
|
|
769
|
-
// THIS command, unlike the top-level `trawl create <url>`, which accepts
|
|
770
|
-
// `[url]` OR `--url` — deliberately left untouched, that either/or needs a
|
|
771
|
-
// human product decision, tracked separately). Before this, the flag's
|
|
772
|
-
// description carried the only "Required." signal, in English prose — the
|
|
773
|
-
// exact thing docs/agent-quickstart.md tells an agent not to parse.
|
|
774
|
-
// Commander itself now enforces it, so `trawl spec --json`'s published
|
|
775
|
-
// `mandatory:true` is honest instead of a second, driftable copy of this
|
|
776
|
-
// same fact.
|
|
777
410
|
.requiredOption('-u, --url <url>', 'Target URL — the site the scrap targets, not necessarily the exact URL fetched at run time (a request script can compute that, e.g. a templated search query).')
|
|
778
411
|
.option('-r, --request <request>', 'Request/query')
|
|
779
412
|
.option('-d, --description <text>', 'Scrap description')
|
|
@@ -784,10 +417,6 @@ scraps
|
|
|
784
417
|
usageError(`Invalid --tier "${opts.tier}" (allowed: ${VALID_TIERS.join(', ')})`, { json: opts.json });
|
|
785
418
|
return;
|
|
786
419
|
}
|
|
787
|
-
// #163 — the API now requires a parseable `url` on create (trawl_node
|
|
788
|
-
// #1823/#1828); validate client-side BEFORE the request goes out so a
|
|
789
|
-
// missing/unparseable value surfaces as a readable usage error instead
|
|
790
|
-
// of a raw server 400/422.
|
|
791
420
|
const url = tryValidateUrl(opts.url, opts);
|
|
792
421
|
if (url === undefined)
|
|
793
422
|
return;
|
|
@@ -798,9 +427,6 @@ scraps
|
|
|
798
427
|
...(opts.description !== undefined && { description: opts.description }),
|
|
799
428
|
...(opts.tier !== undefined && { proxyTier: opts.tier }),
|
|
800
429
|
}), { text: 'Creating scrap…', successText: (d) => `Scrap created: ${chalk.bold(d._id)}` });
|
|
801
|
-
// #86 finding 4a — read _tierOverride back from the POST response and
|
|
802
|
-
// render it exactly like `update` does; never echo the requested tier as
|
|
803
|
-
// if it were applied when an older server doesn't confirm it.
|
|
804
430
|
const tierWasRequested = opts.tier !== undefined;
|
|
805
431
|
warnIfUnconfirmedTier(data, tierWasRequested, data._id);
|
|
806
432
|
const refused = Boolean(data._tierOverride?.refused);
|
|
@@ -817,7 +443,6 @@ scraps
|
|
|
817
443
|
if (refused)
|
|
818
444
|
process.exitCode = 1;
|
|
819
445
|
});
|
|
820
|
-
// update
|
|
821
446
|
scraps
|
|
822
447
|
.command('update <id>')
|
|
823
448
|
.description('Update an existing scrap')
|
|
@@ -849,9 +474,6 @@ scraps
|
|
|
849
474
|
const body = {};
|
|
850
475
|
if (opts.title !== undefined)
|
|
851
476
|
body.title = opts.title;
|
|
852
|
-
// #163 — omitting --url stays a no-op (the API keeps that allowed on
|
|
853
|
-
// update), but a value that IS supplied must still be a parseable URL —
|
|
854
|
-
// validate it client-side before the request goes out, same as create.
|
|
855
477
|
if (opts.url !== undefined) {
|
|
856
478
|
const url = tryValidateUrl(opts.url, opts);
|
|
857
479
|
if (url === undefined)
|
|
@@ -900,7 +522,6 @@ scraps
|
|
|
900
522
|
if (opts.tier !== undefined)
|
|
901
523
|
body.proxyTier = opts.tier;
|
|
902
524
|
if (opts.forceTier !== undefined) {
|
|
903
|
-
// Raise the ceiling; also start the run at that tier unless --tier says otherwise.
|
|
904
525
|
body.proxyMaxTier = opts.forceTier;
|
|
905
526
|
if (opts.tier === undefined)
|
|
906
527
|
body.proxyTier = opts.forceTier;
|
|
@@ -917,10 +538,6 @@ scraps
|
|
|
917
538
|
text: 'Updating scrap…',
|
|
918
539
|
successText: (d) => `Scrap updated: ${chalk.bold(d._id)}`,
|
|
919
540
|
});
|
|
920
|
-
// #1559 / #86 findings 4b/5 — surface the effective tier + clamp/refuse
|
|
921
|
-
// reason (fixes the silent-clamp: the server may persist a lower tier
|
|
922
|
-
// than requested), and NEVER echo the requested value as applied when
|
|
923
|
-
// the server doesn't confirm it (old-server fallback below).
|
|
924
541
|
const tierWasRequested = opts.tier !== undefined || opts.forceTier !== undefined;
|
|
925
542
|
warnIfUnconfirmedTier(data, tierWasRequested, id);
|
|
926
543
|
const refused = Boolean(data._tierOverride?.refused);
|
|
@@ -934,10 +551,6 @@ scraps
|
|
|
934
551
|
}
|
|
935
552
|
const shown = data;
|
|
936
553
|
for (const key of Object.keys(body)) {
|
|
937
|
-
// Tier keys are rendered exclusively by renderTierOverrideHuman /
|
|
938
|
-
// warnIfUnconfirmedTier above — never echo them here, whether or not
|
|
939
|
-
// _tierOverride came back (an old-server echo of the REQUESTED value
|
|
940
|
-
// is exactly the silent-clamp lie #1559 fixed).
|
|
941
554
|
if (key === 'proxyTier' || key === 'proxyMaxTier')
|
|
942
555
|
continue;
|
|
943
556
|
const src = key in shown ? shown[key] : body[key];
|
|
@@ -947,7 +560,6 @@ scraps
|
|
|
947
560
|
if (refused)
|
|
948
561
|
process.exitCode = 1;
|
|
949
562
|
});
|
|
950
|
-
// run — promoted to a top-level verb (#108)
|
|
951
563
|
export function attachRunCommand(parent, attachOpts = {}) {
|
|
952
564
|
return parent
|
|
953
565
|
.command('run <id>', attachOpts)
|
|
@@ -956,66 +568,25 @@ export function attachRunCommand(parent, attachOpts = {}) {
|
|
|
956
568
|
.option('--json', 'Output the raw launch payload as JSON')
|
|
957
569
|
.action(async (id, opts) => {
|
|
958
570
|
validateObjectId(id);
|
|
959
|
-
// #91 P1 / #93 item 1 — captured BEFORE launching so pollRunProgress can
|
|
960
|
-
// tell "the run that's about to finish" apart from whatever the last run
|
|
961
|
-
// happened to be (including a dedup onto an already-in-flight run).
|
|
962
571
|
const beforeRun = opts.watch ? await captureBeforeRunState(id) : undefined;
|
|
963
|
-
// #91 P0 — GET /api/scraps/load/:id runs the scrap synchronously
|
|
964
|
-
// server-side (30-250s); the 30s default was aborting it mid-flight.
|
|
965
572
|
const call = () => api.get(`/api/scraps/load/${id}`, { timeoutMs: LONG_RUN_TIMEOUT_MS });
|
|
966
|
-
// #107 — under --json the stdout path stays pure: no spinner channel at
|
|
967
|
-
// all, mirroring `trawl create`'s own --json handling.
|
|
968
573
|
const data = opts.json
|
|
969
574
|
? await call()
|
|
970
575
|
: await spin(call, { text: 'Launching scrap…', successText: 'Scrap launched' });
|
|
971
576
|
if (opts.json)
|
|
972
577
|
json(data);
|
|
973
578
|
else if (!opts.watch) {
|
|
974
|
-
/* #166 — see confirmNonTTY. Two deliberate choices here:
|
|
975
|
-
*
|
|
976
|
-
* "run complete", not "launched": `run` is the SYNCHRONOUS verb —
|
|
977
|
-
* GET /api/scraps/load/:id executes the scrap server-side (30-250s)
|
|
978
|
-
* and returns the finished result, so by the time this prints the run
|
|
979
|
-
* is over. The spinner's successText says "Scrap launched", but that
|
|
980
|
-
* is stderr chrome for a human watching it happen; THIS line is the
|
|
981
|
-
* machine surface, where a script reading "launched" would poll for a
|
|
982
|
-
* completion that already happened. `trigger` already draws the same
|
|
983
|
-
* distinction ("Worker triggered" async vs "Worker run complete" under
|
|
984
|
-
* --wait) — this matches that vocabulary.
|
|
985
|
-
*
|
|
986
|
-
* `--watch` is excluded because pollRunProgress below does its own
|
|
987
|
-
* reporting; without that guard a watched run would print this line
|
|
988
|
-
* and then narrate the same run. */
|
|
989
579
|
confirmNonTTY(`Scrap ${id} run complete`);
|
|
990
580
|
}
|
|
991
581
|
if (opts.watch) {
|
|
992
|
-
// #107 — under --json, pollRunProgress suppresses its own intermediate
|
|
993
|
-
// console.log calls and instead emits exactly ONE final NDJSON outcome
|
|
994
|
-
// line (+ sets process.exitCode honestly) once the watch reaches a
|
|
995
|
-
// terminal status, a timeout, or a persistent poll error (review F1) —
|
|
996
|
-
// `run --json --watch` never again exits 0 after dead air regardless
|
|
997
|
-
// of what the watched run actually did.
|
|
998
582
|
await pollRunProgress(id, beforeRun, { json: opts.json });
|
|
999
583
|
}
|
|
1000
|
-
// #151 — throttled post-run referral tip. Reached only when the
|
|
1001
|
-
// launch call above didn't throw (a hard failure rejects it, unwinding
|
|
1002
|
-
// straight to index.ts's central catch before this line ever runs)
|
|
1003
|
-
// — and, when --watch polled to a terminal state, only when
|
|
1004
|
-
// pollRunProgress didn't flag a genuine run failure/timeout/poll-error
|
|
1005
|
-
// via a non-zero process.exitCode (#107 review F1). See lib/tips.ts
|
|
1006
|
-
// for the throttle/TTY/json/opt-out/server-flag gating itself. #153 —
|
|
1007
|
-
// now async (a due tip fetches the server's userFacing flag before
|
|
1008
|
-
// printing), so this must be awaited: otherwise commander's
|
|
1009
|
-
// parseAsync-driven action would resolve before the tip's own
|
|
1010
|
-
// await settles, racing the process's natural exit.
|
|
1011
584
|
const runFailed = typeof process.exitCode === 'number' && process.exitCode !== 0;
|
|
1012
585
|
if (!runFailed)
|
|
1013
586
|
await maybeShowReferralTip({ json: opts.json });
|
|
1014
587
|
});
|
|
1015
588
|
}
|
|
1016
589
|
attachRunCommand(scraps, { hidden: true });
|
|
1017
|
-
// #70 — render an items array either as a table summary or --json. Shared by
|
|
1018
|
-
// both the default (persisted read) and --fresh (live execute) paths of `data`.
|
|
1019
590
|
function renderScrapItems(items, asJson) {
|
|
1020
591
|
if (asJson) {
|
|
1021
592
|
json(items);
|
|
@@ -1028,43 +599,15 @@ function renderScrapItems(items, asJson) {
|
|
|
1028
599
|
}
|
|
1029
600
|
console.log(chalk.dim(' Use --json for full output.'));
|
|
1030
601
|
}
|
|
1031
|
-
/**
|
|
1032
|
-
* #86 finding 6 — `data`'s honest empty-vs-error distinction, for both --json
|
|
1033
|
-
* and human prose. Before this, EVERY non-array outcome (never run, last run
|
|
1034
|
-
* failed, or the payload aged out of retention) collapsed to the SAME `[]` /
|
|
1035
|
-
* "No data yet." — indistinguishable from a genuine zero-item successful run.
|
|
1036
|
-
* That's a lie under --json: an agent can't tell "nothing to show" from "go
|
|
1037
|
-
* look at what actually happened". Mirrors reportError's dual json/human
|
|
1038
|
-
* shape (human line to stderr always; --json ALSO gets a machine envelope on
|
|
1039
|
-
* stdout) with a caller-chosen exit code + kind, since these states are not
|
|
1040
|
-
* all "usage" (2) — never-run / aged-out-of-retention are not_found (4), a
|
|
1041
|
-
* failed last run is a business-logic failure (1).
|
|
1042
|
-
*/
|
|
1043
|
-
/**
|
|
1044
|
-
* #170 — `retryable`/`next` follow the SAME frozen `RETRY_POLICY` map
|
|
1045
|
-
* `classifyError` reads (errors.ts), keyed by `kind` — this path never
|
|
1046
|
-
* builds its own copy. `retryOverride` is for the one case here where the
|
|
1047
|
-
* kind alone is genuinely wrong: `in_progress`'s caller passes
|
|
1048
|
-
* `{retryable:true, next:['trawl data <id>']}` since retrying `data` (not
|
|
1049
|
-
* `--fresh`, which would 429 against the lock already held) IS worth doing
|
|
1050
|
-
* once the run finishes. Every other kind here (`not_found`, `run_failed`)
|
|
1051
|
-
* isn't in `RETRY_POLICY` at all — `retryFieldsFor` is total, so those fall
|
|
1052
|
-
* back to its conservative "not retryable, nothing to suggest" default.
|
|
1053
|
-
*/
|
|
1054
602
|
function reportDataState(message, exitCode, kind, wantsJson, retryOverride) {
|
|
1055
603
|
console.error(chalk.red(`✗ ${message}`));
|
|
1056
604
|
if (wantsJson) {
|
|
1057
605
|
const { retryable, next } = retryOverride ?? retryFieldsFor(kind);
|
|
1058
|
-
// #170 review F3 — typed as ErrorEnvelope so the compiler enforces
|
|
1059
|
-
// `retryable` here too (this was one of two emitters a mutation test
|
|
1060
|
-
// proved `tsc --noEmit` never actually guarded before this annotation —
|
|
1061
|
-
// the untyped literal let `retryable` be omitted silently).
|
|
1062
606
|
const envelope = { message, kind, retryable, ...(next ? { next } : {}) };
|
|
1063
607
|
console.log(JSON.stringify({ error: envelope }));
|
|
1064
608
|
}
|
|
1065
609
|
process.exitCode = exitCode;
|
|
1066
610
|
}
|
|
1067
|
-
// data — promoted to a top-level verb (#108)
|
|
1068
611
|
export function attachDataCommand(parent, attachOpts = {}) {
|
|
1069
612
|
return parent
|
|
1070
613
|
.command('data <id>', attachOpts)
|
|
@@ -1074,15 +617,9 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1074
617
|
.option('--fresh', 'Launch a fresh run instead of reading the last persisted payload (consumes execute quota, same as `run`)')
|
|
1075
618
|
.action(async (id, opts) => {
|
|
1076
619
|
validateObjectId(id);
|
|
1077
|
-
// --errors: fetch full run + fix detail via fetchRunAndFix (DRY with doctor).
|
|
1078
|
-
// Read-only: GET /api/scraps/:id + GET /api/historys/:hid, no quota, no run lock.
|
|
1079
620
|
if (opts.errors) {
|
|
1080
621
|
const result = await fetchRunAndFix(id);
|
|
1081
622
|
if (!result) {
|
|
1082
|
-
// #88 item 7 — unified no-runs shape with `doctor --json`: a bare
|
|
1083
|
-
// `null` was indistinguishable from any other absent-payload state
|
|
1084
|
-
// (a scrap CAN legitimately have a null-ish result elsewhere); an
|
|
1085
|
-
// explicit `{status:"no_runs"}` object is unambiguous everywhere.
|
|
1086
623
|
if (opts.json) {
|
|
1087
624
|
json({ status: 'no_runs' });
|
|
1088
625
|
return;
|
|
@@ -1090,13 +627,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1090
627
|
console.log(chalk.dim('No runs yet.'));
|
|
1091
628
|
return;
|
|
1092
629
|
}
|
|
1093
|
-
// --json is honored for BOTH outcomes (success or failure) — an agent
|
|
1094
|
-
// parsing `data --errors --json` must always get the flat run object,
|
|
1095
|
-
// never prose gated behind a status check. (#71 finding 13)
|
|
1096
|
-
// #185 — `docs` rides alongside as a sibling field on the same flat
|
|
1097
|
-
// object (never nested under `run`, since there's no wrapper shape
|
|
1098
|
-
// here), present only for a live mapped failureKind (see
|
|
1099
|
-
// runDocsField's doc comment).
|
|
1100
630
|
if (opts.json)
|
|
1101
631
|
return json({ ...pickRun(result.run), ...runDocsField(result.run) });
|
|
1102
632
|
if (result.run.status === true) {
|
|
@@ -1104,25 +634,18 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1104
634
|
return;
|
|
1105
635
|
}
|
|
1106
636
|
console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
|
|
637
|
+
if (isAuthWall(result.run)) {
|
|
638
|
+
maybeSuggestSkillsForAuthWall();
|
|
639
|
+
}
|
|
1107
640
|
return;
|
|
1108
641
|
}
|
|
1109
|
-
// #70 — --fresh is the explicit opt-in for a LIVE run. This is what the
|
|
1110
|
-
// default path used to do silently: GET /api/scraps/load/:id executes a
|
|
1111
|
-
// fresh scrap run server-side (requireQuota('scraps','execute') + a
|
|
1112
|
-
// distributed run lock — trawl_node modules/scraps/routes/scraps.routes.js),
|
|
1113
|
-
// burning execute quota and 429ing if a run is already in flight. A user
|
|
1114
|
-
// or agent "just reading data" must never trigger that by accident.
|
|
1115
642
|
if (opts.fresh) {
|
|
1116
|
-
// #91 P0 — same long-run endpoint as `run` (30-250s server-side);
|
|
1117
|
-
// the 30s default was aborting it mid-flight.
|
|
1118
643
|
const loaded = await spin(() => api.get(`/api/scraps/load/${id}`, { timeoutMs: LONG_RUN_TIMEOUT_MS }), {
|
|
1119
644
|
text: 'Launching a fresh scrap run (consumes execute quota)…',
|
|
1120
645
|
successText: 'Fresh run complete',
|
|
1121
646
|
});
|
|
1122
647
|
const items = loaded?.result?.data;
|
|
1123
648
|
if (!Array.isArray(items)) {
|
|
1124
|
-
// --json always returns an array from `data` — [] is the honest
|
|
1125
|
-
// "no items" signal instead of prose breaking JSON parsing. (#71)
|
|
1126
649
|
if (opts.json) {
|
|
1127
650
|
json([]);
|
|
1128
651
|
return;
|
|
@@ -1130,18 +653,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1130
653
|
console.log(chalk.dim('No data yet. Run the scrap first.'));
|
|
1131
654
|
return;
|
|
1132
655
|
}
|
|
1133
|
-
// #93 item 2 — --fresh used to render items without ever checking for a
|
|
1134
|
-
// regression, unlike the persisted `data` path below (#88 item 2). The
|
|
1135
|
-
// load() response's OWN embedded `scrap.history[0]` can't be trusted for
|
|
1136
|
-
// this (see the ScrapLoadResult comment above): it may be the previous
|
|
1137
|
-
// run's row, and even the right row never carries the regression flip.
|
|
1138
|
-
// `--fresh` runs synchronously to completion server-side though — by the
|
|
1139
|
-
// time this call returns, node has already awaited the regression patch
|
|
1140
|
-
// — so a fresh GET /api/scraps/:id (the SAME read the persisted path
|
|
1141
|
-
// below already trusts) reliably observes the finalized DB state.
|
|
1142
|
-
// Best-effort: never fail --fresh's real output over this side check,
|
|
1143
|
-
// and never gate on it — the items returned ARE this run's real,
|
|
1144
|
-
// synchronously-computed data regardless of what this check finds.
|
|
1145
656
|
try {
|
|
1146
657
|
const fresh = await api.get(`/api/scraps/${id}`);
|
|
1147
658
|
if (fresh.history?.[0]?.statusDetail === 'regression') {
|
|
@@ -1149,56 +660,20 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1149
660
|
}
|
|
1150
661
|
}
|
|
1151
662
|
catch {
|
|
1152
|
-
// best-effort — the fresh run's items are still valid without this check
|
|
1153
663
|
}
|
|
1154
664
|
renderScrapItems(items, opts.json);
|
|
1155
665
|
return;
|
|
1156
666
|
}
|
|
1157
|
-
// Default: read the last PERSISTED run payload — no quota, no run lock.
|
|
1158
|
-
// History.data (trawl_node modules/historys/services/historys.service.js
|
|
1159
|
-
// `update()`) is a JSON-stringified clone of the worker result, so it
|
|
1160
|
-
// carries the same `.data` items array the live load endpoint returns —
|
|
1161
|
-
// it's just pulled from the most recent history row instead of a fresh
|
|
1162
|
-
// run. Retention keeps this only for the newest row per (scrap, status)
|
|
1163
|
-
// bucket (config.trawl.keepData, default 1); older rows null it out.
|
|
1164
|
-
//
|
|
1165
|
-
// #86 finding 6 — [] is reserved for a GENUINE zero-item successful run.
|
|
1166
|
-
// Every other outcome below is an honest error envelope instead: never
|
|
1167
|
-
// run (not_found/4), last run failed (1), or the payload aged out of
|
|
1168
|
-
// retention (not_found/4) all used to collapse into the same silent [].
|
|
1169
667
|
const scrap = await api.get(`/api/scraps/${id}`);
|
|
1170
668
|
const last = scrap.history?.[0];
|
|
1171
669
|
if (!last?._id) {
|
|
1172
670
|
reportDataState(`Scrap ${id} has never run. Run it first (trawl run ${id}) or pass --fresh to launch one now.`, 4, 'not_found', opts.json);
|
|
1173
671
|
return;
|
|
1174
672
|
}
|
|
1175
|
-
// #88 item 1 — status:null is an IN-FLIGHT run (node persists
|
|
1176
|
-
// {status:null, statusDetail:null, inFlight:true} the moment a run
|
|
1177
|
-
// starts, and only flips status/statusDetail once it finishes). That is
|
|
1178
|
-
// neither "never run" nor "the last run failed" — a caller reading data
|
|
1179
|
-
// mid-run needs an honest "wait" signal. Never suggest --fresh here: a
|
|
1180
|
-
// run already holds the server-side distributed lock, so --fresh would
|
|
1181
|
-
// just 429 against it.
|
|
1182
673
|
if (last.status === null) {
|
|
1183
|
-
// #170 — retrying `trawl data <id>` (unchanged) once the run
|
|
1184
|
-
// finishes IS worth it; `--fresh` never is (it would 429 against the
|
|
1185
|
-
// lock this very run already holds) — never suggested here.
|
|
1186
674
|
reportDataState(`Run in progress for ${id} — retry shortly.`, 1, 'in_progress', opts.json, { retryable: true, next: [`trawl data ${id}`] });
|
|
1187
675
|
return;
|
|
1188
676
|
}
|
|
1189
|
-
// #86 review — node persists status=false for a GENUINE zero-item run
|
|
1190
|
-
// too (historys schema: status boolean|null + statusDetail
|
|
1191
|
-
// success/error/empty/regression; a zero-item run is status=false +
|
|
1192
|
-
// statusDetail='empty', and the embedded history rows from GET
|
|
1193
|
-
// /api/scraps/:id include statusDetail via the repository populate
|
|
1194
|
-
// select). An 'empty' run is the one case [] is FOR — only a real
|
|
1195
|
-
// failure (error/unknown detail) gets the run_failed envelope.
|
|
1196
|
-
//
|
|
1197
|
-
// #88 item 2 — statusDetail='regression' is ALSO status=false (an async
|
|
1198
|
-
// patch flips it after item count dropped vs baseline), but the row's
|
|
1199
|
-
// `data` still holds REAL, non-empty items — the write that persisted
|
|
1200
|
-
// them succeeded before the regression was even detected. Treating it as
|
|
1201
|
-
// run_failed would hide genuine data behind a false negative.
|
|
1202
677
|
const isEmptyRun = last.status === false && last.statusDetail === 'empty';
|
|
1203
678
|
const isRegression = last.status === false && last.statusDetail === 'regression';
|
|
1204
679
|
if (last.status === false && !isEmptyRun && !isRegression) {
|
|
@@ -1209,11 +684,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1209
684
|
let items;
|
|
1210
685
|
if (typeof detail?.data === 'string' && detail.data) {
|
|
1211
686
|
try {
|
|
1212
|
-
// #159 — `detail.data` is the same JSON-string-within-JSON shape
|
|
1213
|
-
// (`history.data`) the create --json bug lived in: a raw control
|
|
1214
|
-
// character here would otherwise throw and silently report "aged
|
|
1215
|
-
// out of retention" below for data that is actually present and
|
|
1216
|
-
// recoverable, burning a --fresh re-run + quota for nothing.
|
|
1217
687
|
items = parseServerJson(detail.data)?.data;
|
|
1218
688
|
}
|
|
1219
689
|
catch {
|
|
@@ -1222,23 +692,12 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1222
692
|
}
|
|
1223
693
|
if (!Array.isArray(items)) {
|
|
1224
694
|
if (isEmptyRun) {
|
|
1225
|
-
// A genuine zero-item run whose payload is '[]' or absent — both are
|
|
1226
|
-
// the SAME honest answer: no items, exit 0. Never the retention
|
|
1227
|
-
// message (nothing aged out; there was nothing to persist).
|
|
1228
695
|
renderScrapItems([], opts.json);
|
|
1229
696
|
return;
|
|
1230
697
|
}
|
|
1231
|
-
// #88 item 2 — a regression row whose payload aged out of retention has
|
|
1232
|
-
// nothing left to show either; fall through to the SAME honest
|
|
1233
|
-
// aged-out envelope a normal successful row would get (never fabricate
|
|
1234
|
-
// items, never silently succeed).
|
|
1235
698
|
reportDataState(`No persisted data for the last run of ${id} — it aged out of retention. Pass --fresh to launch a new run.`, 4, 'not_found', opts.json);
|
|
1236
699
|
return;
|
|
1237
700
|
}
|
|
1238
|
-
// #88 item 2 — a regression row's items are REAL (the write succeeded
|
|
1239
|
-
// before the async patch flagged the drop) — return them on stdout
|
|
1240
|
-
// (exit 0, both modes) with an honest stderr warning pointing at the
|
|
1241
|
-
// diagnostic command, instead of hiding genuine data behind run_failed.
|
|
1242
701
|
if (isRegression) {
|
|
1243
702
|
console.error(chalk.yellow(`⚠ Item count regressed vs baseline for the last run of ${id} — see: trawl scraps doctor ${id}`));
|
|
1244
703
|
}
|
|
@@ -1246,7 +705,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
|
|
|
1246
705
|
});
|
|
1247
706
|
}
|
|
1248
707
|
attachDataCommand(scraps, { hidden: true });
|
|
1249
|
-
// history — list past runs for a scrap — promoted to a top-level verb (#108)
|
|
1250
708
|
export function attachHistoryCommand(parent, attachOpts = {}) {
|
|
1251
709
|
return parent
|
|
1252
710
|
.command('history <id>', attachOpts)
|
|
@@ -1267,8 +725,6 @@ export function attachHistoryCommand(parent, attachOpts = {}) {
|
|
|
1267
725
|
time: h.time ?? null,
|
|
1268
726
|
tier: h.proxyTier ?? null,
|
|
1269
727
|
failureKind: h.failureKind ?? null,
|
|
1270
|
-
// trawl_node#1950 renamed blockType to block.kind; fall back to the
|
|
1271
|
-
// deprecated flat field for a prod backend not yet on that tag.
|
|
1272
728
|
blockType: h.block?.kind ?? h.blockType ?? null,
|
|
1273
729
|
createdAt: h.createdAt ?? null,
|
|
1274
730
|
}));
|
|
@@ -1280,7 +736,6 @@ export function attachHistoryCommand(parent, attachOpts = {}) {
|
|
|
1280
736
|
});
|
|
1281
737
|
}
|
|
1282
738
|
attachHistoryCommand(scraps, { hidden: true });
|
|
1283
|
-
// run-info — single-run detail by history id — promoted to a top-level verb (#108)
|
|
1284
739
|
export function attachRunInfoCommand(parent, attachOpts = {}) {
|
|
1285
740
|
return parent
|
|
1286
741
|
.command('run-info <hid>', attachOpts)
|
|
@@ -1295,36 +750,24 @@ export function attachRunInfoCommand(parent, attachOpts = {}) {
|
|
|
1295
750
|
time: h.time ?? null,
|
|
1296
751
|
tier: h.proxyTier ?? null,
|
|
1297
752
|
failureKind: h.failureKind ?? null,
|
|
1298
|
-
// trawl_node#1950 renamed blockType to block.kind; fall back to the
|
|
1299
|
-
// deprecated flat field for a prod backend not yet on that tag.
|
|
1300
753
|
blockType: h.block?.kind ?? h.blockType ?? null,
|
|
1301
754
|
errorMessage: h.errorSnapshot?.errorMessage ?? null,
|
|
1302
755
|
selector: h.errorSnapshot?.selector ?? null,
|
|
1303
756
|
emptyContext: h.emptyContext ?? null,
|
|
1304
757
|
createdAt: h.createdAt ?? null,
|
|
1305
|
-
// #185 — present only for a live mapped failureKind (see
|
|
1306
|
-
// runDocsField's doc comment); omitted, never `docs: undefined`, for
|
|
1307
|
-
// every other run so this object's own JSON.stringify never grows an
|
|
1308
|
-
// unexpected key on existing consumers.
|
|
1309
758
|
...runDocsField(h),
|
|
1310
759
|
};
|
|
1311
760
|
if (opts.json) {
|
|
1312
761
|
json(info);
|
|
1313
762
|
return;
|
|
1314
763
|
}
|
|
1315
|
-
// The blob is an object; the table needs one line. --json keeps it whole.
|
|
1316
764
|
table([{ ...info, emptyContext: summarizeEmptyContext(info.emptyContext) }], ['hid', 'status', 'time', 'tier', 'failureKind', 'blockType', 'errorMessage', 'selector', 'emptyContext', 'createdAt']);
|
|
1317
|
-
// #184 safety-net — the run just shown carries a live login-wall
|
|
1318
|
-
// verdict; if Claude's skills aren't installed either, point at the
|
|
1319
|
-
// command that installs them too (never under --json, see the early
|
|
1320
|
-
// return above).
|
|
1321
765
|
if (isAuthWall(h)) {
|
|
1322
766
|
maybeSuggestSkillsForAuthWall();
|
|
1323
767
|
}
|
|
1324
768
|
});
|
|
1325
769
|
}
|
|
1326
770
|
attachRunInfoCommand(scraps, { hidden: true });
|
|
1327
|
-
// delete
|
|
1328
771
|
scraps
|
|
1329
772
|
.command('delete <id>')
|
|
1330
773
|
.alias('rm')
|
|
@@ -1333,16 +776,6 @@ scraps
|
|
|
1333
776
|
.option('--json', 'Output as JSON')
|
|
1334
777
|
.action(async (id, opts) => {
|
|
1335
778
|
validateObjectId(id);
|
|
1336
|
-
// #107 — never blocks on a y/N under --json or a non-TTY invocation
|
|
1337
|
-
// (agent/CI subprocess); -f/--force always pre-confirms.
|
|
1338
|
-
//
|
|
1339
|
-
// #107 review F2 — `message` (the plain-id form) feeds the refusal
|
|
1340
|
-
// UsageError, which can land verbatim in the `--json` error envelope;
|
|
1341
|
-
// `chalk.bold(id)` is passed ONLY as `promptMessage`, shown solely on
|
|
1342
|
-
// the interactive TTY `[y/N]` prompt. Before this split, the styled
|
|
1343
|
-
// string was the ONLY message confirmDestructive had, so a non-TTY/
|
|
1344
|
-
// --json refusal on `scraps delete X --json` emitted raw ANSI escape
|
|
1345
|
-
// bytes inside the JSON string.
|
|
1346
779
|
const { proceed, blocked } = await confirmDestructive(`Delete scrap ${id}?`, {
|
|
1347
780
|
force: opts.force,
|
|
1348
781
|
json: opts.json,
|
|
@@ -1361,9 +794,8 @@ scraps
|
|
|
1361
794
|
return;
|
|
1362
795
|
}
|
|
1363
796
|
await spin(call, { text: 'Deleting…', successText: 'Scrap deleted' });
|
|
1364
|
-
confirmNonTTY(`Scrap ${id} deleted`);
|
|
797
|
+
confirmNonTTY(`Scrap ${id} deleted`);
|
|
1365
798
|
});
|
|
1366
|
-
// banner
|
|
1367
799
|
scraps
|
|
1368
800
|
.command('banner <id>')
|
|
1369
801
|
.description('Upload a banner image for a scrap')
|
|
@@ -1385,9 +817,6 @@ scraps
|
|
|
1385
817
|
jpeg: 'image/jpeg',
|
|
1386
818
|
webp: 'image/webp',
|
|
1387
819
|
};
|
|
1388
|
-
// #91 P2 — an unsupported/unknown extension silently became image/png
|
|
1389
|
-
// (uploading the raw bytes of, say, a .gif or .pdf under an image/png
|
|
1390
|
-
// Content-Type — a lie about the actual file's format). Refuse instead.
|
|
1391
820
|
const mimeType = mimeMap[ext];
|
|
1392
821
|
if (!mimeType) {
|
|
1393
822
|
usageError(`Unsupported image type "${ext ? `.${ext}` : filename}" — use png, jpg, or webp.`, { json: opts.json });
|
|
@@ -1407,10 +836,8 @@ scraps
|
|
|
1407
836
|
text: 'Uploading banner…',
|
|
1408
837
|
successText: `Banner uploaded for scrap ${chalk.bold(id)}`,
|
|
1409
838
|
});
|
|
1410
|
-
// #166 — plain, unstyled: chalk.bold would emit escape codes into a pipe.
|
|
1411
839
|
confirmNonTTY(`Banner uploaded for scrap ${id}`);
|
|
1412
840
|
});
|
|
1413
|
-
// watch (stream activities)
|
|
1414
841
|
scraps
|
|
1415
842
|
.command('watch <id>')
|
|
1416
843
|
.description('Stream scrap activities in real-time')
|
|
@@ -1419,7 +846,6 @@ scraps
|
|
|
1419
846
|
validateObjectId(id);
|
|
1420
847
|
await watchActivities(id, opts.json);
|
|
1421
848
|
});
|
|
1422
|
-
// trigger — promoted to a top-level verb (#108)
|
|
1423
849
|
export function attachTriggerCommand(parent, attachOpts = {}) {
|
|
1424
850
|
return parent
|
|
1425
851
|
.command('trigger <id>', attachOpts)
|
|
@@ -1429,23 +855,9 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
|
|
|
1429
855
|
.option('--json', 'Output the raw trigger payload as JSON')
|
|
1430
856
|
.action(async (id, opts) => {
|
|
1431
857
|
validateObjectId(id);
|
|
1432
|
-
// #91 P1 / #93 item 1 — captured BEFORE triggering so pollRunProgress can
|
|
1433
|
-
// tell "the run we just triggered" apart from whatever the last run
|
|
1434
|
-
// happened to be. This is the dedup-prone path: `trigger`'s method:'worker'
|
|
1435
|
-
// collapses onto an already-pending/running worker job for the same scrap
|
|
1436
|
-
// (ScrapJobsService, LIVE_STATUSES) instead of creating a new history row —
|
|
1437
|
-
// captureBeforeRunState records that so pollRunProgress can still track it.
|
|
1438
858
|
const beforeRun = opts.watch ? await captureBeforeRunState(id) : undefined;
|
|
1439
|
-
// #50 — default async: the backend (#1313) kicks off the run and returns a
|
|
1440
|
-
// 'queued' envelope immediately instead of holding the connection for the
|
|
1441
|
-
// whole run. --wait restores the old synchronous round-trip.
|
|
1442
859
|
const path = opts.wait ? `/api/scraps/worker/${id}` : `/api/scraps/worker/${id}?wait=false`;
|
|
1443
|
-
// #91 P0 — the synchronous --wait branch runs the scrap server-side
|
|
1444
|
-
// (30-250s), same as `run`; the 30s default was aborting it
|
|
1445
|
-
// mid-flight. The async (default) POST returns almost immediately, so it
|
|
1446
|
-
// keeps the 30s default.
|
|
1447
860
|
const call = () => (opts.wait ? api.post(path, undefined, { timeoutMs: LONG_RUN_TIMEOUT_MS }) : api.post(path));
|
|
1448
|
-
// #107 — under --json the stdout path stays pure: no spinner channel.
|
|
1449
861
|
const data = opts.json
|
|
1450
862
|
? await call()
|
|
1451
863
|
: await spin(call, {
|
|
@@ -1456,26 +868,16 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
|
|
|
1456
868
|
json(data);
|
|
1457
869
|
}
|
|
1458
870
|
else {
|
|
1459
|
-
/* #160, folded onto the shared helper by #166 — the rationale and the
|
|
1460
|
-
* known residual now live once, on `confirmNonTTY` itself, instead of
|
|
1461
|
-
* in the one call site that happened to be fixed first. Wording is
|
|
1462
|
-
* unchanged on purpose: the trawl-internal QA runbook asserts on the
|
|
1463
|
-
* exact "Worker triggered" / "Worker run complete" substrings. */
|
|
1464
871
|
confirmNonTTY(opts.wait ? `Worker run complete for scrap ${id}` : `Worker triggered for scrap ${id}`);
|
|
1465
872
|
}
|
|
1466
|
-
// #107 — see the matching comment on `run`'s --watch call above (review
|
|
1467
|
-
// F1): honest final NDJSON line + exit code under --json, human mode
|
|
1468
|
-
// gets the same honest exit code too.
|
|
1469
873
|
if (opts.watch)
|
|
1470
874
|
await pollRunProgress(id, beforeRun, { json: opts.json });
|
|
1471
875
|
});
|
|
1472
876
|
}
|
|
1473
877
|
attachTriggerCommand(scraps, { hidden: true });
|
|
1474
|
-
// account subcommand group
|
|
1475
878
|
const account = scraps
|
|
1476
879
|
.command('account')
|
|
1477
880
|
.description('Manage scrap account credentials and session');
|
|
1478
|
-
// helper: prompt for a value with readline (visible input)
|
|
1479
881
|
async function promptLine(prompt) {
|
|
1480
882
|
const { createInterface } = await import('readline');
|
|
1481
883
|
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
@@ -1488,7 +890,6 @@ async function promptLine(prompt) {
|
|
|
1488
890
|
rl.close();
|
|
1489
891
|
}
|
|
1490
892
|
}
|
|
1491
|
-
// account set
|
|
1492
893
|
account
|
|
1493
894
|
.command('set <id>')
|
|
1494
895
|
.description('Set credentials for a scrap account')
|
|
@@ -1500,13 +901,8 @@ account
|
|
|
1500
901
|
let username = opts.username || '';
|
|
1501
902
|
let password = opts.password || '';
|
|
1502
903
|
if (opts.password) {
|
|
1503
|
-
// #107 — this advisory is stderr-only: stdout must stay pure under
|
|
1504
|
-
// --json, and every other advisory in this CLI already follows that
|
|
1505
|
-
// rule (warnIfUnconfirmedTier, token.ts's expiry hints, …).
|
|
1506
904
|
console.error(chalk.yellow('⚠ Passing --password on the command line is insecure and may be stored in shell history.'));
|
|
1507
905
|
}
|
|
1508
|
-
// #107 — never blocks on a readline prompt under --json or a non-TTY
|
|
1509
|
-
// invocation (agent/CI subprocess); pass -u/-p instead.
|
|
1510
906
|
const interactive = isInteractive({ json: opts.json });
|
|
1511
907
|
if (!username) {
|
|
1512
908
|
if (!interactive) {
|
|
@@ -1541,7 +937,6 @@ account
|
|
|
1541
937
|
console.log(chalk.dim(' Credentials: ') + (acc.hasCredentials ? chalk.green('✓ configured') : chalk.dim('not set')));
|
|
1542
938
|
console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('active') : chalk.dim('none')));
|
|
1543
939
|
});
|
|
1544
|
-
// account delete
|
|
1545
940
|
account
|
|
1546
941
|
.command('delete <id>')
|
|
1547
942
|
.description('Delete account credentials for a scrap')
|
|
@@ -1549,10 +944,6 @@ account
|
|
|
1549
944
|
.option('--json', 'Output as JSON')
|
|
1550
945
|
.action(async (id, opts) => {
|
|
1551
946
|
validateObjectId(id);
|
|
1552
|
-
// #107 — never blocks on a y/N under --json or a non-TTY invocation.
|
|
1553
|
-
// #107 review F2 — plain-id `message` for the refusal/JSON envelope,
|
|
1554
|
-
// styled `promptMessage` for the interactive TTY prompt only (see the
|
|
1555
|
-
// matching comment on `scraps delete` above).
|
|
1556
947
|
const { proceed, blocked } = await confirmDestructive(`Delete account credentials for scrap ${id}?`, { force: opts.force, json: opts.json, promptMessage: `Delete account credentials for scrap ${chalk.bold(id)}?` });
|
|
1557
948
|
if (blocked)
|
|
1558
949
|
return;
|
|
@@ -1570,9 +961,8 @@ account
|
|
|
1570
961
|
text: 'Deleting credentials…',
|
|
1571
962
|
successText: 'Account credentials deleted',
|
|
1572
963
|
});
|
|
1573
|
-
confirmNonTTY(`Account credentials deleted for scrap ${id}`);
|
|
964
|
+
confirmNonTTY(`Account credentials deleted for scrap ${id}`);
|
|
1574
965
|
});
|
|
1575
|
-
// account clear-session
|
|
1576
966
|
account
|
|
1577
967
|
.command('clear-session <id>')
|
|
1578
968
|
.description('Clear the saved session for a scrap account')
|
|
@@ -1589,38 +979,64 @@ account
|
|
|
1589
979
|
text: 'Clearing session…',
|
|
1590
980
|
successText: 'Session cleared',
|
|
1591
981
|
});
|
|
1592
|
-
confirmNonTTY(`Session cleared for scrap ${id}`);
|
|
982
|
+
confirmNonTTY(`Session cleared for scrap ${id}`);
|
|
1593
983
|
});
|
|
1594
|
-
// account session subcommand group
|
|
1595
984
|
const accountSession = account
|
|
1596
985
|
.command('session')
|
|
1597
986
|
.description('Manage scrap account session cookies (flavour B BYO-cookies)');
|
|
1598
|
-
|
|
987
|
+
function isValidOriginsShape(value) {
|
|
988
|
+
return (Array.isArray(value) &&
|
|
989
|
+
value.every((o) => o &&
|
|
990
|
+
typeof o === 'object' &&
|
|
991
|
+
typeof o.origin === 'string' &&
|
|
992
|
+
Array.isArray(o.localStorage) &&
|
|
993
|
+
o.localStorage.every((kv) => kv && typeof kv === 'object' && typeof kv.name === 'string' && typeof kv.value === 'string')));
|
|
994
|
+
}
|
|
1599
995
|
accountSession
|
|
1600
996
|
.command('set <id>')
|
|
1601
|
-
.description('Upload browser session
|
|
1602
|
-
.requiredOption('-c, --cookies <file>', 'Path to a
|
|
997
|
+
.description('Upload a browser session for a scrap — a cookie JSON array, or a { cookies, origins } storageState file (see `session capture`)')
|
|
998
|
+
.requiredOption('-c, --cookies <file>', 'Path to a cookie JSON array, or a storageState file ({ cookies, origins? })')
|
|
1603
999
|
.option('--json', 'Output as JSON')
|
|
1604
1000
|
.action(async (id, opts) => {
|
|
1605
1001
|
validateObjectId(id);
|
|
1002
|
+
const secureTransportError = assertSecureTransport(getApiUrl());
|
|
1003
|
+
if (secureTransportError) {
|
|
1004
|
+
usageError(secureTransportError, { json: opts.json });
|
|
1005
|
+
return;
|
|
1006
|
+
}
|
|
1606
1007
|
const { existsSync, readFileSync } = await import('fs');
|
|
1607
1008
|
if (!existsSync(opts.cookies)) {
|
|
1608
1009
|
usageError(`File not found: ${opts.cookies}`, { json: opts.json });
|
|
1609
1010
|
return;
|
|
1610
1011
|
}
|
|
1611
1012
|
let cookies;
|
|
1013
|
+
let origins;
|
|
1612
1014
|
try {
|
|
1613
1015
|
const raw = readFileSync(opts.cookies, 'utf-8');
|
|
1614
|
-
|
|
1016
|
+
const parsed = JSON.parse(raw);
|
|
1017
|
+
if (Array.isArray(parsed)) {
|
|
1018
|
+
cookies = parsed;
|
|
1019
|
+
}
|
|
1020
|
+
else if (parsed && typeof parsed === 'object' && Array.isArray(parsed.cookies)) {
|
|
1021
|
+
const shaped = parsed;
|
|
1022
|
+
cookies = shaped.cookies;
|
|
1023
|
+
if (shaped.origins !== undefined) {
|
|
1024
|
+
if (!isValidOriginsShape(shaped.origins)) {
|
|
1025
|
+
usageError('origins must be an array of { origin: string, localStorage: [{name, value}] }', { json: opts.json });
|
|
1026
|
+
return;
|
|
1027
|
+
}
|
|
1028
|
+
origins = shaped.origins;
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
else {
|
|
1032
|
+
usageError('Cookies file must contain a JSON array, or a { cookies, origins? } storageState object', { json: opts.json });
|
|
1033
|
+
return;
|
|
1034
|
+
}
|
|
1615
1035
|
}
|
|
1616
1036
|
catch (e) {
|
|
1617
1037
|
usageError(`Failed to parse cookies file: ${e.message}`, { json: opts.json });
|
|
1618
1038
|
return;
|
|
1619
1039
|
}
|
|
1620
|
-
if (!Array.isArray(cookies)) {
|
|
1621
|
-
usageError('Cookies file must contain a JSON array', { json: opts.json });
|
|
1622
|
-
return;
|
|
1623
|
-
}
|
|
1624
1040
|
if (cookies.length === 0) {
|
|
1625
1041
|
usageError('Cookies array must not be empty', { json: opts.json });
|
|
1626
1042
|
return;
|
|
@@ -1629,7 +1045,10 @@ accountSession
|
|
|
1629
1045
|
usageError('Each cookie must have a name (string) and value (string)', { json: opts.json });
|
|
1630
1046
|
return;
|
|
1631
1047
|
}
|
|
1632
|
-
const
|
|
1048
|
+
const body = { cookies };
|
|
1049
|
+
if (origins)
|
|
1050
|
+
body.origins = origins;
|
|
1051
|
+
const call = () => api.put(`/api/scraps/${id}/account/session`, body);
|
|
1633
1052
|
if (opts.json) {
|
|
1634
1053
|
const data = await call();
|
|
1635
1054
|
json(data);
|
|
@@ -1642,7 +1061,97 @@ accountSession
|
|
|
1642
1061
|
const acc = data.account;
|
|
1643
1062
|
console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('✓ active') : chalk.dim('none')));
|
|
1644
1063
|
});
|
|
1645
|
-
|
|
1064
|
+
function reportCaptureFailure(result, opts) {
|
|
1065
|
+
if (result.reason === 'no_cookies_in_scope') {
|
|
1066
|
+
process.exitCode = reportError(new RefusalError(result.message), { json: opts.json });
|
|
1067
|
+
return;
|
|
1068
|
+
}
|
|
1069
|
+
if (result.reason === 'capture_failed') {
|
|
1070
|
+
process.exitCode = reportError(new Error(result.message), { json: opts.json });
|
|
1071
|
+
return;
|
|
1072
|
+
}
|
|
1073
|
+
usageError(result.message, { json: opts.json });
|
|
1074
|
+
}
|
|
1075
|
+
accountSession
|
|
1076
|
+
.command('capture <id>')
|
|
1077
|
+
.description('Open a headed Chrome to the scrap\'s target URL, wait for you to log in (2FA included), and capture the session over CDP — cookies AND localStorage. Requires a local interactive terminal with a display; does not work headless, in CI, or over a plain SSH session. You stay authenticated as yourself — Trawl never sees your credentials. Responsibility for lawful use of the captured session stays with you.')
|
|
1078
|
+
.option('--chrome <path>', 'Path to a Chrome/Chromium executable (auto-detected if omitted)')
|
|
1079
|
+
.option('--json', 'Output as JSON (counts only — a session is a bearer secret and is never printed, in any mode)')
|
|
1080
|
+
.action(async (id, opts) => {
|
|
1081
|
+
validateObjectId(id);
|
|
1082
|
+
const secureTransportError = assertSecureTransport(getApiUrl());
|
|
1083
|
+
if (secureTransportError) {
|
|
1084
|
+
usageError(secureTransportError, { json: opts.json });
|
|
1085
|
+
return;
|
|
1086
|
+
}
|
|
1087
|
+
const scrap = await api.get(`/api/scraps/${id}`);
|
|
1088
|
+
if (!scrap.url) {
|
|
1089
|
+
usageError(`Scrap ${id} has no target URL configured — nothing to open a browser to. A URL can be set with: trawl scraps update ${id} -u <url>`, { json: opts.json });
|
|
1090
|
+
return;
|
|
1091
|
+
}
|
|
1092
|
+
if (opts.chrome) {
|
|
1093
|
+
const { existsSync, accessSync, constants } = await import('fs');
|
|
1094
|
+
if (!existsSync(opts.chrome)) {
|
|
1095
|
+
usageError(`Chrome not found at: ${opts.chrome}`, { json: opts.json });
|
|
1096
|
+
return;
|
|
1097
|
+
}
|
|
1098
|
+
try {
|
|
1099
|
+
accessSync(opts.chrome, constants.X_OK);
|
|
1100
|
+
}
|
|
1101
|
+
catch {
|
|
1102
|
+
usageError(`Chrome exists at ${opts.chrome} but is not executable.`, { json: opts.json });
|
|
1103
|
+
return;
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
else if (process.env.TRAWL_CHROME_PATH) {
|
|
1107
|
+
const trawlChromePath = process.env.TRAWL_CHROME_PATH;
|
|
1108
|
+
const { existsSync, accessSync, constants } = await import('fs');
|
|
1109
|
+
if (existsSync(trawlChromePath)) {
|
|
1110
|
+
try {
|
|
1111
|
+
accessSync(trawlChromePath, constants.X_OK);
|
|
1112
|
+
}
|
|
1113
|
+
catch {
|
|
1114
|
+
usageError(`Chrome exists at ${trawlChromePath} (TRAWL_CHROME_PATH) but is not executable.`, { json: opts.json });
|
|
1115
|
+
return;
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
const nonInteractiveReason = checkInteractiveEnvironment();
|
|
1120
|
+
if (nonInteractiveReason) {
|
|
1121
|
+
reportCaptureFailure({ ok: false, reason: 'non_interactive', message: nonInteractiveReason }, opts);
|
|
1122
|
+
return;
|
|
1123
|
+
}
|
|
1124
|
+
console.error(chalk.dim(`Chrome will open at ${scrap.url} for you to log in there (2FA included); capture completes when you press Enter back in this terminal.`));
|
|
1125
|
+
const result = await captureSession(scrap.url, opts.chrome ? { findChrome: () => opts.chrome ?? null } : {});
|
|
1126
|
+
if (!result.ok) {
|
|
1127
|
+
reportCaptureFailure(result, opts);
|
|
1128
|
+
return;
|
|
1129
|
+
}
|
|
1130
|
+
const call = () => api.put(`/api/scraps/${id}/account/session`, {
|
|
1131
|
+
cookies: result.storageState.cookies,
|
|
1132
|
+
origins: result.storageState.origins,
|
|
1133
|
+
});
|
|
1134
|
+
if (opts.json) {
|
|
1135
|
+
const data = await call();
|
|
1136
|
+
json({ account: data.account, targetDomain: result.targetDomain, capture: result.counts });
|
|
1137
|
+
return;
|
|
1138
|
+
}
|
|
1139
|
+
const data = await spin(call, {
|
|
1140
|
+
text: `Uploading captured session for scrap ${chalk.bold(id)}…`,
|
|
1141
|
+
successText: `Session captured for scrap ${chalk.bold(id)}: ${result.counts.cookiesCaptured} cookie(s), ${result.counts.originsCaptured} origin(s) in scope for ${result.targetDomain}`,
|
|
1142
|
+
});
|
|
1143
|
+
const acc = data.account;
|
|
1144
|
+
console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('✓ active') : chalk.dim('none')));
|
|
1145
|
+
if (result.counts.cookiesDroppedOutOfScope > 0 || result.counts.originsDroppedOutOfScope > 0) {
|
|
1146
|
+
console.log(chalk.dim(` Scoped to ${result.targetDomain}: dropped ${result.counts.cookiesDroppedOutOfScope} cookie(s) and ${result.counts.originsDroppedOutOfScope} origin(s) outside that domain.`));
|
|
1147
|
+
}
|
|
1148
|
+
if (result.counts.closedEarly) {
|
|
1149
|
+
console.log(chalk.dim(' The browser window was closed before Enter — localStorage was not captured (cookies only).'));
|
|
1150
|
+
}
|
|
1151
|
+
if (result.counts.originsUnreadable > 0) {
|
|
1152
|
+
console.log(chalk.yellow(` ⚠ ${result.counts.originsUnreadable} origin(s) could not be captured — threw reading localStorage, or gave no response within ${LOCALSTORAGE_READ_TIMEOUT_MS / 1000}s (a blocked tab). Not the same as being empty; cookies were still captured.`));
|
|
1153
|
+
}
|
|
1154
|
+
});
|
|
1646
1155
|
account
|
|
1647
1156
|
.command('status <id>')
|
|
1648
1157
|
.description('Show account credentials and session status for a scrap')
|
|
@@ -1652,8 +1161,6 @@ account
|
|
|
1652
1161
|
const data = await spin(() => api.get(`/api/scraps/${id}`), 'Fetching scrap…');
|
|
1653
1162
|
const acc = data.account;
|
|
1654
1163
|
if (opts.json) {
|
|
1655
|
-
// #86 finding 12 — `json` is already statically imported at the top of
|
|
1656
|
-
// this file; the dynamic import here was pure dead weight.
|
|
1657
1164
|
return json(acc ?? null);
|
|
1658
1165
|
}
|
|
1659
1166
|
if (!acc) {
|
|
@@ -1682,7 +1189,6 @@ account
|
|
|
1682
1189
|
}
|
|
1683
1190
|
console.log(`${credLine} | ${sessionLine}`);
|
|
1684
1191
|
});
|
|
1685
|
-
// doctor — diagnose last run of a scrap (error, failed selector, block status, page state, autofix)
|
|
1686
1192
|
scraps
|
|
1687
1193
|
.command('doctor <id>')
|
|
1688
1194
|
.description('Diagnose the last run (error, failed selector, block status, page state, autofix outcome)')
|
|
@@ -1699,24 +1205,16 @@ scraps
|
|
|
1699
1205
|
console.log(chalk.dim('No runs yet.'));
|
|
1700
1206
|
return;
|
|
1701
1207
|
}
|
|
1702
|
-
// #185 — `docs` sits alongside `run`/`fix` (present only for a live
|
|
1703
|
-
// mapped failureKind — see runDocsField's doc comment), never nested
|
|
1704
|
-
// inside the allowlisted `run` object.
|
|
1705
1208
|
if (opts.json)
|
|
1706
1209
|
return json({ run: pickRun(result.run), fix: pickFix(result.fix), ...runDocsField(result.run) });
|
|
1707
1210
|
console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
|
|
1708
1211
|
if (opts.autofix && result.fix) {
|
|
1709
1212
|
console.log('\n' + formatAutofix(result.fix));
|
|
1710
1213
|
}
|
|
1711
|
-
// #184 safety-net — formatDoctor already prints the login-wall hint
|
|
1712
|
-
// (#182) above when this is a live auth wall; if Claude's skills aren't
|
|
1713
|
-
// installed either, point at the command that installs them too (never
|
|
1714
|
-
// under --json, see the early return above).
|
|
1715
1214
|
if (isAuthWall(result.run)) {
|
|
1716
1215
|
maybeSuggestSkillsForAuthWall();
|
|
1717
1216
|
}
|
|
1718
1217
|
});
|
|
1719
|
-
// autofix — show full auto-fix attempt detail (diff, dry-run, knowledge)
|
|
1720
1218
|
scraps
|
|
1721
1219
|
.command('autofix <id>')
|
|
1722
1220
|
.description('Show the last auto-fix attempt for a scrap (decision, diff, dry-run, knowledge)')
|
|
@@ -1725,9 +1223,6 @@ scraps
|
|
|
1725
1223
|
validateObjectId(id);
|
|
1726
1224
|
const result = await fetchRunAndFix(id);
|
|
1727
1225
|
if (!result) {
|
|
1728
|
-
// #88 item 7 — unified no-runs shape with `doctor --json` / `data
|
|
1729
|
-
// --errors --json`: a never-run scrap is a distinct, nameable state,
|
|
1730
|
-
// not the same bare `null` a run-with-no-fix-attempt returns below.
|
|
1731
1226
|
if (opts.json) {
|
|
1732
1227
|
json({ status: 'no_runs' });
|
|
1733
1228
|
return;
|
|
@@ -1736,8 +1231,6 @@ scraps
|
|
|
1736
1231
|
return;
|
|
1737
1232
|
}
|
|
1738
1233
|
if (!result.fix) {
|
|
1739
|
-
// A run DID happen, it just had no autofix attempt — genuinely "no
|
|
1740
|
-
// data", unlike the never-run case above.
|
|
1741
1234
|
if (opts.json) {
|
|
1742
1235
|
json(null);
|
|
1743
1236
|
return;
|
|
@@ -1749,7 +1242,6 @@ scraps
|
|
|
1749
1242
|
return json(result.fix);
|
|
1750
1243
|
console.log(formatAutofix(result.fix));
|
|
1751
1244
|
});
|
|
1752
|
-
// snapshot — download the captured page HTML (or error-path HTML) for the last run
|
|
1753
1245
|
scraps
|
|
1754
1246
|
.command('snapshot <id>')
|
|
1755
1247
|
.description('Download captured page HTML for the last run of a scrap')
|
|
@@ -1761,16 +1253,10 @@ scraps
|
|
|
1761
1253
|
const scrap = await api.get(`/api/scraps/${id}`);
|
|
1762
1254
|
const hid = scrap.history?.[0]?._id;
|
|
1763
1255
|
if (!hid) {
|
|
1764
|
-
// #91 P2 — same no-runs contract as `doctor`/`autofix`/`data --errors`
|
|
1765
|
-
// (`{status:"no_runs"}`, exit 0) under --json.
|
|
1766
1256
|
if (opts.json) {
|
|
1767
1257
|
json({ status: 'no_runs' });
|
|
1768
1258
|
return;
|
|
1769
1259
|
}
|
|
1770
|
-
// `-o` explicitly asked for a file to be written. Silently exiting 0
|
|
1771
|
-
// with nothing written (and only a dim console line) is indistinguishable
|
|
1772
|
-
// from success to a script checking the exit code alone — give it the
|
|
1773
|
-
// SAME not_found envelope `scraps data`'s own never-run case uses.
|
|
1774
1260
|
if (opts.out) {
|
|
1775
1261
|
reportDataState(`Scrap ${id} has never run — nothing to write to ${opts.out}.`, 4, 'not_found', false);
|
|
1776
1262
|
return;
|