@trawlme/cli 3.12.0 → 3.12.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +2 -2
  2. package/dist/commands/create.d.ts +0 -28
  3. package/dist/commands/create.js +0 -89
  4. package/dist/commands/doctor.d.ts +0 -79
  5. package/dist/commands/doctor.js +1 -187
  6. package/dist/commands/login.js +0 -67
  7. package/dist/commands/ping.d.ts +0 -15
  8. package/dist/commands/ping.js +0 -15
  9. package/dist/commands/scraps.d.ts +0 -120
  10. package/dist/commands/scraps.js +10 -724
  11. package/dist/commands/skills.js +0 -22
  12. package/dist/commands/spec.d.ts +0 -85
  13. package/dist/commands/spec.js +0 -67
  14. package/dist/commands/telemetry.js +0 -4
  15. package/dist/commands/token.js +0 -28
  16. package/dist/commands/upgrade.js +0 -22
  17. package/dist/commands/whoami.d.ts +0 -12
  18. package/dist/commands/whoami.js +0 -6
  19. package/dist/index.d.ts +0 -188
  20. package/dist/index.js +0 -349
  21. package/dist/lib/api.d.ts +0 -78
  22. package/dist/lib/api.js +1 -320
  23. package/dist/lib/cdp-pipe.d.ts +0 -72
  24. package/dist/lib/cdp-pipe.js +1 -81
  25. package/dist/lib/chrome-discovery.d.ts +0 -11
  26. package/dist/lib/chrome-discovery.js +0 -19
  27. package/dist/lib/chrome-launch.d.ts +0 -40
  28. package/dist/lib/chrome-launch.js +0 -69
  29. package/dist/lib/config.d.ts +0 -53
  30. package/dist/lib/config.js +0 -55
  31. package/dist/lib/confirm.d.ts +0 -55
  32. package/dist/lib/confirm.js +0 -47
  33. package/dist/lib/docs.d.ts +0 -123
  34. package/dist/lib/docs.js +0 -169
  35. package/dist/lib/errors.d.ts +0 -134
  36. package/dist/lib/errors.js +0 -151
  37. package/dist/lib/format.d.ts +0 -6
  38. package/dist/lib/format.js +0 -6
  39. package/dist/lib/json.d.ts +0 -35
  40. package/dist/lib/json.js +0 -48
  41. package/dist/lib/jwt.d.ts +0 -7
  42. package/dist/lib/jwt.js +0 -7
  43. package/dist/lib/pinch.d.ts +0 -53
  44. package/dist/lib/pinch.js +6 -112
  45. package/dist/lib/pinchAnimation.d.ts +0 -16
  46. package/dist/lib/pinchAnimation.js +8 -29
  47. package/dist/lib/posthog.d.ts +0 -9
  48. package/dist/lib/posthog.js +0 -23
  49. package/dist/lib/prompt.js +1 -20
  50. package/dist/lib/secure-transport.d.ts +0 -7
  51. package/dist/lib/secure-transport.js +0 -24
  52. package/dist/lib/session-capture-guard.d.ts +0 -15
  53. package/dist/lib/session-capture-guard.js +0 -5
  54. package/dist/lib/session-capture.d.ts +0 -125
  55. package/dist/lib/session-capture.js +0 -281
  56. package/dist/lib/skills.d.ts +0 -175
  57. package/dist/lib/skills.js +1 -216
  58. package/dist/lib/skillsNudge.d.ts +0 -17
  59. package/dist/lib/skillsNudge.js +0 -83
  60. package/dist/lib/spinner.d.ts +0 -39
  61. package/dist/lib/spinner.js +0 -40
  62. package/dist/lib/storage-state.d.ts +0 -112
  63. package/dist/lib/storage-state.js +0 -131
  64. package/dist/lib/tips.d.ts +0 -38
  65. package/dist/lib/tips.js +0 -77
  66. package/dist/lib/updateCheckWorker.js +0 -14
  67. package/dist/lib/updateNotifier.d.ts +0 -17
  68. package/dist/lib/updateNotifier.js +0 -53
  69. package/dist/lib/validate.d.ts +0 -8
  70. package/dist/lib/validate.js +0 -8
  71. package/dist/lib/version.d.ts +0 -12
  72. package/dist/lib/version.js +1 -13
  73. package/docs/agent-quickstart.md +2 -2
  74. package/package.json +2 -2
@@ -16,56 +16,16 @@ import { captureSession, checkInteractiveEnvironment, LOCALSTORAGE_READ_TIMEOUT_
16
16
  import { assertSecureTransport } from '../lib/secure-transport.js';
17
17
  import { getApiUrl } from '../lib/config.js';
18
18
  import { resolveDocsUrls, resolveFailureKindDocsUrl } from '../lib/docs.js';
19
- /**
20
- * #185 — `{ docs }` (spreadable, empty object when nothing to add) for a
21
- * run's JSON payload (`doctor`/`data --errors`/`run-info`), keyed on
22
- * trawl_node's `failureKind` (NOT errors.ts's unrelated `ErrorEnvelope.kind`
23
- * — see docs.ts's `FAILURE_KIND_DOC_PATHS` doc comment for why those two
24
- * `'auth'`s must never be conflated). Reuses `isAuthWall` — the SAME
25
- * staleness guard `formatDoctor`'s human hint and the #184 skills nudge
26
- * already share (a `failureKind:'auth'` stamp survives a later
27
- * success/regression patch, see `isAuthWall`'s own doc comment) — so a run
28
- * that ultimately succeeded or degraded never carries a stale `docs` link
29
- * either, exactly the invariant `detectWallVendor`/`isAuthWall` already
30
- * enforce for the human-facing surfaces. One call site, one rule, reused by
31
- * all three JSON surfaces below instead of three copies that could drift.
32
- */
33
19
  function runDocsField(run) {
34
20
  if (!isAuthWall(run))
35
21
  return {};
36
- // Whole resolved object (never just `.docsUrl`) so resolveFailureKindDocsUrl
37
- // can gate on rung provenance — see docs.ts's DocsUrls.docsUrlIsDerived.
38
- // This call never supplies `externalDocsUrl`, so `docsUrl` here can only
39
- // ever be rung-2-derived or absent — never a rung-1 server-declared root —
40
- // but the gate stays explicit rather than relying on that as an invariant.
41
22
  const resolved = resolveDocsUrls({ apiBaseUrl: getApiUrl() });
42
23
  const docs = resolveFailureKindDocsUrl(run.failureKind, resolved);
43
24
  return docs ? { docs } : {};
44
25
  }
45
- /**
46
- * Print a usage/validation error consistently: human text to stderr, or a
47
- * machine envelope on stdout under --json (never both — reportError is the
48
- * single formatting path shared with index.ts's central catch, #86 finding
49
- * 9). Sets exit code 2 (usage) — distinct from a business-logic refusal
50
- * (which stays 1) or an unmapped ApiError/NetworkError (handled centrally in
51
- * index.ts). (#71)
52
- */
53
26
  function usageError(message, opts = {}) {
54
27
  process.exitCode = reportError(new UsageError(message), { json: opts.json });
55
28
  }
56
- /**
57
- * #163 — client-side URL validation shared by `create`/`update`, mirroring
58
- * the server's now-required-and-parseable `url` field (trawl_node#1823,
59
- * PR #1828: `z.string().trim().default('')` -> `z.string().trim().url()`).
60
- * Gives a readable message BEFORE the request goes out instead of a raw
61
- * 400/422 from the server. Reuses the exact same `requireUrl` helper
62
- * `trawl create`/`trawl login --url` already validate against (lib/validate.js),
63
- * just routed through this file's own return-style `usageError()` convention
64
- * (never throw — every other validation in this file, tier/limit/page/params/
65
- * status, follows the same pattern) instead of the throw-style those two
66
- * commands use. Returns `undefined` (never throws) on a bad value — the
67
- * caller must check for that and `return` without calling the API.
68
- */
69
29
  function tryValidateUrl(value, opts) {
70
30
  try {
71
31
  return requireUrl(value, '--url');
@@ -78,28 +38,13 @@ function tryValidateUrl(value, opts) {
78
38
  throw err;
79
39
  }
80
40
  }
81
- /**
82
- * One-line rendering of an emptyContext blob for the run-info table. Two
83
- * buckets are actionable and must stay distinct: selectors that matched zero
84
- * nodes (the page changed) and selectors the browser rejected outright
85
- * (`-1`, a bug in the scrap script). Folding the second into "all matched"
86
- * would report a broken selector as healthy. `--json` still emits the full
87
- * object, `page.topClasses` / `topAnchorPaths` included.
88
- */
89
41
  function summarizeEmptyContext(ctx) {
90
42
  if (!ctx || typeof ctx !== 'object')
91
43
  return null;
92
44
  const parts = [];
93
45
  const selectors = ctx.selectors && typeof ctx.selectors === 'object' ? ctx.selectors : {};
94
46
  const names = Object.keys(selectors);
95
- // Selectors may themselves contain a comma (`div, span`), so quote each one
96
- // — a raw join renders one grouping selector as if it were two.
97
47
  const list = (ns) => `${ns.slice(0, 3).map((s) => `\`${s}\``).join(', ')}${ns.length > 3 ? ` (+${ns.length - 3})` : ''}`;
98
- // Four disjoint, exhaustive buckets — every selector lands in exactly one, so
99
- // no count is ever dropped and none is summarised as something it is not.
100
- // `selectors` comes from a Mongoose Mixed field, so a count may be `null` or
101
- // a string: that is `unreadable`, which is neither healthy nor zero-match and
102
- // must be named as its own thing next to the others, not collapsed into them.
103
48
  const isCount = (v) => typeof v === 'number' && Number.isFinite(v);
104
49
  const zero = names.filter((s) => selectors[s] === 0);
105
50
  const invalid = names.filter((s) => selectors[s] === -1);
@@ -113,15 +58,8 @@ function summarizeEmptyContext(ctx) {
113
58
  if (unreadable.length)
114
59
  parts.push(`unreadable: ${list(unreadable)}`);
115
60
  if (matched.length) {
116
- // Named even in a mixed set: "3 of the selectors still work" is the
117
- // difference between a page that moved and a page that vanished.
118
61
  parts.push(mixed ? `${matched.length} matched` : `${matched.length} selector(s), all matched`);
119
62
  }
120
- // Same Mixed-field caution on the title: coerce before calling a string
121
- // method (a numeric title would otherwise crash `run-info` on the very run
122
- // it exists to explain), and test against null/undefined rather than
123
- // truthiness so a falsy-but-real title like `0` still shows. Inner double
124
- // quotes would close the wrapper early and corrupt the line.
125
63
  const title = ctx.page?.title;
126
64
  if (title !== undefined && title !== null && String(title) !== '') {
127
65
  parts.push(`title="${String(title).replace(/"/g, "'")}"`);
@@ -134,25 +72,16 @@ function lastStatus(scrap) {
134
72
  const last = scrap.history?.[0];
135
73
  if (!last)
136
74
  return 'never';
137
- // status:null means a run is IN FLIGHT (node persists {status:null,
138
- // statusDetail:null, inFlight:true} the moment a run starts) — that is NOT
139
- // the same thing as "this scrap has never run" (no history row at all).
140
- // (#88 item 1)
141
75
  if (last.status === null)
142
76
  return 'running';
143
77
  if (last.status === undefined)
144
78
  return 'never';
145
- // #91 LOW — same regression-as-failure bucketing bug as doctor's badge
146
- // (formatDoctor): a regression row's write actually succeeded (item count
147
- // just dropped vs baseline, flagged by an async patch afterward — #88 item
148
- // 2), so it must not collapse into the same 'failure' bucket a genuine
149
- // failed run gets.
150
79
  if (last.status === false && last.statusDetail === 'regression')
151
80
  return 'regression';
152
81
  return last.status === true ? 'success' : 'failure';
153
82
  }
154
83
  function lastRun(scrap) {
155
- return formatDate(scrap.history?.[0]?.createdAt); // #119 — unified DD/MM/YY HH:mm
84
+ return formatDate(scrap.history?.[0]?.createdAt);
156
85
  }
157
86
  function statusIcon(status) {
158
87
  if (status === 'success')
@@ -161,28 +90,10 @@ function statusIcon(status) {
161
90
  return chalk.red('✗');
162
91
  if (status === 'running')
163
92
  return chalk.cyan('↻');
164
- // #91 LOW — distinct amber icon, never the red ✗ a genuine failure gets
165
- // (mirrors doctor's "● regression" badge — see formatDoctor).
166
93
  if (status === 'regression')
167
94
  return chalk.hex('#FFA500')('▼');
168
95
  return chalk.dim('—');
169
96
  }
170
- /**
171
- * #179 — health badge, next to the existing status icon. Two distinct
172
- * sources, deliberately never blended into one derived verdict:
173
- * 1. `consecutiveFailedRuns`/`unhealthySince` — present ONLY when `--unhealthy`
174
- * asked the API to attach them (see attachListCommand). This is the
175
- * live, exact server verdict — shown whenever available, since it is
176
- * strictly more informative than the breadcrumb below.
177
- * 2. `lastCronOutcome === 'skipped_unhealthy'` — always present (a normal
178
- * scrap field, no extra query cost), but a STALE breadcrumb of the last
179
- * scheduled tick, not a live read: never set on a no-cron scrap, lags
180
- * by up to one tick, and flips back to 'ok'/'error' once the recovery
181
- * probe fires. Labeled as what it is ("cron paused"), never upgraded to
182
- * "unhealthy" — that word is reserved for the live verdict above.
183
- * Never re-derives either signal from `history[]` — both come straight off
184
- * the server (#1952 in trawl_node).
185
- */
186
97
  function healthBadge(scrap) {
187
98
  if (typeof scrap.consecutiveFailedRuns === 'number' && scrap.consecutiveFailedRuns > 0) {
188
99
  const n = scrap.consecutiveFailedRuns;
@@ -193,19 +104,6 @@ function healthBadge(scrap) {
193
104
  }
194
105
  return chalk.dim('—');
195
106
  }
196
- /**
197
- * #179 — old-server fallback for `list --unhealthy`, same shape as
198
- * `warnIfUnconfirmedTier` above (#86 finding 4b): an older trawl_node that
199
- * predates #1952 ignores the unknown `minConsecutiveFailures` query param
200
- * entirely and returns the FULL unfiltered list — silently presenting every
201
- * scrap as "unhealthy" is exactly the kind of unconfirmed-outcome lie that
202
- * precedent exists to prevent. A confirming server attaches
203
- * `consecutiveFailedRuns` to EVERY item once the param is set (the
204
- * controller writes `s.consecutiveFailedRuns || 0` unconditionally in that
205
- * branch — see trawl_node scraps.controller.js), so its total absence across
206
- * a non-empty response is the honest tell. Stderr only (safe under --json —
207
- * stdout purity is untouched), never blocks the (unreliable) results below.
208
- */
209
107
  function healthFilterUnconfirmed(data, unhealthyWasRequested) {
210
108
  if (!unhealthyWasRequested || data.length === 0)
211
109
  return false;
@@ -214,35 +112,19 @@ function healthFilterUnconfirmed(data, unhealthyWasRequested) {
214
112
  function warnIfUnconfirmedHealthFilter(data, unhealthyWasRequested) {
215
113
  if (!healthFilterUnconfirmed(data, unhealthyWasRequested))
216
114
  return;
217
- console.error(chalk.yellow('⚠ Server did not confirm the --unhealthy filter (older server, predates trawl_node#1952) — results below are UNFILTERED, not just the unhealthy scraps.'));
115
+ console.error(chalk.yellow('⚠ Server did not confirm the --unhealthy filter (older server version, predates this filter) — results below are UNFILTERED, not just the unhealthy scraps.'));
218
116
  }
219
- /**
220
- * #179 — the machine-readable counterpart to the stderr warning above, in the
221
- * exact shape `withTierUnconfirmed` already uses for the tier case. A --json
222
- * caller has no reliable reason to read stderr, so without this a script sees
223
- * a full unfiltered list and cannot tell it is not the unhealthy set.
224
- * Emitted ONLY when the filter went unconfirmed — never fabricated otherwise.
225
- */
226
117
  function withHealthFilterUnconfirmed(rows, data, unhealthyWasRequested) {
227
118
  if (!healthFilterUnconfirmed(data, unhealthyWasRequested))
228
119
  return rows;
229
120
  return { scraps: rows, _healthFilterUnconfirmed: true };
230
121
  }
231
122
  export const scraps = new Command('scraps').description('Manage scraps');
232
- // shared SSE streaming helper. `asJson` (#107) emits one raw JSON object per
233
- // line (NDJSON) on stdout instead of the human-formatted timestamped text —
234
- // a streaming command still needs a pure-stdout machine mode, just line-
235
- // delimited instead of a single blob (there's no single "final" payload to
236
- // wait for).
237
123
  async function watchActivities(id, asJson) {
238
124
  if (!asJson)
239
125
  console.log(chalk.dim('Streaming activities (Ctrl+C to stop)…\n'));
240
126
  for await (const event of api.stream(`/api/scraps/${id}/activities/stream`)) {
241
127
  try {
242
- // #159 — same tolerant parse as api.ts's response-parsing seam: an SSE
243
- // activity line can carry the same server-side raw-control-character
244
- // quirk as any other API payload, and this feeds `--json` NDJSON
245
- // output too (not just create).
246
128
  const activity = parseServerJson(event);
247
129
  if (asJson) {
248
130
  console.log(JSON.stringify(activity));
@@ -252,34 +134,12 @@ async function watchActivities(id, asJson) {
252
134
  console.log(`${chalk.dim(`[${time}]`)} ${activity.message}`);
253
135
  }
254
136
  catch {
255
- // non-JSON lines are heartbeats or SSE comments — skip silently
256
137
  if (process.env['DEBUG'] && event.trim()) {
257
138
  console.error(chalk.dim(`[SSE] skipping non-JSON: ${event}`));
258
139
  }
259
140
  }
260
141
  }
261
142
  }
262
- /**
263
- * #91 P1 / #93 item 1 — snapshot the current top history entry BEFORE
264
- * triggering a run, so pollRunProgress can later tell "the run we just
265
- * launched" apart from whatever the last run happened to be. Node persists a
266
- * fresh {status:null, inFlight:true} row the moment a run starts (see
267
- * HistorysService.create in trawl_node), so a changed history[0]._id is
268
- * normally an honest "the new run has begun" signal — EXCEPT `trigger`
269
- * (method:'worker') dedups onto an already-pending/running worker job
270
- * (ScrapJobsService, LIVE_STATUSES) instead of creating a new row: the top
271
- * row's `_id` never changes even though this launch IS that run. Recording
272
- * `alreadyInFlight` (status===null at capture time) lets pollRunProgress
273
- * recognize that dedup case too, instead of waiting forever for an `_id`
274
- * that will never arrive (#93 item 1).
275
- *
276
- * #97 — a failed lookup used to fall back to `{alreadyInFlight:false}` with
277
- * no `id`, which LOOKED identical to "scrap has never run" — but is not:
278
- * the scrap may well have prior (possibly terminal) history this GET simply
279
- * never observed. `captured:false` flags that distinction explicitly so
280
- * pollRunProgress can defer trusting a baseline instead of risking a stale
281
- * pre-existing row being reported as the run just launched.
282
- */
283
143
  async function captureBeforeRunState(id) {
284
144
  try {
285
145
  const scrap = await api.get(`/api/scraps/${id}`);
@@ -294,115 +154,14 @@ function sleep(ms) {
294
154
  return new Promise((resolve) => setTimeout(resolve, ms));
295
155
  }
296
156
  const POLL_INTERVAL_MS = 2000;
297
- // Mirrors LONG_RUN_TIMEOUT_MS (#91 item 1) — the server-side worst case this
298
- // polls for is the same one that timeout was sized for.
299
157
  const POLL_TIMEOUT_MS = LONG_RUN_TIMEOUT_MS;
300
- // #107 review F1 — a genuinely unreachable API must not run the watch
301
- // silently for the full 300s timeout with dead air; after this many
302
- // CONSECUTIVE (not cumulative — any successful poll resets the counter)
303
- // failed polls, treat it as a persistent error and surface it immediately
304
- // instead of waiting out the clock. At the default 2s interval this gives up
305
- // after ~10s of continuous failures — comfortably longer than any single
306
- // transient blip, nowhere near the 300s ceiling.
307
158
  const MAX_CONSECUTIVE_POLL_ERRORS = 5;
308
- /**
309
- * #91 P1 — replaces "await the run to completion, THEN open the activities
310
- * SSE stream" (which showed NOTHING: the activities SSE
311
- * (GET /api/scraps/:id/activities/stream) is backed by an in-process
312
- * EventEmitter with no backlog — trawl_node
313
- * modules/activities/services/activities.service.js — so by the time a
314
- * synchronous run has already finished there is nothing left to emit; and
315
- * the async `trigger` default enqueues the run onto a durable job queue a
316
- * SEPARATE cron-consumer pod drains, whose in-process emitter never reaches
317
- * the API pod holding the SSE connection at all).
318
- *
319
- * Instead this polls two REST reads that are BOTH Mongo-backed (not
320
- * in-process), so they work no matter which pod actually executed the run:
321
- * - GET /api/scraps/:id/activities?history=<hid>&limit=20 — the SAME
322
- * activities-list endpoint `doctor`'s fetchRunAndFix already calls
323
- * (src/commands/doctor.ts) — prints each new activity line once the new
324
- * run's history id is known.
325
- * - GET /api/scraps/:id — history[0].status/statusDetail, the SAME
326
- * terminal-status signal `lastStatus()` above already trusts (status:null
327
- * === in flight, #88 item 1) to know when the run is done.
328
- *
329
- * #93 item 1 — dedup-race fix. "Is this the run we're watching?" used to be
330
- * a single check: `history[0]._id !== beforeHistoryId`. That's wrong when
331
- * `before.alreadyInFlight` is true (a `trigger` call deduped onto a worker
332
- * job that was ALREADY pending/running at capture time): the top row IS the
333
- * run we're watching, but its `_id` never changes, so the old guard never
334
- * released and the poll ran the full timeout to a false "Timed out". The run
335
- * we're watching is now EITHER a brand-new id (fresh trigger, the common
336
- * case) OR the same id that was already in-flight (status:null) at capture
337
- * (the dedup case) — a same-id row that was already TERMINAL at capture is
338
- * neither, and must not be latched onto as "done" (it's just the previous
339
- * run, still sitting there until a genuinely new run supersedes it).
340
- *
341
- * #97 — capture-failed fallback. The dedup-race fix above assumes `before`
342
- * is trustworthy. When `captureBeforeRunState`'s own GET threw,
343
- * `before.id` is `undefined` — and that is INDISTINGUISHABLE from "the
344
- * scrap has genuinely never run" (also `id: undefined`), which is exactly
345
- * the case `isNewRun` above is designed to match on the very first row that
346
- * ever appears. So on the very first poll, ANY pre-existing history row —
347
- * even the STALE PREVIOUS run, already terminal — satisfied
348
- * `last._id !== undefined` and got reported as "the run we just launched"
349
- * finishing, when it was really just whatever ran before.
350
- *
351
- * Fix: `before.captured === false` defers trusting a baseline at all.
352
- * Instead of comparing against the (unknown) `beforeId` from the start, the
353
- * FIRST successful poll read is treated as the deferred capture itself —
354
- * exactly what `captureBeforeRunState` would have returned had its GET
355
- * succeeded — and only READS from that point on are compared against it,
356
- * via the exact same `isNewRun` / `isDedupOntoInFlight` logic above. A
357
- * pre-existing terminal row observed on that first read becomes `beforeId`
358
- * (not a match for itself), so it correctly falls into "still the stale
359
- * previous run" below and the poll keeps waiting; a row still in flight
360
- * becomes the `alreadyInFlight` baseline, exactly like a successful capture
361
- * would have recorded. Either way this costs at most one extra poll
362
- * interval, bounded by the same deadline as everything else. The one
363
- * remaining edge case — capture failed AND the scrap never ran before AND
364
- * the triggered run already finished by the very first poll — is genuinely
365
- * undecidable from "stale pre-existing row" with no more information than
366
- * this function has, so it resolves to the same honest timeout rather than
367
- * risk reporting a possibly-wrong outcome (never a lie, at worst a timeout
368
- * telling the caller to check `doctor`).
369
- *
370
- * #107 review F1 — before this fix, `run|trigger --json --watch` was
371
- * outcome-blind: `quiet` suppressed ALL output (including "Run finished:
372
- * failure" and the timeout notice), a transient poll error was caught and
373
- * silently retried FOREVER within the deadline, and the process always
374
- * exited 0 after the poll loop regardless of what the watched run actually
375
- * did — dead air, then a clean exit code, even for a failed or timed-out
376
- * run. An agent scripting this CLI had no way to tell success from failure
377
- * from "we gave up". Fixed by:
378
- * - emitting exactly ONE final NDJSON line on stdout under `--json` once
379
- * the watch reaches ANY of its three exits (terminal status, timeout, or
380
- * a persistent poll error) — `{runId,status}` (+`error` for a poll
381
- * error) — while every intermediate progress line stays suppressed
382
- * (unchanged from before);
383
- * - setting `process.exitCode` non-zero on a genuine run failure, a
384
- * timeout, or a persistent poll error, and `0` on a real success — in
385
- * BOTH `--json` and human `--watch` modes (human mode used to exit 0
386
- * unconditionally, the same bug, just silent instead of dishonest);
387
- * - giving up after `MAX_CONSECUTIVE_POLL_ERRORS` consecutive failed reads
388
- * instead of retrying the same dead endpoint for the full 300s.
389
- */
390
159
  export async function pollRunProgress(id, before, opts = {}) {
391
160
  const intervalMs = opts.intervalMs ?? POLL_INTERVAL_MS;
392
161
  const timeoutMs = opts.timeoutMs ?? POLL_TIMEOUT_MS;
393
- // #107 — `run --json --watch` / `trigger --json --watch` must keep stdout
394
- // pure JSON: none of this function's intermediate progress text is safe to
395
- // print once a caller asked for --json. `asJson` suppresses every
396
- // intermediate console.log below (including the pinch celebration frame)
397
- // while the polling/wait logic itself runs unchanged; the ONE exception is
398
- // the single final outcome line emitted right before each return below.
399
162
  const asJson = opts.json ?? false;
400
163
  let beforeId = before?.id;
401
164
  let beforeAlreadyInFlight = before?.alreadyInFlight ?? false;
402
- // #97 — only an EXPLICIT captured:false (capture's GET actually threw)
403
- // defers the baseline. `before` itself being undefined (callers that skip
404
- // capture entirely) or `captured` being true/absent both mean "trust
405
- // beforeId as given", preserving every existing call site's behavior.
406
165
  let baselineEstablished = before?.captured ?? true;
407
166
  if (!asJson) {
408
167
  console.log(chalk.dim('Live activity streaming has no signal for this run (async/cross-pod) — polling for progress instead…\n'));
@@ -422,11 +181,6 @@ export async function pollRunProgress(id, before, opts = {}) {
422
181
  }
423
182
  catch (err) {
424
183
  consecutivePollErrors++;
425
- // #107 review F1 — a single failed read is still a TRANSIENT blip,
426
- // safe to retry within the deadline (unchanged). Only once reads fail
427
- // this many times IN A ROW is the API treated as genuinely
428
- // unreachable — surface that honestly instead of quietly burning the
429
- // full 300s timeout on a dead endpoint.
430
184
  if (consecutivePollErrors >= MAX_CONSECUTIVE_POLL_ERRORS) {
431
185
  const message = err instanceof Error ? err.message : String(err);
432
186
  if (asJson) {
@@ -443,13 +197,8 @@ export async function pollRunProgress(id, before, opts = {}) {
443
197
  }
444
198
  const last = scrap.history?.[0];
445
199
  if (!last?._id)
446
- continue; // no history row recorded yet
200
+ continue;
447
201
  if (!baselineEstablished) {
448
- // #97 — deferred capture: whatever we see on this first successful
449
- // read becomes the reference point, BEFORE the isNewRun check below
450
- // ever runs against it. This must happen before that check, not
451
- // after, so a pre-existing terminal row is recognized as stale on
452
- // this very same iteration rather than one poll late.
453
202
  beforeId = last._id;
454
203
  beforeAlreadyInFlight = last.status === null;
455
204
  baselineEstablished = true;
@@ -457,10 +206,9 @@ export async function pollRunProgress(id, before, opts = {}) {
457
206
  const isNewRun = last._id !== beforeId;
458
207
  const isDedupOntoInFlight = last._id === beforeId && beforeAlreadyInFlight;
459
208
  if (!isNewRun && !isDedupOntoInFlight)
460
- continue; // still the stale previous run
209
+ continue;
461
210
  try {
462
211
  const activities = await api.get(`/api/scraps/${id}/activities?history=${last._id}&limit=20`);
463
- // Server returns newest-first — print unseen ones oldest-first.
464
212
  for (const a of [...(activities ?? [])].reverse()) {
465
213
  const key = a._id ?? `${a.createdAt}:${a.message}`;
466
214
  if (seen.has(key))
@@ -473,16 +221,9 @@ export async function pollRunProgress(id, before, opts = {}) {
473
221
  }
474
222
  }
475
223
  catch {
476
- // best-effort — terminal detection below still works without activity lines
477
224
  }
478
225
  if (last.status !== null) {
479
226
  const outcome = last.statusDetail ?? (last.status ? 'success' : 'failure');
480
- // #107 review F1 — the exit code, in BOTH modes: a regression row's
481
- // WRITE actually succeeded (see lastStatus()'s own #91 comment above,
482
- // and `scraps data`'s isRegression branch) and an 'empty' row is a
483
- // genuine zero-item success — neither is a real failure. Only
484
- // status:false with neither of those details (e.g. 'error', or no
485
- // detail at all) is a genuine failed run.
486
227
  const isGenuineFailure = last.status === false && outcome !== 'empty' && outcome !== 'regression';
487
228
  if (asJson) {
488
229
  const payload = { runId: last._id, status: outcome };
@@ -490,10 +231,6 @@ export async function pollRunProgress(id, before, opts = {}) {
490
231
  }
491
232
  else {
492
233
  console.log(chalk.dim(`Run finished: ${outcome}`));
493
- // Pinch celebrates a clean run finish (#94) — mirrors doctor.ts's own
494
- // `status === true` success definition (regardless of statusDetail),
495
- // never for a failed/regression run. Guarded by pinchEnabled()
496
- // (NO_COLOR/non-TTY) and never under --json (stdout must stay pure).
497
234
  if (last.status === true && pinchEnabled()) {
498
235
  console.log(renderPinch('celebrating'));
499
236
  }
@@ -502,9 +239,6 @@ export async function pollRunProgress(id, before, opts = {}) {
502
239
  return;
503
240
  }
504
241
  }
505
- // #107 review F1 — timeout: honest non-zero exit in both modes, plus the
506
- // machine-readable final line under --json (never silently exit 0 after a
507
- // watch that never actually confirmed what happened).
508
242
  if (asJson) {
509
243
  const outcome = { runId: beforeId, status: 'timeout' };
510
244
  console.log(JSON.stringify(outcome));
@@ -514,20 +248,8 @@ export async function pollRunProgress(id, before, opts = {}) {
514
248
  }
515
249
  process.exitCode = 1;
516
250
  }
517
- // #149 item 3 — same shared-enum shape as VALID_TIERS below (~line 603):
518
- // keeps `--status`'s allowed values and its error message in one place,
519
- // mirrored from lastStatus()'s own return type so the two can never drift.
520
251
  const VALID_LAST_STATUSES = ['success', 'failure', 'never', 'running', 'regression'];
521
- // #179 — mirrors `UNHEALTHY_STREAK_LENGTH` in trawl_node
522
- // modules/scraps/helpers/scrapCronHealth.js. `--unhealthy` is a boolean
523
- // convenience flag for that ONE product-defined threshold (also what the
524
- // server's own cron-skip gate and email use), not a general numeric filter —
525
- // the API's `minConsecutiveFailures` param accepts any threshold, but this
526
- // CLI only ever asks for the one value the rest of the product means by
527
- // "unhealthy". If trawl_node's threshold ever changes, update this constant
528
- // to match.
529
252
  const UNHEALTHY_STREAK_LENGTH = 3;
530
- // list — promoted to a top-level verb (#108, see the AttachOptions comment above)
531
253
  export function attachListCommand(parent, attachOpts = {}) {
532
254
  return parent
533
255
  .command('list', attachOpts)
@@ -535,33 +257,14 @@ export function attachListCommand(parent, attachOpts = {}) {
535
257
  .description('List all scraps')
536
258
  .option('--json', 'Output as JSON')
537
259
  .option('--status <status>', 'Filter by last run status (success|failure|never|running|regression)')
538
- // #88 item 8 — no custom parser here (unlike the old `(v) => parseInt(v,
539
- // 10)`): a bad value like "abc" used to silently become NaN, which then
540
- // sailed straight through `Number.isInteger`-less checks and into
541
- // `.slice(0, NaN)` (silently truncates to 0 rows) or `?page=NaN` (silently
542
- // sent to the server) — never a usage error. Keeping the raw string here
543
- // lets the validation below mirror `history`'s own --limit check exactly
544
- // (~line 660) and report the actual bad input in the error message.
545
260
  .option('--limit <n>', 'Show only the first N results')
546
261
  .option('--page <n>', 'Fetch a specific page only (50 per page, no auto-pagination)')
547
- // #179 — backed by the server's `minConsecutiveFailures` filter
548
- // (trawl_node#1952): restricts the result set server-side to scraps
549
- // whose CURRENT consecutive-failure streak is >= UNHEALTHY_STREAK_LENGTH,
550
- // and annotates each returned scrap with the exact `consecutiveFailedRuns`/
551
- // `unhealthySince` the aggregation computed. Deliberately never derived
552
- // client-side from `history[]` — that would drift from what the web
553
- // dashboard shows (comes-io/trawl_vue#1299 reads the identical filter).
554
262
  .option('--unhealthy', `Only show scraps failing their last ${UNHEALTHY_STREAK_LENGTH}+ runs (server-computed)`)
555
263
  .action(async (opts, cmd) => {
556
- // #149 item 3 — validate --status against its enum the same way
557
- // --tier already validates (usageError + return, checked first, before
558
- // any other guard/async work — mirrors `create`/`update`'s --tier
559
- // check below).
560
264
  if (opts.status !== undefined && !VALID_LAST_STATUSES.includes(opts.status)) {
561
265
  usageError(`Invalid --status "${opts.status}" (allowed: ${VALID_LAST_STATUSES.join(', ')})`, { json: opts.json });
562
266
  return;
563
267
  }
564
- // Guard: --limit and --page are mutually exclusive
565
268
  if (opts.limit !== undefined && opts.page !== undefined) {
566
269
  usageError('--limit and --page are mutually exclusive. Use one or the other.', { json: opts.json });
567
270
  return;
@@ -582,18 +285,13 @@ export function attachListCommand(parent, attachOpts = {}) {
582
285
  return;
583
286
  }
584
287
  }
585
- // #179 — same param on every request this action can issue (both
586
- // branches below), so a filtered result stays filtered across the
587
- // fetch-all pagination loop, not just its first page.
588
288
  const healthFilter = opts.unhealthy ? `&minConsecutiveFailures=${UNHEALTHY_STREAK_LENGTH}` : '';
589
289
  let data;
590
290
  try {
591
291
  data = await spin(async () => {
592
292
  if (page !== undefined) {
593
- // Single-page mode: explicit page requested, no loop
594
293
  return api.get(`/api/scraps?perPage=50&page=${page}${healthFilter}`);
595
294
  }
596
- // Fetch-all mode: paginate until a page returns < 200 items
597
295
  const perPage = 200;
598
296
  let result = [];
599
297
  let pageNum = 1;
@@ -608,16 +306,6 @@ export function attachListCommand(parent, attachOpts = {}) {
608
306
  }, 'Fetching scraps…');
609
307
  }
610
308
  catch (err) {
611
- // Spinner already failed by oraPromise — report with the scrap-specific
612
- // prefix kept, but route through the shared classifier so exit code +
613
- // --json envelope stay consistent with every other command. (#71)
614
- //
615
- // This bespoke catch (kept for the "Failed to fetch scraps:" prefix,
616
- // which the shared reportError() can't add) used to silently swallow
617
- // --debug: unlike the central index.ts catch, it never printed the raw
618
- // stack trace. optsWithGlobals() reads --debug off the ROOT command
619
- // (this leaf has no --debug of its own) so it can honor the flag
620
- // locally instead. (#86 finding 9)
621
309
  const isDebug = Boolean(cmd.optsWithGlobals().debug || process.env['DEBUG']);
622
310
  const { exitCode, envelope } = classifyError(err);
623
311
  const message = `Failed to fetch scraps: ${envelope.message}`;
@@ -632,9 +320,6 @@ export function attachListCommand(parent, attachOpts = {}) {
632
320
  process.exitCode = exitCode;
633
321
  return;
634
322
  }
635
- // #179 — check the RAW response, before --status narrows it further,
636
- // so a genuinely empty (all-healthy) result is never mistaken for an
637
- // unconfirmed filter.
638
323
  warnIfUnconfirmedHealthFilter(data, opts.unhealthy);
639
324
  if (opts.status)
640
325
  data = data.filter((s) => lastStatus(s) === opts.status);
@@ -647,21 +332,17 @@ export function attachListCommand(parent, attachOpts = {}) {
647
332
  title: s.title || '(untitled)',
648
333
  cron: s.cron || '—',
649
334
  status: statusIcon(lastStatus(s)),
650
- // #179 — next to the existing status badge, per the issue's own
651
- // wording (see healthBadge() above for the two source signals).
652
335
  health: healthBadge(s),
653
336
  'last run': lastRun(s),
654
337
  updated: formatDate(s.updatedAt),
655
338
  }));
656
339
  table(tableRows, ['id', 'title', 'cron', 'status', 'health', 'last run', 'updated']);
657
- // Print footer when --limit truncates
658
340
  if (limit !== undefined && rows.length < totalMatched) {
659
341
  console.log(chalk.dim(`Showing ${rows.length} of ${totalMatched} — omit --limit to see all`));
660
342
  }
661
343
  });
662
344
  }
663
345
  attachListCommand(scraps, { hidden: true });
664
- // get — promoted to a top-level verb (#108)
665
346
  export function attachGetCommand(parent, attachOpts = {}) {
666
347
  return parent
667
348
  .command('get <id>', attachOpts)
@@ -676,9 +357,6 @@ export function attachGetCommand(parent, attachOpts = {}) {
676
357
  console.log(chalk.dim(` ID: `) + data._id);
677
358
  console.log(chalk.dim(` Cron: `) + (data.cron || '—'));
678
359
  console.log(chalk.dim(` Status: `) + statusIcon(lastStatus(data)));
679
- // #179 — same badge `list` renders (see healthBadge() above); `get`
680
- // never sends `minConsecutiveFailures`, so this always reads the
681
- // free `lastCronOutcome` breadcrumb, never the live per-run count.
682
360
  console.log(chalk.dim(` Health: `) + healthBadge(data));
683
361
  console.log(chalk.dim(` Last run: `) + lastRun(data));
684
362
  console.log(chalk.dim(` Updated: `) + formatDate(data.updatedAt));
@@ -686,14 +364,6 @@ export function attachGetCommand(parent, attachOpts = {}) {
686
364
  }
687
365
  attachGetCommand(scraps, { hidden: true });
688
366
  const VALID_TIERS = ['tier0', 'tier1', 'tier2', 'tier3', 'tier4'];
689
- /**
690
- * #86 findings 4/5 — shared honest-tier renderer for `create --tier` and
691
- * `update --tier/--force-tier`. Reads the typed `_tierOverride` echoed back
692
- * by the server (#1559) so both commands show the SAME truth: a refusal, an
693
- * allowed ceiling raise (+ spend warning), or a silently clamped proxyTier.
694
- * Human-mode output only — a --json caller gets the same truth for free from
695
- * the full scrap object (which already includes `_tierOverride`).
696
- */
697
367
  function renderTierOverrideHuman(data) {
698
368
  const ov = data._tierOverride;
699
369
  if (!ov)
@@ -718,64 +388,25 @@ function renderTierOverrideHuman(data) {
718
388
  }
719
389
  }
720
390
  }
721
- /**
722
- * #86 finding 4b — old-server fallback. When a tier was requested but the
723
- * response carries no `_tierOverride` at all, the server is too old to
724
- * confirm what actually got applied. Echoing the REQUESTED value as if it
725
- * were the outcome is exactly the silent-clamp lie #1559 fixed — warn
726
- * instead, on stderr (safe under --json too; stdout purity is untouched).
727
- */
728
391
  function warnIfUnconfirmedTier(data, tierWasRequested, id) {
729
392
  if (data._tierOverride || !tierWasRequested)
730
393
  return;
731
394
  console.error(chalk.yellow(` ⚠ Server did not confirm the tier change (older server) — verify with: trawl get ${id}`));
732
395
  }
733
- /**
734
- * #88 item 3 — the --json machine-readable counterpart to
735
- * warnIfUnconfirmedTier's stderr warning above. A --json caller (an agent
736
- * scripting this CLI) has no reliable reason to read stderr — that channel
737
- * is advisory-only everywhere else in this CLI, and stdout must stay the
738
- * sole payload. Without this, the ONLY signal that the server never
739
- * confirmed the tier change was a string on stderr, invisible to any --json
740
- * consumer parsing stdout alone. Adds `_tierUnconfirmed: true` to the
741
- * emitted object under EXACTLY the same condition warnIfUnconfirmedTier
742
- * warns on (tier requested, response carries no `_tierOverride`) — never
743
- * fabricated, never present otherwise.
744
- */
745
396
  function withTierUnconfirmed(data, tierWasRequested) {
746
397
  if (data._tierOverride || !tierWasRequested)
747
398
  return data;
748
399
  return { ...data, _tierUnconfirmed: true };
749
400
  }
750
- /** #86 finding 5 — the standard error envelope for a refused tier override,
751
- * routed through the same reportError() central formatting path used
752
- * everywhere else (exit 1: a business-logic refusal, not a usage error).
753
- *
754
- * #107 review F3 — uses `RefusalError` (kind:"refused"), not a bare `Error`
755
- * (which fell through classifyError's default `kind:"unknown"` bucket,
756
- * indistinguishable from a generic crash even though the README sells `kind`
757
- * as the machine discriminant an agent branches on).
758
- */
759
401
  function reportTierRefusal(data, wantsJson) {
760
402
  const ov = data._tierOverride;
761
403
  const message = `Tier ceiling override refused: ${ov?.reason ?? 'unknown'} (requested ${ov?.requestedMaxTier ?? '—'}; kept the registry cap)`;
762
404
  return reportError(new RefusalError(message), { json: wantsJson });
763
405
  }
764
- // create
765
406
  scraps
766
407
  .command('create')
767
408
  .description('Create a new scrap')
768
409
  .requiredOption('-t, --title <title>', 'Scrap title')
769
- // #170 review F6 — promoted from a plain `.option()` to `.requiredOption()`:
770
- // this is unconditionally required (no positional alternative exists on
771
- // THIS command, unlike the top-level `trawl create <url>`, which accepts
772
- // `[url]` OR `--url` — deliberately left untouched, that either/or needs a
773
- // human product decision, tracked separately). Before this, the flag's
774
- // description carried the only "Required." signal, in English prose — the
775
- // exact thing docs/agent-quickstart.md tells an agent not to parse.
776
- // Commander itself now enforces it, so `trawl spec --json`'s published
777
- // `mandatory:true` is honest instead of a second, driftable copy of this
778
- // same fact.
779
410
  .requiredOption('-u, --url <url>', 'Target URL — the site the scrap targets, not necessarily the exact URL fetched at run time (a request script can compute that, e.g. a templated search query).')
780
411
  .option('-r, --request <request>', 'Request/query')
781
412
  .option('-d, --description <text>', 'Scrap description')
@@ -786,10 +417,6 @@ scraps
786
417
  usageError(`Invalid --tier "${opts.tier}" (allowed: ${VALID_TIERS.join(', ')})`, { json: opts.json });
787
418
  return;
788
419
  }
789
- // #163 — the API now requires a parseable `url` on create (trawl_node
790
- // #1823/#1828); validate client-side BEFORE the request goes out so a
791
- // missing/unparseable value surfaces as a readable usage error instead
792
- // of a raw server 400/422.
793
420
  const url = tryValidateUrl(opts.url, opts);
794
421
  if (url === undefined)
795
422
  return;
@@ -800,9 +427,6 @@ scraps
800
427
  ...(opts.description !== undefined && { description: opts.description }),
801
428
  ...(opts.tier !== undefined && { proxyTier: opts.tier }),
802
429
  }), { text: 'Creating scrap…', successText: (d) => `Scrap created: ${chalk.bold(d._id)}` });
803
- // #86 finding 4a — read _tierOverride back from the POST response and
804
- // render it exactly like `update` does; never echo the requested tier as
805
- // if it were applied when an older server doesn't confirm it.
806
430
  const tierWasRequested = opts.tier !== undefined;
807
431
  warnIfUnconfirmedTier(data, tierWasRequested, data._id);
808
432
  const refused = Boolean(data._tierOverride?.refused);
@@ -819,7 +443,6 @@ scraps
819
443
  if (refused)
820
444
  process.exitCode = 1;
821
445
  });
822
- // update
823
446
  scraps
824
447
  .command('update <id>')
825
448
  .description('Update an existing scrap')
@@ -851,9 +474,6 @@ scraps
851
474
  const body = {};
852
475
  if (opts.title !== undefined)
853
476
  body.title = opts.title;
854
- // #163 — omitting --url stays a no-op (the API keeps that allowed on
855
- // update), but a value that IS supplied must still be a parseable URL —
856
- // validate it client-side before the request goes out, same as create.
857
477
  if (opts.url !== undefined) {
858
478
  const url = tryValidateUrl(opts.url, opts);
859
479
  if (url === undefined)
@@ -902,7 +522,6 @@ scraps
902
522
  if (opts.tier !== undefined)
903
523
  body.proxyTier = opts.tier;
904
524
  if (opts.forceTier !== undefined) {
905
- // Raise the ceiling; also start the run at that tier unless --tier says otherwise.
906
525
  body.proxyMaxTier = opts.forceTier;
907
526
  if (opts.tier === undefined)
908
527
  body.proxyTier = opts.forceTier;
@@ -919,10 +538,6 @@ scraps
919
538
  text: 'Updating scrap…',
920
539
  successText: (d) => `Scrap updated: ${chalk.bold(d._id)}`,
921
540
  });
922
- // #1559 / #86 findings 4b/5 — surface the effective tier + clamp/refuse
923
- // reason (fixes the silent-clamp: the server may persist a lower tier
924
- // than requested), and NEVER echo the requested value as applied when
925
- // the server doesn't confirm it (old-server fallback below).
926
541
  const tierWasRequested = opts.tier !== undefined || opts.forceTier !== undefined;
927
542
  warnIfUnconfirmedTier(data, tierWasRequested, id);
928
543
  const refused = Boolean(data._tierOverride?.refused);
@@ -936,10 +551,6 @@ scraps
936
551
  }
937
552
  const shown = data;
938
553
  for (const key of Object.keys(body)) {
939
- // Tier keys are rendered exclusively by renderTierOverrideHuman /
940
- // warnIfUnconfirmedTier above — never echo them here, whether or not
941
- // _tierOverride came back (an old-server echo of the REQUESTED value
942
- // is exactly the silent-clamp lie #1559 fixed).
943
554
  if (key === 'proxyTier' || key === 'proxyMaxTier')
944
555
  continue;
945
556
  const src = key in shown ? shown[key] : body[key];
@@ -949,7 +560,6 @@ scraps
949
560
  if (refused)
950
561
  process.exitCode = 1;
951
562
  });
952
- // run — promoted to a top-level verb (#108)
953
563
  export function attachRunCommand(parent, attachOpts = {}) {
954
564
  return parent
955
565
  .command('run <id>', attachOpts)
@@ -958,66 +568,25 @@ export function attachRunCommand(parent, attachOpts = {}) {
958
568
  .option('--json', 'Output the raw launch payload as JSON')
959
569
  .action(async (id, opts) => {
960
570
  validateObjectId(id);
961
- // #91 P1 / #93 item 1 — captured BEFORE launching so pollRunProgress can
962
- // tell "the run that's about to finish" apart from whatever the last run
963
- // happened to be (including a dedup onto an already-in-flight run).
964
571
  const beforeRun = opts.watch ? await captureBeforeRunState(id) : undefined;
965
- // #91 P0 — GET /api/scraps/load/:id runs the scrap synchronously
966
- // server-side (30-250s); the 30s default was aborting it mid-flight.
967
572
  const call = () => api.get(`/api/scraps/load/${id}`, { timeoutMs: LONG_RUN_TIMEOUT_MS });
968
- // #107 — under --json the stdout path stays pure: no spinner channel at
969
- // all, mirroring `trawl create`'s own --json handling.
970
573
  const data = opts.json
971
574
  ? await call()
972
575
  : await spin(call, { text: 'Launching scrap…', successText: 'Scrap launched' });
973
576
  if (opts.json)
974
577
  json(data);
975
578
  else if (!opts.watch) {
976
- /* #166 — see confirmNonTTY. Two deliberate choices here:
977
- *
978
- * "run complete", not "launched": `run` is the SYNCHRONOUS verb —
979
- * GET /api/scraps/load/:id executes the scrap server-side (30-250s)
980
- * and returns the finished result, so by the time this prints the run
981
- * is over. The spinner's successText says "Scrap launched", but that
982
- * is stderr chrome for a human watching it happen; THIS line is the
983
- * machine surface, where a script reading "launched" would poll for a
984
- * completion that already happened. `trigger` already draws the same
985
- * distinction ("Worker triggered" async vs "Worker run complete" under
986
- * --wait) — this matches that vocabulary.
987
- *
988
- * `--watch` is excluded because pollRunProgress below does its own
989
- * reporting; without that guard a watched run would print this line
990
- * and then narrate the same run. */
991
579
  confirmNonTTY(`Scrap ${id} run complete`);
992
580
  }
993
581
  if (opts.watch) {
994
- // #107 — under --json, pollRunProgress suppresses its own intermediate
995
- // console.log calls and instead emits exactly ONE final NDJSON outcome
996
- // line (+ sets process.exitCode honestly) once the watch reaches a
997
- // terminal status, a timeout, or a persistent poll error (review F1) —
998
- // `run --json --watch` never again exits 0 after dead air regardless
999
- // of what the watched run actually did.
1000
582
  await pollRunProgress(id, beforeRun, { json: opts.json });
1001
583
  }
1002
- // #151 — throttled post-run referral tip. Reached only when the
1003
- // launch call above didn't throw (a hard failure rejects it, unwinding
1004
- // straight to index.ts's central catch before this line ever runs)
1005
- // — and, when --watch polled to a terminal state, only when
1006
- // pollRunProgress didn't flag a genuine run failure/timeout/poll-error
1007
- // via a non-zero process.exitCode (#107 review F1). See lib/tips.ts
1008
- // for the throttle/TTY/json/opt-out/server-flag gating itself. #153 —
1009
- // now async (a due tip fetches the server's userFacing flag before
1010
- // printing), so this must be awaited: otherwise commander's
1011
- // parseAsync-driven action would resolve before the tip's own
1012
- // await settles, racing the process's natural exit.
1013
584
  const runFailed = typeof process.exitCode === 'number' && process.exitCode !== 0;
1014
585
  if (!runFailed)
1015
586
  await maybeShowReferralTip({ json: opts.json });
1016
587
  });
1017
588
  }
1018
589
  attachRunCommand(scraps, { hidden: true });
1019
- // #70 — render an items array either as a table summary or --json. Shared by
1020
- // both the default (persisted read) and --fresh (live execute) paths of `data`.
1021
590
  function renderScrapItems(items, asJson) {
1022
591
  if (asJson) {
1023
592
  json(items);
@@ -1030,43 +599,15 @@ function renderScrapItems(items, asJson) {
1030
599
  }
1031
600
  console.log(chalk.dim(' Use --json for full output.'));
1032
601
  }
1033
- /**
1034
- * #86 finding 6 — `data`'s honest empty-vs-error distinction, for both --json
1035
- * and human prose. Before this, EVERY non-array outcome (never run, last run
1036
- * failed, or the payload aged out of retention) collapsed to the SAME `[]` /
1037
- * "No data yet." — indistinguishable from a genuine zero-item successful run.
1038
- * That's a lie under --json: an agent can't tell "nothing to show" from "go
1039
- * look at what actually happened". Mirrors reportError's dual json/human
1040
- * shape (human line to stderr always; --json ALSO gets a machine envelope on
1041
- * stdout) with a caller-chosen exit code + kind, since these states are not
1042
- * all "usage" (2) — never-run / aged-out-of-retention are not_found (4), a
1043
- * failed last run is a business-logic failure (1).
1044
- */
1045
- /**
1046
- * #170 — `retryable`/`next` follow the SAME frozen `RETRY_POLICY` map
1047
- * `classifyError` reads (errors.ts), keyed by `kind` — this path never
1048
- * builds its own copy. `retryOverride` is for the one case here where the
1049
- * kind alone is genuinely wrong: `in_progress`'s caller passes
1050
- * `{retryable:true, next:['trawl data <id>']}` since retrying `data` (not
1051
- * `--fresh`, which would 429 against the lock already held) IS worth doing
1052
- * once the run finishes. Every other kind here (`not_found`, `run_failed`)
1053
- * isn't in `RETRY_POLICY` at all — `retryFieldsFor` is total, so those fall
1054
- * back to its conservative "not retryable, nothing to suggest" default.
1055
- */
1056
602
  function reportDataState(message, exitCode, kind, wantsJson, retryOverride) {
1057
603
  console.error(chalk.red(`✗ ${message}`));
1058
604
  if (wantsJson) {
1059
605
  const { retryable, next } = retryOverride ?? retryFieldsFor(kind);
1060
- // #170 review F3 — typed as ErrorEnvelope so the compiler enforces
1061
- // `retryable` here too (this was one of two emitters a mutation test
1062
- // proved `tsc --noEmit` never actually guarded before this annotation —
1063
- // the untyped literal let `retryable` be omitted silently).
1064
606
  const envelope = { message, kind, retryable, ...(next ? { next } : {}) };
1065
607
  console.log(JSON.stringify({ error: envelope }));
1066
608
  }
1067
609
  process.exitCode = exitCode;
1068
610
  }
1069
- // data — promoted to a top-level verb (#108)
1070
611
  export function attachDataCommand(parent, attachOpts = {}) {
1071
612
  return parent
1072
613
  .command('data <id>', attachOpts)
@@ -1076,15 +617,9 @@ export function attachDataCommand(parent, attachOpts = {}) {
1076
617
  .option('--fresh', 'Launch a fresh run instead of reading the last persisted payload (consumes execute quota, same as `run`)')
1077
618
  .action(async (id, opts) => {
1078
619
  validateObjectId(id);
1079
- // --errors: fetch full run + fix detail via fetchRunAndFix (DRY with doctor).
1080
- // Read-only: GET /api/scraps/:id + GET /api/historys/:hid, no quota, no run lock.
1081
620
  if (opts.errors) {
1082
621
  const result = await fetchRunAndFix(id);
1083
622
  if (!result) {
1084
- // #88 item 7 — unified no-runs shape with `doctor --json`: a bare
1085
- // `null` was indistinguishable from any other absent-payload state
1086
- // (a scrap CAN legitimately have a null-ish result elsewhere); an
1087
- // explicit `{status:"no_runs"}` object is unambiguous everywhere.
1088
623
  if (opts.json) {
1089
624
  json({ status: 'no_runs' });
1090
625
  return;
@@ -1092,13 +627,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
1092
627
  console.log(chalk.dim('No runs yet.'));
1093
628
  return;
1094
629
  }
1095
- // --json is honored for BOTH outcomes (success or failure) — an agent
1096
- // parsing `data --errors --json` must always get the flat run object,
1097
- // never prose gated behind a status check. (#71 finding 13)
1098
- // #185 — `docs` rides alongside as a sibling field on the same flat
1099
- // object (never nested under `run`, since there's no wrapper shape
1100
- // here), present only for a live mapped failureKind (see
1101
- // runDocsField's doc comment).
1102
630
  if (opts.json)
1103
631
  return json({ ...pickRun(result.run), ...runDocsField(result.run) });
1104
632
  if (result.run.status === true) {
@@ -1106,25 +634,18 @@ export function attachDataCommand(parent, attachOpts = {}) {
1106
634
  return;
1107
635
  }
1108
636
  console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
637
+ if (isAuthWall(result.run)) {
638
+ maybeSuggestSkillsForAuthWall();
639
+ }
1109
640
  return;
1110
641
  }
1111
- // #70 — --fresh is the explicit opt-in for a LIVE run. This is what the
1112
- // default path used to do silently: GET /api/scraps/load/:id executes a
1113
- // fresh scrap run server-side (requireQuota('scraps','execute') + a
1114
- // distributed run lock — trawl_node modules/scraps/routes/scraps.routes.js),
1115
- // burning execute quota and 429ing if a run is already in flight. A user
1116
- // or agent "just reading data" must never trigger that by accident.
1117
642
  if (opts.fresh) {
1118
- // #91 P0 — same long-run endpoint as `run` (30-250s server-side);
1119
- // the 30s default was aborting it mid-flight.
1120
643
  const loaded = await spin(() => api.get(`/api/scraps/load/${id}`, { timeoutMs: LONG_RUN_TIMEOUT_MS }), {
1121
644
  text: 'Launching a fresh scrap run (consumes execute quota)…',
1122
645
  successText: 'Fresh run complete',
1123
646
  });
1124
647
  const items = loaded?.result?.data;
1125
648
  if (!Array.isArray(items)) {
1126
- // --json always returns an array from `data` — [] is the honest
1127
- // "no items" signal instead of prose breaking JSON parsing. (#71)
1128
649
  if (opts.json) {
1129
650
  json([]);
1130
651
  return;
@@ -1132,18 +653,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
1132
653
  console.log(chalk.dim('No data yet. Run the scrap first.'));
1133
654
  return;
1134
655
  }
1135
- // #93 item 2 — --fresh used to render items without ever checking for a
1136
- // regression, unlike the persisted `data` path below (#88 item 2). The
1137
- // load() response's OWN embedded `scrap.history[0]` can't be trusted for
1138
- // this (see the ScrapLoadResult comment above): it may be the previous
1139
- // run's row, and even the right row never carries the regression flip.
1140
- // `--fresh` runs synchronously to completion server-side though — by the
1141
- // time this call returns, node has already awaited the regression patch
1142
- // — so a fresh GET /api/scraps/:id (the SAME read the persisted path
1143
- // below already trusts) reliably observes the finalized DB state.
1144
- // Best-effort: never fail --fresh's real output over this side check,
1145
- // and never gate on it — the items returned ARE this run's real,
1146
- // synchronously-computed data regardless of what this check finds.
1147
656
  try {
1148
657
  const fresh = await api.get(`/api/scraps/${id}`);
1149
658
  if (fresh.history?.[0]?.statusDetail === 'regression') {
@@ -1151,56 +660,20 @@ export function attachDataCommand(parent, attachOpts = {}) {
1151
660
  }
1152
661
  }
1153
662
  catch {
1154
- // best-effort — the fresh run's items are still valid without this check
1155
663
  }
1156
664
  renderScrapItems(items, opts.json);
1157
665
  return;
1158
666
  }
1159
- // Default: read the last PERSISTED run payload — no quota, no run lock.
1160
- // History.data (trawl_node modules/historys/services/historys.service.js
1161
- // `update()`) is a JSON-stringified clone of the worker result, so it
1162
- // carries the same `.data` items array the live load endpoint returns —
1163
- // it's just pulled from the most recent history row instead of a fresh
1164
- // run. Retention keeps this only for the newest row per (scrap, status)
1165
- // bucket (config.trawl.keepData, default 1); older rows null it out.
1166
- //
1167
- // #86 finding 6 — [] is reserved for a GENUINE zero-item successful run.
1168
- // Every other outcome below is an honest error envelope instead: never
1169
- // run (not_found/4), last run failed (1), or the payload aged out of
1170
- // retention (not_found/4) all used to collapse into the same silent [].
1171
667
  const scrap = await api.get(`/api/scraps/${id}`);
1172
668
  const last = scrap.history?.[0];
1173
669
  if (!last?._id) {
1174
670
  reportDataState(`Scrap ${id} has never run. Run it first (trawl run ${id}) or pass --fresh to launch one now.`, 4, 'not_found', opts.json);
1175
671
  return;
1176
672
  }
1177
- // #88 item 1 — status:null is an IN-FLIGHT run (node persists
1178
- // {status:null, statusDetail:null, inFlight:true} the moment a run
1179
- // starts, and only flips status/statusDetail once it finishes). That is
1180
- // neither "never run" nor "the last run failed" — a caller reading data
1181
- // mid-run needs an honest "wait" signal. Never suggest --fresh here: a
1182
- // run already holds the server-side distributed lock, so --fresh would
1183
- // just 429 against it.
1184
673
  if (last.status === null) {
1185
- // #170 — retrying `trawl data <id>` (unchanged) once the run
1186
- // finishes IS worth it; `--fresh` never is (it would 429 against the
1187
- // lock this very run already holds) — never suggested here.
1188
674
  reportDataState(`Run in progress for ${id} — retry shortly.`, 1, 'in_progress', opts.json, { retryable: true, next: [`trawl data ${id}`] });
1189
675
  return;
1190
676
  }
1191
- // #86 review — node persists status=false for a GENUINE zero-item run
1192
- // too (historys schema: status boolean|null + statusDetail
1193
- // success/error/empty/regression; a zero-item run is status=false +
1194
- // statusDetail='empty', and the embedded history rows from GET
1195
- // /api/scraps/:id include statusDetail via the repository populate
1196
- // select). An 'empty' run is the one case [] is FOR — only a real
1197
- // failure (error/unknown detail) gets the run_failed envelope.
1198
- //
1199
- // #88 item 2 — statusDetail='regression' is ALSO status=false (an async
1200
- // patch flips it after item count dropped vs baseline), but the row's
1201
- // `data` still holds REAL, non-empty items — the write that persisted
1202
- // them succeeded before the regression was even detected. Treating it as
1203
- // run_failed would hide genuine data behind a false negative.
1204
677
  const isEmptyRun = last.status === false && last.statusDetail === 'empty';
1205
678
  const isRegression = last.status === false && last.statusDetail === 'regression';
1206
679
  if (last.status === false && !isEmptyRun && !isRegression) {
@@ -1211,11 +684,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
1211
684
  let items;
1212
685
  if (typeof detail?.data === 'string' && detail.data) {
1213
686
  try {
1214
- // #159 — `detail.data` is the same JSON-string-within-JSON shape
1215
- // (`history.data`) the create --json bug lived in: a raw control
1216
- // character here would otherwise throw and silently report "aged
1217
- // out of retention" below for data that is actually present and
1218
- // recoverable, burning a --fresh re-run + quota for nothing.
1219
687
  items = parseServerJson(detail.data)?.data;
1220
688
  }
1221
689
  catch {
@@ -1224,23 +692,12 @@ export function attachDataCommand(parent, attachOpts = {}) {
1224
692
  }
1225
693
  if (!Array.isArray(items)) {
1226
694
  if (isEmptyRun) {
1227
- // A genuine zero-item run whose payload is '[]' or absent — both are
1228
- // the SAME honest answer: no items, exit 0. Never the retention
1229
- // message (nothing aged out; there was nothing to persist).
1230
695
  renderScrapItems([], opts.json);
1231
696
  return;
1232
697
  }
1233
- // #88 item 2 — a regression row whose payload aged out of retention has
1234
- // nothing left to show either; fall through to the SAME honest
1235
- // aged-out envelope a normal successful row would get (never fabricate
1236
- // items, never silently succeed).
1237
698
  reportDataState(`No persisted data for the last run of ${id} — it aged out of retention. Pass --fresh to launch a new run.`, 4, 'not_found', opts.json);
1238
699
  return;
1239
700
  }
1240
- // #88 item 2 — a regression row's items are REAL (the write succeeded
1241
- // before the async patch flagged the drop) — return them on stdout
1242
- // (exit 0, both modes) with an honest stderr warning pointing at the
1243
- // diagnostic command, instead of hiding genuine data behind run_failed.
1244
701
  if (isRegression) {
1245
702
  console.error(chalk.yellow(`⚠ Item count regressed vs baseline for the last run of ${id} — see: trawl scraps doctor ${id}`));
1246
703
  }
@@ -1248,7 +705,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
1248
705
  });
1249
706
  }
1250
707
  attachDataCommand(scraps, { hidden: true });
1251
- // history — list past runs for a scrap — promoted to a top-level verb (#108)
1252
708
  export function attachHistoryCommand(parent, attachOpts = {}) {
1253
709
  return parent
1254
710
  .command('history <id>', attachOpts)
@@ -1269,8 +725,6 @@ export function attachHistoryCommand(parent, attachOpts = {}) {
1269
725
  time: h.time ?? null,
1270
726
  tier: h.proxyTier ?? null,
1271
727
  failureKind: h.failureKind ?? null,
1272
- // trawl_node#1950 renamed blockType to block.kind; fall back to the
1273
- // deprecated flat field for a prod backend not yet on that tag.
1274
728
  blockType: h.block?.kind ?? h.blockType ?? null,
1275
729
  createdAt: h.createdAt ?? null,
1276
730
  }));
@@ -1282,7 +736,6 @@ export function attachHistoryCommand(parent, attachOpts = {}) {
1282
736
  });
1283
737
  }
1284
738
  attachHistoryCommand(scraps, { hidden: true });
1285
- // run-info — single-run detail by history id — promoted to a top-level verb (#108)
1286
739
  export function attachRunInfoCommand(parent, attachOpts = {}) {
1287
740
  return parent
1288
741
  .command('run-info <hid>', attachOpts)
@@ -1297,36 +750,24 @@ export function attachRunInfoCommand(parent, attachOpts = {}) {
1297
750
  time: h.time ?? null,
1298
751
  tier: h.proxyTier ?? null,
1299
752
  failureKind: h.failureKind ?? null,
1300
- // trawl_node#1950 renamed blockType to block.kind; fall back to the
1301
- // deprecated flat field for a prod backend not yet on that tag.
1302
753
  blockType: h.block?.kind ?? h.blockType ?? null,
1303
754
  errorMessage: h.errorSnapshot?.errorMessage ?? null,
1304
755
  selector: h.errorSnapshot?.selector ?? null,
1305
756
  emptyContext: h.emptyContext ?? null,
1306
757
  createdAt: h.createdAt ?? null,
1307
- // #185 — present only for a live mapped failureKind (see
1308
- // runDocsField's doc comment); omitted, never `docs: undefined`, for
1309
- // every other run so this object's own JSON.stringify never grows an
1310
- // unexpected key on existing consumers.
1311
758
  ...runDocsField(h),
1312
759
  };
1313
760
  if (opts.json) {
1314
761
  json(info);
1315
762
  return;
1316
763
  }
1317
- // The blob is an object; the table needs one line. --json keeps it whole.
1318
764
  table([{ ...info, emptyContext: summarizeEmptyContext(info.emptyContext) }], ['hid', 'status', 'time', 'tier', 'failureKind', 'blockType', 'errorMessage', 'selector', 'emptyContext', 'createdAt']);
1319
- // #184 safety-net — the run just shown carries a live login-wall
1320
- // verdict; if Claude's skills aren't installed either, point at the
1321
- // command that installs them too (never under --json, see the early
1322
- // return above).
1323
765
  if (isAuthWall(h)) {
1324
766
  maybeSuggestSkillsForAuthWall();
1325
767
  }
1326
768
  });
1327
769
  }
1328
770
  attachRunInfoCommand(scraps, { hidden: true });
1329
- // delete
1330
771
  scraps
1331
772
  .command('delete <id>')
1332
773
  .alias('rm')
@@ -1335,16 +776,6 @@ scraps
1335
776
  .option('--json', 'Output as JSON')
1336
777
  .action(async (id, opts) => {
1337
778
  validateObjectId(id);
1338
- // #107 — never blocks on a y/N under --json or a non-TTY invocation
1339
- // (agent/CI subprocess); -f/--force always pre-confirms.
1340
- //
1341
- // #107 review F2 — `message` (the plain-id form) feeds the refusal
1342
- // UsageError, which can land verbatim in the `--json` error envelope;
1343
- // `chalk.bold(id)` is passed ONLY as `promptMessage`, shown solely on
1344
- // the interactive TTY `[y/N]` prompt. Before this split, the styled
1345
- // string was the ONLY message confirmDestructive had, so a non-TTY/
1346
- // --json refusal on `scraps delete X --json` emitted raw ANSI escape
1347
- // bytes inside the JSON string.
1348
779
  const { proceed, blocked } = await confirmDestructive(`Delete scrap ${id}?`, {
1349
780
  force: opts.force,
1350
781
  json: opts.json,
@@ -1363,9 +794,8 @@ scraps
1363
794
  return;
1364
795
  }
1365
796
  await spin(call, { text: 'Deleting…', successText: 'Scrap deleted' });
1366
- confirmNonTTY(`Scrap ${id} deleted`); // #166
797
+ confirmNonTTY(`Scrap ${id} deleted`);
1367
798
  });
1368
- // banner
1369
799
  scraps
1370
800
  .command('banner <id>')
1371
801
  .description('Upload a banner image for a scrap')
@@ -1387,9 +817,6 @@ scraps
1387
817
  jpeg: 'image/jpeg',
1388
818
  webp: 'image/webp',
1389
819
  };
1390
- // #91 P2 — an unsupported/unknown extension silently became image/png
1391
- // (uploading the raw bytes of, say, a .gif or .pdf under an image/png
1392
- // Content-Type — a lie about the actual file's format). Refuse instead.
1393
820
  const mimeType = mimeMap[ext];
1394
821
  if (!mimeType) {
1395
822
  usageError(`Unsupported image type "${ext ? `.${ext}` : filename}" — use png, jpg, or webp.`, { json: opts.json });
@@ -1409,10 +836,8 @@ scraps
1409
836
  text: 'Uploading banner…',
1410
837
  successText: `Banner uploaded for scrap ${chalk.bold(id)}`,
1411
838
  });
1412
- // #166 — plain, unstyled: chalk.bold would emit escape codes into a pipe.
1413
839
  confirmNonTTY(`Banner uploaded for scrap ${id}`);
1414
840
  });
1415
- // watch (stream activities)
1416
841
  scraps
1417
842
  .command('watch <id>')
1418
843
  .description('Stream scrap activities in real-time')
@@ -1421,7 +846,6 @@ scraps
1421
846
  validateObjectId(id);
1422
847
  await watchActivities(id, opts.json);
1423
848
  });
1424
- // trigger — promoted to a top-level verb (#108)
1425
849
  export function attachTriggerCommand(parent, attachOpts = {}) {
1426
850
  return parent
1427
851
  .command('trigger <id>', attachOpts)
@@ -1431,23 +855,9 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
1431
855
  .option('--json', 'Output the raw trigger payload as JSON')
1432
856
  .action(async (id, opts) => {
1433
857
  validateObjectId(id);
1434
- // #91 P1 / #93 item 1 — captured BEFORE triggering so pollRunProgress can
1435
- // tell "the run we just triggered" apart from whatever the last run
1436
- // happened to be. This is the dedup-prone path: `trigger`'s method:'worker'
1437
- // collapses onto an already-pending/running worker job for the same scrap
1438
- // (ScrapJobsService, LIVE_STATUSES) instead of creating a new history row —
1439
- // captureBeforeRunState records that so pollRunProgress can still track it.
1440
858
  const beforeRun = opts.watch ? await captureBeforeRunState(id) : undefined;
1441
- // #50 — default async: the backend (#1313) kicks off the run and returns a
1442
- // 'queued' envelope immediately instead of holding the connection for the
1443
- // whole run. --wait restores the old synchronous round-trip.
1444
859
  const path = opts.wait ? `/api/scraps/worker/${id}` : `/api/scraps/worker/${id}?wait=false`;
1445
- // #91 P0 — the synchronous --wait branch runs the scrap server-side
1446
- // (30-250s), same as `run`; the 30s default was aborting it
1447
- // mid-flight. The async (default) POST returns almost immediately, so it
1448
- // keeps the 30s default.
1449
860
  const call = () => (opts.wait ? api.post(path, undefined, { timeoutMs: LONG_RUN_TIMEOUT_MS }) : api.post(path));
1450
- // #107 — under --json the stdout path stays pure: no spinner channel.
1451
861
  const data = opts.json
1452
862
  ? await call()
1453
863
  : await spin(call, {
@@ -1458,26 +868,16 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
1458
868
  json(data);
1459
869
  }
1460
870
  else {
1461
- /* #160, folded onto the shared helper by #166 — the rationale and the
1462
- * known residual now live once, on `confirmNonTTY` itself, instead of
1463
- * in the one call site that happened to be fixed first. Wording is
1464
- * unchanged on purpose: the trawl-internal QA runbook asserts on the
1465
- * exact "Worker triggered" / "Worker run complete" substrings. */
1466
871
  confirmNonTTY(opts.wait ? `Worker run complete for scrap ${id}` : `Worker triggered for scrap ${id}`);
1467
872
  }
1468
- // #107 — see the matching comment on `run`'s --watch call above (review
1469
- // F1): honest final NDJSON line + exit code under --json, human mode
1470
- // gets the same honest exit code too.
1471
873
  if (opts.watch)
1472
874
  await pollRunProgress(id, beforeRun, { json: opts.json });
1473
875
  });
1474
876
  }
1475
877
  attachTriggerCommand(scraps, { hidden: true });
1476
- // account subcommand group
1477
878
  const account = scraps
1478
879
  .command('account')
1479
880
  .description('Manage scrap account credentials and session');
1480
- // helper: prompt for a value with readline (visible input)
1481
881
  async function promptLine(prompt) {
1482
882
  const { createInterface } = await import('readline');
1483
883
  const rl = createInterface({ input: process.stdin, output: process.stdout });
@@ -1490,7 +890,6 @@ async function promptLine(prompt) {
1490
890
  rl.close();
1491
891
  }
1492
892
  }
1493
- // account set
1494
893
  account
1495
894
  .command('set <id>')
1496
895
  .description('Set credentials for a scrap account')
@@ -1502,13 +901,8 @@ account
1502
901
  let username = opts.username || '';
1503
902
  let password = opts.password || '';
1504
903
  if (opts.password) {
1505
- // #107 — this advisory is stderr-only: stdout must stay pure under
1506
- // --json, and every other advisory in this CLI already follows that
1507
- // rule (warnIfUnconfirmedTier, token.ts's expiry hints, …).
1508
904
  console.error(chalk.yellow('⚠ Passing --password on the command line is insecure and may be stored in shell history.'));
1509
905
  }
1510
- // #107 — never blocks on a readline prompt under --json or a non-TTY
1511
- // invocation (agent/CI subprocess); pass -u/-p instead.
1512
906
  const interactive = isInteractive({ json: opts.json });
1513
907
  if (!username) {
1514
908
  if (!interactive) {
@@ -1543,7 +937,6 @@ account
1543
937
  console.log(chalk.dim(' Credentials: ') + (acc.hasCredentials ? chalk.green('✓ configured') : chalk.dim('not set')));
1544
938
  console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('active') : chalk.dim('none')));
1545
939
  });
1546
- // account delete
1547
940
  account
1548
941
  .command('delete <id>')
1549
942
  .description('Delete account credentials for a scrap')
@@ -1551,10 +944,6 @@ account
1551
944
  .option('--json', 'Output as JSON')
1552
945
  .action(async (id, opts) => {
1553
946
  validateObjectId(id);
1554
- // #107 — never blocks on a y/N under --json or a non-TTY invocation.
1555
- // #107 review F2 — plain-id `message` for the refusal/JSON envelope,
1556
- // styled `promptMessage` for the interactive TTY prompt only (see the
1557
- // matching comment on `scraps delete` above).
1558
947
  const { proceed, blocked } = await confirmDestructive(`Delete account credentials for scrap ${id}?`, { force: opts.force, json: opts.json, promptMessage: `Delete account credentials for scrap ${chalk.bold(id)}?` });
1559
948
  if (blocked)
1560
949
  return;
@@ -1572,9 +961,8 @@ account
1572
961
  text: 'Deleting credentials…',
1573
962
  successText: 'Account credentials deleted',
1574
963
  });
1575
- confirmNonTTY(`Account credentials deleted for scrap ${id}`); // #166
964
+ confirmNonTTY(`Account credentials deleted for scrap ${id}`);
1576
965
  });
1577
- // account clear-session
1578
966
  account
1579
967
  .command('clear-session <id>')
1580
968
  .description('Clear the saved session for a scrap account')
@@ -1591,17 +979,11 @@ account
1591
979
  text: 'Clearing session…',
1592
980
  successText: 'Session cleared',
1593
981
  });
1594
- confirmNonTTY(`Session cleared for scrap ${id}`); // #166
982
+ confirmNonTTY(`Session cleared for scrap ${id}`);
1595
983
  });
1596
- // account session subcommand group
1597
984
  const accountSession = account
1598
985
  .command('session')
1599
986
  .description('Manage scrap account session cookies (flavour B BYO-cookies)');
1600
- /** Structural check only (bare origin format, {name,value} shape) — the
1601
- * server's sessionShape.js does the authoritative validation. No domain
1602
- * scoping here: unlike `capture`'s CDP path, a hand-authored/exported file
1603
- * has no "target URL" to scope against, and the human supplying it already
1604
- * chose what to include. */
1605
987
  function isValidOriginsShape(value) {
1606
988
  return (Array.isArray(value) &&
1607
989
  value.every((o) => o &&
@@ -1610,7 +992,6 @@ function isValidOriginsShape(value) {
1610
992
  Array.isArray(o.localStorage) &&
1611
993
  o.localStorage.every((kv) => kv && typeof kv === 'object' && typeof kv.name === 'string' && typeof kv.value === 'string')));
1612
994
  }
1613
- // account session set
1614
995
  accountSession
1615
996
  .command('set <id>')
1616
997
  .description('Upload a browser session for a scrap — a cookie JSON array, or a { cookies, origins } storageState file (see `session capture`)')
@@ -1618,9 +999,6 @@ accountSession
1618
999
  .option('--json', 'Output as JSON')
1619
1000
  .action(async (id, opts) => {
1620
1001
  validateObjectId(id);
1621
- // A session is a bearer-equivalent secret — refuse to PUT it over
1622
- // plain HTTP (trawl_cli#183 review finding 5). Checked before anything
1623
- // else in this action, including the file read below.
1624
1002
  const secureTransportError = assertSecureTransport(getApiUrl());
1625
1003
  if (secureTransportError) {
1626
1004
  usageError(secureTransportError, { json: opts.json });
@@ -1640,7 +1018,6 @@ accountSession
1640
1018
  cookies = parsed;
1641
1019
  }
1642
1020
  else if (parsed && typeof parsed === 'object' && Array.isArray(parsed.cookies)) {
1643
- // storageState-shaped file — trawl_cli#183's `session capture` output shape.
1644
1021
  const shaped = parsed;
1645
1022
  cookies = shaped.cookies;
1646
1023
  if (shaped.origins !== undefined) {
@@ -1684,32 +1061,6 @@ accountSession
1684
1061
  const acc = data.account;
1685
1062
  console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('✓ active') : chalk.dim('none')));
1686
1063
  });
1687
- /**
1688
- * @desc Map a failed `captureSession()` result onto this CLI's existing
1689
- * exit-code taxonomy (#71/#88/#107). `no_cookies_in_scope` is the one
1690
- * outcome where everything ran correctly but nothing was found to upload —
1691
- * a business-level refusal (exit 1, kind:"refused"), not a usage mistake.
1692
- *
1693
- * `capture_failed` (trawl_cli#183 R4) is a SECOND exception, for the
1694
- * opposite reason: launch + handshake already succeeded — Chrome was
1695
- * reachable over CDP, a page was already open — and the failure is a
1696
- * genuine bug in this CLI (or an unanticipated non-page-scoped failure),
1697
- * never a "cannot run in this environment" problem. Routing it through
1698
- * `usageError`'s exit 2 / "cannot complete in the current environment"
1699
- * framing would misclassify it exactly the way a bare `launch_failed` used
1700
- * to before the R4 fix. Reported instead through `reportError` with a bare
1701
- * `Error` — `classifyError` has no dedicated `kind` for it, so it falls
1702
- * into the generic `kind:"unknown"`/exit 1 bucket, the SAME code an
1703
- * unhandled bug reaching index.ts's own top-level catch would already get.
1704
- * Never `RefusalError` (this is not a business-logic refusal, everything
1705
- * up to this point may have gone fine) and never `UsageError` (nothing
1706
- * about the invocation or the environment was wrong).
1707
- *
1708
- * Every remaining reason (non_interactive, no_chrome, launch_failed, the CDP
1709
- * pipe closing before capture, a corrupt CDP frame tearing it down) means
1710
- * THIS invocation cannot complete in the current environment — exit 2, the
1711
- * same bucket `confirmDestructive`'s own non-interactive refusal uses.
1712
- */
1713
1064
  function reportCaptureFailure(result, opts) {
1714
1065
  if (result.reason === 'no_cookies_in_scope') {
1715
1066
  process.exitCode = reportError(new RefusalError(result.message), { json: opts.json });
@@ -1721,7 +1072,6 @@ function reportCaptureFailure(result, opts) {
1721
1072
  }
1722
1073
  usageError(result.message, { json: opts.json });
1723
1074
  }
1724
- // account session capture — trawl_cli#183
1725
1075
  accountSession
1726
1076
  .command('capture <id>')
1727
1077
  .description('Open a headed Chrome to the scrap\'s target URL, wait for you to log in (2FA included), and capture the session over CDP — cookies AND localStorage. Requires a local interactive terminal with a display; does not work headless, in CI, or over a plain SSH session. You stay authenticated as yourself — Trawl never sees your credentials. Responsibility for lawful use of the captured session stays with you.')
@@ -1729,10 +1079,6 @@ accountSession
1729
1079
  .option('--json', 'Output as JSON (counts only — a session is a bearer secret and is never printed, in any mode)')
1730
1080
  .action(async (id, opts) => {
1731
1081
  validateObjectId(id);
1732
- // A captured session is a bearer-equivalent secret — refuse to PUT it
1733
- // over plain HTTP (trawl_cli#183 review finding 5). Checked before
1734
- // anything else, including the scrap lookup below (which itself
1735
- // already sends the auth token over whatever transport is configured).
1736
1082
  const secureTransportError = assertSecureTransport(getApiUrl());
1737
1083
  if (secureTransportError) {
1738
1084
  usageError(secureTransportError, { json: opts.json });
@@ -1743,14 +1089,6 @@ accountSession
1743
1089
  usageError(`Scrap ${id} has no target URL configured — nothing to open a browser to. A URL can be set with: trawl scraps update ${id} -u <url>`, { json: opts.json });
1744
1090
  return;
1745
1091
  }
1746
- // Validated here, before captureSession ever creates a temp profile:
1747
- // `existsSync` alone doesn't mean `spawn()` can run the file — a
1748
- // non-executable path fails asynchronously deep inside chrome-launch.ts
1749
- // instead. That failure is still caught there (an 'error' listener is
1750
- // the safety net for anything not caught by this pre-flight — a race,
1751
- // a permission change after this check, PUPPETEER_EXECUTABLE_PATH/
1752
- // CHROME_PATH), but a pre-flight check gives a faster, more specific
1753
- // message for the common case: an explicit path the user pointed at.
1754
1092
  if (opts.chrome) {
1755
1093
  const { existsSync, accessSync, constants } = await import('fs');
1756
1094
  if (!existsSync(opts.chrome)) {
@@ -1766,10 +1104,6 @@ accountSession
1766
1104
  }
1767
1105
  }
1768
1106
  else if (process.env.TRAWL_CHROME_PATH) {
1769
- // Same check, same reasoning, for the env-var form of an explicit
1770
- // path — findChrome() gives TRAWL_CHROME_PATH top precedence and
1771
- // would otherwise hand this same non-executable path straight to
1772
- // captureSession.
1773
1107
  const trawlChromePath = process.env.TRAWL_CHROME_PATH;
1774
1108
  const { existsSync, accessSync, constants } = await import('fs');
1775
1109
  if (existsSync(trawlChromePath)) {
@@ -1782,18 +1116,11 @@ accountSession
1782
1116
  }
1783
1117
  }
1784
1118
  }
1785
- // Check the same guard captureSession runs internally BEFORE printing
1786
- // anything about a Chrome window that may never open — otherwise a
1787
- // non-interactive run saw "a window will open" immediately followed by
1788
- // "this cannot work headless" (trawl_cli#183 gap).
1789
1119
  const nonInteractiveReason = checkInteractiveEnvironment();
1790
1120
  if (nonInteractiveReason) {
1791
1121
  reportCaptureFailure({ ok: false, reason: 'non_interactive', message: nonInteractiveReason }, opts);
1792
1122
  return;
1793
1123
  }
1794
- // Progress/prompts are stderr-only, in every mode — stdout under
1795
- // --json must stay a single parseable document (#88/#107's rule,
1796
- // restated for this command by trawl_cli#183's own hard rules).
1797
1124
  console.error(chalk.dim(`Chrome will open at ${scrap.url} for you to log in there (2FA included); capture completes when you press Enter back in this terminal.`));
1798
1125
  const result = await captureSession(scrap.url, opts.chrome ? { findChrome: () => opts.chrome ?? null } : {});
1799
1126
  if (!result.ok) {
@@ -1806,9 +1133,6 @@ accountSession
1806
1133
  });
1807
1134
  if (opts.json) {
1808
1135
  const data = await call();
1809
- // Never the session itself — only what the server echoes back, plus
1810
- // counts (#183 hard rule: no cookie/localStorage VALUE, ever, under
1811
- // --json or otherwise).
1812
1136
  json({ account: data.account, targetDomain: result.targetDomain, capture: result.counts });
1813
1137
  return;
1814
1138
  }
@@ -1825,24 +1149,9 @@ accountSession
1825
1149
  console.log(chalk.dim(' The browser window was closed before Enter — localStorage was not captured (cookies only).'));
1826
1150
  }
1827
1151
  if (result.counts.originsUnreadable > 0) {
1828
- // Impossible to overlook (chalk.yellow + ⚠, not the routine chalk.dim
1829
- // scope-drop note above) — this is a DEGRADED capture: cookies still
1830
- // uploaded, but at least one in-scope origin's localStorage did not.
1831
- // Two distinct causes share this one count: the page threw reading
1832
- // `window.localStorage` (e.g. a SecurityError on partitioned
1833
- // storage), or the read never got a response within
1834
- // LOCALSTORAGE_READ_TIMEOUT_MS (the page's renderer was blocked — a
1835
- // native dialog, a synchronous script, a paused debugger;
1836
- // trawl_cli#183 review finding 3's own pipe-level timeout fix
1837
- // reopened finding 4's exact bug class on this one path, since a
1838
- // timeout does not close the pipe the way a corrupt frame does). The
1839
- // two are not split apart here: both mean the same actionable fact
1840
- // ("re-run once nothing is blocking that tab") and neither one names
1841
- // the page URL/title/exception text — a page controls that content.
1842
1152
  console.log(chalk.yellow(` ⚠ ${result.counts.originsUnreadable} origin(s) could not be captured — threw reading localStorage, or gave no response within ${LOCALSTORAGE_READ_TIMEOUT_MS / 1000}s (a blocked tab). Not the same as being empty; cookies were still captured.`));
1843
1153
  }
1844
1154
  });
1845
- // account status
1846
1155
  account
1847
1156
  .command('status <id>')
1848
1157
  .description('Show account credentials and session status for a scrap')
@@ -1852,8 +1161,6 @@ account
1852
1161
  const data = await spin(() => api.get(`/api/scraps/${id}`), 'Fetching scrap…');
1853
1162
  const acc = data.account;
1854
1163
  if (opts.json) {
1855
- // #86 finding 12 — `json` is already statically imported at the top of
1856
- // this file; the dynamic import here was pure dead weight.
1857
1164
  return json(acc ?? null);
1858
1165
  }
1859
1166
  if (!acc) {
@@ -1882,7 +1189,6 @@ account
1882
1189
  }
1883
1190
  console.log(`${credLine} | ${sessionLine}`);
1884
1191
  });
1885
- // doctor — diagnose last run of a scrap (error, failed selector, block status, page state, autofix)
1886
1192
  scraps
1887
1193
  .command('doctor <id>')
1888
1194
  .description('Diagnose the last run (error, failed selector, block status, page state, autofix outcome)')
@@ -1899,24 +1205,16 @@ scraps
1899
1205
  console.log(chalk.dim('No runs yet.'));
1900
1206
  return;
1901
1207
  }
1902
- // #185 — `docs` sits alongside `run`/`fix` (present only for a live
1903
- // mapped failureKind — see runDocsField's doc comment), never nested
1904
- // inside the allowlisted `run` object.
1905
1208
  if (opts.json)
1906
1209
  return json({ run: pickRun(result.run), fix: pickFix(result.fix), ...runDocsField(result.run) });
1907
1210
  console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
1908
1211
  if (opts.autofix && result.fix) {
1909
1212
  console.log('\n' + formatAutofix(result.fix));
1910
1213
  }
1911
- // #184 safety-net — formatDoctor already prints the login-wall hint
1912
- // (#182) above when this is a live auth wall; if Claude's skills aren't
1913
- // installed either, point at the command that installs them too (never
1914
- // under --json, see the early return above).
1915
1214
  if (isAuthWall(result.run)) {
1916
1215
  maybeSuggestSkillsForAuthWall();
1917
1216
  }
1918
1217
  });
1919
- // autofix — show full auto-fix attempt detail (diff, dry-run, knowledge)
1920
1218
  scraps
1921
1219
  .command('autofix <id>')
1922
1220
  .description('Show the last auto-fix attempt for a scrap (decision, diff, dry-run, knowledge)')
@@ -1925,9 +1223,6 @@ scraps
1925
1223
  validateObjectId(id);
1926
1224
  const result = await fetchRunAndFix(id);
1927
1225
  if (!result) {
1928
- // #88 item 7 — unified no-runs shape with `doctor --json` / `data
1929
- // --errors --json`: a never-run scrap is a distinct, nameable state,
1930
- // not the same bare `null` a run-with-no-fix-attempt returns below.
1931
1226
  if (opts.json) {
1932
1227
  json({ status: 'no_runs' });
1933
1228
  return;
@@ -1936,8 +1231,6 @@ scraps
1936
1231
  return;
1937
1232
  }
1938
1233
  if (!result.fix) {
1939
- // A run DID happen, it just had no autofix attempt — genuinely "no
1940
- // data", unlike the never-run case above.
1941
1234
  if (opts.json) {
1942
1235
  json(null);
1943
1236
  return;
@@ -1949,7 +1242,6 @@ scraps
1949
1242
  return json(result.fix);
1950
1243
  console.log(formatAutofix(result.fix));
1951
1244
  });
1952
- // snapshot — download the captured page HTML (or error-path HTML) for the last run
1953
1245
  scraps
1954
1246
  .command('snapshot <id>')
1955
1247
  .description('Download captured page HTML for the last run of a scrap')
@@ -1961,16 +1253,10 @@ scraps
1961
1253
  const scrap = await api.get(`/api/scraps/${id}`);
1962
1254
  const hid = scrap.history?.[0]?._id;
1963
1255
  if (!hid) {
1964
- // #91 P2 — same no-runs contract as `doctor`/`autofix`/`data --errors`
1965
- // (`{status:"no_runs"}`, exit 0) under --json.
1966
1256
  if (opts.json) {
1967
1257
  json({ status: 'no_runs' });
1968
1258
  return;
1969
1259
  }
1970
- // `-o` explicitly asked for a file to be written. Silently exiting 0
1971
- // with nothing written (and only a dim console line) is indistinguishable
1972
- // from success to a script checking the exit code alone — give it the
1973
- // SAME not_found envelope `scraps data`'s own never-run case uses.
1974
1260
  if (opts.out) {
1975
1261
  reportDataState(`Scrap ${id} has never run — nothing to write to ${opts.out}.`, 4, 'not_found', false);
1976
1262
  return;