@trawlme/cli 3.11.0 → 3.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +5 -2
  2. package/dist/commands/create.d.ts +0 -28
  3. package/dist/commands/create.js +0 -89
  4. package/dist/commands/doctor.d.ts +0 -79
  5. package/dist/commands/doctor.js +1 -187
  6. package/dist/commands/login.js +0 -67
  7. package/dist/commands/ping.d.ts +0 -15
  8. package/dist/commands/ping.js +0 -15
  9. package/dist/commands/scraps.d.ts +0 -120
  10. package/dist/commands/scraps.js +142 -656
  11. package/dist/commands/skills.js +0 -22
  12. package/dist/commands/spec.d.ts +0 -85
  13. package/dist/commands/spec.js +0 -67
  14. package/dist/commands/telemetry.js +0 -4
  15. package/dist/commands/token.js +0 -28
  16. package/dist/commands/upgrade.js +0 -22
  17. package/dist/commands/whoami.d.ts +0 -12
  18. package/dist/commands/whoami.js +0 -6
  19. package/dist/index.d.ts +0 -188
  20. package/dist/index.js +0 -349
  21. package/dist/lib/api.d.ts +0 -78
  22. package/dist/lib/api.js +1 -320
  23. package/dist/lib/cdp-pipe.d.ts +31 -0
  24. package/dist/lib/cdp-pipe.js +141 -0
  25. package/dist/lib/chrome-discovery.d.ts +1 -0
  26. package/dist/lib/chrome-discovery.js +30 -0
  27. package/dist/lib/chrome-launch.d.ts +8 -0
  28. package/dist/lib/chrome-launch.js +53 -0
  29. package/dist/lib/config.d.ts +0 -53
  30. package/dist/lib/config.js +0 -55
  31. package/dist/lib/confirm.d.ts +0 -55
  32. package/dist/lib/confirm.js +0 -47
  33. package/dist/lib/docs.d.ts +0 -123
  34. package/dist/lib/docs.js +0 -169
  35. package/dist/lib/errors.d.ts +0 -134
  36. package/dist/lib/errors.js +0 -151
  37. package/dist/lib/format.d.ts +0 -6
  38. package/dist/lib/format.js +0 -6
  39. package/dist/lib/json.d.ts +0 -35
  40. package/dist/lib/json.js +0 -48
  41. package/dist/lib/jwt.d.ts +0 -7
  42. package/dist/lib/jwt.js +0 -7
  43. package/dist/lib/pinch.d.ts +0 -53
  44. package/dist/lib/pinch.js +6 -112
  45. package/dist/lib/pinchAnimation.d.ts +0 -16
  46. package/dist/lib/pinchAnimation.js +8 -29
  47. package/dist/lib/posthog.d.ts +0 -9
  48. package/dist/lib/posthog.js +0 -23
  49. package/dist/lib/prompt.js +1 -20
  50. package/dist/lib/secure-transport.d.ts +1 -0
  51. package/dist/lib/secure-transport.js +15 -0
  52. package/dist/lib/session-capture-guard.d.ts +6 -0
  53. package/dist/lib/session-capture-guard.js +9 -0
  54. package/dist/lib/session-capture.d.ts +55 -0
  55. package/dist/lib/session-capture.js +319 -0
  56. package/dist/lib/skills.d.ts +0 -175
  57. package/dist/lib/skills.js +1 -216
  58. package/dist/lib/skillsNudge.d.ts +0 -17
  59. package/dist/lib/skillsNudge.js +0 -83
  60. package/dist/lib/spinner.d.ts +0 -39
  61. package/dist/lib/spinner.js +0 -40
  62. package/dist/lib/storage-state.d.ts +55 -0
  63. package/dist/lib/storage-state.js +96 -0
  64. package/dist/lib/tips.d.ts +0 -38
  65. package/dist/lib/tips.js +0 -77
  66. package/dist/lib/updateCheckWorker.js +0 -14
  67. package/dist/lib/updateNotifier.d.ts +0 -17
  68. package/dist/lib/updateNotifier.js +0 -53
  69. package/dist/lib/validate.d.ts +0 -8
  70. package/dist/lib/validate.js +0 -8
  71. package/dist/lib/version.d.ts +0 -12
  72. package/dist/lib/version.js +1 -13
  73. package/package.json +2 -2
@@ -12,58 +12,20 @@ import { formatDoctor, formatAutofix, fetchRunAndFix, pickRun, pickFix, isAuthWa
12
12
  import { renderPinch, pinchEnabled } from '../lib/pinch.js';
13
13
  import { maybeShowReferralTip } from '../lib/tips.js';
14
14
  import { maybeSuggestSkillsForAuthWall } from '../lib/skillsNudge.js';
15
+ import { captureSession, checkInteractiveEnvironment, LOCALSTORAGE_READ_TIMEOUT_MS, } from '../lib/session-capture.js';
16
+ import { assertSecureTransport } from '../lib/secure-transport.js';
15
17
  import { getApiUrl } from '../lib/config.js';
16
18
  import { resolveDocsUrls, resolveFailureKindDocsUrl } from '../lib/docs.js';
17
- /**
18
- * #185 — `{ docs }` (spreadable, empty object when nothing to add) for a
19
- * run's JSON payload (`doctor`/`data --errors`/`run-info`), keyed on
20
- * trawl_node's `failureKind` (NOT errors.ts's unrelated `ErrorEnvelope.kind`
21
- * — see docs.ts's `FAILURE_KIND_DOC_PATHS` doc comment for why those two
22
- * `'auth'`s must never be conflated). Reuses `isAuthWall` — the SAME
23
- * staleness guard `formatDoctor`'s human hint and the #184 skills nudge
24
- * already share (a `failureKind:'auth'` stamp survives a later
25
- * success/regression patch, see `isAuthWall`'s own doc comment) — so a run
26
- * that ultimately succeeded or degraded never carries a stale `docs` link
27
- * either, exactly the invariant `detectWallVendor`/`isAuthWall` already
28
- * enforce for the human-facing surfaces. One call site, one rule, reused by
29
- * all three JSON surfaces below instead of three copies that could drift.
30
- */
31
19
  function runDocsField(run) {
32
20
  if (!isAuthWall(run))
33
21
  return {};
34
- // Whole resolved object (never just `.docsUrl`) so resolveFailureKindDocsUrl
35
- // can gate on rung provenance — see docs.ts's DocsUrls.docsUrlIsDerived.
36
- // This call never supplies `externalDocsUrl`, so `docsUrl` here can only
37
- // ever be rung-2-derived or absent — never a rung-1 server-declared root —
38
- // but the gate stays explicit rather than relying on that as an invariant.
39
22
  const resolved = resolveDocsUrls({ apiBaseUrl: getApiUrl() });
40
23
  const docs = resolveFailureKindDocsUrl(run.failureKind, resolved);
41
24
  return docs ? { docs } : {};
42
25
  }
43
- /**
44
- * Print a usage/validation error consistently: human text to stderr, or a
45
- * machine envelope on stdout under --json (never both — reportError is the
46
- * single formatting path shared with index.ts's central catch, #86 finding
47
- * 9). Sets exit code 2 (usage) — distinct from a business-logic refusal
48
- * (which stays 1) or an unmapped ApiError/NetworkError (handled centrally in
49
- * index.ts). (#71)
50
- */
51
26
  function usageError(message, opts = {}) {
52
27
  process.exitCode = reportError(new UsageError(message), { json: opts.json });
53
28
  }
54
- /**
55
- * #163 — client-side URL validation shared by `create`/`update`, mirroring
56
- * the server's now-required-and-parseable `url` field (trawl_node#1823,
57
- * PR #1828: `z.string().trim().default('')` -> `z.string().trim().url()`).
58
- * Gives a readable message BEFORE the request goes out instead of a raw
59
- * 400/422 from the server. Reuses the exact same `requireUrl` helper
60
- * `trawl create`/`trawl login --url` already validate against (lib/validate.js),
61
- * just routed through this file's own return-style `usageError()` convention
62
- * (never throw — every other validation in this file, tier/limit/page/params/
63
- * status, follows the same pattern) instead of the throw-style those two
64
- * commands use. Returns `undefined` (never throws) on a bad value — the
65
- * caller must check for that and `return` without calling the API.
66
- */
67
29
  function tryValidateUrl(value, opts) {
68
30
  try {
69
31
  return requireUrl(value, '--url');
@@ -76,28 +38,13 @@ function tryValidateUrl(value, opts) {
76
38
  throw err;
77
39
  }
78
40
  }
79
- /**
80
- * One-line rendering of an emptyContext blob for the run-info table. Two
81
- * buckets are actionable and must stay distinct: selectors that matched zero
82
- * nodes (the page changed) and selectors the browser rejected outright
83
- * (`-1`, a bug in the scrap script). Folding the second into "all matched"
84
- * would report a broken selector as healthy. `--json` still emits the full
85
- * object, `page.topClasses` / `topAnchorPaths` included.
86
- */
87
41
  function summarizeEmptyContext(ctx) {
88
42
  if (!ctx || typeof ctx !== 'object')
89
43
  return null;
90
44
  const parts = [];
91
45
  const selectors = ctx.selectors && typeof ctx.selectors === 'object' ? ctx.selectors : {};
92
46
  const names = Object.keys(selectors);
93
- // Selectors may themselves contain a comma (`div, span`), so quote each one
94
- // — a raw join renders one grouping selector as if it were two.
95
47
  const list = (ns) => `${ns.slice(0, 3).map((s) => `\`${s}\``).join(', ')}${ns.length > 3 ? ` (+${ns.length - 3})` : ''}`;
96
- // Four disjoint, exhaustive buckets — every selector lands in exactly one, so
97
- // no count is ever dropped and none is summarised as something it is not.
98
- // `selectors` comes from a Mongoose Mixed field, so a count may be `null` or
99
- // a string: that is `unreadable`, which is neither healthy nor zero-match and
100
- // must be named as its own thing next to the others, not collapsed into them.
101
48
  const isCount = (v) => typeof v === 'number' && Number.isFinite(v);
102
49
  const zero = names.filter((s) => selectors[s] === 0);
103
50
  const invalid = names.filter((s) => selectors[s] === -1);
@@ -111,15 +58,8 @@ function summarizeEmptyContext(ctx) {
111
58
  if (unreadable.length)
112
59
  parts.push(`unreadable: ${list(unreadable)}`);
113
60
  if (matched.length) {
114
- // Named even in a mixed set: "3 of the selectors still work" is the
115
- // difference between a page that moved and a page that vanished.
116
61
  parts.push(mixed ? `${matched.length} matched` : `${matched.length} selector(s), all matched`);
117
62
  }
118
- // Same Mixed-field caution on the title: coerce before calling a string
119
- // method (a numeric title would otherwise crash `run-info` on the very run
120
- // it exists to explain), and test against null/undefined rather than
121
- // truthiness so a falsy-but-real title like `0` still shows. Inner double
122
- // quotes would close the wrapper early and corrupt the line.
123
63
  const title = ctx.page?.title;
124
64
  if (title !== undefined && title !== null && String(title) !== '') {
125
65
  parts.push(`title="${String(title).replace(/"/g, "'")}"`);
@@ -132,25 +72,16 @@ function lastStatus(scrap) {
132
72
  const last = scrap.history?.[0];
133
73
  if (!last)
134
74
  return 'never';
135
- // status:null means a run is IN FLIGHT (node persists {status:null,
136
- // statusDetail:null, inFlight:true} the moment a run starts) — that is NOT
137
- // the same thing as "this scrap has never run" (no history row at all).
138
- // (#88 item 1)
139
75
  if (last.status === null)
140
76
  return 'running';
141
77
  if (last.status === undefined)
142
78
  return 'never';
143
- // #91 LOW — same regression-as-failure bucketing bug as doctor's badge
144
- // (formatDoctor): a regression row's write actually succeeded (item count
145
- // just dropped vs baseline, flagged by an async patch afterward — #88 item
146
- // 2), so it must not collapse into the same 'failure' bucket a genuine
147
- // failed run gets.
148
79
  if (last.status === false && last.statusDetail === 'regression')
149
80
  return 'regression';
150
81
  return last.status === true ? 'success' : 'failure';
151
82
  }
152
83
  function lastRun(scrap) {
153
- return formatDate(scrap.history?.[0]?.createdAt); // #119 — unified DD/MM/YY HH:mm
84
+ return formatDate(scrap.history?.[0]?.createdAt);
154
85
  }
155
86
  function statusIcon(status) {
156
87
  if (status === 'success')
@@ -159,28 +90,10 @@ function statusIcon(status) {
159
90
  return chalk.red('✗');
160
91
  if (status === 'running')
161
92
  return chalk.cyan('↻');
162
- // #91 LOW — distinct amber icon, never the red ✗ a genuine failure gets
163
- // (mirrors doctor's "● regression" badge — see formatDoctor).
164
93
  if (status === 'regression')
165
94
  return chalk.hex('#FFA500')('▼');
166
95
  return chalk.dim('—');
167
96
  }
168
- /**
169
- * #179 — health badge, next to the existing status icon. Two distinct
170
- * sources, deliberately never blended into one derived verdict:
171
- * 1. `consecutiveFailedRuns`/`unhealthySince` — present ONLY when `--unhealthy`
172
- * asked the API to attach them (see attachListCommand). This is the
173
- * live, exact server verdict — shown whenever available, since it is
174
- * strictly more informative than the breadcrumb below.
175
- * 2. `lastCronOutcome === 'skipped_unhealthy'` — always present (a normal
176
- * scrap field, no extra query cost), but a STALE breadcrumb of the last
177
- * scheduled tick, not a live read: never set on a no-cron scrap, lags
178
- * by up to one tick, and flips back to 'ok'/'error' once the recovery
179
- * probe fires. Labeled as what it is ("cron paused"), never upgraded to
180
- * "unhealthy" — that word is reserved for the live verdict above.
181
- * Never re-derives either signal from `history[]` — both come straight off
182
- * the server (#1952 in trawl_node).
183
- */
184
97
  function healthBadge(scrap) {
185
98
  if (typeof scrap.consecutiveFailedRuns === 'number' && scrap.consecutiveFailedRuns > 0) {
186
99
  const n = scrap.consecutiveFailedRuns;
@@ -191,19 +104,6 @@ function healthBadge(scrap) {
191
104
  }
192
105
  return chalk.dim('—');
193
106
  }
194
- /**
195
- * #179 — old-server fallback for `list --unhealthy`, same shape as
196
- * `warnIfUnconfirmedTier` above (#86 finding 4b): an older trawl_node that
197
- * predates #1952 ignores the unknown `minConsecutiveFailures` query param
198
- * entirely and returns the FULL unfiltered list — silently presenting every
199
- * scrap as "unhealthy" is exactly the kind of unconfirmed-outcome lie that
200
- * precedent exists to prevent. A confirming server attaches
201
- * `consecutiveFailedRuns` to EVERY item once the param is set (the
202
- * controller writes `s.consecutiveFailedRuns || 0` unconditionally in that
203
- * branch — see trawl_node scraps.controller.js), so its total absence across
204
- * a non-empty response is the honest tell. Stderr only (safe under --json —
205
- * stdout purity is untouched), never blocks the (unreliable) results below.
206
- */
207
107
  function healthFilterUnconfirmed(data, unhealthyWasRequested) {
208
108
  if (!unhealthyWasRequested || data.length === 0)
209
109
  return false;
@@ -212,35 +112,19 @@ function healthFilterUnconfirmed(data, unhealthyWasRequested) {
212
112
  function warnIfUnconfirmedHealthFilter(data, unhealthyWasRequested) {
213
113
  if (!healthFilterUnconfirmed(data, unhealthyWasRequested))
214
114
  return;
215
- console.error(chalk.yellow('⚠ Server did not confirm the --unhealthy filter (older server, predates trawl_node#1952) — results below are UNFILTERED, not just the unhealthy scraps.'));
115
+ console.error(chalk.yellow('⚠ Server did not confirm the --unhealthy filter (older server version, predates this filter) — results below are UNFILTERED, not just the unhealthy scraps.'));
216
116
  }
217
- /**
218
- * #179 — the machine-readable counterpart to the stderr warning above, in the
219
- * exact shape `withTierUnconfirmed` already uses for the tier case. A --json
220
- * caller has no reliable reason to read stderr, so without this a script sees
221
- * a full unfiltered list and cannot tell it is not the unhealthy set.
222
- * Emitted ONLY when the filter went unconfirmed — never fabricated otherwise.
223
- */
224
117
  function withHealthFilterUnconfirmed(rows, data, unhealthyWasRequested) {
225
118
  if (!healthFilterUnconfirmed(data, unhealthyWasRequested))
226
119
  return rows;
227
120
  return { scraps: rows, _healthFilterUnconfirmed: true };
228
121
  }
229
122
  export const scraps = new Command('scraps').description('Manage scraps');
230
- // shared SSE streaming helper. `asJson` (#107) emits one raw JSON object per
231
- // line (NDJSON) on stdout instead of the human-formatted timestamped text —
232
- // a streaming command still needs a pure-stdout machine mode, just line-
233
- // delimited instead of a single blob (there's no single "final" payload to
234
- // wait for).
235
123
  async function watchActivities(id, asJson) {
236
124
  if (!asJson)
237
125
  console.log(chalk.dim('Streaming activities (Ctrl+C to stop)…\n'));
238
126
  for await (const event of api.stream(`/api/scraps/${id}/activities/stream`)) {
239
127
  try {
240
- // #159 — same tolerant parse as api.ts's response-parsing seam: an SSE
241
- // activity line can carry the same server-side raw-control-character
242
- // quirk as any other API payload, and this feeds `--json` NDJSON
243
- // output too (not just create).
244
128
  const activity = parseServerJson(event);
245
129
  if (asJson) {
246
130
  console.log(JSON.stringify(activity));
@@ -250,34 +134,12 @@ async function watchActivities(id, asJson) {
250
134
  console.log(`${chalk.dim(`[${time}]`)} ${activity.message}`);
251
135
  }
252
136
  catch {
253
- // non-JSON lines are heartbeats or SSE comments — skip silently
254
137
  if (process.env['DEBUG'] && event.trim()) {
255
138
  console.error(chalk.dim(`[SSE] skipping non-JSON: ${event}`));
256
139
  }
257
140
  }
258
141
  }
259
142
  }
260
- /**
261
- * #91 P1 / #93 item 1 — snapshot the current top history entry BEFORE
262
- * triggering a run, so pollRunProgress can later tell "the run we just
263
- * launched" apart from whatever the last run happened to be. Node persists a
264
- * fresh {status:null, inFlight:true} row the moment a run starts (see
265
- * HistorysService.create in trawl_node), so a changed history[0]._id is
266
- * normally an honest "the new run has begun" signal — EXCEPT `trigger`
267
- * (method:'worker') dedups onto an already-pending/running worker job
268
- * (ScrapJobsService, LIVE_STATUSES) instead of creating a new row: the top
269
- * row's `_id` never changes even though this launch IS that run. Recording
270
- * `alreadyInFlight` (status===null at capture time) lets pollRunProgress
271
- * recognize that dedup case too, instead of waiting forever for an `_id`
272
- * that will never arrive (#93 item 1).
273
- *
274
- * #97 — a failed lookup used to fall back to `{alreadyInFlight:false}` with
275
- * no `id`, which LOOKED identical to "scrap has never run" — but is not:
276
- * the scrap may well have prior (possibly terminal) history this GET simply
277
- * never observed. `captured:false` flags that distinction explicitly so
278
- * pollRunProgress can defer trusting a baseline instead of risking a stale
279
- * pre-existing row being reported as the run just launched.
280
- */
281
143
  async function captureBeforeRunState(id) {
282
144
  try {
283
145
  const scrap = await api.get(`/api/scraps/${id}`);
@@ -292,115 +154,14 @@ function sleep(ms) {
292
154
  return new Promise((resolve) => setTimeout(resolve, ms));
293
155
  }
294
156
  const POLL_INTERVAL_MS = 2000;
295
- // Mirrors LONG_RUN_TIMEOUT_MS (#91 item 1) — the server-side worst case this
296
- // polls for is the same one that timeout was sized for.
297
157
  const POLL_TIMEOUT_MS = LONG_RUN_TIMEOUT_MS;
298
- // #107 review F1 — a genuinely unreachable API must not run the watch
299
- // silently for the full 300s timeout with dead air; after this many
300
- // CONSECUTIVE (not cumulative — any successful poll resets the counter)
301
- // failed polls, treat it as a persistent error and surface it immediately
302
- // instead of waiting out the clock. At the default 2s interval this gives up
303
- // after ~10s of continuous failures — comfortably longer than any single
304
- // transient blip, nowhere near the 300s ceiling.
305
158
  const MAX_CONSECUTIVE_POLL_ERRORS = 5;
306
- /**
307
- * #91 P1 — replaces "await the run to completion, THEN open the activities
308
- * SSE stream" (which showed NOTHING: the activities SSE
309
- * (GET /api/scraps/:id/activities/stream) is backed by an in-process
310
- * EventEmitter with no backlog — trawl_node
311
- * modules/activities/services/activities.service.js — so by the time a
312
- * synchronous run has already finished there is nothing left to emit; and
313
- * the async `trigger` default enqueues the run onto a durable job queue a
314
- * SEPARATE cron-consumer pod drains, whose in-process emitter never reaches
315
- * the API pod holding the SSE connection at all).
316
- *
317
- * Instead this polls two REST reads that are BOTH Mongo-backed (not
318
- * in-process), so they work no matter which pod actually executed the run:
319
- * - GET /api/scraps/:id/activities?history=<hid>&limit=20 — the SAME
320
- * activities-list endpoint `doctor`'s fetchRunAndFix already calls
321
- * (src/commands/doctor.ts) — prints each new activity line once the new
322
- * run's history id is known.
323
- * - GET /api/scraps/:id — history[0].status/statusDetail, the SAME
324
- * terminal-status signal `lastStatus()` above already trusts (status:null
325
- * === in flight, #88 item 1) to know when the run is done.
326
- *
327
- * #93 item 1 — dedup-race fix. "Is this the run we're watching?" used to be
328
- * a single check: `history[0]._id !== beforeHistoryId`. That's wrong when
329
- * `before.alreadyInFlight` is true (a `trigger` call deduped onto a worker
330
- * job that was ALREADY pending/running at capture time): the top row IS the
331
- * run we're watching, but its `_id` never changes, so the old guard never
332
- * released and the poll ran the full timeout to a false "Timed out". The run
333
- * we're watching is now EITHER a brand-new id (fresh trigger, the common
334
- * case) OR the same id that was already in-flight (status:null) at capture
335
- * (the dedup case) — a same-id row that was already TERMINAL at capture is
336
- * neither, and must not be latched onto as "done" (it's just the previous
337
- * run, still sitting there until a genuinely new run supersedes it).
338
- *
339
- * #97 — capture-failed fallback. The dedup-race fix above assumes `before`
340
- * is trustworthy. When `captureBeforeRunState`'s own GET threw,
341
- * `before.id` is `undefined` — and that is INDISTINGUISHABLE from "the
342
- * scrap has genuinely never run" (also `id: undefined`), which is exactly
343
- * the case `isNewRun` above is designed to match on the very first row that
344
- * ever appears. So on the very first poll, ANY pre-existing history row —
345
- * even the STALE PREVIOUS run, already terminal — satisfied
346
- * `last._id !== undefined` and got reported as "the run we just launched"
347
- * finishing, when it was really just whatever ran before.
348
- *
349
- * Fix: `before.captured === false` defers trusting a baseline at all.
350
- * Instead of comparing against the (unknown) `beforeId` from the start, the
351
- * FIRST successful poll read is treated as the deferred capture itself —
352
- * exactly what `captureBeforeRunState` would have returned had its GET
353
- * succeeded — and only READS from that point on are compared against it,
354
- * via the exact same `isNewRun` / `isDedupOntoInFlight` logic above. A
355
- * pre-existing terminal row observed on that first read becomes `beforeId`
356
- * (not a match for itself), so it correctly falls into "still the stale
357
- * previous run" below and the poll keeps waiting; a row still in flight
358
- * becomes the `alreadyInFlight` baseline, exactly like a successful capture
359
- * would have recorded. Either way this costs at most one extra poll
360
- * interval, bounded by the same deadline as everything else. The one
361
- * remaining edge case — capture failed AND the scrap never ran before AND
362
- * the triggered run already finished by the very first poll — is genuinely
363
- * undecidable from "stale pre-existing row" with no more information than
364
- * this function has, so it resolves to the same honest timeout rather than
365
- * risk reporting a possibly-wrong outcome (never a lie, at worst a timeout
366
- * telling the caller to check `doctor`).
367
- *
368
- * #107 review F1 — before this fix, `run|trigger --json --watch` was
369
- * outcome-blind: `quiet` suppressed ALL output (including "Run finished:
370
- * failure" and the timeout notice), a transient poll error was caught and
371
- * silently retried FOREVER within the deadline, and the process always
372
- * exited 0 after the poll loop regardless of what the watched run actually
373
- * did — dead air, then a clean exit code, even for a failed or timed-out
374
- * run. An agent scripting this CLI had no way to tell success from failure
375
- * from "we gave up". Fixed by:
376
- * - emitting exactly ONE final NDJSON line on stdout under `--json` once
377
- * the watch reaches ANY of its three exits (terminal status, timeout, or
378
- * a persistent poll error) — `{runId,status}` (+`error` for a poll
379
- * error) — while every intermediate progress line stays suppressed
380
- * (unchanged from before);
381
- * - setting `process.exitCode` non-zero on a genuine run failure, a
382
- * timeout, or a persistent poll error, and `0` on a real success — in
383
- * BOTH `--json` and human `--watch` modes (human mode used to exit 0
384
- * unconditionally, the same bug, just silent instead of dishonest);
385
- * - giving up after `MAX_CONSECUTIVE_POLL_ERRORS` consecutive failed reads
386
- * instead of retrying the same dead endpoint for the full 300s.
387
- */
388
159
  export async function pollRunProgress(id, before, opts = {}) {
389
160
  const intervalMs = opts.intervalMs ?? POLL_INTERVAL_MS;
390
161
  const timeoutMs = opts.timeoutMs ?? POLL_TIMEOUT_MS;
391
- // #107 — `run --json --watch` / `trigger --json --watch` must keep stdout
392
- // pure JSON: none of this function's intermediate progress text is safe to
393
- // print once a caller asked for --json. `asJson` suppresses every
394
- // intermediate console.log below (including the pinch celebration frame)
395
- // while the polling/wait logic itself runs unchanged; the ONE exception is
396
- // the single final outcome line emitted right before each return below.
397
162
  const asJson = opts.json ?? false;
398
163
  let beforeId = before?.id;
399
164
  let beforeAlreadyInFlight = before?.alreadyInFlight ?? false;
400
- // #97 — only an EXPLICIT captured:false (capture's GET actually threw)
401
- // defers the baseline. `before` itself being undefined (callers that skip
402
- // capture entirely) or `captured` being true/absent both mean "trust
403
- // beforeId as given", preserving every existing call site's behavior.
404
165
  let baselineEstablished = before?.captured ?? true;
405
166
  if (!asJson) {
406
167
  console.log(chalk.dim('Live activity streaming has no signal for this run (async/cross-pod) — polling for progress instead…\n'));
@@ -420,11 +181,6 @@ export async function pollRunProgress(id, before, opts = {}) {
420
181
  }
421
182
  catch (err) {
422
183
  consecutivePollErrors++;
423
- // #107 review F1 — a single failed read is still a TRANSIENT blip,
424
- // safe to retry within the deadline (unchanged). Only once reads fail
425
- // this many times IN A ROW is the API treated as genuinely
426
- // unreachable — surface that honestly instead of quietly burning the
427
- // full 300s timeout on a dead endpoint.
428
184
  if (consecutivePollErrors >= MAX_CONSECUTIVE_POLL_ERRORS) {
429
185
  const message = err instanceof Error ? err.message : String(err);
430
186
  if (asJson) {
@@ -441,13 +197,8 @@ export async function pollRunProgress(id, before, opts = {}) {
441
197
  }
442
198
  const last = scrap.history?.[0];
443
199
  if (!last?._id)
444
- continue; // no history row recorded yet
200
+ continue;
445
201
  if (!baselineEstablished) {
446
- // #97 — deferred capture: whatever we see on this first successful
447
- // read becomes the reference point, BEFORE the isNewRun check below
448
- // ever runs against it. This must happen before that check, not
449
- // after, so a pre-existing terminal row is recognized as stale on
450
- // this very same iteration rather than one poll late.
451
202
  beforeId = last._id;
452
203
  beforeAlreadyInFlight = last.status === null;
453
204
  baselineEstablished = true;
@@ -455,10 +206,9 @@ export async function pollRunProgress(id, before, opts = {}) {
455
206
  const isNewRun = last._id !== beforeId;
456
207
  const isDedupOntoInFlight = last._id === beforeId && beforeAlreadyInFlight;
457
208
  if (!isNewRun && !isDedupOntoInFlight)
458
- continue; // still the stale previous run
209
+ continue;
459
210
  try {
460
211
  const activities = await api.get(`/api/scraps/${id}/activities?history=${last._id}&limit=20`);
461
- // Server returns newest-first — print unseen ones oldest-first.
462
212
  for (const a of [...(activities ?? [])].reverse()) {
463
213
  const key = a._id ?? `${a.createdAt}:${a.message}`;
464
214
  if (seen.has(key))
@@ -471,16 +221,9 @@ export async function pollRunProgress(id, before, opts = {}) {
471
221
  }
472
222
  }
473
223
  catch {
474
- // best-effort — terminal detection below still works without activity lines
475
224
  }
476
225
  if (last.status !== null) {
477
226
  const outcome = last.statusDetail ?? (last.status ? 'success' : 'failure');
478
- // #107 review F1 — the exit code, in BOTH modes: a regression row's
479
- // WRITE actually succeeded (see lastStatus()'s own #91 comment above,
480
- // and `scraps data`'s isRegression branch) and an 'empty' row is a
481
- // genuine zero-item success — neither is a real failure. Only
482
- // status:false with neither of those details (e.g. 'error', or no
483
- // detail at all) is a genuine failed run.
484
227
  const isGenuineFailure = last.status === false && outcome !== 'empty' && outcome !== 'regression';
485
228
  if (asJson) {
486
229
  const payload = { runId: last._id, status: outcome };
@@ -488,10 +231,6 @@ export async function pollRunProgress(id, before, opts = {}) {
488
231
  }
489
232
  else {
490
233
  console.log(chalk.dim(`Run finished: ${outcome}`));
491
- // Pinch celebrates a clean run finish (#94) — mirrors doctor.ts's own
492
- // `status === true` success definition (regardless of statusDetail),
493
- // never for a failed/regression run. Guarded by pinchEnabled()
494
- // (NO_COLOR/non-TTY) and never under --json (stdout must stay pure).
495
234
  if (last.status === true && pinchEnabled()) {
496
235
  console.log(renderPinch('celebrating'));
497
236
  }
@@ -500,9 +239,6 @@ export async function pollRunProgress(id, before, opts = {}) {
500
239
  return;
501
240
  }
502
241
  }
503
- // #107 review F1 — timeout: honest non-zero exit in both modes, plus the
504
- // machine-readable final line under --json (never silently exit 0 after a
505
- // watch that never actually confirmed what happened).
506
242
  if (asJson) {
507
243
  const outcome = { runId: beforeId, status: 'timeout' };
508
244
  console.log(JSON.stringify(outcome));
@@ -512,20 +248,8 @@ export async function pollRunProgress(id, before, opts = {}) {
512
248
  }
513
249
  process.exitCode = 1;
514
250
  }
515
- // #149 item 3 — same shared-enum shape as VALID_TIERS below (~line 603):
516
- // keeps `--status`'s allowed values and its error message in one place,
517
- // mirrored from lastStatus()'s own return type so the two can never drift.
518
251
  const VALID_LAST_STATUSES = ['success', 'failure', 'never', 'running', 'regression'];
519
- // #179 — mirrors `UNHEALTHY_STREAK_LENGTH` in trawl_node
520
- // modules/scraps/helpers/scrapCronHealth.js. `--unhealthy` is a boolean
521
- // convenience flag for that ONE product-defined threshold (also what the
522
- // server's own cron-skip gate and email use), not a general numeric filter —
523
- // the API's `minConsecutiveFailures` param accepts any threshold, but this
524
- // CLI only ever asks for the one value the rest of the product means by
525
- // "unhealthy". If trawl_node's threshold ever changes, update this constant
526
- // to match.
527
252
  const UNHEALTHY_STREAK_LENGTH = 3;
528
- // list — promoted to a top-level verb (#108, see the AttachOptions comment above)
529
253
  export function attachListCommand(parent, attachOpts = {}) {
530
254
  return parent
531
255
  .command('list', attachOpts)
@@ -533,33 +257,14 @@ export function attachListCommand(parent, attachOpts = {}) {
533
257
  .description('List all scraps')
534
258
  .option('--json', 'Output as JSON')
535
259
  .option('--status <status>', 'Filter by last run status (success|failure|never|running|regression)')
536
- // #88 item 8 — no custom parser here (unlike the old `(v) => parseInt(v,
537
- // 10)`): a bad value like "abc" used to silently become NaN, which then
538
- // sailed straight through `Number.isInteger`-less checks and into
539
- // `.slice(0, NaN)` (silently truncates to 0 rows) or `?page=NaN` (silently
540
- // sent to the server) — never a usage error. Keeping the raw string here
541
- // lets the validation below mirror `history`'s own --limit check exactly
542
- // (~line 660) and report the actual bad input in the error message.
543
260
  .option('--limit <n>', 'Show only the first N results')
544
261
  .option('--page <n>', 'Fetch a specific page only (50 per page, no auto-pagination)')
545
- // #179 — backed by the server's `minConsecutiveFailures` filter
546
- // (trawl_node#1952): restricts the result set server-side to scraps
547
- // whose CURRENT consecutive-failure streak is >= UNHEALTHY_STREAK_LENGTH,
548
- // and annotates each returned scrap with the exact `consecutiveFailedRuns`/
549
- // `unhealthySince` the aggregation computed. Deliberately never derived
550
- // client-side from `history[]` — that would drift from what the web
551
- // dashboard shows (comes-io/trawl_vue#1299 reads the identical filter).
552
262
  .option('--unhealthy', `Only show scraps failing their last ${UNHEALTHY_STREAK_LENGTH}+ runs (server-computed)`)
553
263
  .action(async (opts, cmd) => {
554
- // #149 item 3 — validate --status against its enum the same way
555
- // --tier already validates (usageError + return, checked first, before
556
- // any other guard/async work — mirrors `create`/`update`'s --tier
557
- // check below).
558
264
  if (opts.status !== undefined && !VALID_LAST_STATUSES.includes(opts.status)) {
559
265
  usageError(`Invalid --status "${opts.status}" (allowed: ${VALID_LAST_STATUSES.join(', ')})`, { json: opts.json });
560
266
  return;
561
267
  }
562
- // Guard: --limit and --page are mutually exclusive
563
268
  if (opts.limit !== undefined && opts.page !== undefined) {
564
269
  usageError('--limit and --page are mutually exclusive. Use one or the other.', { json: opts.json });
565
270
  return;
@@ -580,18 +285,13 @@ export function attachListCommand(parent, attachOpts = {}) {
580
285
  return;
581
286
  }
582
287
  }
583
- // #179 — same param on every request this action can issue (both
584
- // branches below), so a filtered result stays filtered across the
585
- // fetch-all pagination loop, not just its first page.
586
288
  const healthFilter = opts.unhealthy ? `&minConsecutiveFailures=${UNHEALTHY_STREAK_LENGTH}` : '';
587
289
  let data;
588
290
  try {
589
291
  data = await spin(async () => {
590
292
  if (page !== undefined) {
591
- // Single-page mode: explicit page requested, no loop
592
293
  return api.get(`/api/scraps?perPage=50&page=${page}${healthFilter}`);
593
294
  }
594
- // Fetch-all mode: paginate until a page returns < 200 items
595
295
  const perPage = 200;
596
296
  let result = [];
597
297
  let pageNum = 1;
@@ -606,16 +306,6 @@ export function attachListCommand(parent, attachOpts = {}) {
606
306
  }, 'Fetching scraps…');
607
307
  }
608
308
  catch (err) {
609
- // Spinner already failed by oraPromise — report with the scrap-specific
610
- // prefix kept, but route through the shared classifier so exit code +
611
- // --json envelope stay consistent with every other command. (#71)
612
- //
613
- // This bespoke catch (kept for the "Failed to fetch scraps:" prefix,
614
- // which the shared reportError() can't add) used to silently swallow
615
- // --debug: unlike the central index.ts catch, it never printed the raw
616
- // stack trace. optsWithGlobals() reads --debug off the ROOT command
617
- // (this leaf has no --debug of its own) so it can honor the flag
618
- // locally instead. (#86 finding 9)
619
309
  const isDebug = Boolean(cmd.optsWithGlobals().debug || process.env['DEBUG']);
620
310
  const { exitCode, envelope } = classifyError(err);
621
311
  const message = `Failed to fetch scraps: ${envelope.message}`;
@@ -630,9 +320,6 @@ export function attachListCommand(parent, attachOpts = {}) {
630
320
  process.exitCode = exitCode;
631
321
  return;
632
322
  }
633
- // #179 — check the RAW response, before --status narrows it further,
634
- // so a genuinely empty (all-healthy) result is never mistaken for an
635
- // unconfirmed filter.
636
323
  warnIfUnconfirmedHealthFilter(data, opts.unhealthy);
637
324
  if (opts.status)
638
325
  data = data.filter((s) => lastStatus(s) === opts.status);
@@ -645,21 +332,17 @@ export function attachListCommand(parent, attachOpts = {}) {
645
332
  title: s.title || '(untitled)',
646
333
  cron: s.cron || '—',
647
334
  status: statusIcon(lastStatus(s)),
648
- // #179 — next to the existing status badge, per the issue's own
649
- // wording (see healthBadge() above for the two source signals).
650
335
  health: healthBadge(s),
651
336
  'last run': lastRun(s),
652
337
  updated: formatDate(s.updatedAt),
653
338
  }));
654
339
  table(tableRows, ['id', 'title', 'cron', 'status', 'health', 'last run', 'updated']);
655
- // Print footer when --limit truncates
656
340
  if (limit !== undefined && rows.length < totalMatched) {
657
341
  console.log(chalk.dim(`Showing ${rows.length} of ${totalMatched} — omit --limit to see all`));
658
342
  }
659
343
  });
660
344
  }
661
345
  attachListCommand(scraps, { hidden: true });
662
- // get — promoted to a top-level verb (#108)
663
346
  export function attachGetCommand(parent, attachOpts = {}) {
664
347
  return parent
665
348
  .command('get <id>', attachOpts)
@@ -674,9 +357,6 @@ export function attachGetCommand(parent, attachOpts = {}) {
674
357
  console.log(chalk.dim(` ID: `) + data._id);
675
358
  console.log(chalk.dim(` Cron: `) + (data.cron || '—'));
676
359
  console.log(chalk.dim(` Status: `) + statusIcon(lastStatus(data)));
677
- // #179 — same badge `list` renders (see healthBadge() above); `get`
678
- // never sends `minConsecutiveFailures`, so this always reads the
679
- // free `lastCronOutcome` breadcrumb, never the live per-run count.
680
360
  console.log(chalk.dim(` Health: `) + healthBadge(data));
681
361
  console.log(chalk.dim(` Last run: `) + lastRun(data));
682
362
  console.log(chalk.dim(` Updated: `) + formatDate(data.updatedAt));
@@ -684,14 +364,6 @@ export function attachGetCommand(parent, attachOpts = {}) {
684
364
  }
685
365
  attachGetCommand(scraps, { hidden: true });
686
366
  const VALID_TIERS = ['tier0', 'tier1', 'tier2', 'tier3', 'tier4'];
687
- /**
688
- * #86 findings 4/5 — shared honest-tier renderer for `create --tier` and
689
- * `update --tier/--force-tier`. Reads the typed `_tierOverride` echoed back
690
- * by the server (#1559) so both commands show the SAME truth: a refusal, an
691
- * allowed ceiling raise (+ spend warning), or a silently clamped proxyTier.
692
- * Human-mode output only — a --json caller gets the same truth for free from
693
- * the full scrap object (which already includes `_tierOverride`).
694
- */
695
367
  function renderTierOverrideHuman(data) {
696
368
  const ov = data._tierOverride;
697
369
  if (!ov)
@@ -716,64 +388,25 @@ function renderTierOverrideHuman(data) {
716
388
  }
717
389
  }
718
390
  }
719
- /**
720
- * #86 finding 4b — old-server fallback. When a tier was requested but the
721
- * response carries no `_tierOverride` at all, the server is too old to
722
- * confirm what actually got applied. Echoing the REQUESTED value as if it
723
- * were the outcome is exactly the silent-clamp lie #1559 fixed — warn
724
- * instead, on stderr (safe under --json too; stdout purity is untouched).
725
- */
726
391
  function warnIfUnconfirmedTier(data, tierWasRequested, id) {
727
392
  if (data._tierOverride || !tierWasRequested)
728
393
  return;
729
394
  console.error(chalk.yellow(` ⚠ Server did not confirm the tier change (older server) — verify with: trawl get ${id}`));
730
395
  }
731
- /**
732
- * #88 item 3 — the --json machine-readable counterpart to
733
- * warnIfUnconfirmedTier's stderr warning above. A --json caller (an agent
734
- * scripting this CLI) has no reliable reason to read stderr — that channel
735
- * is advisory-only everywhere else in this CLI, and stdout must stay the
736
- * sole payload. Without this, the ONLY signal that the server never
737
- * confirmed the tier change was a string on stderr, invisible to any --json
738
- * consumer parsing stdout alone. Adds `_tierUnconfirmed: true` to the
739
- * emitted object under EXACTLY the same condition warnIfUnconfirmedTier
740
- * warns on (tier requested, response carries no `_tierOverride`) — never
741
- * fabricated, never present otherwise.
742
- */
743
396
  function withTierUnconfirmed(data, tierWasRequested) {
744
397
  if (data._tierOverride || !tierWasRequested)
745
398
  return data;
746
399
  return { ...data, _tierUnconfirmed: true };
747
400
  }
748
- /** #86 finding 5 — the standard error envelope for a refused tier override,
749
- * routed through the same reportError() central formatting path used
750
- * everywhere else (exit 1: a business-logic refusal, not a usage error).
751
- *
752
- * #107 review F3 — uses `RefusalError` (kind:"refused"), not a bare `Error`
753
- * (which fell through classifyError's default `kind:"unknown"` bucket,
754
- * indistinguishable from a generic crash even though the README sells `kind`
755
- * as the machine discriminant an agent branches on).
756
- */
757
401
  function reportTierRefusal(data, wantsJson) {
758
402
  const ov = data._tierOverride;
759
403
  const message = `Tier ceiling override refused: ${ov?.reason ?? 'unknown'} (requested ${ov?.requestedMaxTier ?? '—'}; kept the registry cap)`;
760
404
  return reportError(new RefusalError(message), { json: wantsJson });
761
405
  }
762
- // create
763
406
  scraps
764
407
  .command('create')
765
408
  .description('Create a new scrap')
766
409
  .requiredOption('-t, --title <title>', 'Scrap title')
767
- // #170 review F6 — promoted from a plain `.option()` to `.requiredOption()`:
768
- // this is unconditionally required (no positional alternative exists on
769
- // THIS command, unlike the top-level `trawl create <url>`, which accepts
770
- // `[url]` OR `--url` — deliberately left untouched, that either/or needs a
771
- // human product decision, tracked separately). Before this, the flag's
772
- // description carried the only "Required." signal, in English prose — the
773
- // exact thing docs/agent-quickstart.md tells an agent not to parse.
774
- // Commander itself now enforces it, so `trawl spec --json`'s published
775
- // `mandatory:true` is honest instead of a second, driftable copy of this
776
- // same fact.
777
410
  .requiredOption('-u, --url <url>', 'Target URL — the site the scrap targets, not necessarily the exact URL fetched at run time (a request script can compute that, e.g. a templated search query).')
778
411
  .option('-r, --request <request>', 'Request/query')
779
412
  .option('-d, --description <text>', 'Scrap description')
@@ -784,10 +417,6 @@ scraps
784
417
  usageError(`Invalid --tier "${opts.tier}" (allowed: ${VALID_TIERS.join(', ')})`, { json: opts.json });
785
418
  return;
786
419
  }
787
- // #163 — the API now requires a parseable `url` on create (trawl_node
788
- // #1823/#1828); validate client-side BEFORE the request goes out so a
789
- // missing/unparseable value surfaces as a readable usage error instead
790
- // of a raw server 400/422.
791
420
  const url = tryValidateUrl(opts.url, opts);
792
421
  if (url === undefined)
793
422
  return;
@@ -798,9 +427,6 @@ scraps
798
427
  ...(opts.description !== undefined && { description: opts.description }),
799
428
  ...(opts.tier !== undefined && { proxyTier: opts.tier }),
800
429
  }), { text: 'Creating scrap…', successText: (d) => `Scrap created: ${chalk.bold(d._id)}` });
801
- // #86 finding 4a — read _tierOverride back from the POST response and
802
- // render it exactly like `update` does; never echo the requested tier as
803
- // if it were applied when an older server doesn't confirm it.
804
430
  const tierWasRequested = opts.tier !== undefined;
805
431
  warnIfUnconfirmedTier(data, tierWasRequested, data._id);
806
432
  const refused = Boolean(data._tierOverride?.refused);
@@ -817,7 +443,6 @@ scraps
817
443
  if (refused)
818
444
  process.exitCode = 1;
819
445
  });
820
- // update
821
446
  scraps
822
447
  .command('update <id>')
823
448
  .description('Update an existing scrap')
@@ -849,9 +474,6 @@ scraps
849
474
  const body = {};
850
475
  if (opts.title !== undefined)
851
476
  body.title = opts.title;
852
- // #163 — omitting --url stays a no-op (the API keeps that allowed on
853
- // update), but a value that IS supplied must still be a parseable URL —
854
- // validate it client-side before the request goes out, same as create.
855
477
  if (opts.url !== undefined) {
856
478
  const url = tryValidateUrl(opts.url, opts);
857
479
  if (url === undefined)
@@ -900,7 +522,6 @@ scraps
900
522
  if (opts.tier !== undefined)
901
523
  body.proxyTier = opts.tier;
902
524
  if (opts.forceTier !== undefined) {
903
- // Raise the ceiling; also start the run at that tier unless --tier says otherwise.
904
525
  body.proxyMaxTier = opts.forceTier;
905
526
  if (opts.tier === undefined)
906
527
  body.proxyTier = opts.forceTier;
@@ -917,10 +538,6 @@ scraps
917
538
  text: 'Updating scrap…',
918
539
  successText: (d) => `Scrap updated: ${chalk.bold(d._id)}`,
919
540
  });
920
- // #1559 / #86 findings 4b/5 — surface the effective tier + clamp/refuse
921
- // reason (fixes the silent-clamp: the server may persist a lower tier
922
- // than requested), and NEVER echo the requested value as applied when
923
- // the server doesn't confirm it (old-server fallback below).
924
541
  const tierWasRequested = opts.tier !== undefined || opts.forceTier !== undefined;
925
542
  warnIfUnconfirmedTier(data, tierWasRequested, id);
926
543
  const refused = Boolean(data._tierOverride?.refused);
@@ -934,10 +551,6 @@ scraps
934
551
  }
935
552
  const shown = data;
936
553
  for (const key of Object.keys(body)) {
937
- // Tier keys are rendered exclusively by renderTierOverrideHuman /
938
- // warnIfUnconfirmedTier above — never echo them here, whether or not
939
- // _tierOverride came back (an old-server echo of the REQUESTED value
940
- // is exactly the silent-clamp lie #1559 fixed).
941
554
  if (key === 'proxyTier' || key === 'proxyMaxTier')
942
555
  continue;
943
556
  const src = key in shown ? shown[key] : body[key];
@@ -947,7 +560,6 @@ scraps
947
560
  if (refused)
948
561
  process.exitCode = 1;
949
562
  });
950
- // run — promoted to a top-level verb (#108)
951
563
  export function attachRunCommand(parent, attachOpts = {}) {
952
564
  return parent
953
565
  .command('run <id>', attachOpts)
@@ -956,66 +568,25 @@ export function attachRunCommand(parent, attachOpts = {}) {
956
568
  .option('--json', 'Output the raw launch payload as JSON')
957
569
  .action(async (id, opts) => {
958
570
  validateObjectId(id);
959
- // #91 P1 / #93 item 1 — captured BEFORE launching so pollRunProgress can
960
- // tell "the run that's about to finish" apart from whatever the last run
961
- // happened to be (including a dedup onto an already-in-flight run).
962
571
  const beforeRun = opts.watch ? await captureBeforeRunState(id) : undefined;
963
- // #91 P0 — GET /api/scraps/load/:id runs the scrap synchronously
964
- // server-side (30-250s); the 30s default was aborting it mid-flight.
965
572
  const call = () => api.get(`/api/scraps/load/${id}`, { timeoutMs: LONG_RUN_TIMEOUT_MS });
966
- // #107 — under --json the stdout path stays pure: no spinner channel at
967
- // all, mirroring `trawl create`'s own --json handling.
968
573
  const data = opts.json
969
574
  ? await call()
970
575
  : await spin(call, { text: 'Launching scrap…', successText: 'Scrap launched' });
971
576
  if (opts.json)
972
577
  json(data);
973
578
  else if (!opts.watch) {
974
- /* #166 — see confirmNonTTY. Two deliberate choices here:
975
- *
976
- * "run complete", not "launched": `run` is the SYNCHRONOUS verb —
977
- * GET /api/scraps/load/:id executes the scrap server-side (30-250s)
978
- * and returns the finished result, so by the time this prints the run
979
- * is over. The spinner's successText says "Scrap launched", but that
980
- * is stderr chrome for a human watching it happen; THIS line is the
981
- * machine surface, where a script reading "launched" would poll for a
982
- * completion that already happened. `trigger` already draws the same
983
- * distinction ("Worker triggered" async vs "Worker run complete" under
984
- * --wait) — this matches that vocabulary.
985
- *
986
- * `--watch` is excluded because pollRunProgress below does its own
987
- * reporting; without that guard a watched run would print this line
988
- * and then narrate the same run. */
989
579
  confirmNonTTY(`Scrap ${id} run complete`);
990
580
  }
991
581
  if (opts.watch) {
992
- // #107 — under --json, pollRunProgress suppresses its own intermediate
993
- // console.log calls and instead emits exactly ONE final NDJSON outcome
994
- // line (+ sets process.exitCode honestly) once the watch reaches a
995
- // terminal status, a timeout, or a persistent poll error (review F1) —
996
- // `run --json --watch` never again exits 0 after dead air regardless
997
- // of what the watched run actually did.
998
582
  await pollRunProgress(id, beforeRun, { json: opts.json });
999
583
  }
1000
- // #151 — throttled post-run referral tip. Reached only when the
1001
- // launch call above didn't throw (a hard failure rejects it, unwinding
1002
- // straight to index.ts's central catch before this line ever runs)
1003
- // — and, when --watch polled to a terminal state, only when
1004
- // pollRunProgress didn't flag a genuine run failure/timeout/poll-error
1005
- // via a non-zero process.exitCode (#107 review F1). See lib/tips.ts
1006
- // for the throttle/TTY/json/opt-out/server-flag gating itself. #153 —
1007
- // now async (a due tip fetches the server's userFacing flag before
1008
- // printing), so this must be awaited: otherwise commander's
1009
- // parseAsync-driven action would resolve before the tip's own
1010
- // await settles, racing the process's natural exit.
1011
584
  const runFailed = typeof process.exitCode === 'number' && process.exitCode !== 0;
1012
585
  if (!runFailed)
1013
586
  await maybeShowReferralTip({ json: opts.json });
1014
587
  });
1015
588
  }
1016
589
  attachRunCommand(scraps, { hidden: true });
1017
- // #70 — render an items array either as a table summary or --json. Shared by
1018
- // both the default (persisted read) and --fresh (live execute) paths of `data`.
1019
590
  function renderScrapItems(items, asJson) {
1020
591
  if (asJson) {
1021
592
  json(items);
@@ -1028,43 +599,15 @@ function renderScrapItems(items, asJson) {
1028
599
  }
1029
600
  console.log(chalk.dim(' Use --json for full output.'));
1030
601
  }
1031
- /**
1032
- * #86 finding 6 — `data`'s honest empty-vs-error distinction, for both --json
1033
- * and human prose. Before this, EVERY non-array outcome (never run, last run
1034
- * failed, or the payload aged out of retention) collapsed to the SAME `[]` /
1035
- * "No data yet." — indistinguishable from a genuine zero-item successful run.
1036
- * That's a lie under --json: an agent can't tell "nothing to show" from "go
1037
- * look at what actually happened". Mirrors reportError's dual json/human
1038
- * shape (human line to stderr always; --json ALSO gets a machine envelope on
1039
- * stdout) with a caller-chosen exit code + kind, since these states are not
1040
- * all "usage" (2) — never-run / aged-out-of-retention are not_found (4), a
1041
- * failed last run is a business-logic failure (1).
1042
- */
1043
- /**
1044
- * #170 — `retryable`/`next` follow the SAME frozen `RETRY_POLICY` map
1045
- * `classifyError` reads (errors.ts), keyed by `kind` — this path never
1046
- * builds its own copy. `retryOverride` is for the one case here where the
1047
- * kind alone is genuinely wrong: `in_progress`'s caller passes
1048
- * `{retryable:true, next:['trawl data <id>']}` since retrying `data` (not
1049
- * `--fresh`, which would 429 against the lock already held) IS worth doing
1050
- * once the run finishes. Every other kind here (`not_found`, `run_failed`)
1051
- * isn't in `RETRY_POLICY` at all — `retryFieldsFor` is total, so those fall
1052
- * back to its conservative "not retryable, nothing to suggest" default.
1053
- */
1054
602
  function reportDataState(message, exitCode, kind, wantsJson, retryOverride) {
1055
603
  console.error(chalk.red(`✗ ${message}`));
1056
604
  if (wantsJson) {
1057
605
  const { retryable, next } = retryOverride ?? retryFieldsFor(kind);
1058
- // #170 review F3 — typed as ErrorEnvelope so the compiler enforces
1059
- // `retryable` here too (this was one of two emitters a mutation test
1060
- // proved `tsc --noEmit` never actually guarded before this annotation —
1061
- // the untyped literal let `retryable` be omitted silently).
1062
606
  const envelope = { message, kind, retryable, ...(next ? { next } : {}) };
1063
607
  console.log(JSON.stringify({ error: envelope }));
1064
608
  }
1065
609
  process.exitCode = exitCode;
1066
610
  }
1067
- // data — promoted to a top-level verb (#108)
1068
611
  export function attachDataCommand(parent, attachOpts = {}) {
1069
612
  return parent
1070
613
  .command('data <id>', attachOpts)
@@ -1074,15 +617,9 @@ export function attachDataCommand(parent, attachOpts = {}) {
1074
617
  .option('--fresh', 'Launch a fresh run instead of reading the last persisted payload (consumes execute quota, same as `run`)')
1075
618
  .action(async (id, opts) => {
1076
619
  validateObjectId(id);
1077
- // --errors: fetch full run + fix detail via fetchRunAndFix (DRY with doctor).
1078
- // Read-only: GET /api/scraps/:id + GET /api/historys/:hid, no quota, no run lock.
1079
620
  if (opts.errors) {
1080
621
  const result = await fetchRunAndFix(id);
1081
622
  if (!result) {
1082
- // #88 item 7 — unified no-runs shape with `doctor --json`: a bare
1083
- // `null` was indistinguishable from any other absent-payload state
1084
- // (a scrap CAN legitimately have a null-ish result elsewhere); an
1085
- // explicit `{status:"no_runs"}` object is unambiguous everywhere.
1086
623
  if (opts.json) {
1087
624
  json({ status: 'no_runs' });
1088
625
  return;
@@ -1090,13 +627,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
1090
627
  console.log(chalk.dim('No runs yet.'));
1091
628
  return;
1092
629
  }
1093
- // --json is honored for BOTH outcomes (success or failure) — an agent
1094
- // parsing `data --errors --json` must always get the flat run object,
1095
- // never prose gated behind a status check. (#71 finding 13)
1096
- // #185 — `docs` rides alongside as a sibling field on the same flat
1097
- // object (never nested under `run`, since there's no wrapper shape
1098
- // here), present only for a live mapped failureKind (see
1099
- // runDocsField's doc comment).
1100
630
  if (opts.json)
1101
631
  return json({ ...pickRun(result.run), ...runDocsField(result.run) });
1102
632
  if (result.run.status === true) {
@@ -1104,25 +634,18 @@ export function attachDataCommand(parent, attachOpts = {}) {
1104
634
  return;
1105
635
  }
1106
636
  console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
637
+ if (isAuthWall(result.run)) {
638
+ maybeSuggestSkillsForAuthWall();
639
+ }
1107
640
  return;
1108
641
  }
1109
- // #70 — --fresh is the explicit opt-in for a LIVE run. This is what the
1110
- // default path used to do silently: GET /api/scraps/load/:id executes a
1111
- // fresh scrap run server-side (requireQuota('scraps','execute') + a
1112
- // distributed run lock — trawl_node modules/scraps/routes/scraps.routes.js),
1113
- // burning execute quota and 429ing if a run is already in flight. A user
1114
- // or agent "just reading data" must never trigger that by accident.
1115
642
  if (opts.fresh) {
1116
- // #91 P0 — same long-run endpoint as `run` (30-250s server-side);
1117
- // the 30s default was aborting it mid-flight.
1118
643
  const loaded = await spin(() => api.get(`/api/scraps/load/${id}`, { timeoutMs: LONG_RUN_TIMEOUT_MS }), {
1119
644
  text: 'Launching a fresh scrap run (consumes execute quota)…',
1120
645
  successText: 'Fresh run complete',
1121
646
  });
1122
647
  const items = loaded?.result?.data;
1123
648
  if (!Array.isArray(items)) {
1124
- // --json always returns an array from `data` — [] is the honest
1125
- // "no items" signal instead of prose breaking JSON parsing. (#71)
1126
649
  if (opts.json) {
1127
650
  json([]);
1128
651
  return;
@@ -1130,18 +653,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
1130
653
  console.log(chalk.dim('No data yet. Run the scrap first.'));
1131
654
  return;
1132
655
  }
1133
- // #93 item 2 — --fresh used to render items without ever checking for a
1134
- // regression, unlike the persisted `data` path below (#88 item 2). The
1135
- // load() response's OWN embedded `scrap.history[0]` can't be trusted for
1136
- // this (see the ScrapLoadResult comment above): it may be the previous
1137
- // run's row, and even the right row never carries the regression flip.
1138
- // `--fresh` runs synchronously to completion server-side though — by the
1139
- // time this call returns, node has already awaited the regression patch
1140
- // — so a fresh GET /api/scraps/:id (the SAME read the persisted path
1141
- // below already trusts) reliably observes the finalized DB state.
1142
- // Best-effort: never fail --fresh's real output over this side check,
1143
- // and never gate on it — the items returned ARE this run's real,
1144
- // synchronously-computed data regardless of what this check finds.
1145
656
  try {
1146
657
  const fresh = await api.get(`/api/scraps/${id}`);
1147
658
  if (fresh.history?.[0]?.statusDetail === 'regression') {
@@ -1149,56 +660,20 @@ export function attachDataCommand(parent, attachOpts = {}) {
1149
660
  }
1150
661
  }
1151
662
  catch {
1152
- // best-effort — the fresh run's items are still valid without this check
1153
663
  }
1154
664
  renderScrapItems(items, opts.json);
1155
665
  return;
1156
666
  }
1157
- // Default: read the last PERSISTED run payload — no quota, no run lock.
1158
- // History.data (trawl_node modules/historys/services/historys.service.js
1159
- // `update()`) is a JSON-stringified clone of the worker result, so it
1160
- // carries the same `.data` items array the live load endpoint returns —
1161
- // it's just pulled from the most recent history row instead of a fresh
1162
- // run. Retention keeps this only for the newest row per (scrap, status)
1163
- // bucket (config.trawl.keepData, default 1); older rows null it out.
1164
- //
1165
- // #86 finding 6 — [] is reserved for a GENUINE zero-item successful run.
1166
- // Every other outcome below is an honest error envelope instead: never
1167
- // run (not_found/4), last run failed (1), or the payload aged out of
1168
- // retention (not_found/4) all used to collapse into the same silent [].
1169
667
  const scrap = await api.get(`/api/scraps/${id}`);
1170
668
  const last = scrap.history?.[0];
1171
669
  if (!last?._id) {
1172
670
  reportDataState(`Scrap ${id} has never run. Run it first (trawl run ${id}) or pass --fresh to launch one now.`, 4, 'not_found', opts.json);
1173
671
  return;
1174
672
  }
1175
- // #88 item 1 — status:null is an IN-FLIGHT run (node persists
1176
- // {status:null, statusDetail:null, inFlight:true} the moment a run
1177
- // starts, and only flips status/statusDetail once it finishes). That is
1178
- // neither "never run" nor "the last run failed" — a caller reading data
1179
- // mid-run needs an honest "wait" signal. Never suggest --fresh here: a
1180
- // run already holds the server-side distributed lock, so --fresh would
1181
- // just 429 against it.
1182
673
  if (last.status === null) {
1183
- // #170 — retrying `trawl data <id>` (unchanged) once the run
1184
- // finishes IS worth it; `--fresh` never is (it would 429 against the
1185
- // lock this very run already holds) — never suggested here.
1186
674
  reportDataState(`Run in progress for ${id} — retry shortly.`, 1, 'in_progress', opts.json, { retryable: true, next: [`trawl data ${id}`] });
1187
675
  return;
1188
676
  }
1189
- // #86 review — node persists status=false for a GENUINE zero-item run
1190
- // too (historys schema: status boolean|null + statusDetail
1191
- // success/error/empty/regression; a zero-item run is status=false +
1192
- // statusDetail='empty', and the embedded history rows from GET
1193
- // /api/scraps/:id include statusDetail via the repository populate
1194
- // select). An 'empty' run is the one case [] is FOR — only a real
1195
- // failure (error/unknown detail) gets the run_failed envelope.
1196
- //
1197
- // #88 item 2 — statusDetail='regression' is ALSO status=false (an async
1198
- // patch flips it after item count dropped vs baseline), but the row's
1199
- // `data` still holds REAL, non-empty items — the write that persisted
1200
- // them succeeded before the regression was even detected. Treating it as
1201
- // run_failed would hide genuine data behind a false negative.
1202
677
  const isEmptyRun = last.status === false && last.statusDetail === 'empty';
1203
678
  const isRegression = last.status === false && last.statusDetail === 'regression';
1204
679
  if (last.status === false && !isEmptyRun && !isRegression) {
@@ -1209,11 +684,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
1209
684
  let items;
1210
685
  if (typeof detail?.data === 'string' && detail.data) {
1211
686
  try {
1212
- // #159 — `detail.data` is the same JSON-string-within-JSON shape
1213
- // (`history.data`) the create --json bug lived in: a raw control
1214
- // character here would otherwise throw and silently report "aged
1215
- // out of retention" below for data that is actually present and
1216
- // recoverable, burning a --fresh re-run + quota for nothing.
1217
687
  items = parseServerJson(detail.data)?.data;
1218
688
  }
1219
689
  catch {
@@ -1222,23 +692,12 @@ export function attachDataCommand(parent, attachOpts = {}) {
1222
692
  }
1223
693
  if (!Array.isArray(items)) {
1224
694
  if (isEmptyRun) {
1225
- // A genuine zero-item run whose payload is '[]' or absent — both are
1226
- // the SAME honest answer: no items, exit 0. Never the retention
1227
- // message (nothing aged out; there was nothing to persist).
1228
695
  renderScrapItems([], opts.json);
1229
696
  return;
1230
697
  }
1231
- // #88 item 2 — a regression row whose payload aged out of retention has
1232
- // nothing left to show either; fall through to the SAME honest
1233
- // aged-out envelope a normal successful row would get (never fabricate
1234
- // items, never silently succeed).
1235
698
  reportDataState(`No persisted data for the last run of ${id} — it aged out of retention. Pass --fresh to launch a new run.`, 4, 'not_found', opts.json);
1236
699
  return;
1237
700
  }
1238
- // #88 item 2 — a regression row's items are REAL (the write succeeded
1239
- // before the async patch flagged the drop) — return them on stdout
1240
- // (exit 0, both modes) with an honest stderr warning pointing at the
1241
- // diagnostic command, instead of hiding genuine data behind run_failed.
1242
701
  if (isRegression) {
1243
702
  console.error(chalk.yellow(`⚠ Item count regressed vs baseline for the last run of ${id} — see: trawl scraps doctor ${id}`));
1244
703
  }
@@ -1246,7 +705,6 @@ export function attachDataCommand(parent, attachOpts = {}) {
1246
705
  });
1247
706
  }
1248
707
  attachDataCommand(scraps, { hidden: true });
1249
- // history — list past runs for a scrap — promoted to a top-level verb (#108)
1250
708
  export function attachHistoryCommand(parent, attachOpts = {}) {
1251
709
  return parent
1252
710
  .command('history <id>', attachOpts)
@@ -1267,8 +725,6 @@ export function attachHistoryCommand(parent, attachOpts = {}) {
1267
725
  time: h.time ?? null,
1268
726
  tier: h.proxyTier ?? null,
1269
727
  failureKind: h.failureKind ?? null,
1270
- // trawl_node#1950 renamed blockType to block.kind; fall back to the
1271
- // deprecated flat field for a prod backend not yet on that tag.
1272
728
  blockType: h.block?.kind ?? h.blockType ?? null,
1273
729
  createdAt: h.createdAt ?? null,
1274
730
  }));
@@ -1280,7 +736,6 @@ export function attachHistoryCommand(parent, attachOpts = {}) {
1280
736
  });
1281
737
  }
1282
738
  attachHistoryCommand(scraps, { hidden: true });
1283
- // run-info — single-run detail by history id — promoted to a top-level verb (#108)
1284
739
  export function attachRunInfoCommand(parent, attachOpts = {}) {
1285
740
  return parent
1286
741
  .command('run-info <hid>', attachOpts)
@@ -1295,36 +750,24 @@ export function attachRunInfoCommand(parent, attachOpts = {}) {
1295
750
  time: h.time ?? null,
1296
751
  tier: h.proxyTier ?? null,
1297
752
  failureKind: h.failureKind ?? null,
1298
- // trawl_node#1950 renamed blockType to block.kind; fall back to the
1299
- // deprecated flat field for a prod backend not yet on that tag.
1300
753
  blockType: h.block?.kind ?? h.blockType ?? null,
1301
754
  errorMessage: h.errorSnapshot?.errorMessage ?? null,
1302
755
  selector: h.errorSnapshot?.selector ?? null,
1303
756
  emptyContext: h.emptyContext ?? null,
1304
757
  createdAt: h.createdAt ?? null,
1305
- // #185 — present only for a live mapped failureKind (see
1306
- // runDocsField's doc comment); omitted, never `docs: undefined`, for
1307
- // every other run so this object's own JSON.stringify never grows an
1308
- // unexpected key on existing consumers.
1309
758
  ...runDocsField(h),
1310
759
  };
1311
760
  if (opts.json) {
1312
761
  json(info);
1313
762
  return;
1314
763
  }
1315
- // The blob is an object; the table needs one line. --json keeps it whole.
1316
764
  table([{ ...info, emptyContext: summarizeEmptyContext(info.emptyContext) }], ['hid', 'status', 'time', 'tier', 'failureKind', 'blockType', 'errorMessage', 'selector', 'emptyContext', 'createdAt']);
1317
- // #184 safety-net — the run just shown carries a live login-wall
1318
- // verdict; if Claude's skills aren't installed either, point at the
1319
- // command that installs them too (never under --json, see the early
1320
- // return above).
1321
765
  if (isAuthWall(h)) {
1322
766
  maybeSuggestSkillsForAuthWall();
1323
767
  }
1324
768
  });
1325
769
  }
1326
770
  attachRunInfoCommand(scraps, { hidden: true });
1327
- // delete
1328
771
  scraps
1329
772
  .command('delete <id>')
1330
773
  .alias('rm')
@@ -1333,16 +776,6 @@ scraps
1333
776
  .option('--json', 'Output as JSON')
1334
777
  .action(async (id, opts) => {
1335
778
  validateObjectId(id);
1336
- // #107 — never blocks on a y/N under --json or a non-TTY invocation
1337
- // (agent/CI subprocess); -f/--force always pre-confirms.
1338
- //
1339
- // #107 review F2 — `message` (the plain-id form) feeds the refusal
1340
- // UsageError, which can land verbatim in the `--json` error envelope;
1341
- // `chalk.bold(id)` is passed ONLY as `promptMessage`, shown solely on
1342
- // the interactive TTY `[y/N]` prompt. Before this split, the styled
1343
- // string was the ONLY message confirmDestructive had, so a non-TTY/
1344
- // --json refusal on `scraps delete X --json` emitted raw ANSI escape
1345
- // bytes inside the JSON string.
1346
779
  const { proceed, blocked } = await confirmDestructive(`Delete scrap ${id}?`, {
1347
780
  force: opts.force,
1348
781
  json: opts.json,
@@ -1361,9 +794,8 @@ scraps
1361
794
  return;
1362
795
  }
1363
796
  await spin(call, { text: 'Deleting…', successText: 'Scrap deleted' });
1364
- confirmNonTTY(`Scrap ${id} deleted`); // #166
797
+ confirmNonTTY(`Scrap ${id} deleted`);
1365
798
  });
1366
- // banner
1367
799
  scraps
1368
800
  .command('banner <id>')
1369
801
  .description('Upload a banner image for a scrap')
@@ -1385,9 +817,6 @@ scraps
1385
817
  jpeg: 'image/jpeg',
1386
818
  webp: 'image/webp',
1387
819
  };
1388
- // #91 P2 — an unsupported/unknown extension silently became image/png
1389
- // (uploading the raw bytes of, say, a .gif or .pdf under an image/png
1390
- // Content-Type — a lie about the actual file's format). Refuse instead.
1391
820
  const mimeType = mimeMap[ext];
1392
821
  if (!mimeType) {
1393
822
  usageError(`Unsupported image type "${ext ? `.${ext}` : filename}" — use png, jpg, or webp.`, { json: opts.json });
@@ -1407,10 +836,8 @@ scraps
1407
836
  text: 'Uploading banner…',
1408
837
  successText: `Banner uploaded for scrap ${chalk.bold(id)}`,
1409
838
  });
1410
- // #166 — plain, unstyled: chalk.bold would emit escape codes into a pipe.
1411
839
  confirmNonTTY(`Banner uploaded for scrap ${id}`);
1412
840
  });
1413
- // watch (stream activities)
1414
841
  scraps
1415
842
  .command('watch <id>')
1416
843
  .description('Stream scrap activities in real-time')
@@ -1419,7 +846,6 @@ scraps
1419
846
  validateObjectId(id);
1420
847
  await watchActivities(id, opts.json);
1421
848
  });
1422
- // trigger — promoted to a top-level verb (#108)
1423
849
  export function attachTriggerCommand(parent, attachOpts = {}) {
1424
850
  return parent
1425
851
  .command('trigger <id>', attachOpts)
@@ -1429,23 +855,9 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
1429
855
  .option('--json', 'Output the raw trigger payload as JSON')
1430
856
  .action(async (id, opts) => {
1431
857
  validateObjectId(id);
1432
- // #91 P1 / #93 item 1 — captured BEFORE triggering so pollRunProgress can
1433
- // tell "the run we just triggered" apart from whatever the last run
1434
- // happened to be. This is the dedup-prone path: `trigger`'s method:'worker'
1435
- // collapses onto an already-pending/running worker job for the same scrap
1436
- // (ScrapJobsService, LIVE_STATUSES) instead of creating a new history row —
1437
- // captureBeforeRunState records that so pollRunProgress can still track it.
1438
858
  const beforeRun = opts.watch ? await captureBeforeRunState(id) : undefined;
1439
- // #50 — default async: the backend (#1313) kicks off the run and returns a
1440
- // 'queued' envelope immediately instead of holding the connection for the
1441
- // whole run. --wait restores the old synchronous round-trip.
1442
859
  const path = opts.wait ? `/api/scraps/worker/${id}` : `/api/scraps/worker/${id}?wait=false`;
1443
- // #91 P0 — the synchronous --wait branch runs the scrap server-side
1444
- // (30-250s), same as `run`; the 30s default was aborting it
1445
- // mid-flight. The async (default) POST returns almost immediately, so it
1446
- // keeps the 30s default.
1447
860
  const call = () => (opts.wait ? api.post(path, undefined, { timeoutMs: LONG_RUN_TIMEOUT_MS }) : api.post(path));
1448
- // #107 — under --json the stdout path stays pure: no spinner channel.
1449
861
  const data = opts.json
1450
862
  ? await call()
1451
863
  : await spin(call, {
@@ -1456,26 +868,16 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
1456
868
  json(data);
1457
869
  }
1458
870
  else {
1459
- /* #160, folded onto the shared helper by #166 — the rationale and the
1460
- * known residual now live once, on `confirmNonTTY` itself, instead of
1461
- * in the one call site that happened to be fixed first. Wording is
1462
- * unchanged on purpose: the trawl-internal QA runbook asserts on the
1463
- * exact "Worker triggered" / "Worker run complete" substrings. */
1464
871
  confirmNonTTY(opts.wait ? `Worker run complete for scrap ${id}` : `Worker triggered for scrap ${id}`);
1465
872
  }
1466
- // #107 — see the matching comment on `run`'s --watch call above (review
1467
- // F1): honest final NDJSON line + exit code under --json, human mode
1468
- // gets the same honest exit code too.
1469
873
  if (opts.watch)
1470
874
  await pollRunProgress(id, beforeRun, { json: opts.json });
1471
875
  });
1472
876
  }
1473
877
  attachTriggerCommand(scraps, { hidden: true });
1474
- // account subcommand group
1475
878
  const account = scraps
1476
879
  .command('account')
1477
880
  .description('Manage scrap account credentials and session');
1478
- // helper: prompt for a value with readline (visible input)
1479
881
  async function promptLine(prompt) {
1480
882
  const { createInterface } = await import('readline');
1481
883
  const rl = createInterface({ input: process.stdin, output: process.stdout });
@@ -1488,7 +890,6 @@ async function promptLine(prompt) {
1488
890
  rl.close();
1489
891
  }
1490
892
  }
1491
- // account set
1492
893
  account
1493
894
  .command('set <id>')
1494
895
  .description('Set credentials for a scrap account')
@@ -1500,13 +901,8 @@ account
1500
901
  let username = opts.username || '';
1501
902
  let password = opts.password || '';
1502
903
  if (opts.password) {
1503
- // #107 — this advisory is stderr-only: stdout must stay pure under
1504
- // --json, and every other advisory in this CLI already follows that
1505
- // rule (warnIfUnconfirmedTier, token.ts's expiry hints, …).
1506
904
  console.error(chalk.yellow('⚠ Passing --password on the command line is insecure and may be stored in shell history.'));
1507
905
  }
1508
- // #107 — never blocks on a readline prompt under --json or a non-TTY
1509
- // invocation (agent/CI subprocess); pass -u/-p instead.
1510
906
  const interactive = isInteractive({ json: opts.json });
1511
907
  if (!username) {
1512
908
  if (!interactive) {
@@ -1541,7 +937,6 @@ account
1541
937
  console.log(chalk.dim(' Credentials: ') + (acc.hasCredentials ? chalk.green('✓ configured') : chalk.dim('not set')));
1542
938
  console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('active') : chalk.dim('none')));
1543
939
  });
1544
- // account delete
1545
940
  account
1546
941
  .command('delete <id>')
1547
942
  .description('Delete account credentials for a scrap')
@@ -1549,10 +944,6 @@ account
1549
944
  .option('--json', 'Output as JSON')
1550
945
  .action(async (id, opts) => {
1551
946
  validateObjectId(id);
1552
- // #107 — never blocks on a y/N under --json or a non-TTY invocation.
1553
- // #107 review F2 — plain-id `message` for the refusal/JSON envelope,
1554
- // styled `promptMessage` for the interactive TTY prompt only (see the
1555
- // matching comment on `scraps delete` above).
1556
947
  const { proceed, blocked } = await confirmDestructive(`Delete account credentials for scrap ${id}?`, { force: opts.force, json: opts.json, promptMessage: `Delete account credentials for scrap ${chalk.bold(id)}?` });
1557
948
  if (blocked)
1558
949
  return;
@@ -1570,9 +961,8 @@ account
1570
961
  text: 'Deleting credentials…',
1571
962
  successText: 'Account credentials deleted',
1572
963
  });
1573
- confirmNonTTY(`Account credentials deleted for scrap ${id}`); // #166
964
+ confirmNonTTY(`Account credentials deleted for scrap ${id}`);
1574
965
  });
1575
- // account clear-session
1576
966
  account
1577
967
  .command('clear-session <id>')
1578
968
  .description('Clear the saved session for a scrap account')
@@ -1589,38 +979,64 @@ account
1589
979
  text: 'Clearing session…',
1590
980
  successText: 'Session cleared',
1591
981
  });
1592
- confirmNonTTY(`Session cleared for scrap ${id}`); // #166
982
+ confirmNonTTY(`Session cleared for scrap ${id}`);
1593
983
  });
1594
- // account session subcommand group
1595
984
  const accountSession = account
1596
985
  .command('session')
1597
986
  .description('Manage scrap account session cookies (flavour B BYO-cookies)');
1598
- // account session set
987
+ function isValidOriginsShape(value) {
988
+ return (Array.isArray(value) &&
989
+ value.every((o) => o &&
990
+ typeof o === 'object' &&
991
+ typeof o.origin === 'string' &&
992
+ Array.isArray(o.localStorage) &&
993
+ o.localStorage.every((kv) => kv && typeof kv === 'object' && typeof kv.name === 'string' && typeof kv.value === 'string')));
994
+ }
1599
995
  accountSession
1600
996
  .command('set <id>')
1601
- .description('Upload browser session cookies for a scrap (Puppeteer cookie JSON array)')
1602
- .requiredOption('-c, --cookies <file>', 'Path to a Puppeteer cookie JSON array file')
997
+ .description('Upload a browser session for a scrap — a cookie JSON array, or a { cookies, origins } storageState file (see `session capture`)')
998
+ .requiredOption('-c, --cookies <file>', 'Path to a cookie JSON array, or a storageState file ({ cookies, origins? })')
1603
999
  .option('--json', 'Output as JSON')
1604
1000
  .action(async (id, opts) => {
1605
1001
  validateObjectId(id);
1002
+ const secureTransportError = assertSecureTransport(getApiUrl());
1003
+ if (secureTransportError) {
1004
+ usageError(secureTransportError, { json: opts.json });
1005
+ return;
1006
+ }
1606
1007
  const { existsSync, readFileSync } = await import('fs');
1607
1008
  if (!existsSync(opts.cookies)) {
1608
1009
  usageError(`File not found: ${opts.cookies}`, { json: opts.json });
1609
1010
  return;
1610
1011
  }
1611
1012
  let cookies;
1013
+ let origins;
1612
1014
  try {
1613
1015
  const raw = readFileSync(opts.cookies, 'utf-8');
1614
- cookies = JSON.parse(raw);
1016
+ const parsed = JSON.parse(raw);
1017
+ if (Array.isArray(parsed)) {
1018
+ cookies = parsed;
1019
+ }
1020
+ else if (parsed && typeof parsed === 'object' && Array.isArray(parsed.cookies)) {
1021
+ const shaped = parsed;
1022
+ cookies = shaped.cookies;
1023
+ if (shaped.origins !== undefined) {
1024
+ if (!isValidOriginsShape(shaped.origins)) {
1025
+ usageError('origins must be an array of { origin: string, localStorage: [{name, value}] }', { json: opts.json });
1026
+ return;
1027
+ }
1028
+ origins = shaped.origins;
1029
+ }
1030
+ }
1031
+ else {
1032
+ usageError('Cookies file must contain a JSON array, or a { cookies, origins? } storageState object', { json: opts.json });
1033
+ return;
1034
+ }
1615
1035
  }
1616
1036
  catch (e) {
1617
1037
  usageError(`Failed to parse cookies file: ${e.message}`, { json: opts.json });
1618
1038
  return;
1619
1039
  }
1620
- if (!Array.isArray(cookies)) {
1621
- usageError('Cookies file must contain a JSON array', { json: opts.json });
1622
- return;
1623
- }
1624
1040
  if (cookies.length === 0) {
1625
1041
  usageError('Cookies array must not be empty', { json: opts.json });
1626
1042
  return;
@@ -1629,7 +1045,10 @@ accountSession
1629
1045
  usageError('Each cookie must have a name (string) and value (string)', { json: opts.json });
1630
1046
  return;
1631
1047
  }
1632
- const call = () => api.put(`/api/scraps/${id}/account/session`, { cookies });
1048
+ const body = { cookies };
1049
+ if (origins)
1050
+ body.origins = origins;
1051
+ const call = () => api.put(`/api/scraps/${id}/account/session`, body);
1633
1052
  if (opts.json) {
1634
1053
  const data = await call();
1635
1054
  json(data);
@@ -1642,7 +1061,97 @@ accountSession
1642
1061
  const acc = data.account;
1643
1062
  console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('✓ active') : chalk.dim('none')));
1644
1063
  });
1645
- // account status
1064
+ function reportCaptureFailure(result, opts) {
1065
+ if (result.reason === 'no_cookies_in_scope') {
1066
+ process.exitCode = reportError(new RefusalError(result.message), { json: opts.json });
1067
+ return;
1068
+ }
1069
+ if (result.reason === 'capture_failed') {
1070
+ process.exitCode = reportError(new Error(result.message), { json: opts.json });
1071
+ return;
1072
+ }
1073
+ usageError(result.message, { json: opts.json });
1074
+ }
1075
+ accountSession
1076
+ .command('capture <id>')
1077
+ .description('Open a headed Chrome to the scrap\'s target URL, wait for you to log in (2FA included), and capture the session over CDP — cookies AND localStorage. Requires a local interactive terminal with a display; does not work headless, in CI, or over a plain SSH session. You stay authenticated as yourself — Trawl never sees your credentials. Responsibility for lawful use of the captured session stays with you.')
1078
+ .option('--chrome <path>', 'Path to a Chrome/Chromium executable (auto-detected if omitted)')
1079
+ .option('--json', 'Output as JSON (counts only — a session is a bearer secret and is never printed, in any mode)')
1080
+ .action(async (id, opts) => {
1081
+ validateObjectId(id);
1082
+ const secureTransportError = assertSecureTransport(getApiUrl());
1083
+ if (secureTransportError) {
1084
+ usageError(secureTransportError, { json: opts.json });
1085
+ return;
1086
+ }
1087
+ const scrap = await api.get(`/api/scraps/${id}`);
1088
+ if (!scrap.url) {
1089
+ usageError(`Scrap ${id} has no target URL configured — nothing to open a browser to. A URL can be set with: trawl scraps update ${id} -u <url>`, { json: opts.json });
1090
+ return;
1091
+ }
1092
+ if (opts.chrome) {
1093
+ const { existsSync, accessSync, constants } = await import('fs');
1094
+ if (!existsSync(opts.chrome)) {
1095
+ usageError(`Chrome not found at: ${opts.chrome}`, { json: opts.json });
1096
+ return;
1097
+ }
1098
+ try {
1099
+ accessSync(opts.chrome, constants.X_OK);
1100
+ }
1101
+ catch {
1102
+ usageError(`Chrome exists at ${opts.chrome} but is not executable.`, { json: opts.json });
1103
+ return;
1104
+ }
1105
+ }
1106
+ else if (process.env.TRAWL_CHROME_PATH) {
1107
+ const trawlChromePath = process.env.TRAWL_CHROME_PATH;
1108
+ const { existsSync, accessSync, constants } = await import('fs');
1109
+ if (existsSync(trawlChromePath)) {
1110
+ try {
1111
+ accessSync(trawlChromePath, constants.X_OK);
1112
+ }
1113
+ catch {
1114
+ usageError(`Chrome exists at ${trawlChromePath} (TRAWL_CHROME_PATH) but is not executable.`, { json: opts.json });
1115
+ return;
1116
+ }
1117
+ }
1118
+ }
1119
+ const nonInteractiveReason = checkInteractiveEnvironment();
1120
+ if (nonInteractiveReason) {
1121
+ reportCaptureFailure({ ok: false, reason: 'non_interactive', message: nonInteractiveReason }, opts);
1122
+ return;
1123
+ }
1124
+ console.error(chalk.dim(`Chrome will open at ${scrap.url} for you to log in there (2FA included); capture completes when you press Enter back in this terminal.`));
1125
+ const result = await captureSession(scrap.url, opts.chrome ? { findChrome: () => opts.chrome ?? null } : {});
1126
+ if (!result.ok) {
1127
+ reportCaptureFailure(result, opts);
1128
+ return;
1129
+ }
1130
+ const call = () => api.put(`/api/scraps/${id}/account/session`, {
1131
+ cookies: result.storageState.cookies,
1132
+ origins: result.storageState.origins,
1133
+ });
1134
+ if (opts.json) {
1135
+ const data = await call();
1136
+ json({ account: data.account, targetDomain: result.targetDomain, capture: result.counts });
1137
+ return;
1138
+ }
1139
+ const data = await spin(call, {
1140
+ text: `Uploading captured session for scrap ${chalk.bold(id)}…`,
1141
+ successText: `Session captured for scrap ${chalk.bold(id)}: ${result.counts.cookiesCaptured} cookie(s), ${result.counts.originsCaptured} origin(s) in scope for ${result.targetDomain}`,
1142
+ });
1143
+ const acc = data.account;
1144
+ console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('✓ active') : chalk.dim('none')));
1145
+ if (result.counts.cookiesDroppedOutOfScope > 0 || result.counts.originsDroppedOutOfScope > 0) {
1146
+ console.log(chalk.dim(` Scoped to ${result.targetDomain}: dropped ${result.counts.cookiesDroppedOutOfScope} cookie(s) and ${result.counts.originsDroppedOutOfScope} origin(s) outside that domain.`));
1147
+ }
1148
+ if (result.counts.closedEarly) {
1149
+ console.log(chalk.dim(' The browser window was closed before Enter — localStorage was not captured (cookies only).'));
1150
+ }
1151
+ if (result.counts.originsUnreadable > 0) {
1152
+ console.log(chalk.yellow(` ⚠ ${result.counts.originsUnreadable} origin(s) could not be captured — threw reading localStorage, or gave no response within ${LOCALSTORAGE_READ_TIMEOUT_MS / 1000}s (a blocked tab). Not the same as being empty; cookies were still captured.`));
1153
+ }
1154
+ });
1646
1155
  account
1647
1156
  .command('status <id>')
1648
1157
  .description('Show account credentials and session status for a scrap')
@@ -1652,8 +1161,6 @@ account
1652
1161
  const data = await spin(() => api.get(`/api/scraps/${id}`), 'Fetching scrap…');
1653
1162
  const acc = data.account;
1654
1163
  if (opts.json) {
1655
- // #86 finding 12 — `json` is already statically imported at the top of
1656
- // this file; the dynamic import here was pure dead weight.
1657
1164
  return json(acc ?? null);
1658
1165
  }
1659
1166
  if (!acc) {
@@ -1682,7 +1189,6 @@ account
1682
1189
  }
1683
1190
  console.log(`${credLine} | ${sessionLine}`);
1684
1191
  });
1685
- // doctor — diagnose last run of a scrap (error, failed selector, block status, page state, autofix)
1686
1192
  scraps
1687
1193
  .command('doctor <id>')
1688
1194
  .description('Diagnose the last run (error, failed selector, block status, page state, autofix outcome)')
@@ -1699,24 +1205,16 @@ scraps
1699
1205
  console.log(chalk.dim('No runs yet.'));
1700
1206
  return;
1701
1207
  }
1702
- // #185 — `docs` sits alongside `run`/`fix` (present only for a live
1703
- // mapped failureKind — see runDocsField's doc comment), never nested
1704
- // inside the allowlisted `run` object.
1705
1208
  if (opts.json)
1706
1209
  return json({ run: pickRun(result.run), fix: pickFix(result.fix), ...runDocsField(result.run) });
1707
1210
  console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
1708
1211
  if (opts.autofix && result.fix) {
1709
1212
  console.log('\n' + formatAutofix(result.fix));
1710
1213
  }
1711
- // #184 safety-net — formatDoctor already prints the login-wall hint
1712
- // (#182) above when this is a live auth wall; if Claude's skills aren't
1713
- // installed either, point at the command that installs them too (never
1714
- // under --json, see the early return above).
1715
1214
  if (isAuthWall(result.run)) {
1716
1215
  maybeSuggestSkillsForAuthWall();
1717
1216
  }
1718
1217
  });
1719
- // autofix — show full auto-fix attempt detail (diff, dry-run, knowledge)
1720
1218
  scraps
1721
1219
  .command('autofix <id>')
1722
1220
  .description('Show the last auto-fix attempt for a scrap (decision, diff, dry-run, knowledge)')
@@ -1725,9 +1223,6 @@ scraps
1725
1223
  validateObjectId(id);
1726
1224
  const result = await fetchRunAndFix(id);
1727
1225
  if (!result) {
1728
- // #88 item 7 — unified no-runs shape with `doctor --json` / `data
1729
- // --errors --json`: a never-run scrap is a distinct, nameable state,
1730
- // not the same bare `null` a run-with-no-fix-attempt returns below.
1731
1226
  if (opts.json) {
1732
1227
  json({ status: 'no_runs' });
1733
1228
  return;
@@ -1736,8 +1231,6 @@ scraps
1736
1231
  return;
1737
1232
  }
1738
1233
  if (!result.fix) {
1739
- // A run DID happen, it just had no autofix attempt — genuinely "no
1740
- // data", unlike the never-run case above.
1741
1234
  if (opts.json) {
1742
1235
  json(null);
1743
1236
  return;
@@ -1749,7 +1242,6 @@ scraps
1749
1242
  return json(result.fix);
1750
1243
  console.log(formatAutofix(result.fix));
1751
1244
  });
1752
- // snapshot — download the captured page HTML (or error-path HTML) for the last run
1753
1245
  scraps
1754
1246
  .command('snapshot <id>')
1755
1247
  .description('Download captured page HTML for the last run of a scrap')
@@ -1761,16 +1253,10 @@ scraps
1761
1253
  const scrap = await api.get(`/api/scraps/${id}`);
1762
1254
  const hid = scrap.history?.[0]?._id;
1763
1255
  if (!hid) {
1764
- // #91 P2 — same no-runs contract as `doctor`/`autofix`/`data --errors`
1765
- // (`{status:"no_runs"}`, exit 0) under --json.
1766
1256
  if (opts.json) {
1767
1257
  json({ status: 'no_runs' });
1768
1258
  return;
1769
1259
  }
1770
- // `-o` explicitly asked for a file to be written. Silently exiting 0
1771
- // with nothing written (and only a dim console line) is indistinguishable
1772
- // from success to a script checking the exit code alone — give it the
1773
- // SAME not_found envelope `scraps data`'s own never-run case uses.
1774
1260
  if (opts.out) {
1775
1261
  reportDataState(`Scrap ${id} has never run — nothing to write to ${opts.out}.`, 4, 'not_found', false);
1776
1262
  return;