@trawlme/cli 3.7.2 → 3.7.5

package/README.md

@@ -79,7 +79,7 @@ trawl ping [--json]                            Health/version handshake against
 ### Scrap management
 
 ```
-trawl scraps create -t <title> [-u <url>] [-r <request>] [-d <description>] [--tier <tier0|tier1|tier2|tier3|tier4>] [--json]
+trawl scraps create -t <title> -u <url> [-r <request>] [-d <description>] [--tier <tier0|tier1|tier2|tier3|tier4>] [--json]
 trawl scraps update <id> [-t <title>] [-u <url>] [-r <request>] [-d <description>] [--cron <expr>|--no-cron] [--alert <email>|--no-alert] [--autofix|--no-autofix] [-p <json>|--params-file <path>] [--tier <tier0|tier1|tier2|tier3|tier4>] [--force-tier <tier0|tier1|tier2|tier3|tier4>] [--json]
 trawl scraps watch <id> [--json]
 trawl scraps doctor <id> [--json] [--autofix]
@@ -89,6 +89,7 @@ trawl scraps banner <id> -f <file> [--json]
 trawl scraps delete <id> [--force] [--json]         Alias: rm
 ```
 
+- `-u/--url` is **the site the scrap targets**, not necessarily the exact URL fetched at run time — a scrap whose real target is computed inside its `-r/--request` script (a templated search query, an interpolated item id, e.g. `https://www.ebay.com/sch/i.html?_nkw=${query}`) should still supply a base/site URL. Every domain-level policy (proxy tier ceiling, provider routing, per-domain memoize) keys off this field, so a blank value silently disables all of it. `create` requires it — validated client-side (must parse as an `http`/`https` URL) so a missing or malformed value fails fast with a readable usage error (exit 2) instead of a raw server 400/422; `update` keeps it optional (omitting it is still a no-op, matching the API) but a value that **is** passed is validated the same way.
 - `--tier` forces a proxy tier; `--force-tier` raises the proxy-tier ceiling past the auto-cap (history-gated: may be refused or cost more). `create --json`/`update --json` print the full scrap object (including the `_tierOverride` outcome) on stdout; a refused tier override exits 1 with a standard `--json` error envelope (`kind:"refused"` — distinct from `"unknown"`, so a script can branch on "the server said no"). When a tier was requested but the server's response carries no `_tierOverride` at all (an older server that can't confirm what actually got applied), a stderr warning is printed either way, and under `--json` the emitted object also carries `"_tierUnconfirmed": true` — the machine-readable counterpart to that warning, since a `--json` caller has no reliable reason to read stderr.
 - `scraps doctor` diagnoses the last run (error, failed selector, block status, page state, autofix outcome); `--autofix` includes the full autofix diff/dry-run/knowledge. A run that is still in flight (`status: null`, server-side) shows a `running` badge, never `failed`; a run whose item count regressed vs baseline (`statusDetail: "regression"`) shows its own amber `regression` badge, never `failed` either.
 - `scraps autofix` shows the last auto-fix attempt on its own (decision, diff, dry-run, knowledge). `--json` on a scrap that has **never run** returns `{"status":"no_runs"}` (exit 0) — distinct from `null`, which means a run exists but had no auto-fix attempt.

package/dist/commands/scraps.js

@@ -1,11 +1,11 @@
 import { Command } from 'commander';
 import chalk from 'chalk';
-import { spin } from '../lib/spinner.js';
+import { spin, confirmNonTTY } from '../lib/spinner.js';
 import { api, LONG_RUN_TIMEOUT_MS } from '../lib/api.js';
 import { table, json, formatDate } from '../lib/format.js';
 import { parseServerJson } from '../lib/json.js';
 import { promptPassword } from '../lib/prompt.js';
-import { validateObjectId } from '../lib/validate.js';
+import { validateObjectId, requireUrl } from '../lib/validate.js';
 import { classifyError, reportError, UsageError, RefusalError } from '../lib/errors.js';
 import { confirmDestructive, isInteractive } from '../lib/confirm.js';
 import { formatDoctor, formatAutofix, fetchRunAndFix, pickRun, pickFix } from './doctor.js';
@@ -22,6 +22,31 @@ import { maybeShowReferralTip } from '../lib/tips.js';
 function usageError(message, opts = {}) {
     process.exitCode = reportError(new UsageError(message), { json: opts.json });
 }
+/**
+ * #163 — client-side URL validation shared by `create`/`update`, mirroring
+ * the server's now-required-and-parseable `url` field (trawl_node#1823,
+ * PR #1828: `z.string().trim().default('')` -> `z.string().trim().url()`).
+ * Gives a readable message BEFORE the request goes out instead of a raw
+ * 400/422 from the server. Reuses the exact same `requireUrl` helper
+ * `trawl create`/`trawl login --url` already validate against (lib/validate.js),
+ * just routed through this file's own return-style `usageError()` convention
+ * (never throw — every other validation in this file, tier/limit/page/params/
+ * status, follows the same pattern) instead of the throw-style those two
+ * commands use. Returns `undefined` (never throws) on a bad value — the
+ * caller must check for that and `return` without calling the API.
+ */
+function tryValidateUrl(value, opts) {
+    try {
+        return requireUrl(value, '--url');
+    }
+    catch (err) {
+        if (err instanceof UsageError) {
+            usageError(err.message, { json: opts.json });
+            return undefined;
+        }
+        throw err;
+    }
+}
 function lastStatus(scrap) {
     const last = scrap.history?.[0];
     if (!last)
@@ -565,7 +590,7 @@ scraps
     .command('create')
     .description('Create a new scrap')
     .requiredOption('-t, --title <title>', 'Scrap title')
-    .option('-u, --url <url>', 'Target URL')
+    .option('-u, --url <url>', 'Target URL — the site the scrap targets, not necessarily the exact URL fetched at run time (a request script can compute that, e.g. a templated search query). Required.')
     .option('-r, --request <request>', 'Request/query')
     .option('-d, --description <text>', 'Scrap description')
     .option('--tier <tier>', `Force proxy tier (${VALID_TIERS.join('|')})`)
@@ -575,9 +600,16 @@ scraps
         usageError(`Invalid --tier "${opts.tier}" (allowed: ${VALID_TIERS.join(', ')})`, { json: opts.json });
         return;
     }
+    // #163 — the API now requires a parseable `url` on create (trawl_node
+    // #1823/#1828); validate client-side BEFORE the request goes out so a
+    // missing/unparseable value surfaces as a readable usage error instead
+    // of a raw server 400/422.
+    const url = tryValidateUrl(opts.url, opts);
+    if (url === undefined)
+        return;
     const data = await spin(() => api.post('/api/scraps', {
         title: opts.title,
-        ...(opts.url && { url: opts.url }),
+        url,
         request: opts.request || '',
         ...(opts.description !== undefined && { description: opts.description }),
         ...(opts.tier !== undefined && { proxyTier: opts.tier }),
@@ -606,7 +638,7 @@ scraps
     .command('update <id>')
     .description('Update an existing scrap')
     .option('-t, --title <title>', 'New title')
-    .option('-u, --url <url>', 'New target URL')
+    .option('-u, --url <url>', 'New target URL — the site the scrap targets. Optional: omitting it leaves the current value unchanged (a no-op, matching the API); when passed it must still be a parseable URL.')
     .option('-r, --request <request>', 'New request')
     .option('-d, --description <text>', 'New description')
     .option('--cron <expression>', 'Cron expression (empty string to disable)')
@@ -633,8 +665,15 @@ scraps
     const body = {};
     if (opts.title !== undefined)
         body.title = opts.title;
-    if (opts.url !== undefined)
-        body.url = opts.url;
+    // #163 — omitting --url stays a no-op (the API keeps that allowed on
+    // update), but a value that IS supplied must still be a parseable URL —
+    // validate it client-side before the request goes out, same as create.
+    if (opts.url !== undefined) {
+        const url = tryValidateUrl(opts.url, opts);
+        if (url === undefined)
+            return;
+        body.url = url;
+    }
     if (opts.request !== undefined)
         body.request = opts.request;
     if (opts.description !== undefined)
@@ -747,6 +786,24 @@ export function attachRunCommand(parent, attachOpts = {}) {
             : await spin(call, { text: 'Launching scrap…', successText: 'Scrap launched' });
         if (opts.json)
             json(data);
+        else if (!opts.watch) {
+            /* #166 — see confirmNonTTY. Two deliberate choices here:
+             *
+             * "run complete", not "launched": `run` is the SYNCHRONOUS verb —
+             * GET /api/scraps/load/:id executes the scrap server-side (30-250s)
+             * and returns the finished result, so by the time this prints the run
+             * is over. The spinner's successText says "Scrap launched", but that
+             * is stderr chrome for a human watching it happen; THIS line is the
+             * machine surface, where a script reading "launched" would poll for a
+             * completion that already happened. `trigger` already draws the same
+             * distinction ("Worker triggered" async vs "Worker run complete" under
+             * --wait) — this matches that vocabulary.
+             *
+             * `--watch` is excluded because pollRunProgress below does its own
+             * reporting; without that guard a watched run would print this line
+             * and then narrate the same run. */
+            confirmNonTTY(`Scrap ${id} run complete`);
+        }
         if (opts.watch) {
             // #107 — under --json, pollRunProgress suppresses its own intermediate
             // console.log calls and instead emits exactly ONE final NDJSON outcome
@@ -1079,6 +1136,7 @@ scraps
         return;
     }
     await spin(call, { text: 'Deleting…', successText: 'Scrap deleted' });
+    confirmNonTTY(`Scrap ${id} deleted`); // #166
 });
 // banner
 scraps
@@ -1124,6 +1182,8 @@ scraps
         text: 'Uploading banner…',
         successText: `Banner uploaded for scrap ${chalk.bold(id)}`,
     });
+    // #166 — plain, unstyled: chalk.bold would emit escape codes into a pipe.
+    confirmNonTTY(`Banner uploaded for scrap ${id}`);
 });
 // watch (stream activities)
 scraps
@@ -1167,8 +1227,17 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
                 text: opts.wait ? 'Running worker…' : 'Triggering worker…',
                 successText: opts.wait ? 'Worker run complete' : 'Worker triggered',
             });
-        if (opts.json)
+        if (opts.json) {
             json(data);
+        }
+        else {
+            /* #160, folded onto the shared helper by #166 — the rationale and the
+             * known residual now live once, on `confirmNonTTY` itself, instead of
+             * in the one call site that happened to be fixed first. Wording is
+             * unchanged on purpose: the trawl-internal QA runbook asserts on the
+             * exact "Worker triggered" / "Worker run complete" substrings. */
+            confirmNonTTY(opts.wait ? `Worker run complete for scrap ${id}` : `Worker triggered for scrap ${id}`);
+        }
         // #107 — see the matching comment on `run`'s --watch call above (review
         // F1): honest final NDJSON line + exit code under --json, human mode
         // gets the same honest exit code too.
@@ -1276,6 +1345,7 @@ account
         text: 'Deleting credentials…',
         successText: 'Account credentials deleted',
     });
+    confirmNonTTY(`Account credentials deleted for scrap ${id}`); // #166
 });
 // account clear-session
 account
@@ -1294,6 +1364,7 @@ account
         text: 'Clearing session…',
         successText: 'Session cleared',
     });
+    confirmNonTTY(`Session cleared for scrap ${id}`); // #166
 });
 // account session subcommand group
 const accountSession = account

package/dist/lib/spinner.d.ts

@@ -15,4 +15,35 @@ type SpinOptions<T> = string | {
  * to stderr, stdout stays clean either way).
  */
 export declare function spin<T>(action: Action<T>, options?: SpinOptions<T>): Promise<T>;
+/**
+ * The counterpart to `spin()`'s silence — print a plain-text confirmation on
+ * stdout when, and only when, stdout is not a TTY.
+ *
+ * #160 then #166. `spin()` above deliberately emits nothing when stderr is not
+ * a TTY, so for every command whose only non-`--json` feedback was its
+ * `successText`, a piped or redirected invocation wrote **zero bytes to both
+ * streams while exiting 0**. Our ICP is agent builders, and agents pipe stdout:
+ * a verb that writes nothing on success is unusable from a script, because the
+ * caller cannot tell success from a no-op. Two of the affected commands were
+ * deletes, where that ambiguity is at its worst.
+ *
+ * Lives here, next to the silence it compensates for, because #160 fixed
+ * `trigger` by inlining this rule and #166 then found five siblings carrying
+ * the identical defect — five more inlined copies is how the next one gets
+ * missed. Callers pass the message; the gate lives in one place.
+ *
+ * Gated on **stdout**, not stderr: stdout is the stream a script actually reads
+ * (`$(trawl …)`, `> out.txt`, `| jq`), independent of whatever `spin()` decides
+ * about stderr. So an interactive session, where the ora spinner already
+ * confirmed on stderr, never gets a duplicate line here.
+ *
+ * Never call this on a `--json` path: under `--json`, stdout must stay exactly
+ * one parseable document.
+ *
+ * Known residual, unchanged from #160: stdout attached to a real terminal while
+ * stderr is separately redirected still prints nothing on either stream. That is
+ * not the reported or common shape (full redirection, or stdout-only capture),
+ * and widening the gate would put a duplicate line in front of interactive users.
+ */
+export declare function confirmNonTTY(message: string): void;
 export {};

package/dist/lib/spinner.js

@@ -15,3 +15,37 @@ export function spin(action, options) {
     // oraPromise's own overloads accept (action, string) and (action, options).
     return oraPromise(action, options);
 }
+/**
+ * The counterpart to `spin()`'s silence — print a plain-text confirmation on
+ * stdout when, and only when, stdout is not a TTY.
+ *
+ * #160 then #166. `spin()` above deliberately emits nothing when stderr is not
+ * a TTY, so for every command whose only non-`--json` feedback was its
+ * `successText`, a piped or redirected invocation wrote **zero bytes to both
+ * streams while exiting 0**. Our ICP is agent builders, and agents pipe stdout:
+ * a verb that writes nothing on success is unusable from a script, because the
+ * caller cannot tell success from a no-op. Two of the affected commands were
+ * deletes, where that ambiguity is at its worst.
+ *
+ * Lives here, next to the silence it compensates for, because #160 fixed
+ * `trigger` by inlining this rule and #166 then found five siblings carrying
+ * the identical defect — five more inlined copies is how the next one gets
+ * missed. Callers pass the message; the gate lives in one place.
+ *
+ * Gated on **stdout**, not stderr: stdout is the stream a script actually reads
+ * (`$(trawl …)`, `> out.txt`, `| jq`), independent of whatever `spin()` decides
+ * about stderr. So an interactive session, where the ora spinner already
+ * confirmed on stderr, never gets a duplicate line here.
+ *
+ * Never call this on a `--json` path: under `--json`, stdout must stay exactly
+ * one parseable document.
+ *
+ * Known residual, unchanged from #160: stdout attached to a real terminal while
+ * stderr is separately redirected still prints nothing on either stream. That is
+ * not the reported or common shape (full redirection, or stdout-only capture),
+ * and widening the gate would put a duplicate line in front of interactive users.
+ */
+export function confirmNonTTY(message) {
+    if (!process.stdout.isTTY)
+        console.log(message);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trawlme/cli",
-  "version": "3.7.2",
+  "version": "3.7.5",
   "description": "Trawl CLI — manage scraps from the terminal",
   "type": "module",
   "bin": {