@trawlme/cli 3.7.2 → 3.7.4

package/README.md

@@ -79,7 +79,7 @@ trawl ping [--json]                            Health/version handshake against
 ### Scrap management
 
 ```
-trawl scraps create -t <title> [-u <url>] [-r <request>] [-d <description>] [--tier <tier0|tier1|tier2|tier3|tier4>] [--json]
+trawl scraps create -t <title> -u <url> [-r <request>] [-d <description>] [--tier <tier0|tier1|tier2|tier3|tier4>] [--json]
 trawl scraps update <id> [-t <title>] [-u <url>] [-r <request>] [-d <description>] [--cron <expr>|--no-cron] [--alert <email>|--no-alert] [--autofix|--no-autofix] [-p <json>|--params-file <path>] [--tier <tier0|tier1|tier2|tier3|tier4>] [--force-tier <tier0|tier1|tier2|tier3|tier4>] [--json]
 trawl scraps watch <id> [--json]
 trawl scraps doctor <id> [--json] [--autofix]
@@ -89,6 +89,7 @@ trawl scraps banner <id> -f <file> [--json]
 trawl scraps delete <id> [--force] [--json]         Alias: rm
 ```
 
+- `-u/--url` is **the site the scrap targets**, not necessarily the exact URL fetched at run time — a scrap whose real target is computed inside its `-r/--request` script (a templated search query, an interpolated item id, e.g. `https://www.ebay.com/sch/i.html?_nkw=${query}`) should still supply a base/site URL. Every domain-level policy (proxy tier ceiling, provider routing, per-domain memoize) keys off this field, so a blank value silently disables all of it. `create` requires it — validated client-side (must parse as an `http`/`https` URL) so a missing or malformed value fails fast with a readable usage error (exit 2) instead of a raw server 400/422; `update` keeps it optional (omitting it is still a no-op, matching the API) but a value that **is** passed is validated the same way.
 - `--tier` forces a proxy tier; `--force-tier` raises the proxy-tier ceiling past the auto-cap (history-gated: may be refused or cost more). `create --json`/`update --json` print the full scrap object (including the `_tierOverride` outcome) on stdout; a refused tier override exits 1 with a standard `--json` error envelope (`kind:"refused"` — distinct from `"unknown"`, so a script can branch on "the server said no"). When a tier was requested but the server's response carries no `_tierOverride` at all (an older server that can't confirm what actually got applied), a stderr warning is printed either way, and under `--json` the emitted object also carries `"_tierUnconfirmed": true` — the machine-readable counterpart to that warning, since a `--json` caller has no reliable reason to read stderr.
 - `scraps doctor` diagnoses the last run (error, failed selector, block status, page state, autofix outcome); `--autofix` includes the full autofix diff/dry-run/knowledge. A run that is still in flight (`status: null`, server-side) shows a `running` badge, never `failed`; a run whose item count regressed vs baseline (`statusDetail: "regression"`) shows its own amber `regression` badge, never `failed` either.
 - `scraps autofix` shows the last auto-fix attempt on its own (decision, diff, dry-run, knowledge). `--json` on a scrap that has **never run** returns `{"status":"no_runs"}` (exit 0) — distinct from `null`, which means a run exists but had no auto-fix attempt.

package/dist/commands/scraps.js

@@ -5,7 +5,7 @@ import { api, LONG_RUN_TIMEOUT_MS } from '../lib/api.js';
 import { table, json, formatDate } from '../lib/format.js';
 import { parseServerJson } from '../lib/json.js';
 import { promptPassword } from '../lib/prompt.js';
-import { validateObjectId } from '../lib/validate.js';
+import { validateObjectId, requireUrl } from '../lib/validate.js';
 import { classifyError, reportError, UsageError, RefusalError } from '../lib/errors.js';
 import { confirmDestructive, isInteractive } from '../lib/confirm.js';
 import { formatDoctor, formatAutofix, fetchRunAndFix, pickRun, pickFix } from './doctor.js';
@@ -22,6 +22,31 @@ import { maybeShowReferralTip } from '../lib/tips.js';
 function usageError(message, opts = {}) {
     process.exitCode = reportError(new UsageError(message), { json: opts.json });
 }
+/**
+ * #163 — client-side URL validation shared by `create`/`update`, mirroring
+ * the server's now-required-and-parseable `url` field (trawl_node#1823,
+ * PR #1828: `z.string().trim().default('')` -> `z.string().trim().url()`).
+ * Gives a readable message BEFORE the request goes out instead of a raw
+ * 400/422 from the server. Reuses the exact same `requireUrl` helper
+ * `trawl create`/`trawl login --url` already validate against (lib/validate.js),
+ * just routed through this file's own return-style `usageError()` convention
+ * (never throw — every other validation in this file, tier/limit/page/params/
+ * status, follows the same pattern) instead of the throw-style those two
+ * commands use. Returns `undefined` (never throws) on a bad value — the
+ * caller must check for that and `return` without calling the API.
+ */
+function tryValidateUrl(value, opts) {
+    try {
+        return requireUrl(value, '--url');
+    }
+    catch (err) {
+        if (err instanceof UsageError) {
+            usageError(err.message, { json: opts.json });
+            return undefined;
+        }
+        throw err;
+    }
+}
 function lastStatus(scrap) {
     const last = scrap.history?.[0];
     if (!last)
@@ -565,7 +590,7 @@ scraps
     .command('create')
     .description('Create a new scrap')
     .requiredOption('-t, --title <title>', 'Scrap title')
-    .option('-u, --url <url>', 'Target URL')
+    .option('-u, --url <url>', 'Target URL — the site the scrap targets, not necessarily the exact URL fetched at run time (a request script can compute that, e.g. a templated search query). Required.')
     .option('-r, --request <request>', 'Request/query')
     .option('-d, --description <text>', 'Scrap description')
     .option('--tier <tier>', `Force proxy tier (${VALID_TIERS.join('|')})`)
@@ -575,9 +600,16 @@ scraps
         usageError(`Invalid --tier "${opts.tier}" (allowed: ${VALID_TIERS.join(', ')})`, { json: opts.json });
         return;
     }
+    // #163 — the API now requires a parseable `url` on create (trawl_node
+    // #1823/#1828); validate client-side BEFORE the request goes out so a
+    // missing/unparseable value surfaces as a readable usage error instead
+    // of a raw server 400/422.
+    const url = tryValidateUrl(opts.url, opts);
+    if (url === undefined)
+        return;
     const data = await spin(() => api.post('/api/scraps', {
         title: opts.title,
-        ...(opts.url && { url: opts.url }),
+        url,
         request: opts.request || '',
         ...(opts.description !== undefined && { description: opts.description }),
         ...(opts.tier !== undefined && { proxyTier: opts.tier }),
@@ -606,7 +638,7 @@ scraps
     .command('update <id>')
     .description('Update an existing scrap')
     .option('-t, --title <title>', 'New title')
-    .option('-u, --url <url>', 'New target URL')
+    .option('-u, --url <url>', 'New target URL — the site the scrap targets. Optional: omitting it leaves the current value unchanged (a no-op, matching the API); when passed it must still be a parseable URL.')
     .option('-r, --request <request>', 'New request')
     .option('-d, --description <text>', 'New description')
     .option('--cron <expression>', 'Cron expression (empty string to disable)')
@@ -633,8 +665,15 @@ scraps
     const body = {};
     if (opts.title !== undefined)
         body.title = opts.title;
-    if (opts.url !== undefined)
-        body.url = opts.url;
+    // #163 — omitting --url stays a no-op (the API keeps that allowed on
+    // update), but a value that IS supplied must still be a parseable URL —
+    // validate it client-side before the request goes out, same as create.
+    if (opts.url !== undefined) {
+        const url = tryValidateUrl(opts.url, opts);
+        if (url === undefined)
+            return;
+        body.url = url;
+    }
     if (opts.request !== undefined)
         body.request = opts.request;
     if (opts.description !== undefined)
@@ -1167,8 +1206,28 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
                 text: opts.wait ? 'Running worker…' : 'Triggering worker…',
                 successText: opts.wait ? 'Worker run complete' : 'Worker triggered',
             });
-        if (opts.json)
+        if (opts.json) {
             json(data);
+        }
+        else if (!process.stdout.isTTY) {
+            // #160 — spin() (lib/spinner.ts, #119) intentionally emits NOTHING
+            // when stderr isn't a TTY, so a piped/redirected `trigger` wrote zero
+            // bytes to both stdout AND stderr while exiting 0: a caller driving
+            // this from a script has no way to tell success from a no-op. Gated
+            // on STDOUT specifically — that's the stream a script actually reads
+            // (`$(trawl trigger …)`, `> out.txt`, `| jq`), independent of
+            // whatever spin() decides about stderr — so an interactive TTY
+            // session, where the ora spinner already confirmed this on stderr,
+            // never sees a duplicate line here. Keeps the exact "Worker
+            // triggered"/"Worker run complete" substrings the trawl-internal QA
+            // runbook asserts on, plus the scrap id so a script has something to
+            // parse. Known residual: stdout attached to a real terminal while
+            // stderr is separately redirected (rare) still prints nothing on
+            // either stream, same as pre-#160 — not the reported/common shape
+            // (full redirection or stdout-only capture), left alone to keep this
+            // fix narrowly scoped to the stream it actually touches.
+            console.log(opts.wait ? `Worker run complete for scrap ${id}` : `Worker triggered for scrap ${id}`);
+        }
         // #107 — see the matching comment on `run`'s --watch call above (review
         // F1): honest final NDJSON line + exit code under --json, human mode
         // gets the same honest exit code too.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trawlme/cli",
-  "version": "3.7.2",
+  "version": "3.7.4",
   "description": "Trawl CLI — manage scraps from the terminal",
   "type": "module",
   "bin": {