@trawlme/cli 3.7.1 → 3.7.4

package/README.md

@@ -79,7 +79,7 @@ trawl ping [--json]                            Health/version handshake against
 ### Scrap management
 
 ```
-trawl scraps create -t <title> [-u <url>] [-r <request>] [-d <description>] [--tier <tier0|tier1|tier2|tier3|tier4>] [--json]
+trawl scraps create -t <title> -u <url> [-r <request>] [-d <description>] [--tier <tier0|tier1|tier2|tier3|tier4>] [--json]
 trawl scraps update <id> [-t <title>] [-u <url>] [-r <request>] [-d <description>] [--cron <expr>|--no-cron] [--alert <email>|--no-alert] [--autofix|--no-autofix] [-p <json>|--params-file <path>] [--tier <tier0|tier1|tier2|tier3|tier4>] [--force-tier <tier0|tier1|tier2|tier3|tier4>] [--json]
 trawl scraps watch <id> [--json]
 trawl scraps doctor <id> [--json] [--autofix]
@@ -89,6 +89,7 @@ trawl scraps banner <id> -f <file> [--json]
 trawl scraps delete <id> [--force] [--json]         Alias: rm
 ```
 
+- `-u/--url` is **the site the scrap targets**, not necessarily the exact URL fetched at run time — a scrap whose real target is computed inside its `-r/--request` script (a templated search query, an interpolated item id, e.g. `https://www.ebay.com/sch/i.html?_nkw=${query}`) should still supply a base/site URL. Every domain-level policy (proxy tier ceiling, provider routing, per-domain memoize) keys off this field, so a blank value silently disables all of it. `create` requires it — validated client-side (must parse as an `http`/`https` URL) so a missing or malformed value fails fast with a readable usage error (exit 2) instead of a raw server 400/422; `update` keeps it optional (omitting it is still a no-op, matching the API) but a value that **is** passed is validated the same way.
 - `--tier` forces a proxy tier; `--force-tier` raises the proxy-tier ceiling past the auto-cap (history-gated: may be refused or cost more). `create --json`/`update --json` print the full scrap object (including the `_tierOverride` outcome) on stdout; a refused tier override exits 1 with a standard `--json` error envelope (`kind:"refused"` — distinct from `"unknown"`, so a script can branch on "the server said no"). When a tier was requested but the server's response carries no `_tierOverride` at all (an older server that can't confirm what actually got applied), a stderr warning is printed either way, and under `--json` the emitted object also carries `"_tierUnconfirmed": true` — the machine-readable counterpart to that warning, since a `--json` caller has no reliable reason to read stderr.
 - `scraps doctor` diagnoses the last run (error, failed selector, block status, page state, autofix outcome); `--autofix` includes the full autofix diff/dry-run/knowledge. A run that is still in flight (`status: null`, server-side) shows a `running` badge, never `failed`; a run whose item count regressed vs baseline (`statusDetail: "regression"`) shows its own amber `regression` badge, never `failed` either.
 - `scraps autofix` shows the last auto-fix attempt on its own (decision, diff, dry-run, knowledge). `--json` on a scrap that has **never run** returns `{"status":"no_runs"}` (exit 0) — distinct from `null`, which means a run exists but had no auto-fix attempt.

package/dist/commands/create.js

@@ -3,6 +3,7 @@ import chalk from 'chalk';
 import { spin } from '../lib/spinner.js';
 import { api, LONG_RUN_TIMEOUT_MS, NetworkError } from '../lib/api.js';
 import { json } from '../lib/format.js';
+import { parseServerJson } from '../lib/json.js';
 import { requireUrl, requireString } from '../lib/validate.js';
 import { UsageError } from '../lib/errors.js';
 import { renderPinch, pinchEnabled } from '../lib/pinch.js';
@@ -67,7 +68,10 @@ async function printDataSample(historyId) {
         const detail = await api.get(`/api/historys/${historyId}`);
         if (typeof detail?.data !== 'string' || !detail.data)
             return;
-        const items = JSON.parse(detail.data)?.data;
+        // #159 — same JSON-string-within-JSON shape as scrap.history[0].data;
+        // tolerate the documented raw-control-char server quirk here too instead
+        // of silently losing the sample to the outer try/catch below.
+        const items = parseServerJson(detail.data)?.data;
         if (!Array.isArray(items) || items.length === 0)
             return;
         console.log(chalk.dim(`  Sample:     `) + `${items.length} item${items.length === 1 ? '' : 's'}`);

package/dist/commands/scraps.js

@@ -3,8 +3,9 @@ import chalk from 'chalk';
 import { spin } from '../lib/spinner.js';
 import { api, LONG_RUN_TIMEOUT_MS } from '../lib/api.js';
 import { table, json, formatDate } from '../lib/format.js';
+import { parseServerJson } from '../lib/json.js';
 import { promptPassword } from '../lib/prompt.js';
-import { validateObjectId } from '../lib/validate.js';
+import { validateObjectId, requireUrl } from '../lib/validate.js';
 import { classifyError, reportError, UsageError, RefusalError } from '../lib/errors.js';
 import { confirmDestructive, isInteractive } from '../lib/confirm.js';
 import { formatDoctor, formatAutofix, fetchRunAndFix, pickRun, pickFix } from './doctor.js';
@@ -21,6 +22,31 @@ import { maybeShowReferralTip } from '../lib/tips.js';
 function usageError(message, opts = {}) {
     process.exitCode = reportError(new UsageError(message), { json: opts.json });
 }
+/**
+ * #163 — client-side URL validation shared by `create`/`update`, mirroring
+ * the server's now-required-and-parseable `url` field (trawl_node#1823,
+ * PR #1828: `z.string().trim().default('')` -> `z.string().trim().url()`).
+ * Gives a readable message BEFORE the request goes out instead of a raw
+ * 400/422 from the server. Reuses the exact same `requireUrl` helper
+ * `trawl create`/`trawl login --url` already validate against (lib/validate.js),
+ * just routed through this file's own return-style `usageError()` convention
+ * (never throw — every other validation in this file, tier/limit/page/params/
+ * status, follows the same pattern) instead of the throw-style those two
+ * commands use. Returns `undefined` (never throws) on a bad value — the
+ * caller must check for that and `return` without calling the API.
+ */
+function tryValidateUrl(value, opts) {
+    try {
+        return requireUrl(value, '--url');
+    }
+    catch (err) {
+        if (err instanceof UsageError) {
+            usageError(err.message, { json: opts.json });
+            return undefined;
+        }
+        throw err;
+    }
+}
 function lastStatus(scrap) {
     const last = scrap.history?.[0];
     if (!last)
@@ -69,7 +95,11 @@ async function watchActivities(id, asJson) {
         console.log(chalk.dim('Streaming activities (Ctrl+C to stop)…\n'));
     for await (const event of api.stream(`/api/scraps/${id}/activities/stream`)) {
         try {
-            const activity = JSON.parse(event);
+            // #159 — same tolerant parse as api.ts's response-parsing seam: an SSE
+            // activity line can carry the same server-side raw-control-character
+            // quirk as any other API payload, and this feeds `--json` NDJSON
+            // output too (not just create).
+            const activity = parseServerJson(event);
             if (asJson) {
                 console.log(JSON.stringify(activity));
                 continue;
@@ -560,7 +590,7 @@ scraps
     .command('create')
     .description('Create a new scrap')
     .requiredOption('-t, --title <title>', 'Scrap title')
-    .option('-u, --url <url>', 'Target URL')
+    .option('-u, --url <url>', 'Target URL — the site the scrap targets, not necessarily the exact URL fetched at run time (a request script can compute that, e.g. a templated search query). Required.')
     .option('-r, --request <request>', 'Request/query')
     .option('-d, --description <text>', 'Scrap description')
     .option('--tier <tier>', `Force proxy tier (${VALID_TIERS.join('|')})`)
@@ -570,9 +600,16 @@ scraps
         usageError(`Invalid --tier "${opts.tier}" (allowed: ${VALID_TIERS.join(', ')})`, { json: opts.json });
         return;
     }
+    // #163 — the API now requires a parseable `url` on create (trawl_node
+    // #1823/#1828); validate client-side BEFORE the request goes out so a
+    // missing/unparseable value surfaces as a readable usage error instead
+    // of a raw server 400/422.
+    const url = tryValidateUrl(opts.url, opts);
+    if (url === undefined)
+        return;
     const data = await spin(() => api.post('/api/scraps', {
         title: opts.title,
-        ...(opts.url && { url: opts.url }),
+        url,
         request: opts.request || '',
         ...(opts.description !== undefined && { description: opts.description }),
         ...(opts.tier !== undefined && { proxyTier: opts.tier }),
@@ -601,7 +638,7 @@ scraps
     .command('update <id>')
     .description('Update an existing scrap')
     .option('-t, --title <title>', 'New title')
-    .option('-u, --url <url>', 'New target URL')
+    .option('-u, --url <url>', 'New target URL — the site the scrap targets. Optional: omitting it leaves the current value unchanged (a no-op, matching the API); when passed it must still be a parseable URL.')
     .option('-r, --request <request>', 'New request')
     .option('-d, --description <text>', 'New description')
     .option('--cron <expression>', 'Cron expression (empty string to disable)')
@@ -628,8 +665,15 @@ scraps
     const body = {};
     if (opts.title !== undefined)
         body.title = opts.title;
-    if (opts.url !== undefined)
-        body.url = opts.url;
+    // #163 — omitting --url stays a no-op (the API keeps that allowed on
+    // update), but a value that IS supplied must still be a parseable URL —
+    // validate it client-side before the request goes out, same as create.
+    if (opts.url !== undefined) {
+        const url = tryValidateUrl(opts.url, opts);
+        if (url === undefined)
+            return;
+        body.url = url;
+    }
     if (opts.request !== undefined)
         body.request = opts.request;
     if (opts.description !== undefined)
@@ -939,7 +983,12 @@ export function attachDataCommand(parent, attachOpts = {}) {
         let items;
         if (typeof detail?.data === 'string' && detail.data) {
             try {
-                items = JSON.parse(detail.data)?.data;
+                // #159 — `detail.data` is the same JSON-string-within-JSON shape
+                // (`history.data`) the create --json bug lived in: a raw control
+                // character here would otherwise throw and silently report "aged
+                // out of retention" below for data that is actually present and
+                // recoverable, burning a --fresh re-run + quota for nothing.
+                items = parseServerJson(detail.data)?.data;
             }
             catch {
                 items = undefined;
@@ -1157,8 +1206,28 @@ export function attachTriggerCommand(parent, attachOpts = {}) {
                 text: opts.wait ? 'Running worker…' : 'Triggering worker…',
                 successText: opts.wait ? 'Worker run complete' : 'Worker triggered',
             });
-        if (opts.json)
+        if (opts.json) {
             json(data);
+        }
+        else if (!process.stdout.isTTY) {
+            // #160 — spin() (lib/spinner.ts, #119) intentionally emits NOTHING
+            // when stderr isn't a TTY, so a piped/redirected `trigger` wrote zero
+            // bytes to both stdout AND stderr while exiting 0: a caller driving
+            // this from a script has no way to tell success from a no-op. Gated
+            // on STDOUT specifically — that's the stream a script actually reads
+            // (`$(trawl trigger …)`, `> out.txt`, `| jq`), independent of
+            // whatever spin() decides about stderr — so an interactive TTY
+            // session, where the ora spinner already confirmed this on stderr,
+            // never sees a duplicate line here. Keeps the exact "Worker
+            // triggered"/"Worker run complete" substrings the trawl-internal QA
+            // runbook asserts on, plus the scrap id so a script has something to
+            // parse. Known residual: stdout attached to a real terminal while
+            // stderr is separately redirected (rare) still prints nothing on
+            // either stream, same as pre-#160 — not the reported/common shape
+            // (full redirection or stdout-only capture), left alone to keep this
+            // fix narrowly scoped to the stream it actually touches.
+            console.log(opts.wait ? `Worker run complete for scrap ${id}` : `Worker triggered for scrap ${id}`);
+        }
         // #107 — see the matching comment on `run`'s --watch call above (review
         // F1): honest final NDJSON line + exit code under --json, human mode
         // gets the same honest exit code too.

package/dist/lib/api.js

@@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { dirname, resolve } from 'node:path';
 import { getApiUrl, getToken } from './config.js';
+import { parseServerJson } from './json.js';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const pkg = JSON.parse(readFileSync(resolve(__dirname, '../../package.json'), 'utf8'));
 const USER_AGENT = `@trawlme/cli/${pkg.version}`;
@@ -135,13 +136,17 @@ function extractErrorMessage(raw, statusText) {
     if (!raw)
         return '';
     try {
-        const parsed = JSON.parse(raw);
+        // #159 — an error envelope's message/description/nested `error` string can
+        // carry the same raw-control-char server quirk as a success payload;
+        // recover it here too instead of falling all the way back to the raw,
+        // unparsed blob as the "error message".
+        const parsed = parseServerJson(raw);
         if (parsed && typeof parsed === 'object') {
             const env = parsed;
             const nested = typeof env.error === 'string' ? env.error : null;
             if (nested) {
                 try {
-                    const inner = JSON.parse(nested);
+                    const inner = parseServerJson(nested);
                     const details = inner.details;
                     if (details && typeof details.message === 'string')
                         return details.message;
@@ -178,7 +183,7 @@ function extractErrorMessage(raw, statusText) {
  */
 function extractUpgradeUrl(raw) {
     try {
-        const parsed = JSON.parse(raw);
+        const parsed = parseServerJson(raw);
         if (typeof parsed.upgradeUrl === 'string')
             return parsed.upgradeUrl;
         const details = parsed.details;
@@ -187,7 +192,7 @@ function extractUpgradeUrl(raw) {
         }
         if (typeof parsed.error === 'string') {
             try {
-                const inner = JSON.parse(parsed.error);
+                const inner = parseServerJson(parsed.error);
                 if (typeof inner.upgradeUrl === 'string')
                     return inner.upgradeUrl;
                 const innerDetails = inner.details;
@@ -252,7 +257,7 @@ async function request(path, options = {}, reqOpts = {}) {
     try {
         if (!text)
             return {};
-        const parsed = JSON.parse(text);
+        const parsed = parseServerJson(text);
         // Unwrap API envelope { type, message, data: T }
         if (parsed !== null && typeof parsed === 'object' && 'data' in parsed) {
             return parsed.data;
@@ -284,7 +289,7 @@ async function upload(path, formData, reqOpts = {}) {
     try {
         if (!text)
             return {};
-        const parsed = JSON.parse(text);
+        const parsed = parseServerJson(text);
         // Unwrap API envelope { type, message, data: T }
         if (parsed !== null && typeof parsed === 'object' && 'data' in parsed) {
             return parsed.data;
@@ -307,7 +312,7 @@ async function publicPost(path, body, baseUrlOverride, reqOpts = {}) {
     await throwIfError(res, true);
     const text = await res.text();
     try {
-        const data = text ? JSON.parse(text) : {};
+        const data = text ? parseServerJson(text) : {};
         return { data, headers: res.headers };
     }
     catch {
@@ -344,7 +349,7 @@ async function publicGet(path, reqOpts = {}) {
     try {
         if (!text)
             return {};
-        const parsed = JSON.parse(text);
+        const parsed = parseServerJson(text);
         // Unwrap API envelope { type, message, data: T }
         if (parsed !== null && typeof parsed === 'object' && 'data' in parsed) {
             return parsed.data;

package/dist/lib/json.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Shared, tolerant JSON parsing for server response bodies (#159).
+ *
+ * The Trawl API is known to occasionally emit a raw, unescaped control
+ * character (U+0000–U+001F — a literal newline, NUL, BEL, …) inside a JSON
+ * string value, most often nested inside `scrap.history[0].data` (a
+ * worker-envelope blob embedded as a string field of the outer response —
+ * the exact server-side emission point is not yet pinned down; tracked in
+ * comes-io/trawl_node#1768). That is invalid per RFC 8259 — our own QA
+ * tooling already works around it by parsing raw API responses with
+ * Python's `json.loads(text, strict=False)` — but native `JSON.parse`
+ * has no equivalent lenient mode: it throws outright, and until this fix
+ * every one of `api.ts`'s response-parsing call sites (`request`, `upload`,
+ * `publicGet`, `publicPost`) turned that into a bare "Invalid JSON in server
+ * response" error, discarding the ENTIRE response — even a successful
+ * create/run whose scrap was already persisted server-side. Exactly the
+ * "payload-dependent, not a constant break" shape reported in #159: it only
+ * fires when the specific scraped content/log text happens to carry a raw
+ * control byte.
+ *
+ * `parseServerJson` recovers from that ONE known shape — nothing else. It
+ * tries a normal strict `JSON.parse` first (the common case, zero extra
+ * cost) and only falls back to a sanitizing re-parse when that throws.
+ * Genuinely malformed JSON (truncated body, stray token, …) still throws
+ * after the fallback also fails — this is a targeted recovery, not a
+ * general-purpose lenient parser.
+ */
+export declare function escapeRawControlCharsInJsonStrings(text: string): string;
+/**
+ * Parse a server response body, tolerating the known #159 quirk above. Never
+ * silently drops data: a byte that reaches here as a real control character
+ * comes back out as the same character (via the escape → JSON.parse
+ * unescape round trip) once parsed, and JSON.stringify guarantees it is
+ * re-escaped correctly on the way back out through the shared `--json`
+ * output seam (`lib/format.ts#json`).
+ */
+export declare function parseServerJson(text: string): unknown;

package/dist/lib/json.js

@@ -0,0 +1,104 @@
+/**
+ * Shared, tolerant JSON parsing for server response bodies (#159).
+ *
+ * The Trawl API is known to occasionally emit a raw, unescaped control
+ * character (U+0000–U+001F — a literal newline, NUL, BEL, …) inside a JSON
+ * string value, most often nested inside `scrap.history[0].data` (a
+ * worker-envelope blob embedded as a string field of the outer response —
+ * the exact server-side emission point is not yet pinned down; tracked in
+ * comes-io/trawl_node#1768). That is invalid per RFC 8259 — our own QA
+ * tooling already works around it by parsing raw API responses with
+ * Python's `json.loads(text, strict=False)` — but native `JSON.parse`
+ * has no equivalent lenient mode: it throws outright, and until this fix
+ * every one of `api.ts`'s response-parsing call sites (`request`, `upload`,
+ * `publicGet`, `publicPost`) turned that into a bare "Invalid JSON in server
+ * response" error, discarding the ENTIRE response — even a successful
+ * create/run whose scrap was already persisted server-side. Exactly the
+ * "payload-dependent, not a constant break" shape reported in #159: it only
+ * fires when the specific scraped content/log text happens to carry a raw
+ * control byte.
+ *
+ * `parseServerJson` recovers from that ONE known shape — nothing else. It
+ * tries a normal strict `JSON.parse` first (the common case, zero extra
+ * cost) and only falls back to a sanitizing re-parse when that throws.
+ * Genuinely malformed JSON (truncated body, stray token, …) still throws
+ * after the fallback also fails — this is a targeted recovery, not a
+ * general-purpose lenient parser.
+ */
+/**
+ * Escape any raw control character (U+0000–U+001F) found INSIDE a JSON
+ * string literal, leaving everything else — including legitimate raw
+ * whitespace between tokens (space/tab/CR/LF are valid there per RFC 8259) —
+ * untouched. A small state machine tracks whether the scan is currently
+ * inside a `"…"` string and whether the previous character was an unescaped
+ * backslash; it does not otherwise validate structure, so text that is
+ * invalid JSON for any OTHER reason still fails the subsequent `JSON.parse`
+ * call unchanged.
+ */
+const NAMED_CONTROL_CHAR_ESCAPES = {
+    '\n': '\\n',
+    '\r': '\\r',
+    '\t': '\\t',
+    '\b': '\\b',
+    '\f': '\\f',
+};
+export function escapeRawControlCharsInJsonStrings(text) {
+    let out = '';
+    let inString = false;
+    let escaped = false;
+    for (let i = 0; i < text.length; i += 1) {
+        const ch = text[i];
+        const code = text.charCodeAt(i);
+        if (!inString) {
+            if (ch === '"')
+                inString = true;
+            out += ch;
+            continue;
+        }
+        if (escaped) {
+            out += ch;
+            escaped = false;
+            continue;
+        }
+        if (ch === '\\') {
+            out += ch;
+            escaped = true;
+            continue;
+        }
+        if (ch === '"') {
+            inString = false;
+            out += ch;
+            continue;
+        }
+        if (code <= 0x1f) {
+            // Raw control character inside a string literal — recover the same
+            // way JSON.stringify would have escaped it on the way out, instead of
+            // losing the whole response to a SyntaxError.
+            out += NAMED_CONTROL_CHAR_ESCAPES[ch] ?? `\\u${code.toString(16).padStart(4, '0')}`;
+            continue;
+        }
+        out += ch;
+    }
+    return out;
+}
+/**
+ * Parse a server response body, tolerating the known #159 quirk above. Never
+ * silently drops data: a byte that reaches here as a real control character
+ * comes back out as the same character (via the escape → JSON.parse
+ * unescape round trip) once parsed, and JSON.stringify guarantees it is
+ * re-escaped correctly on the way back out through the shared `--json`
+ * output seam (`lib/format.ts#json`).
+ */
+export function parseServerJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch (firstErr) {
+        try {
+            return JSON.parse(escapeRawControlCharsInJsonStrings(text));
+        }
+        catch {
+            throw firstErr;
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trawlme/cli",
-  "version": "3.7.1",
+  "version": "3.7.4",
   "description": "Trawl CLI — manage scraps from the terminal",
   "type": "module",
   "bin": {