@trawlme/cli 3.7.1 → 3.7.2

package/dist/commands/create.js

@@ -3,6 +3,7 @@ import chalk from 'chalk';
 import { spin } from '../lib/spinner.js';
 import { api, LONG_RUN_TIMEOUT_MS, NetworkError } from '../lib/api.js';
 import { json } from '../lib/format.js';
+import { parseServerJson } from '../lib/json.js';
 import { requireUrl, requireString } from '../lib/validate.js';
 import { UsageError } from '../lib/errors.js';
 import { renderPinch, pinchEnabled } from '../lib/pinch.js';
@@ -67,7 +68,10 @@ async function printDataSample(historyId) {
         const detail = await api.get(`/api/historys/${historyId}`);
         if (typeof detail?.data !== 'string' || !detail.data)
             return;
-        const items = JSON.parse(detail.data)?.data;
+        // #159 — same JSON-string-within-JSON shape as scrap.history[0].data;
+        // tolerate the documented raw-control-char server quirk here too instead
+        // of silently losing the sample to the outer try/catch below.
+        const items = parseServerJson(detail.data)?.data;
         if (!Array.isArray(items) || items.length === 0)
             return;
         console.log(chalk.dim(`  Sample:     `) + `${items.length} item${items.length === 1 ? '' : 's'}`);

package/dist/commands/scraps.js

@@ -3,6 +3,7 @@ import chalk from 'chalk';
 import { spin } from '../lib/spinner.js';
 import { api, LONG_RUN_TIMEOUT_MS } from '../lib/api.js';
 import { table, json, formatDate } from '../lib/format.js';
+import { parseServerJson } from '../lib/json.js';
 import { promptPassword } from '../lib/prompt.js';
 import { validateObjectId } from '../lib/validate.js';
 import { classifyError, reportError, UsageError, RefusalError } from '../lib/errors.js';
@@ -69,7 +70,11 @@ async function watchActivities(id, asJson) {
         console.log(chalk.dim('Streaming activities (Ctrl+C to stop)…\n'));
     for await (const event of api.stream(`/api/scraps/${id}/activities/stream`)) {
         try {
-            const activity = JSON.parse(event);
+            // #159 — same tolerant parse as api.ts's response-parsing seam: an SSE
+            // activity line can carry the same server-side raw-control-character
+            // quirk as any other API payload, and this feeds `--json` NDJSON
+            // output too (not just create).
+            const activity = parseServerJson(event);
             if (asJson) {
                 console.log(JSON.stringify(activity));
                 continue;
@@ -939,7 +944,12 @@ export function attachDataCommand(parent, attachOpts = {}) {
         let items;
         if (typeof detail?.data === 'string' && detail.data) {
             try {
-                items = JSON.parse(detail.data)?.data;
+                // #159 — `detail.data` is the same JSON-string-within-JSON shape
+                // (`history.data`) the create --json bug lived in: a raw control
+                // character here would otherwise throw and silently report "aged
+                // out of retention" below for data that is actually present and
+                // recoverable, burning a --fresh re-run + quota for nothing.
+                items = parseServerJson(detail.data)?.data;
             }
             catch {
                 items = undefined;

package/dist/lib/api.js

@@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { dirname, resolve } from 'node:path';
 import { getApiUrl, getToken } from './config.js';
+import { parseServerJson } from './json.js';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const pkg = JSON.parse(readFileSync(resolve(__dirname, '../../package.json'), 'utf8'));
 const USER_AGENT = `@trawlme/cli/${pkg.version}`;
@@ -135,13 +136,17 @@ function extractErrorMessage(raw, statusText) {
     if (!raw)
         return '';
     try {
-        const parsed = JSON.parse(raw);
+        // #159 — an error envelope's message/description/nested `error` string can
+        // carry the same raw-control-char server quirk as a success payload;
+        // recover it here too instead of falling all the way back to the raw,
+        // unparsed blob as the "error message".
+        const parsed = parseServerJson(raw);
         if (parsed && typeof parsed === 'object') {
             const env = parsed;
             const nested = typeof env.error === 'string' ? env.error : null;
             if (nested) {
                 try {
-                    const inner = JSON.parse(nested);
+                    const inner = parseServerJson(nested);
                     const details = inner.details;
                     if (details && typeof details.message === 'string')
                         return details.message;
@@ -178,7 +183,7 @@ function extractErrorMessage(raw, statusText) {
  */
 function extractUpgradeUrl(raw) {
     try {
-        const parsed = JSON.parse(raw);
+        const parsed = parseServerJson(raw);
         if (typeof parsed.upgradeUrl === 'string')
             return parsed.upgradeUrl;
         const details = parsed.details;
@@ -187,7 +192,7 @@ function extractUpgradeUrl(raw) {
         }
         if (typeof parsed.error === 'string') {
             try {
-                const inner = JSON.parse(parsed.error);
+                const inner = parseServerJson(parsed.error);
                 if (typeof inner.upgradeUrl === 'string')
                     return inner.upgradeUrl;
                 const innerDetails = inner.details;
@@ -252,7 +257,7 @@ async function request(path, options = {}, reqOpts = {}) {
     try {
         if (!text)
             return {};
-        const parsed = JSON.parse(text);
+        const parsed = parseServerJson(text);
         // Unwrap API envelope { type, message, data: T }
         if (parsed !== null && typeof parsed === 'object' && 'data' in parsed) {
             return parsed.data;
@@ -284,7 +289,7 @@ async function upload(path, formData, reqOpts = {}) {
     try {
         if (!text)
             return {};
-        const parsed = JSON.parse(text);
+        const parsed = parseServerJson(text);
         // Unwrap API envelope { type, message, data: T }
         if (parsed !== null && typeof parsed === 'object' && 'data' in parsed) {
             return parsed.data;
@@ -307,7 +312,7 @@ async function publicPost(path, body, baseUrlOverride, reqOpts = {}) {
     await throwIfError(res, true);
     const text = await res.text();
     try {
-        const data = text ? JSON.parse(text) : {};
+        const data = text ? parseServerJson(text) : {};
         return { data, headers: res.headers };
     }
     catch {
@@ -344,7 +349,7 @@ async function publicGet(path, reqOpts = {}) {
     try {
         if (!text)
             return {};
-        const parsed = JSON.parse(text);
+        const parsed = parseServerJson(text);
         // Unwrap API envelope { type, message, data: T }
         if (parsed !== null && typeof parsed === 'object' && 'data' in parsed) {
             return parsed.data;

package/dist/lib/json.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Shared, tolerant JSON parsing for server response bodies (#159).
+ *
+ * The Trawl API is known to occasionally emit a raw, unescaped control
+ * character (U+0000–U+001F — a literal newline, NUL, BEL, …) inside a JSON
+ * string value, most often nested inside `scrap.history[0].data` (a
+ * worker-envelope blob embedded as a string field of the outer response —
+ * the exact server-side emission point is not yet pinned down; tracked in
+ * comes-io/trawl_node#1768). That is invalid per RFC 8259 — our own QA
+ * tooling already works around it by parsing raw API responses with
+ * Python's `json.loads(text, strict=False)` — but native `JSON.parse`
+ * has no equivalent lenient mode: it throws outright, and until this fix
+ * every one of `api.ts`'s response-parsing call sites (`request`, `upload`,
+ * `publicGet`, `publicPost`) turned that into a bare "Invalid JSON in server
+ * response" error, discarding the ENTIRE response — even a successful
+ * create/run whose scrap was already persisted server-side. Exactly the
+ * "payload-dependent, not a constant break" shape reported in #159: it only
+ * fires when the specific scraped content/log text happens to carry a raw
+ * control byte.
+ *
+ * `parseServerJson` recovers from that ONE known shape — nothing else. It
+ * tries a normal strict `JSON.parse` first (the common case, zero extra
+ * cost) and only falls back to a sanitizing re-parse when that throws.
+ * Genuinely malformed JSON (truncated body, stray token, …) still throws
+ * after the fallback also fails — this is a targeted recovery, not a
+ * general-purpose lenient parser.
+ */
+export declare function escapeRawControlCharsInJsonStrings(text: string): string;
+/**
+ * Parse a server response body, tolerating the known #159 quirk above. Never
+ * silently drops data: a byte that reaches here as a real control character
+ * comes back out as the same character (via the escape → JSON.parse
+ * unescape round trip) once parsed, and JSON.stringify guarantees it is
+ * re-escaped correctly on the way back out through the shared `--json`
+ * output seam (`lib/format.ts#json`).
+ */
+export declare function parseServerJson(text: string): unknown;

package/dist/lib/json.js

@@ -0,0 +1,104 @@
+/**
+ * Shared, tolerant JSON parsing for server response bodies (#159).
+ *
+ * The Trawl API is known to occasionally emit a raw, unescaped control
+ * character (U+0000–U+001F — a literal newline, NUL, BEL, …) inside a JSON
+ * string value, most often nested inside `scrap.history[0].data` (a
+ * worker-envelope blob embedded as a string field of the outer response —
+ * the exact server-side emission point is not yet pinned down; tracked in
+ * comes-io/trawl_node#1768). That is invalid per RFC 8259 — our own QA
+ * tooling already works around it by parsing raw API responses with
+ * Python's `json.loads(text, strict=False)` — but native `JSON.parse`
+ * has no equivalent lenient mode: it throws outright, and until this fix
+ * every one of `api.ts`'s response-parsing call sites (`request`, `upload`,
+ * `publicGet`, `publicPost`) turned that into a bare "Invalid JSON in server
+ * response" error, discarding the ENTIRE response — even a successful
+ * create/run whose scrap was already persisted server-side. Exactly the
+ * "payload-dependent, not a constant break" shape reported in #159: it only
+ * fires when the specific scraped content/log text happens to carry a raw
+ * control byte.
+ *
+ * `parseServerJson` recovers from that ONE known shape — nothing else. It
+ * tries a normal strict `JSON.parse` first (the common case, zero extra
+ * cost) and only falls back to a sanitizing re-parse when that throws.
+ * Genuinely malformed JSON (truncated body, stray token, …) still throws
+ * after the fallback also fails — this is a targeted recovery, not a
+ * general-purpose lenient parser.
+ */
+/**
+ * Escape any raw control character (U+0000–U+001F) found INSIDE a JSON
+ * string literal, leaving everything else — including legitimate raw
+ * whitespace between tokens (space/tab/CR/LF are valid there per RFC 8259) —
+ * untouched. A small state machine tracks whether the scan is currently
+ * inside a `"…"` string and whether the previous character was an unescaped
+ * backslash; it does not otherwise validate structure, so text that is
+ * invalid JSON for any OTHER reason still fails the subsequent `JSON.parse`
+ * call unchanged.
+ */
+const NAMED_CONTROL_CHAR_ESCAPES = {
+    '\n': '\\n',
+    '\r': '\\r',
+    '\t': '\\t',
+    '\b': '\\b',
+    '\f': '\\f',
+};
+export function escapeRawControlCharsInJsonStrings(text) {
+    let out = '';
+    let inString = false;
+    let escaped = false;
+    for (let i = 0; i < text.length; i += 1) {
+        const ch = text[i];
+        const code = text.charCodeAt(i);
+        if (!inString) {
+            if (ch === '"')
+                inString = true;
+            out += ch;
+            continue;
+        }
+        if (escaped) {
+            out += ch;
+            escaped = false;
+            continue;
+        }
+        if (ch === '\\') {
+            out += ch;
+            escaped = true;
+            continue;
+        }
+        if (ch === '"') {
+            inString = false;
+            out += ch;
+            continue;
+        }
+        if (code <= 0x1f) {
+            // Raw control character inside a string literal — recover the same
+            // way JSON.stringify would have escaped it on the way out, instead of
+            // losing the whole response to a SyntaxError.
+            out += NAMED_CONTROL_CHAR_ESCAPES[ch] ?? `\\u${code.toString(16).padStart(4, '0')}`;
+            continue;
+        }
+        out += ch;
+    }
+    return out;
+}
+/**
+ * Parse a server response body, tolerating the known #159 quirk above. Never
+ * silently drops data: a byte that reaches here as a real control character
+ * comes back out as the same character (via the escape → JSON.parse
+ * unescape round trip) once parsed, and JSON.stringify guarantees it is
+ * re-escaped correctly on the way back out through the shared `--json`
+ * output seam (`lib/format.ts#json`).
+ */
+export function parseServerJson(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch (firstErr) {
+        try {
+            return JSON.parse(escapeRawControlCharsInJsonStrings(text));
+        }
+        catch {
+            throw firstErr;
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trawlme/cli",
-  "version": "3.7.1",
+  "version": "3.7.2",
   "description": "Trawl CLI — manage scraps from the terminal",
   "type": "module",
   "bin": {