@trawlme/cli 1.16.0 → 1.18.0

package/README.md

@@ -26,27 +26,43 @@ Custom API URL: `trawl login --url https://self-hosted.example.com`
 
 ## Commands
 
+All commands accept a global `--debug` flag to show full error stack traces on failure.
+
 ### Auth
 
 ```
 trawl login [--url <url>] [--token <jwt>] [--email <email>] [--password <pass>]
 trawl logout
+trawl token                                  Print the stored session JWT (for MCP Bearer auth)
 ```
 
 ### Scraps
 
 ```
-trawl scraps list [--json] [--status <success|failure|never>]
+trawl scraps list [--json] [--status <success|failure|never>] [--limit <n>] [--page <n>]
 trawl scraps get <id> [--json]
-trawl scraps create -t <title> [-u <url>] [-r <request>] [-s <scrapper>]
-trawl scraps update <id> [-t title] [-u <url>] [-r request] [-s scrapper] [--cron <expr>|--no-cron] [--alert <email>|--no-alert] [-p <json>|--params-file <path>]
+trawl scraps create -t <title> [-u <url>] [-r <request>] [-d <description>] [--tier <tier0|tier1|tier2|tier3|tier4>]
+trawl scraps update <id> [-t <title>] [-u <url>] [-r <request>] [-d <description>] [--cron <expr>|--no-cron] [--alert <email>|--no-alert] [--autofix|--no-autofix] [-p <json>|--params-file <path>] [--tier <tier0|tier1|tier2|tier3|tier4>] [--force-tier <tier0|tier1|tier2|tier3|tier4>]
 trawl scraps run <id> [--watch]
-trawl scraps trigger <id> [--watch]
+trawl scraps trigger <id> [--watch] [--wait]
 trawl scraps watch <id>
-trawl scraps data <id> [--json]
+trawl scraps data <id> [--json] [--fresh] [--errors]
+trawl scraps history <id> [--json] [-n <limit>]
+trawl scraps run-info <hid> [--json]
+trawl scraps doctor <id> [--json] [--autofix]
+trawl scraps autofix <id> [--json]
+trawl scraps snapshot <id> [--error] [-o <file>]
+trawl scraps banner <id> -f <file>
 trawl scraps rm <id> [--force]
 ```
 
+- `--tier` forces a proxy tier; `--force-tier` raises the proxy-tier ceiling past the auto-cap (history-gated: may be refused or cost more).
+- `scraps doctor` diagnoses the last run (error, failed selector, block status, page state, autofix outcome); `--autofix` includes the full autofix diff/dry-run/knowledge.
+- `scraps autofix` shows the last auto-fix attempt on its own (decision, diff, dry-run, knowledge).
+- `scraps snapshot --error` fetches the error-path snapshot instead of the normal one; `-o <file>` writes to a file instead of stdout.
+- `scraps history` lists past runs (newest first); `scraps run-info <hid>` shows details of a single run from that history.
+- `scraps data` returns the last persisted run payload (no execute quota); `--fresh` runs the scrap live instead (consumes execute quota); `--errors` shows the last run's error detail.
+
 ### Scrap accounts
 
 ```
@@ -54,8 +70,11 @@ trawl scraps account set <id> [-u <username>] [-p <password>]
 trawl scraps account delete <id> [--force]
 trawl scraps account clear-session <id>
 trawl scraps account status <id> [--json]
+trawl scraps account session set <id> -c <file>
 ```
 
+`account session set` uploads a Puppeteer cookie JSON array to bootstrap a logged-in session without storing credentials (BYO-cookies).
+
 ### Claude Code skills
 
 The CLI bundles a Claude Code skill that teaches Claude how to use `trawl`. Once installed, Claude can manage scraps for you via prompts.

package/dist/commands/doctor.d.ts

@@ -25,6 +25,7 @@ export interface Run {
         selectors?: Record<string, number>;
     } | null;
     blocked?: boolean;
+    blockType?: string | null;
     proxyTier?: string | null;
     regressionDetected?: boolean;
     baselineLength?: number | null;
@@ -33,6 +34,14 @@ export interface Run {
     time?: number | null;
     triggeredBy?: string | null;
 }
+/**
+ * Resolve a known anti-bot vendor (or auth) name from a worker `blockType`
+ * string. Returns null when the run isn't blocked, or when it's blocked by
+ * something other than a known vendor (e.g. `proxy-domain-gate`,
+ * `rate_limited_per_host`) — those stay on the genuine-error path since we
+ * can't honestly attribute them to a specific "no reliable bypass" wall.
+ */
+export declare function detectWallVendor(run: Pick<Run, 'blocked' | 'blockType'>): string | null;
 /**
  * Autofix activity metadata — from the persisted ai_fix_end activity.
  * aiUsage (cost) is stripped server-side; all diagnostics are kept.

package/dist/commands/doctor.js

@@ -1,5 +1,48 @@
 import { api } from '../lib/api.js';
 import chalk from 'chalk';
+/**
+ * Known anti-bot vendors (+ auth) that the worker's `blockType` field may name.
+ * Ordered by first-match; `blockType` is a freeform worker string, not an enum
+ * (see the `Run.blockType` doc comment), so this is a best-effort substring
+ * match against the real field — never an invented/mocked value.
+ *
+ * `datadome`/`perimeterx`/`akamai`/`cloudflare` are literal substrings the
+ * current worker emits (detect.js). `kasada` and `auth` are forward-compatible:
+ * the issue (#62 / wall-registry WS-7) names them as genuine walls, but the
+ * current worker vocabulary does not yet emit them — these branches light up
+ * automatically if/when a future worker classifier does, without a CLI change.
+ *
+ * The `auth` pattern is delimiter-anchored (`^`/`$`/`-_:`) rather than a bare
+ * substring so it matches worker-style tokens (`auth`, `auth-wall`, `auth_wall`,
+ * `login:auth`) without false-matching `oauth`/`authorization`/`author`. It uses
+ * `[-_:]` (not `\b`) because `_` is a JS `\w` char, so `\bauth\b` would miss the
+ * underscore-delimited `auth_wall` form the worker's `rate_limited_per_host`-style
+ * naming favors.
+ */
+const WALL_VENDOR_PATTERNS = [
+    [/datadome/i, 'DataDome'],
+    [/kasada/i, 'Kasada'],
+    [/perimeterx/i, 'PerimeterX'],
+    [/akamai/i, 'Akamai'],
+    [/cloudflare/i, 'Cloudflare'],
+    [/(?:^|[-_:])auth(?:$|[-_:])/i, 'auth'],
+];
+/**
+ * Resolve a known anti-bot vendor (or auth) name from a worker `blockType`
+ * string. Returns null when the run isn't blocked, or when it's blocked by
+ * something other than a known vendor (e.g. `proxy-domain-gate`,
+ * `rate_limited_per_host`) — those stay on the genuine-error path since we
+ * can't honestly attribute them to a specific "no reliable bypass" wall.
+ */
+export function detectWallVendor(run) {
+    if (run.blocked !== true || !run.blockType)
+        return null;
+    for (const [pattern, label] of WALL_VENDOR_PATTERNS) {
+        if (pattern.test(run.blockType))
+            return label;
+    }
+    return null;
+}
 const TIER_LABELS = {
     tier0: 'Tier 0',
     tier1: 'Tier 1',
@@ -9,7 +52,7 @@ const TIER_LABELS = {
 };
 const RUN_ALLOWLIST = [
     '_id', 'status', 'statusDetail', 'length', 'errorMessage', 'errorSnapshot',
-    'emptyContext', 'blocked', 'proxyTier', 'regressionDetected', 'baselineLength',
+    'emptyContext', 'blocked', 'blockType', 'proxyTier', 'regressionDetected', 'baselineLength',
     'fixVersionId', 'createdAt', 'time', 'triggeredBy',
 ];
 const FIX_ALLOWLIST = [
@@ -49,9 +92,15 @@ export function formatDoctor(scrapTitle, run, fix = null, scrapId) {
             : chalk.red('● failed');
     lines.push(`${chalk.bold(scrapTitle)}  ${badge}${run.statusDetail ? ` (${run.statusDetail})` : ''}`);
     lines.push(chalk.dim(`  Run ID:  ${run._id}`));
-    // Error message
+    // Error message — an honest accept-wall string for known-walled scraps
+    // (DataDome/Kasada/PerimeterX/Akamai/auth terminal verdict from the worker),
+    // otherwise the real error (genuine transient failure).
+    const wallVendor = detectWallVendor(run);
     const errMsg = run.errorMessage ?? run.errorSnapshot?.errorMessage;
-    if (errMsg) {
+    if (wallVendor) {
+        lines.push(chalk.dim('  Error:   ') + chalk.red(`walled: ${wallVendor} — no reliable bypass`));
+    }
+    else if (errMsg) {
         lines.push(chalk.dim('  Error:   ') + chalk.red(errMsg));
     }
     // Failed selector

package/dist/commands/login.js

@@ -2,14 +2,23 @@ import { Command } from 'commander';
 import chalk from 'chalk';
 import config, { getApiUrl } from '../lib/config.js';
 import { api } from '../lib/api.js';
-import { requireJwt, requireUrl } from '../lib/validate.js';
+import { requireFreshJwt, requireUrl } from '../lib/validate.js';
 import { promptPassword } from '../lib/prompt.js';
 async function promptEmail() {
     const { createInterface } = await import('readline');
     const rl = createInterface({ input: process.stdin, output: process.stdout });
     try {
-        return await new Promise((resolve) => {
-            rl.question('Email: ', (answer) => resolve(answer.trim()));
+        return await new Promise((resolve, reject) => {
+            let answered = false;
+            rl.question('Email: ', (answer) => {
+                answered = true;
+                resolve(answer.trim());
+            });
+            rl.on('close', () => {
+                if (!answered) {
+                    reject(new Error('No input received — pass -e/-p or set TRAWL_TOKEN for non-interactive use'));
+                }
+            });
         });
     }
     finally {
@@ -23,12 +32,21 @@ export const login = new Command('login')
     .option('-e, --email <email>', 'Email address')
     .option('-p, --password <password>', 'Password (CI only — visible in process list and shell history)')
     .action(async (opts) => {
-    if (opts.url)
-        config.set('apiUrl', requireUrl(opts.url, '--url'));
+    // Validate --url but do NOT persist it yet. A failed signin must not
+    // brick the config by pointing it at an unreachable/wrong host while the
+    // OLD token stays stored (and would then be sent to that new host on the
+    // next command). Target this run via `pendingUrl` and persist only once
+    // a token has actually been obtained. (#68)
+    const pendingUrl = opts.url ? requireUrl(opts.url, '--url') : undefined;
+    const persistUrlIfPending = () => {
+        if (pendingUrl)
+            config.set('apiUrl', pendingUrl);
+    };
     // Check environment variable override first
     const envToken = process.env['TRAWL_TOKEN'];
     if (envToken) {
-        config.set('token', requireJwt(envToken, 'TRAWL_TOKEN'));
+        config.set('token', requireFreshJwt(envToken, 'TRAWL_TOKEN'));
+        persistUrlIfPending();
         console.log(chalk.green('✓ Logged in'));
         console.log(chalk.dim(`  API: ${getApiUrl()}`));
         console.log(chalk.dim(`  Config: ${config.path}`));
@@ -36,7 +54,8 @@ export const login = new Command('login')
     }
     // --token flag: direct JWT (CI / retrocompat)
     if (opts.token) {
-        config.set('token', requireJwt(opts.token, '--token'));
+        config.set('token', requireFreshJwt(opts.token, '--token'));
+        persistUrlIfPending();
         console.log(chalk.green('✓ Logged in'));
         console.log(chalk.dim(`  API: ${getApiUrl()}`));
         console.log(chalk.dim(`  Config: ${config.path}`));
@@ -48,7 +67,7 @@ export const login = new Command('login')
     }
     const email = opts.email ?? (await promptEmail());
     const password = opts.password ?? (await promptPassword('Password: '));
-    const { data, headers } = await api.publicPost('/api/auth/signin', { email, password });
+    const { data, headers } = await api.publicPost('/api/auth/signin', { email, password }, pendingUrl);
     // Try token from response body first, then fall back to Set-Cookie header
     let raw = typeof data === 'string' ? data : data.token;
     if (!raw) {
@@ -60,8 +79,9 @@ export const login = new Command('login')
     if (!raw || typeof raw !== 'string') {
         throw new Error(`Invalid response from API: no token received. Got: ${JSON.stringify(data)}`);
     }
-    const token = requireJwt(raw, 'token');
+    const token = requireFreshJwt(raw, 'token');
     config.set('token', token);
+    persistUrlIfPending();
     console.log(chalk.green(`✓ Logged in as ${email}`));
     console.log(chalk.dim(`  API: ${getApiUrl()}`));
     console.log(chalk.dim(`  Config: ${config.path}`));