@trawlme/cli 1.21.0 → 2.0.0

package/dist/index.d.ts

@@ -15,6 +15,14 @@ import { Command } from 'commander';
  * chain — matching the pre-existing convention that a direct child of the
  * root (e.g. `scraps list`, `telemetry on`) is named relative to its
  * immediate group, never prefixed with the program name.
+ *
+ * #108 note: promoting a verb to a top-level command (see `createProgram`
+ * below) renamed ITS resolved telemetry name from `scraps <verb>` to
+ * `<verb>` — the canonical top-level attach and the legacy hidden
+ * `scraps <verb>` attach are two separate Command instances (scraps.ts's
+ * double-attach factories), each with its own parent chain, so they
+ * resolve to two different names here even though they run the same
+ * handler. Intentional (the canonical command IS now `<verb>`), not a bug.
  */
 export declare function resolveCommandName(actionCommand: Command | undefined): string;
 /**

package/dist/index.js

@@ -4,11 +4,11 @@ import { readFileSync, realpathSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
 import { login, logout } from './commands/login.js';
-import { scraps } from './commands/scraps.js';
+import { scraps, attachListCommand, attachGetCommand, attachRunCommand, attachDataCommand, attachHistoryCommand, attachRunInfoCommand, attachTriggerCommand, } from './commands/scraps.js';
 import { skills } from './commands/skills.js';
 import { telemetry } from './commands/telemetry.js';
 import { token } from './commands/token.js';
-import { fetchUrl } from './commands/fetch.js';
+import { create } from './commands/create.js';
 import { whoami } from './commands/whoami.js';
 import { ping } from './commands/ping.js';
 import { autoUpdateInstalledSkills } from './lib/skills.js';
@@ -32,6 +32,14 @@ const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf8
  * chain — matching the pre-existing convention that a direct child of the
  * root (e.g. `scraps list`, `telemetry on`) is named relative to its
  * immediate group, never prefixed with the program name.
+ *
+ * #108 note: promoting a verb to a top-level command (see `createProgram`
+ * below) renamed ITS resolved telemetry name from `scraps <verb>` to
+ * `<verb>` — the canonical top-level attach and the legacy hidden
+ * `scraps <verb>` attach are two separate Command instances (scraps.ts's
+ * double-attach factories), each with its own parent chain, so they
+ * resolve to two different names here even though they run the same
+ * handler. Intentional (the canonical command IS now `<verb>`), not a bug.
  */
 export function resolveCommandName(actionCommand) {
     if (!actionCommand)
@@ -63,21 +71,55 @@ export function collectCommandNames(root) {
     walk(root);
     return names;
 }
+/**
+ * #108 — surface reorg into two `trawl --help` tiers. Core verbs are
+ * agent+human, `--json` first-class, non-interactive; Management is the
+ * existing human/CI surface, kept but grouped so top-level help reads
+ * simple. Commander v14's native per-command help group (`.commandsGroup()`
+ * sets the default a subsequently-registered command inherits via
+ * `.helpGroup()`) drives the section headings — group ORDER in the printed
+ * help follows first-seen insertion order into `program.commands`, so every
+ * Core command is registered below before any Management one.
+ */
+const CORE_GROUP = 'Core commands (agent + human):';
+const MANAGEMENT_GROUP = 'Management commands (human/CI):';
 export function createProgram() {
     const program = new Command()
         .name('trawl')
         .description('Trawl CLI — manage scraps from the terminal')
         .version(pkg.version)
         .option('--debug', 'Show full error stack traces');
-    program.addCommand(login);
-    program.addCommand(logout);
+    // Core verbs (#108) — promoted/listed first: create, run, list, get, data,
+    // history, run-info, trigger, whoami, ping. `list`/`get`/`run`/`data`/
+    // `history`/`run-info`/`trigger` are built via scraps.ts's exported
+    // attachXCommand() factories — the SAME definition also stays wired
+    // (hidden) under `scraps` there, so every pre-#108 `trawl scraps <verb>`
+    // invocation keeps resolving (no breaking change).
+    //
+    // #114 — `create` replaced `fetch` in this slot: `POST /api/ai/wizard`
+    // (AI-generate + persist + first-run + autofix), a distinct command from
+    // the still-untouched `trawl scraps create` (raw-script management verb).
+    program.commandsGroup(CORE_GROUP);
+    program.addCommand(create);
+    attachRunCommand(program);
+    attachListCommand(program);
+    attachGetCommand(program);
+    attachDataCommand(program);
+    attachHistoryCommand(program);
+    attachRunInfoCommand(program);
+    attachTriggerCommand(program);
+    program.addCommand(whoami);
+    program.addCommand(ping);
+    // Management (#108) — human/CI surface, grouped below. `scraps` still
+    // holds every pre-#108 management command (create/update/delete/banner/
+    // watch/account.*/session.*/doctor/autofix/snapshot) exactly as before.
+    program.commandsGroup(MANAGEMENT_GROUP);
     program.addCommand(scraps);
     program.addCommand(skills);
-    program.addCommand(telemetry);
+    program.addCommand(login);
+    program.addCommand(logout);
     program.addCommand(token);
-    program.addCommand(fetchUrl);
-    program.addCommand(whoami);
-    program.addCommand(ping);
+    program.addCommand(telemetry);
     return program;
 }
 /**

package/dist/lib/api.d.ts

@@ -43,12 +43,15 @@ export declare function notLoggedInError(): AuthError;
  * dry-run retries). The generic 30s default was aborting those mid-flight
  * and surfacing a fabricated `NetworkError timed out` (exit 5) for a request
  * that was always going to succeed given enough time. Passed as the per-call
- * `{timeoutMs}` override at exactly the 3 call sites that hit those
- * endpoints (src/commands/scraps.ts): `scraps run` / `data --fresh`
- * (GET /api/scraps/load/:id) and `scraps trigger --wait`
- * (POST /api/scraps/worker/:id, synchronous branch only — the default async
- * `?wait=false` POST returns almost immediately and keeps the 30s default).
- * 300s leaves margin over the ~250s worst case without being unboundedly long.
+ * `{timeoutMs}` override at exactly the 4 call sites that hit those
+ * endpoints: `scraps run` / `data --fresh` (GET /api/scraps/load/:id) and
+ * `scraps trigger --wait` (POST /api/scraps/worker/:id, synchronous branch
+ * only — the default async `?wait=false` POST returns almost immediately and
+ * keeps the 30s default) — all three in src/commands/scraps.ts — plus (#114)
+ * `create` (POST /api/ai/wizard, src/commands/create.ts), whose AI-generation
+ * + scrap creation + first run + autofix pipeline runs the same 30–250s+
+ * server-side. 300s leaves margin over the ~250s worst case without being
+ * unboundedly long.
  */
 export declare const LONG_RUN_TIMEOUT_MS = 300000;
 export interface RequestOptions {

package/dist/lib/api.js

@@ -63,12 +63,15 @@ const DEFAULT_TIMEOUT_MS = 30_000;
  * dry-run retries). The generic 30s default was aborting those mid-flight
  * and surfacing a fabricated `NetworkError timed out` (exit 5) for a request
  * that was always going to succeed given enough time. Passed as the per-call
- * `{timeoutMs}` override at exactly the 3 call sites that hit those
- * endpoints (src/commands/scraps.ts): `scraps run` / `data --fresh`
- * (GET /api/scraps/load/:id) and `scraps trigger --wait`
- * (POST /api/scraps/worker/:id, synchronous branch only — the default async
- * `?wait=false` POST returns almost immediately and keeps the 30s default).
- * 300s leaves margin over the ~250s worst case without being unboundedly long.
+ * `{timeoutMs}` override at exactly the 4 call sites that hit those
+ * endpoints: `scraps run` / `data --fresh` (GET /api/scraps/load/:id) and
+ * `scraps trigger --wait` (POST /api/scraps/worker/:id, synchronous branch
+ * only — the default async `?wait=false` POST returns almost immediately and
+ * keeps the 30s default) — all three in src/commands/scraps.ts — plus (#114)
+ * `create` (POST /api/ai/wizard, src/commands/create.ts), whose AI-generation
+ * + scrap creation + first run + autofix pipeline runs the same 30–250s+
+ * server-side. 300s leaves margin over the ~250s worst case without being
+ * unboundedly long.
  */
 export const LONG_RUN_TIMEOUT_MS = 300_000;
 /**

package/docs/agent-quickstart.md

@@ -0,0 +1,121 @@
+# Trawl CLI — Agent Quickstart
+
+The minimal surface an AI agent needs to drive `@trawlme/cli` non-interactively.
+For the full command reference (management surface, Claude Code skills,
+telemetry, etc.) see the [main README](../README.md) — the human/CI guide.
+
+## Auth — zero prompts
+
+Set `TRAWL_TOKEN` and every command authenticates without ever touching a
+prompt:
+
+```bash
+export TRAWL_TOKEN=<jwt>
+trawl whoami --json
+```
+
+(Interactive `trawl login` and the `--url`/config-file flow are documented in
+the README's [Authentication](../README.md#authentication) section — an
+agent should never need them.)
+
+## Core commands (agent + human)
+
+These ten commands are the CLI's agent+human surface — `--json` is
+first-class on every one, and none of them ever blocks on a prompt (see
+[Non-interactive contract](#non-interactive-contract) below):
+
+```
+trawl create <url> --prompt <goal> [--no-autofix] [--json]   Create a persistent, self-healing scrap from a URL + a goal (AI-generated)
+trawl run <id> [--watch] [--json]              Run a scrap
+trawl list|ls [--json] [--status <s>] [--limit <n>] [--page <n>]   List all scraps
+trawl get <id> [--json]                        Get scrap details
+trawl data <id> [--json] [--fresh] [--errors]  Get scrap data (last persisted run, or --fresh to launch one)
+trawl history <id> [--json] [-n <limit>]       List past runs for a scrap
+trawl run-info <hid> [--json]                  Show details of a single run
+trawl trigger <id> [--watch] [--wait] [--json] Launch a scrap as a background worker
+trawl whoami [--json]                          Show the authenticated user's identity
+trawl ping [--json]                            Health/version handshake against the Trawl API
+```
+
+`trawl create` is the closest primitive to "turn this URL + goal into a
+working scrap" — it runs the AI wizard server-side (`POST /api/ai/wizard`):
+generate scrap code from `--prompt` via LLM, persist the scrap, trigger its
+first run, and auto-fix on failure (default on; `--no-autofix` disables it).
+`success` is an honest outcome of that first run, not "did the HTTP call
+succeed" — a failed first run is still a 200 response (the scrap was still
+created), and the CLI exits `1` in that case even though `--json` always
+prints the raw payload verbatim. The call legitimately runs 30–250s+
+server-side (AI generation + a real run) — the CLI arms the same long-run
+timeout `run`/`data --fresh`/`trigger --wait` use instead of the generic 30s
+default.
+
+> **Not idempotent, and not a one-off run.** A client-side timeout (exit `5`,
+> a `NetworkError`) does not mean the wizard failed server-side — scrap
+> creation + the first run keep going after the CLI gives up waiting, so the
+> scrap may already exist. Check `trawl list --json` for a matching URL/title
+> **before** retrying — a blind retry creates a duplicate scrap and burns
+> quota a second time for the same goal. Separately, the created scrap is
+> scheduled to re-run every day at 07:00 UTC by default
+> (`cron: "0 7 * * *"`, unrelated to `--no-autofix`) — each recurring run
+> consumes execute quota. Disable or change it once you've reviewed the
+> scrap: `trawl scraps update <id> --no-cron` (or `--cron <expr>`). Finally,
+> if `TRAWL_TIMEOUT` is set globally for a tighter budget than 300s, it
+> clamps `create`'s ceiling too (env always wins) — unset it or raise it
+> before calling `create`.
+
+> **No breaking change:** every verb above is also still reachable under its
+> pre-reorg path, `trawl scraps <verb>` (e.g. `trawl scraps list`) — kept as
+> a hidden alias. Prefer the bare top-level form above; it's what
+> `trawl --help` now shows.
+
+For the full flag reference (tier overrides on `scraps create`/`scraps
+update` — the core `create` verb above has no `--tier` of its own, the
+`--watch` polling mechanics, retention/regression semantics on `data`, …)
+see the README's [Core commands](../README.md#core-commands-agent--human) section
+— this doc intentionally stays minimal.
+
+## `--json` contract
+
+Every command above supports `--json`: a single structured payload on
+stdout, nothing else. Two narrow exceptions carried over from the human
+surface: a `--watch` poll emits exactly one final NDJSON line once the run
+reaches a terminal state (not the whole progress stream), and there is no
+JSON form of an HTML page (irrelevant to the core verbs above — that only
+applies to the management-only `scraps snapshot`).
+
+On failure, `--json` emits a single error envelope on stdout instead of
+prose — `{"error":{"message","status?","kind"}}` — and the human-readable
+line goes to stderr, never stdout. `kind` is the machine-readable
+discriminant (`"usage"`/`"auth"`/`"not_found"`/`"network"`/`"api"`/
+`"refused"`/`"unknown"`) a script should switch on.
+
+## Non-interactive contract
+
+No core verb ever blocks waiting for a prompt. When stdin/stdout isn't a
+real TTY (any subprocess-driven invocation) — or `--json` is set — any
+command that would otherwise ask a `[y/N]` confirmation or a missing value
+instead fails fast with a structured usage error (exit `2`) rather than
+hanging. Full rule + rationale: README's
+[Non-interactive rule](../README.md#non-interactive-rule).
+
+## Exit codes
+
+| Code | Meaning |
+|------|---------|
+| `0`  | Success |
+| `1`  | Unknown/generic error, or a business-logic outcome (e.g. `create`'s honest `success:false` first-run outcome, `data`'s `run_failed`/`in_progress`) |
+| `2`  | Usage error (bad flag/value, invalid ID, missing required argument, or the non-interactive guard refusing to prompt) |
+| `3`  | Auth error (not logged in, or the session token is expired/invalid) |
+| `4`  | Not found (no such resource, or no persisted payload to read) |
+| `5`  | Network error (API host unreachable, DNS/connection/TLS failure, or timeout) |
+
+This table is the stable contract; per-command nuance and overloads (e.g.
+`create`'s domain-level failure sharing exit `1` with an unmapped bug) are
+documented once, in the README's [Exit codes](../README.md#exit-codes)
+section — treat that as canonical if the two ever seem to disagree.
+
+## Minimal example
+
+```bash
+TRAWL_TOKEN=<jwt> trawl create https://example.com --prompt "Extract the article title and body text" --json
+```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trawlme/cli",
-  "version": "1.21.0",
+  "version": "2.0.0",
   "description": "Trawl CLI — manage scraps from the terminal",
   "type": "module",
   "bin": {
@@ -8,6 +8,7 @@
   },
   "files": [
     "dist",
+    "docs",
     "README.md",
     "LICENSE"
   ],

package/dist/commands/fetch.d.ts

@@ -1,31 +0,0 @@
-import { Command } from 'commander';
-/**
- * `POST /api/scraps/fetch-url` response contract (#1697, trawl_node —
- * shipped, contract LOCKED). REST counterpart of the MCP `trawl_fetch_url`
- * tool: both run through the same shared engine
- * (scraps.fetchUrl.service.js#runEphemeralUrlFetch), so the outcome shape is
- * identical regardless of transport.
- *
- *  - `status`: 'completed' | 'failed' | 'empty' | 'blocked' — an HONEST
- *    outcome, not a bare "it ran". A failed/blocked/empty run is still a
- *    200 response (this is not an HTTP error), so callers must branch on
- *    `status`, never assume 2xx means "got data".
- *  - `error` is present ONLY when `status === 'failed'`.
- *  - `truncated: true` when the 100KB response-payload cap tripped — the
- *    server drops `result` in that case and repoints `url` at the full
- *    history row (`/api/historys/:runId`) instead of echoing the fetched
- *    target.
- */
-export interface FetchUrlResponse {
-    url: string;
-    status: 'completed' | 'failed' | 'empty' | 'blocked';
-    runId?: string | null;
-    statusDetail?: string | null;
-    blocked?: boolean;
-    length?: number | null;
-    result?: unknown;
-    truncated?: boolean;
-    reason?: string;
-    error?: string;
-}
-export declare const fetchUrl: Command;

package/dist/commands/fetch.js

@@ -1,95 +0,0 @@
-import { Command } from 'commander';
-import chalk from 'chalk';
-import { oraPromise } from 'ora';
-import { api, LONG_RUN_TIMEOUT_MS } from '../lib/api.js';
-import { json } from '../lib/format.js';
-import { requireUrl } from '../lib/validate.js';
-function statusIcon(status) {
-    if (status === 'completed')
-        return chalk.green('✓');
-    if (status === 'failed')
-        return chalk.red('✗');
-    if (status === 'blocked')
-        return chalk.yellow('⚠');
-    return chalk.dim('•'); // empty
-}
-/** Best-effort human summary of `result` — same "count + first-item keys"
- * shape scraps.ts's renderScrapItems uses, never a full dump (that's what
- * --json is for). */
-function renderResultSummary(result) {
-    if (!Array.isArray(result))
-        return;
-    console.log(chalk.dim(`  Items:      `) + result.length);
-    const first = result[0];
-    if (result.length > 0 && first && typeof first === 'object') {
-        console.log(chalk.dim(`  First keys: `) + Object.keys(first).join(', '));
-    }
-}
-export const fetchUrl = new Command('fetch')
-    .description('One-shot fetch + extract readable content from a public URL (no scrap needed)')
-    .argument('<url>', 'Target public HTTPS URL')
-    .option('--json', 'Output the raw API payload')
-    .option('--reason <reason>', 'Audit-trail reason for this fetch (logged server-side, max 500 chars)')
-    .action(async (rawUrl, opts) => {
-    // Fast, local usage-error (exit 2) on an obviously malformed URL — never
-    // a round-trip to the server for something we can already tell is bad.
-    // The server still re-validates (SSRF guard) — this is a UX fast-path,
-    // not a security boundary.
-    const url = requireUrl(rawUrl, 'url');
-    const body = {
-        url,
-        ...(opts.reason !== undefined && { reason: opts.reason }),
-    };
-    // #106 review F1 — this endpoint runs the FULL worker pipeline (browser
-    // `goto` networkidle2 + extraction + persistence), legitimately 30-250s
-    // server-side — same #91 P0 pattern as `scraps run` / `data --fresh` /
-    // `trigger --wait` (src/commands/scraps.ts). The 30s DEFAULT_TIMEOUT_MS
-    // was aborting it mid-flight and would have surfaced a fabricated
-    // NetworkError timeout for a request that was always going to succeed.
-    const call = () => api.post('/api/scraps/fetch-url', body, { timeoutMs: LONG_RUN_TIMEOUT_MS });
-    // #106 review F2 — under --json the stdout path must be provably pure:
-    // no spinner channel at all. Only the human path gets the ora progress
-    // indicator; --json calls the API directly.
-    const data = opts.json
-        ? await call()
-        : await oraPromise(call, {
-            text: `Fetching ${url}…`,
-            // #106 review F3 — no successText verdict here. ora's success
-            // symbol only means "the HTTP call didn't throw", not "the fetch
-            // succeeded" — a failed/blocked domain `status` is still a 200
-            // response. The real outcome is rendered below via statusIcon +
-            // the Status: line; a green check here would contradict a
-            // red/yellow icon printed right after it.
-            successText: 'Request complete',
-        });
-    if (opts.json) {
-        json(data);
-    }
-    else {
-        console.log(`${statusIcon(data.status)} ${chalk.bold(data.url)}`);
-        console.log(chalk.dim(`  Status:     `) + data.status);
-        if (data.runId)
-            console.log(chalk.dim(`  Run ID:     `) + data.runId);
-        if (data.statusDetail)
-            console.log(chalk.dim(`  Detail:     `) + data.statusDetail);
-        if (data.length != null)
-            console.log(chalk.dim(`  Length:     `) + data.length);
-        if (data.truncated) {
-            console.log(chalk.yellow(`  ⚠ Truncated (payload too large) — full result: ${data.url}`));
-        }
-        if (data.status === 'failed' && data.error) {
-            console.log(chalk.red(`  Error:      `) + data.error);
-        }
-        renderResultSummary(data.result);
-    }
-    // Honest exit code alongside the honest payload — a --json caller gets
-    // the raw body regardless (never wrapped/altered), but a script checking
-    // the exit code alone must be able to tell "no usable data" from "ran
-    // fine" without parsing. `blocked` (#106 review F4 — an antibot wall) is
-    // a failure to get data exactly like `failed`: an agent scripting
-    // `trawl fetch ... || handle` must see non-zero for either. `empty`
-    // stays 0 on purpose — the fetch genuinely ran to completion, there was
-    // just nothing extractable at that URL; that's not an error.
-    if (data.status === 'failed' || data.status === 'blocked')
-        process.exitCode = 1;
-});