@trawlme/cli 1.19.0 → 1.20.0

package/README.md

@@ -38,6 +38,16 @@ trawl logout
 trawl token                                  Print the stored session JWT (for MCP Bearer auth)
 ```
 
+### Agent core verbs
+
+```
+trawl fetch <url> [--json] [--reason <text>]  One-shot fetch + extract readable content from a public URL (no scrap needed)
+trawl whoami [--json]                         Show the authenticated user's identity
+trawl ping [--json]                           Health/version handshake against the Trawl API
+```
+
+Fully non-interactive — all three read auth only from `TRAWL_TOKEN`/the stored login token, never prompt. `trawl fetch` is the REST counterpart of the MCP `trawl_fetch_url` tool (same shared engine, `POST /api/scraps/fetch-url`): `status` is an honest outcome (`completed`/`failed`/`empty`/`blocked`), not "did the HTTP call succeed" — a failed fetch is still a 200 response with `status:'failed'` + `error`, and the CLI exits 1 in that case (both human and `--json` modes) even though `--json` always prints the raw payload verbatim. `truncated:true` means the 100KB response cap tripped; `result` is dropped and `url` repoints at the full history row instead. `trawl whoami`/`trawl ping` mirror the MCP `trawl_whoami`/`trawl_health_ping` tools as closely as the REST surface allows (`GET /api/users/me` / `GET /api/health`) — `ping`'s `--json` payload is admin-enriched (version/uptime/db) and just `{"status":"ok"}` for anyone else.
+
 ### Scraps
 
 ```
@@ -83,7 +93,7 @@ trawl scraps account session set <id> -c <file>
 
 ### Claude Code skills
 
-The CLI bundles a Claude Code skill that teaches Claude how to use `trawl`. Once installed, Claude can manage scraps for you via prompts.
+The CLI bundles 5 Claude Code skills that teach Claude how to use `trawl`. Once installed, Claude can manage scraps for you via prompts.
 
 ```
 trawl skills list                                     List bundled skills and install status

package/dist/commands/fetch.d.ts

@@ -0,0 +1,31 @@
+import { Command } from 'commander';
+/**
+ * `POST /api/scraps/fetch-url` response contract (#1697, trawl_node —
+ * shipped, contract LOCKED). REST counterpart of the MCP `trawl_fetch_url`
+ * tool: both run through the same shared engine
+ * (scraps.fetchUrl.service.js#runEphemeralUrlFetch), so the outcome shape is
+ * identical regardless of transport.
+ *
+ *  - `status`: 'completed' | 'failed' | 'empty' | 'blocked' — an HONEST
+ *    outcome, not a bare "it ran". A failed/blocked/empty run is still a
+ *    200 response (this is not an HTTP error), so callers must branch on
+ *    `status`, never assume 2xx means "got data".
+ *  - `error` is present ONLY when `status === 'failed'`.
+ *  - `truncated: true` when the 100KB response-payload cap tripped — the
+ *    server drops `result` in that case and repoints `url` at the full
+ *    history row (`/api/historys/:runId`) instead of echoing the fetched
+ *    target.
+ */
+export interface FetchUrlResponse {
+    url: string;
+    status: 'completed' | 'failed' | 'empty' | 'blocked';
+    runId?: string | null;
+    statusDetail?: string | null;
+    blocked?: boolean;
+    length?: number | null;
+    result?: unknown;
+    truncated?: boolean;
+    reason?: string;
+    error?: string;
+}
+export declare const fetchUrl: Command;

package/dist/commands/fetch.js

@@ -0,0 +1,95 @@
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { oraPromise } from 'ora';
+import { api, LONG_RUN_TIMEOUT_MS } from '../lib/api.js';
+import { json } from '../lib/format.js';
+import { requireUrl } from '../lib/validate.js';
+function statusIcon(status) {
+    if (status === 'completed')
+        return chalk.green('✓');
+    if (status === 'failed')
+        return chalk.red('✗');
+    if (status === 'blocked')
+        return chalk.yellow('⚠');
+    return chalk.dim('•'); // empty
+}
+/** Best-effort human summary of `result` — same "count + first-item keys"
+ * shape scraps.ts's renderScrapItems uses, never a full dump (that's what
+ * --json is for). */
+function renderResultSummary(result) {
+    if (!Array.isArray(result))
+        return;
+    console.log(chalk.dim(`  Items:      `) + result.length);
+    const first = result[0];
+    if (result.length > 0 && first && typeof first === 'object') {
+        console.log(chalk.dim(`  First keys: `) + Object.keys(first).join(', '));
+    }
+}
+export const fetchUrl = new Command('fetch')
+    .description('One-shot fetch + extract readable content from a public URL (no scrap needed)')
+    .argument('<url>', 'Target public HTTPS URL')
+    .option('--json', 'Output the raw API payload')
+    .option('--reason <reason>', 'Audit-trail reason for this fetch (logged server-side, max 500 chars)')
+    .action(async (rawUrl, opts) => {
+    // Fast, local usage-error (exit 2) on an obviously malformed URL — never
+    // a round-trip to the server for something we can already tell is bad.
+    // The server still re-validates (SSRF guard) — this is a UX fast-path,
+    // not a security boundary.
+    const url = requireUrl(rawUrl, 'url');
+    const body = {
+        url,
+        ...(opts.reason !== undefined && { reason: opts.reason }),
+    };
+    // #106 review F1 — this endpoint runs the FULL worker pipeline (browser
+    // `goto` networkidle2 + extraction + persistence), legitimately 30-250s
+    // server-side — same #91 P0 pattern as `scraps run` / `data --fresh` /
+    // `trigger --wait` (src/commands/scraps.ts). The 30s DEFAULT_TIMEOUT_MS
+    // was aborting it mid-flight and would have surfaced a fabricated
+    // NetworkError timeout for a request that was always going to succeed.
+    const call = () => api.post('/api/scraps/fetch-url', body, { timeoutMs: LONG_RUN_TIMEOUT_MS });
+    // #106 review F2 — under --json the stdout path must be provably pure:
+    // no spinner channel at all. Only the human path gets the ora progress
+    // indicator; --json calls the API directly.
+    const data = opts.json
+        ? await call()
+        : await oraPromise(call, {
+            text: `Fetching ${url}…`,
+            // #106 review F3 — no successText verdict here. ora's success
+            // symbol only means "the HTTP call didn't throw", not "the fetch
+            // succeeded" — a failed/blocked domain `status` is still a 200
+            // response. The real outcome is rendered below via statusIcon +
+            // the Status: line; a green check here would contradict a
+            // red/yellow icon printed right after it.
+            successText: 'Request complete',
+        });
+    if (opts.json) {
+        json(data);
+    }
+    else {
+        console.log(`${statusIcon(data.status)} ${chalk.bold(data.url)}`);
+        console.log(chalk.dim(`  Status:     `) + data.status);
+        if (data.runId)
+            console.log(chalk.dim(`  Run ID:     `) + data.runId);
+        if (data.statusDetail)
+            console.log(chalk.dim(`  Detail:     `) + data.statusDetail);
+        if (data.length != null)
+            console.log(chalk.dim(`  Length:     `) + data.length);
+        if (data.truncated) {
+            console.log(chalk.yellow(`  ⚠ Truncated (payload too large) — full result: ${data.url}`));
+        }
+        if (data.status === 'failed' && data.error) {
+            console.log(chalk.red(`  Error:      `) + data.error);
+        }
+        renderResultSummary(data.result);
+    }
+    // Honest exit code alongside the honest payload — a --json caller gets
+    // the raw body regardless (never wrapped/altered), but a script checking
+    // the exit code alone must be able to tell "no usable data" from "ran
+    // fine" without parsing. `blocked` (#106 review F4 — an antibot wall) is
+    // a failure to get data exactly like `failed`: an agent scripting
+    // `trawl fetch ... || handle` must see non-zero for either. `empty`
+    // stays 0 on purpose — the fetch genuinely ran to completion, there was
+    // just nothing extractable at that URL; that's not an error.
+    if (data.status === 'failed' || data.status === 'blocked')
+        process.exitCode = 1;
+});

package/dist/commands/ping.d.ts

@@ -0,0 +1,24 @@
+import { Command } from 'commander';
+/**
+ * `GET /api/health` response shape (home.controller.js#health,
+ * home.service.js#getHealthStatus) — MCP `trawl_health_ping` parity
+ * (`{ok, version, uptime}`). The REST route enriches the payload with
+ * `version`/`uptime`/`db`/`memory` ONLY for an admin caller
+ * (home.controller.js's `isAdmin` gate); a non-admin JWT gets `{status:'ok'}`
+ * alone on a healthy server. A non-2xx response (degraded, `status:503`)
+ * throws an ApiError like any other failed request — but a 2xx response is
+ * NOT a guarantee that `status === 'ok'` (#106 review F5): a soft-degraded
+ * signal (e.g. a partial dependency outage) could still come back as a 200
+ * with a non-"ok" `status`. The HTTP call not throwing only means the
+ * transport succeeded, never that the reported health is good — so the
+ * human-mode `✓ OK` line is gated on the actual field, not assumed from a
+ * successful round-trip.
+ */
+export interface PingResponse {
+    status: string;
+    db?: string;
+    uptime?: number;
+    version?: string;
+    memory?: unknown;
+}
+export declare const ping: Command;

package/dist/commands/ping.js

@@ -0,0 +1,29 @@
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { api } from '../lib/api.js';
+import { json } from '../lib/format.js';
+export const ping = new Command('ping')
+    .description('Health/version handshake against the Trawl API')
+    .option('--json', 'Output as JSON')
+    .action(async (opts) => {
+    const data = await api.get('/api/health');
+    // #106 review (kimi) — honest exit code: a soft-degraded 200 (status !== 'ok')
+    // must exit non-zero so `trawl ping || handle_degraded` doesn't treat a
+    // degraded API as healthy. Applies in BOTH --json and human modes, alongside
+    // the raw/decorated output below.
+    if (data.status !== 'ok')
+        process.exitCode = 1;
+    if (opts.json) {
+        json(data);
+        return;
+    }
+    const versionSuffix = data.version ? ` (v${data.version})` : '';
+    if (data.status === 'ok') {
+        console.log(chalk.green('✓ OK') + versionSuffix);
+    }
+    else {
+        // #106 review F5 — never green-light a degraded API just because the
+        // HTTP call didn't throw; show the real reported status instead.
+        console.log(chalk.yellow(`⚠ ${data.status}`) + versionSuffix);
+    }
+});

package/dist/commands/whoami.d.ts

@@ -0,0 +1,27 @@
+import { Command } from 'commander';
+/**
+ * `GET /api/users/me` response shape (users.account.controller.js#me).
+ *
+ * Chosen over the MCP `trawl_whoami` tool's own payload
+ * (userId/organizationId/organizationName/scrapIdsAllowlist/mode) because
+ * that shape lives entirely in the MCP per-call auth-bridge context
+ * (developers.mcp.js) — there is no equivalent REST route exposing
+ * organization/scope/mode for a session-JWT caller today. `/api/users/me`
+ * is the closest REST parity match: real identity, already used by every
+ * other trawl_node client. `roles` is the nearest analogue to "scopes";
+ * there is currently no "plan" field on this endpoint.
+ */
+export interface WhoamiResponse {
+    id?: string;
+    email?: string;
+    provider?: string;
+    roles?: string[];
+    firstName?: string;
+    lastName?: string;
+    avatar?: string;
+    complementary?: unknown;
+    bio?: string;
+    position?: string;
+    terms?: unknown;
+}
+export declare const whoami: Command;

package/dist/commands/whoami.js

@@ -0,0 +1,24 @@
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { api } from '../lib/api.js';
+import { json } from '../lib/format.js';
+export const whoami = new Command('whoami')
+    .description("Show the authenticated user's identity")
+    .option('--json', 'Output as JSON')
+    .action(async (opts) => {
+    const data = await api.get('/api/users/me');
+    if (opts.json) {
+        json(data);
+        return;
+    }
+    if (!data) {
+        console.log(chalk.dim('No identity information available.'));
+        return;
+    }
+    const name = [data.firstName, data.lastName].filter(Boolean).join(' ');
+    console.log(chalk.bold(name || data.email || '(unknown)'));
+    console.log(chalk.dim(`  ID:       `) + (data.id ?? '—'));
+    console.log(chalk.dim(`  Email:    `) + (data.email ?? '—'));
+    console.log(chalk.dim(`  Provider: `) + (data.provider ?? '—'));
+    console.log(chalk.dim(`  Roles:    `) + (data.roles?.length ? data.roles.join(', ') : '—'));
+});

package/dist/index.d.ts

@@ -24,7 +24,18 @@ export declare function resolveCommandName(actionCommand: Command | undefined):
  */
 export declare function collectCommandNames(root: Command): string[];
 export declare function createProgram(): Command;
-/** True when this module is the process entrypoint (not merely imported by a test). */
+/**
+ * True when this module is the process entrypoint (not merely imported by a test).
+ *
+ * npm installs the global bin as a SYMLINK (`bin/trawl` -> `dist/index.js`). For an
+ * ESM entrypoint Node realpaths the module URL (`import.meta.url` = the real
+ * `dist/index.js`) but leaves `process.argv[1]` as the symlink path — so a raw
+ * `moduleUrl === pathToFileURL(argv1).href` compare is FALSE for the normal global
+ * invocation, the guard never fires, and `runCli()` never runs (silent no-op, #103).
+ * Canonicalise both sides with `realpathSync` before comparing. A genuine entrypoint
+ * was just loaded by Node, so both realpath calls resolve; the catch only trips for a
+ * non-file module URL (exotic loaders) — correctly "not the entrypoint".
+ */
 export declare function isEntryPoint(argv1: string | undefined, moduleUrl: string): boolean;
 /**
  * True when the invocation is a pure `--help`/`--version` query, a bare

package/dist/index.js

@@ -1,13 +1,16 @@
 #!/usr/bin/env node
 import { Command, CommanderError } from 'commander';
-import { readFileSync } from 'node:fs';
-import { fileURLToPath, pathToFileURL } from 'node:url';
+import { readFileSync, realpathSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
 import { login, logout } from './commands/login.js';
 import { scraps } from './commands/scraps.js';
 import { skills } from './commands/skills.js';
 import { telemetry } from './commands/telemetry.js';
 import { token } from './commands/token.js';
+import { fetchUrl } from './commands/fetch.js';
+import { whoami } from './commands/whoami.js';
+import { ping } from './commands/ping.js';
 import { autoUpdateInstalledSkills } from './lib/skills.js';
 import { initPostHog, captureCommand, shutdown, registerAllowedCommands } from './lib/posthog.js';
 import { classifyError, reportError } from './lib/errors.js';
@@ -72,11 +75,32 @@ export function createProgram() {
     program.addCommand(skills);
     program.addCommand(telemetry);
     program.addCommand(token);
+    program.addCommand(fetchUrl);
+    program.addCommand(whoami);
+    program.addCommand(ping);
     return program;
 }
-/** True when this module is the process entrypoint (not merely imported by a test). */
+/**
+ * True when this module is the process entrypoint (not merely imported by a test).
+ *
+ * npm installs the global bin as a SYMLINK (`bin/trawl` -> `dist/index.js`). For an
+ * ESM entrypoint Node realpaths the module URL (`import.meta.url` = the real
+ * `dist/index.js`) but leaves `process.argv[1]` as the symlink path — so a raw
+ * `moduleUrl === pathToFileURL(argv1).href` compare is FALSE for the normal global
+ * invocation, the guard never fires, and `runCli()` never runs (silent no-op, #103).
+ * Canonicalise both sides with `realpathSync` before comparing. A genuine entrypoint
+ * was just loaded by Node, so both realpath calls resolve; the catch only trips for a
+ * non-file module URL (exotic loaders) — correctly "not the entrypoint".
+ */
 export function isEntryPoint(argv1, moduleUrl) {
-    return argv1 !== undefined && moduleUrl === pathToFileURL(argv1).href;
+    if (argv1 === undefined)
+        return false;
+    try {
+        return realpathSync(fileURLToPath(moduleUrl)) === realpathSync(argv1);
+    }
+    catch {
+        return false;
+    }
 }
 /**
  * True when the invocation is a pure `--help`/`--version` query, a bare

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trawlme/cli",
-  "version": "1.19.0",
+  "version": "1.20.0",
   "description": "Trawl CLI — manage scraps from the terminal",
   "type": "module",
   "bin": {