@zenrows/cli 1.4.2 → 1.5.0

package/dist/src/cli/commands/crawl.js

@@ -0,0 +1,535 @@
+/**
+ * `zenrows crawl` — Crawl API (new).
+ *
+ * Give Crawl one start URL and read back the URLs it finds behind it, optionally
+ * with each page's HTML. A crawl is an async job: `create` returns at once
+ * (or waits with `--follow`), then `get` / `wait` / `results` / `content` /
+ * `download` read it and `stop` ends it. An account without Crawl access gets
+ * 403 REQS008 → CRAWL_NOT_ENABLED.
+ */
+import { log } from "../../core/logger.js";
+import { requireApiKey } from "../../core/auth.js";
+import { ensureApiKey } from "../../core/ensure-key.js";
+import { assertUsable } from "../../core/capabilities.js";
+import { assertDomainAllowed, assertWithinLimits, loadPolicy } from "../../core/policy.js";
+import { newRunId, writeRun } from "../../core/artifacts.js";
+import { contentIdOf, createCrawl, downloadCrawl, getCrawl, getCrawlContent, isTerminal, listAllResults, listCrawls, stopCrawl, waitForCrawl, } from "../../core/crawl-api.js";
+import { asNumber, asString, parse } from "../command.js";
+import { ToolkitError } from "../../core/errors.js";
+import { printError, writeOut, writeOutStream } from "../output.js";
+/** The API's default for `max_pages` (and `max_items`) when the flag is unset. */
+const DEFAULT_MAX_PAGES = 100;
+/** Largest `depth` / `max_items` / `max_pages` the API takes. */
+const MAX_LIMIT = 100_000;
+export const crawl = {
+    name: "crawl",
+    summary: "Crawl a site from one start URL and collect the URLs behind it (new).",
+    usage: "zenrows crawl <create <url> [--depth N]|get <id>|list|results <id>|content <id> <content_id>|download <id>|stop <id>|wait <id>>",
+    help: [
+        "Crawl is new. Cloud (needs a key with Crawl access):",
+        "  create <url> [flags]              start a crawl from one URL (returns at once)",
+        "    --depth <n>                     link hops to follow from the start URL (1 to 100000);",
+        "                                    leave it out for no depth limit (the caps below stop the crawl)",
+        "    --max-items <n>                 stop after keeping n URLs (API default 100)",
+        "    --max-pages <n>                 stop after fetching n pages (API default 100; bounds cost);",
+        "                                    the local policy max_pages_per_run caps it (default 1000)",
+        "    --include-patterns <p>          keep only URLs containing this substring (repeatable)",
+        "    --exclude-patterns <p>          drop URLs containing this substring (repeatable)",
+        "    --output-format html            also fetch each kept URL's page HTML (read with content/download);",
+        "                                    each kept page is one more fetch counted by --max-pages",
+        "    --follow                        wait until the crawl ends (Ctrl-C stops the wait, not the crawl)",
+        "    --timeout <s>                   with --follow: stop waiting after s seconds (default 600)",
+        "    --idempotency-key <key>         so a retried create starts no second crawl",
+        "    --no-signup                     do not auto-create a Free plan account if no key exists",
+        "  get <id> [--cursor <c>] [--limit <n>]",
+        "                                    the crawl and one page of its results (limit up to 10000)",
+        "  list [--cursor <c>] [--limit <n>] your crawls, newest first (limit up to 100)",
+        "  results <id> [--limit <n>]        every kept URL as JSONL; on a running crawl, those kept so far",
+        "                                    (partial). --limit is the page size per request",
+        "    --out <file>                    write the results to a file instead of printing",
+        "  content <id> <content_id>         print one kept URL's page (content_id or content_url)",
+        "    --out <file>                    write it to a file instead",
+        "  download <id> [--out <file>]      save the NDJSON export (with page HTML for html crawls),",
+        "                                    default <id>.jsonl; partial while the crawl runs",
+        "  stop <id>                         stop a running crawl (an ended crawl answers as it ended)",
+        "  wait <id> [--timeout <s>]         poll until the crawl ends or s seconds (default 600) run out;",
+        "                                    a crawl still running then is printed, not stopped",
+        "  --json                            print structured output: create/get/wait/stop print",
+        "                                    {ok, crawl_id, status, crawl}; get adds results and next_cursor;",
+        "                                    a wait that runs out adds note",
+    ].join("\n"),
+    async run(argv, ctx) {
+        const [sub, ...rest] = argv;
+        try {
+            switch (sub) {
+                case "create":
+                    return await createCmd(rest, ctx);
+                case "get":
+                    return await getCmd(rest, ctx);
+                case "list":
+                    return await listCmd(rest, ctx);
+                case "results":
+                    return await resultsCmd(rest, ctx);
+                case "content":
+                    return await contentCmd(rest, ctx);
+                case "download":
+                    return await downloadCmd(rest, ctx);
+                case "stop":
+                    return await stopCmd(rest, ctx);
+                case "wait":
+                    return await waitCmd(rest, ctx);
+                default:
+                    throw new ToolkitError({
+                        code: "INVALID_USAGE",
+                        message: `Unknown crawl subcommand: ${sub ?? "(none)"}`,
+                        likely_cause: "Subcommand not recognized.",
+                        next_action: "Use create <url> [--depth N] | get <id> | list | results <id> | content <id> <content_id> | download <id> | stop <id> | wait <id>.",
+                        suggested_commands: ["zenrows crawl --help"],
+                    });
+            }
+        }
+        catch (err) {
+            printError(err, ctx.json);
+            return 1;
+        }
+    },
+};
+async function createCmd(rest, ctx) {
+    const { values, positionals } = parse(rest, {
+        depth: { type: "string" },
+        "max-items": { type: "string" },
+        "max-pages": { type: "string" },
+        "include-patterns": { type: "string", multiple: true },
+        "exclude-patterns": { type: "string", multiple: true },
+        "output-format": { type: "string" },
+        follow: { type: "boolean" },
+        timeout: { type: "string" },
+        "idempotency-key": { type: "string" },
+        "no-signup": { type: "boolean" },
+        json: { type: "boolean" },
+    });
+    const json = ctx.json || values.json === true;
+    const url = positionals[0];
+    if (!url) {
+        throw new ToolkitError({
+            code: "INVALID_USAGE",
+            message: "Provide the start URL.",
+            likely_cause: "No <url> positional was given.",
+            next_action: "Usage: zenrows crawl create <url> [--depth N]",
+            suggested_commands: ["zenrows crawl create https://example.com/ --depth 1"],
+        });
+    }
+    const params = { url };
+    const depth = limitFlag("depth", values.depth);
+    if (depth !== undefined)
+        params.depth = depth;
+    const maxItems = limitFlag("max-items", values["max-items"]);
+    if (maxItems !== undefined)
+        params.max_items = maxItems;
+    const maxPages = limitFlag("max-pages", values["max-pages"]);
+    if (maxPages !== undefined)
+        params.max_pages = maxPages;
+    const include = patterns(values["include-patterns"]);
+    if (include.length)
+        params.include_patterns = include;
+    const exclude = patterns(values["exclude-patterns"]);
+    if (exclude.length)
+        params.exclude_patterns = exclude;
+    const outputFormat = asString(values["output-format"]);
+    if (outputFormat !== undefined) {
+        if (outputFormat !== "html") {
+            throw new ToolkitError({
+                code: "INVALID_USAGE",
+                message: `Invalid --output-format value '${outputFormat}'.`,
+                likely_cause: "--output-format takes html only. Without it, a crawl returns URLs only.",
+                next_action: "Pass --output-format html, or leave it out.",
+            });
+        }
+        params.output_format = outputFormat;
+    }
+    const timeoutMs = timeoutFlag(values.timeout);
+    const idempotencyKey = asString(values["idempotency-key"]);
+    assertUsable("crawl");
+    // Pre-flight governance, all local: the start URL must pass the domain policy,
+    // and max_pages (each page is one fetch) must fit the per-run page cap.
+    const policy = loadPolicy();
+    assertDomainAllowed(url, policy);
+    assertWithinLimits({ pages: maxPages ?? DEFAULT_MAX_PAGES }, policy, "crawl");
+    const apiKey = await ensureApiKey(values["no-signup"] ? { ...policy, auto_signup: false } : policy, {
+        onProvision: (a) => {
+            log.info("No API key found — created a Zenrows Free plan account for you.");
+            log.dim(`Claim it anytime (keeps your usage): ${a.claimUrl}`);
+        },
+    });
+    const runId = newRunId();
+    const startedAt = new Date().toISOString();
+    log.step(`Starting crawl of ${url} (${depth === undefined ? "no depth limit" : `depth ${depth}`})…`);
+    try {
+        const created = await createCrawl(params, { apiKey, idempotencyKey });
+        const follow = values.follow === true;
+        const finished = follow ? await waitUntilEnd(created.crawl_id, { apiKey, timeoutMs }) : created;
+        const failure = crawlFailure(finished);
+        const runDir = writeRun({
+            runId,
+            command: "zenrows crawl create",
+            capability: "crawl",
+            url,
+            startedAt,
+            finishedAt: new Date().toISOString(),
+            status: failure ? "error" : "ok",
+            request: { ...params },
+            result: { crawlId: created.crawl_id, status: finished.status, coverage: finished.coverage },
+            ...(failure ? { error: failure.toJSON() } : {}),
+        });
+        const code = follow
+            ? printCrawl(finished, json, waitHeadline(finished), { note: waitNote(finished) })
+            : printCrawl(finished, json, `Started crawl ${created.crawl_id}`);
+        if (!json && runDir)
+            log.dim(`  artifact: ${runDir}`);
+        return code;
+    }
+    catch (err) {
+        writeRun({
+            runId,
+            command: "zenrows crawl create",
+            capability: "crawl",
+            url,
+            startedAt,
+            finishedAt: new Date().toISOString(),
+            status: "error",
+            request: { ...params },
+            error: err instanceof ToolkitError ? err.toJSON() : { message: String(err) },
+        });
+        throw err;
+    }
+}
+async function getCmd(rest, ctx) {
+    const { values, positionals } = parse(rest, {
+        cursor: { type: "string" },
+        limit: { type: "string" },
+        json: { type: "boolean" },
+    });
+    const json = ctx.json || values.json === true;
+    const id = requireId(positionals[0], "get");
+    const cursor = asString(values.cursor);
+    const limit = pageLimit(values.limit, 10_000);
+    assertUsable("crawl");
+    const apiKey = requireApiKey();
+    const page = await getCrawl(id, { apiKey, cursor, limit });
+    const results = page.results ?? [];
+    const nextCursor = page.next_cursor ?? null;
+    const code = printCrawl(page, json, `Crawl ${id}`, { results, next_cursor: nextCursor });
+    if (!json) {
+        log.info(`${results.length} result(s) on this page:`);
+        if (results.length)
+            log.out(toJsonl(results).trimEnd());
+        log.info(`next_cursor: ${nextCursor ?? "null (last page)"}`);
+    }
+    return code;
+}
+async function waitCmd(rest, ctx) {
+    const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } });
+    const json = ctx.json || values.json === true;
+    const id = requireId(positionals[0], "wait");
+    const timeoutMs = timeoutFlag(values.timeout);
+    assertUsable("crawl");
+    const apiKey = requireApiKey();
+    const c = await waitUntilEnd(id, { apiKey, timeoutMs });
+    return printCrawl(c, json, waitHeadline(c), { note: waitNote(c) });
+}
+function waitHeadline(c) {
+    return isTerminal(c.status) ? `Crawl ${c.crawl_id} finished` : `Stopped waiting: crawl ${c.crawl_id} is still running`;
+}
+/** Set only when a wait ran out with the crawl still running. */
+function waitNote(c) {
+    if (isTerminal(c.status))
+        return undefined;
+    return `The wait ran out and the crawl is still running. Run zenrows crawl wait ${c.crawl_id}.`;
+}
+async function resultsCmd(rest, ctx) {
+    const { values, positionals } = parse(rest, {
+        out: { type: "string" },
+        limit: { type: "string" },
+        json: { type: "boolean" },
+    });
+    const json = ctx.json || values.json === true;
+    const id = requireId(positionals[0], "results");
+    const outFile = asString(values.out);
+    const pageSize = pageLimit(values.limit, 10_000);
+    assertUsable("crawl");
+    const apiKey = requireApiKey();
+    const { crawl: c, results, partial } = await listAllResults(id, { apiKey, pageSize });
+    return printResults(id, c.status, results, { outFile, json, partial });
+}
+async function downloadCmd(rest, ctx) {
+    const { values, positionals } = parse(rest, { out: { type: "string" }, json: { type: "boolean" } });
+    const json = ctx.json || values.json === true;
+    const id = requireId(positionals[0], "download");
+    const file = asString(values.out) ?? `${id.replace(/[^A-Za-z0-9._-]+/g, "_")}.jsonl`;
+    assertUsable("crawl");
+    const apiKey = requireApiKey();
+    const { status, count } = await downloadCrawl(id, (ndjson) => writeOutStream(file, ndjson), { apiKey });
+    const partial = status === "running";
+    if (json) {
+        log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, count, file }, null, 2));
+    }
+    else {
+        log.success(`Downloaded ${count} result(s) → ${file}`);
+        if (partial)
+            log.warn("  The crawl is still running: this file is partial. Download again once it ends.");
+    }
+    return 0;
+}
+/** Write or print results. Under --json, always print the envelope (`file` replaces `results` with --out). */
+function printResults(id, status, results, o) {
+    if (o.outFile)
+        writeOut(o.outFile, toJsonl(results));
+    if (o.json) {
+        const rows = o.outFile ? { file: o.outFile } : { results };
+        log.out(JSON.stringify({ ok: true, crawl_id: id, status, count: results.length, ...rows, partial: o.partial }, null, 2));
+        return 0;
+    }
+    if (o.outFile) {
+        log.success(`Wrote ${results.length} result(s) → ${o.outFile}`);
+    }
+    else {
+        log.info(`${results.length} result(s) for crawl ${id} (status: ${status}):`);
+        log.out(toJsonl(results).trimEnd());
+    }
+    if (o.partial)
+        log.warn("  The crawl is still running: these results are partial. Read them again once it ends.");
+    return 0;
+}
+async function contentCmd(rest, ctx) {
+    const { values, positionals } = parse(rest, { out: { type: "string" }, json: { type: "boolean" } });
+    const json = ctx.json || values.json === true;
+    const id = requireId(positionals[0], "content");
+    const ref = positionals[1];
+    const contentId = ref ? contentIdOfRef(id, ref) : undefined;
+    if (!contentId) {
+        throw new ToolkitError({
+            code: "INVALID_USAGE",
+            message: "Missing content id.",
+            likely_cause: "No <content_id> positional was provided.",
+            next_action: "Pass the id from a fetched result's content_url (the last path segment), e.g. `zenrows crawl content <id> ct_…`. Only crawls created with --output-format html have content.",
+            suggested_commands: [`zenrows crawl results ${id}`],
+        });
+    }
+    assertUsable("crawl");
+    const apiKey = requireApiKey();
+    const { contentType, body } = await getCrawlContent(id, contentId, { apiKey });
+    const outFile = asString(values.out);
+    if (outFile) {
+        writeOut(outFile, body);
+        if (json) {
+            log.out(JSON.stringify({ ok: true, crawl_id: id, content_id: contentId, content_type: contentType, bytes: Buffer.byteLength(body), file: outFile }, null, 2));
+        }
+        else {
+            log.success(`Wrote ${Buffer.byteLength(body)} bytes (${contentType || "unknown type"}) → ${outFile}`);
+        }
+    }
+    else if (json) {
+        log.out(JSON.stringify({ ok: true, crawl_id: id, content_id: contentId, content_type: contentType, bytes: Buffer.byteLength(body), content: body }, null, 2));
+    }
+    else {
+        log.out(body);
+    }
+    return 0;
+}
+/** The content id or the result's whole content_url, as the content id. */
+function contentIdOfRef(id, ref) {
+    try {
+        return contentIdOf({ url: "", content_url: ref }) ?? ref;
+    }
+    catch (err) {
+        if (!(err instanceof URIError))
+            throw err;
+        throw new ToolkitError({
+            code: "INVALID_USAGE",
+            message: `Invalid content id '${ref}'.`,
+            likely_cause: "It has a % that does not start a valid percent-encoded character.",
+            next_action: "Pass the id or content_url exactly as a fetched result shows it.",
+            suggested_commands: [`zenrows crawl results ${id}`],
+        });
+    }
+}
+async function listCmd(rest, ctx) {
+    const { values } = parse(rest, { limit: { type: "string" }, cursor: { type: "string" }, json: { type: "boolean" } });
+    const json = ctx.json || values.json === true;
+    const limit = pageLimit(values.limit, 100);
+    const cursor = asString(values.cursor);
+    assertUsable("crawl");
+    const apiKey = requireApiKey();
+    const page = await listCrawls({ apiKey, limit, cursor });
+    const crawls = page.crawls ?? [];
+    if (json) {
+        log.out(JSON.stringify({ ok: true, count: crawls.length, crawls, next_cursor: page.next_cursor ?? null }, null, 2));
+        return 0;
+    }
+    if (!crawls.length)
+        log.info("No crawls.");
+    for (const c of crawls) {
+        const cov = c.coverage;
+        log.out(`${c.crawl_id}  ${c.status.padEnd(9)}  ${c.created_at}  ${cov ? `${cov.items_found} items/${cov.pages_fetched} pages` : ""}  ${c.url}`);
+    }
+    if (page.next_cursor)
+        log.dim(`  more: zenrows crawl list --cursor ${page.next_cursor}`);
+    return 0;
+}
+async function stopCmd(rest, ctx) {
+    const { values, positionals } = parse(rest, { json: { type: "boolean" } });
+    const id = requireId(positionals[0], "stop");
+    assertUsable("crawl");
+    const apiKey = requireApiKey();
+    const s = await stopCrawl(id, { apiKey });
+    const headline = s.status === "stopped" ? `Stopped crawl ${id}` : `Nothing to stop: crawl ${id} had already ended`;
+    return printCrawl(s, ctx.json || values.json === true, headline);
+}
+/**
+ * Wait for a crawl to end. Ctrl-C ends the wait, not the crawl: say so, give
+ * the command that resumes the wait, and exit 130 as shells do on SIGINT.
+ */
+async function waitUntilEnd(id, opts) {
+    const onSigint = () => {
+        log.warn(`Stopped waiting. Crawl ${id} is still running; it was not stopped.`);
+        log.dim(`  resume: zenrows crawl wait ${id}`);
+        process.exit(130);
+    };
+    process.once("SIGINT", onSigint);
+    try {
+        return await waitForCrawl(id, opts);
+    }
+    finally {
+        process.off("SIGINT", onSigint);
+    }
+}
+/**
+ * The error for a crawl that ended `failed`, or null for any other state. The
+ * crawl's own `error.code` picks the next action.
+ */
+export function crawlFailure(c) {
+    if (c.status !== "failed")
+        return null;
+    const code = c.error?.code;
+    const why = [code, c.error?.detail].filter(Boolean).join(": ");
+    const next = {
+        insufficient_credits: "The account ran out of credits. Check `zenrows usage`, add credits, then start a new crawl.",
+        seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch <url>`.",
+        no_items_found: "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise or drop --depth, or check the start page with `zenrows fetch <url>`.",
+    };
+    return new ToolkitError({
+        code: "CRAWL_FAILED",
+        message: `Crawl ${c.crawl_id} failed.`,
+        likely_cause: why || "The crawl ended in status failed without a reason.",
+        next_action: (code && next[code]) || "Read the error, fix the cause, then start a new crawl.",
+        suggested_commands: [`zenrows crawl get ${c.crawl_id}`],
+        server_code: code,
+        crawl_id: c.crawl_id,
+    });
+}
+/**
+ * Print a crawl (or a stop answer), structured under --json, and return the exit
+ * code: 1 when it ended `failed`, else 0. `stopped` is a deliberate outcome, so
+ * it exits 0 but prints as a warning. `extra` carries the results a `get` read,
+ * or the note of a wait that ran out.
+ */
+function printCrawl(c, json, headline, extra = {}) {
+    const err = crawlFailure(c);
+    const crawlFields = { ...c };
+    // A wait polls with one result to stay cheap; that page is not "the results".
+    delete crawlFields.results;
+    delete crawlFields.next_cursor;
+    if (json) {
+        log.out(JSON.stringify({
+            ok: !err,
+            crawl_id: c.crawl_id,
+            status: c.status,
+            crawl: crawlFields,
+            ...extra,
+            ...(err ? { error: err.toJSON() } : {}),
+        }, null, 2));
+        return err ? 1 : 0;
+    }
+    const line = `${headline} · status: ${c.status ?? "unknown"}${c.stop_reason ? ` (${c.stop_reason})` : ""}`;
+    if (err)
+        log.error(line);
+    else if (c.status === "stopped")
+        log.warn(line);
+    else
+        log.success(line);
+    const cov = c.coverage;
+    if (cov) {
+        log.info(`  ${cov.items_found} item(s) found · ${cov.pages_fetched} page(s) fetched · ${cov.pages_failed} failed`);
+    }
+    if (extra.note)
+        log.dim(`  next: zenrows crawl wait ${c.crawl_id}`);
+    if (err)
+        printError(err, false);
+    return err ? 1 : 0;
+}
+/**
+ * A whole-number flag up to the API's 100,000 ceiling; undefined when unset. The lower bound is
+ * the API's to judge: 0 or less comes back as CRAWL_INVALID_REQUEST, naming the parameter.
+ */
+function limitFlag(name, v) {
+    if (v === undefined)
+        return undefined;
+    const n = asNumber(v);
+    if (n === undefined || !Number.isInteger(n) || n > MAX_LIMIT) {
+        throw new ToolkitError({
+            code: "INVALID_USAGE",
+            message: `Invalid --${name} value '${String(v)}'.`,
+            likely_cause: `--${name} takes a whole number up to ${MAX_LIMIT}.`,
+            next_action: `Pass e.g. --${name} ${name === "depth" ? 1 : 10}.`,
+        });
+    }
+    return n;
+}
+/** `--timeout` in seconds, returned in milliseconds; undefined when unset. */
+function timeoutFlag(v) {
+    if (v === undefined)
+        return undefined;
+    const seconds = asNumber(v);
+    if (seconds === undefined || seconds <= 0) {
+        throw new ToolkitError({
+            code: "INVALID_USAGE",
+            message: `Invalid --timeout value '${String(v)}'.`,
+            likely_cause: "--timeout takes a positive number of seconds.",
+            next_action: "Pass seconds, e.g. --timeout 600 for ten minutes.",
+        });
+    }
+    return seconds * 1000;
+}
+/** `--limit` for a paged read: a whole number from 1 to `max`. */
+function pageLimit(v, max) {
+    if (v === undefined)
+        return undefined;
+    const n = asNumber(v);
+    if (n === undefined || !Number.isInteger(n) || n < 1 || n > max) {
+        throw new ToolkitError({
+            code: "INVALID_USAGE",
+            message: `Invalid --limit value '${String(v)}'.`,
+            likely_cause: `--limit takes a whole number from 1 to ${max}.`,
+            next_action: `Pass e.g. --limit ${Math.min(max, 100)}.`,
+        });
+    }
+    return n;
+}
+function patterns(v) {
+    const list = Array.isArray(v) ? v : typeof v === "string" ? [v] : [];
+    return list.filter((p) => typeof p === "string" && p.length > 0);
+}
+function toJsonl(rows) {
+    return rows.map((r) => JSON.stringify(r)).join("\n") + (rows.length ? "\n" : "");
+}
+function requireId(id, sub) {
+    if (id)
+        return id;
+    throw new ToolkitError({
+        code: "INVALID_USAGE",
+        message: "Missing crawl id.",
+        likely_cause: "No <id> positional was provided.",
+        next_action: `Usage: zenrows crawl ${sub} <id>. List your crawls with \`zenrows crawl list\`.`,
+        suggested_commands: ["zenrows crawl list"],
+    });
+}

package/dist/src/cli/index.js

@@ -25,6 +25,7 @@ import { policy } from "./commands/policy.js";
 import { fetch_ } from "./commands/fetch.js";
 import { extract } from "./commands/extract.js";
 import { batch } from "./commands/batch.js";
+import { crawl } from "./commands/crawl.js";
 import { browser } from "./commands/browser.js";
 import { mcp } from "./commands/mcp.js";
 import { plugin } from "./commands/plugin.js";
@@ -38,7 +39,7 @@ const evalCmd = makeAssetCommand("eval", "Reproducible benchmark/test suites.");
 /** Ordered groups for help output. */
 const GROUPS = [
     { title: "Setup", commands: [init, signup, account, login, logout, status, usage, update, uninstall, config, policy] },
-    { title: "Primitives", commands: [fetch_, extract, batch, browser] },
+    { title: "Primitives", commands: [fetch_, extract, batch, crawl, browser] },
     { title: "Distribution", commands: [mcp, plugin, skill, template, workflow, recipe, evalCmd, trace] },
 ];
 const COMMANDS = {};

package/dist/src/cli/output.js

@@ -2,7 +2,8 @@
  * Output helpers: render normalized errors for humans and machines, and a tiny
  * key/value printer used by `status`, `config`, `policy`.
  */
-import { statSync, writeFileSync } from "node:fs";
+import { createWriteStream, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { pipeline } from "node:stream/promises";
 import { ToolkitError } from "../core/errors.js";
 import { ANSI, c, log } from "../core/logger.js";
 /**
@@ -11,6 +12,32 @@ import { ANSI, c, log } from "../core/logger.js";
  * parent directory doesn't exist (ENOENT). Everything else propagates.
  */
 export function writeOut(filePath, data) {
+    assertNotDirectory(filePath);
+    try {
+        writeFileSync(filePath, data);
+    }
+    catch (err) {
+        throw outError(filePath, err);
+    }
+}
+/**
+ * `writeOut` for a body too large to hold in memory: streams into
+ * `<file>.part`, then renames it into place, so a failed stream leaves any
+ * earlier file intact.
+ */
+export async function writeOutStream(filePath, chunks) {
+    assertNotDirectory(filePath);
+    const part = `${filePath}.part`;
+    try {
+        await pipeline(chunks, createWriteStream(part));
+        renameSync(part, filePath);
+    }
+    catch (err) {
+        rmSync(part, { force: true });
+        throw outError(filePath, err);
+    }
+}
+function assertNotDirectory(filePath) {
     if (statSync(filePath, { throwIfNoEntry: false })?.isDirectory()) {
         throw new ToolkitError({
             code: "INVALID_USAGE",
@@ -19,20 +46,16 @@ export function writeOut(filePath, data) {
             next_action: "Pass a file path, e.g. --out results.jsonl.",
         });
     }
-    try {
-        writeFileSync(filePath, data);
-    }
-    catch (err) {
-        if (err?.code === "ENOENT") {
-            throw new ToolkitError({
-                code: "INVALID_USAGE",
-                message: `Cannot write --out '${filePath}': the parent directory does not exist.`,
-                likely_cause: "A directory in the output path is missing.",
-                next_action: "Create the directory first, or choose an existing one.",
-            });
-        }
-        throw err;
-    }
+}
+function outError(filePath, err) {
+    if (err?.code !== "ENOENT")
+        return err;
+    return new ToolkitError({
+        code: "INVALID_USAGE",
+        message: `Cannot write --out '${filePath}': the parent directory does not exist.`,
+        likely_cause: "A directory in the output path is missing.",
+        next_action: "Create the directory first, or choose an existing one.",
+    });
 }
 /** Print a ToolkitError. In --json mode emits the structured shape to stdout. */
 export function printError(err, json = false) {

package/dist/src/core/config.js

@@ -14,7 +14,7 @@ export const CONFIG_VERSION = "0.1.0";
  * entrypoint) so core modules — telemetry, signup provenance — can read it
  * without an import cycle. `VERSION` in `cli/index.ts` re-exports this.
  */
-export const CLI_VERSION = "1.4.2";
+export const CLI_VERSION = "1.5.0";
 /** The User-Agent on every request to a Zenrows API. The gateway reads the version from it. */
 export const CLI_USER_AGENT = `zenrows-cli/${CLI_VERSION}`;
 /** Env var to override the Fetch and Extract API base (local/staging testing). */