@zenrows/cli 1.5.5 → 1.6.0

package/dist/src/cli/commands/crawl.js

@@ -35,8 +35,11 @@ export const crawl = {
         "                                    the local policy max_pages_per_run caps it (default 1000)",
         "    --include-patterns <p>          keep only URLs containing this substring (repeatable)",
         "    --exclude-patterns <p>          drop URLs containing this substring (repeatable)",
-        "    --output-format html            also fetch each kept URL's page HTML (read with content/download);",
-        "                                    each kept page is one more fetch counted by --max-pages",
+        "    --output-format <html|markdown> also fetch each kept URL's page, as HTML or Markdown",
+        "                                    (read with content/download); each kept page is one more",
+        "                                    fetch counted by --max-pages",
+        "    --discovery links               how the crawl finds the URLs it follows; links (the",
+        "                                    default) follows the links in each page's HTML",
         "    --follow                        wait until the crawl ends (Ctrl-C stops the wait, not the crawl)",
         "    --timeout <s>                   with --follow: stop waiting after s seconds (default 600)",
         "    --idempotency-key <key>         so a retried create starts no second crawl",
@@ -44,13 +47,15 @@ export const crawl = {
         "  get <id> [--cursor <c>] [--limit <n>]",
         "                                    the crawl and one page of its results (limit up to 10000)",
         "  list [--cursor <c>] [--limit <n>] your crawls, newest first (limit up to 100)",
-        "  results <id> [--limit <n>]        every kept URL as JSONL; on a running crawl, those kept so far",
-        "                                    (partial). --limit is the page size per request",
+        "  results <id> [--limit <n>]        every kept URL as JSONL; partial while more can come (a running",
+        "                                    crawl, or pages still in flight after a stop). --limit is the page size",
+        "                                    per request",
         "    --out <file>                    write the results to a file instead of printing",
         "  content <id> <content_id>         print one kept URL's page (content_id or content_url)",
         "    --out <file>                    write it to a file instead",
         "  download <id> [--out <file>]      save the NDJSON export (with page HTML for html crawls),",
-        "                                    default <id>.jsonl; partial while the crawl runs",
+        "                                    default <id>.jsonl; partial while more can come (a running",
+        "                                    crawl, or pages still in flight after a stop)",
         "  stop <id>                         stop a running crawl (an ended crawl answers as it ended)",
         "  wait <id> [--timeout <s>]         poll until the crawl ends or s seconds (default 600) run out;",
         "                                    a crawl still running then is printed, not stopped",
@@ -102,6 +107,7 @@ async function createCmd(rest, ctx) {
         "include-patterns": { type: "string", multiple: true },
         "exclude-patterns": { type: "string", multiple: true },
         "output-format": { type: "string" },
+        discovery: { type: "string", multiple: true },
         follow: { type: "boolean" },
         timeout: { type: "string" },
         "idempotency-key": { type: "string" },
@@ -137,16 +143,29 @@ async function createCmd(rest, ctx) {
         params.exclude_patterns = exclude;
     const outputFormat = asString(values["output-format"]);
     if (outputFormat !== undefined) {
-        if (outputFormat !== "html") {
+        if (outputFormat !== "html" && outputFormat !== "markdown") {
             throw new ToolkitError({
                 code: "INVALID_USAGE",
                 message: `Invalid --output-format value '${outputFormat}'.`,
-                likely_cause: "--output-format takes html only. Without it, a crawl returns URLs only.",
-                next_action: "Pass --output-format html, or leave it out.",
+                likely_cause: "--output-format takes html or markdown. Without it, a crawl returns URLs only.",
+                next_action: "Pass --output-format html or --output-format markdown, or leave it out.",
             });
         }
         params.output_format = outputFormat;
     }
+    const discovery = patterns(values.discovery);
+    if (discovery.length) {
+        const other = discovery.find((d) => d !== "links");
+        if (other !== undefined) {
+            throw new ToolkitError({
+                code: "INVALID_USAGE",
+                message: `Invalid --discovery value '${other}'.`,
+                likely_cause: "--discovery takes links. Without it, the crawl follows the links in each page's HTML.",
+                next_action: "Pass --discovery links, or leave it out.",
+            });
+        }
+        params.discovery = ["links"];
+    }
     const timeoutMs = timeoutFlag(values.timeout);
     const idempotencyKey = asString(values["idempotency-key"]);
     assertUsable("crawl");
@@ -269,14 +288,17 @@ async function downloadCmd(rest, ctx) {
     assertUsable("crawl");
     const apiKey = requireApiKey();
     const { status, count } = await downloadCrawl(id, (ndjson) => writeOutStream(file, ndjson), { apiKey });
-    const partial = status === "running";
+    // A stop reads `stopped` at once, but the API still reads the pages in flight for up to 10
+    // minutes and builds the file from what it has kept until then. Its next_cursor says when it is done.
+    const partial = status === "running" || (status === "stopped" && (await listAllResults(id, { apiKey, pageSize: 10_000 })).partial);
+    const note = partial ? downloadNote(status) : undefined;
     if (json) {
-        log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, count, file }, null, 2));
+        log.out(JSON.stringify({ ok: true, crawl_id: id, status: status ?? null, partial, ...(note ? { note } : {}), count, file }, null, 2));
     }
     else {
         log.success(`Downloaded ${count} result(s) → ${file}`);
-        if (partial)
-            log.warn("  The crawl is still running: this file is partial. Download again once it ends.");
+        if (note)
+            log.warn(`  ${note}`);
     }
     return 0;
 }
@@ -284,9 +306,10 @@ async function downloadCmd(rest, ctx) {
 function printResults(id, status, results, o) {
     if (o.outFile)
         writeOut(o.outFile, toJsonl(results));
+    const note = o.partial ? partialNote(status) : undefined;
     if (o.json) {
         const rows = o.outFile ? { file: o.outFile } : { results };
-        log.out(JSON.stringify({ ok: true, crawl_id: id, status, count: results.length, ...rows, partial: o.partial }, null, 2));
+        log.out(JSON.stringify({ ok: true, crawl_id: id, status, count: results.length, ...rows, partial: o.partial, ...(note ? { note } : {}) }, null, 2));
         return 0;
     }
     if (o.outFile) {
@@ -296,10 +319,20 @@ function printResults(id, status, results, o) {
         log.info(`${results.length} result(s) for crawl ${id} (status: ${status}):`);
         log.out(toJsonl(results).trimEnd());
     }
-    if (o.partial)
-        log.warn("  The crawl is still running: these results are partial. Read them again once it ends.");
+    if (note)
+        log.warn(`  ${note}`);
     return 0;
 }
+function downloadNote(status) {
+    return status === "running"
+        ? "The crawl is still running: this file is partial. Download again once it ends."
+        : "The crawl was stopped, but pages fetched before the stop can still add lines for up to 10 minutes: download again later.";
+}
+function partialNote(status) {
+    return isTerminal(status)
+        ? "The crawl ended, but pages fetched before the end can still add URLs for up to 10 minutes: read the results again later."
+        : "The crawl is still running: these results are partial. Read them again once it ends.";
+}
 async function contentCmd(rest, ctx) {
     const { values, positionals } = parse(rest, { out: { type: "string" }, json: { type: "boolean" } });
     const json = ctx.json || values.json === true;

package/dist/src/core/batch-api.js

@@ -21,6 +21,7 @@ import { readAccount } from "./agent-account.js";
 import { registerSecret } from "./logger.js";
 import { CLI_USER_AGENT } from "./config.js";
 import { agentClientHeader } from "./agent-client.js";
+import { BILLING_URL } from "./open-url.js";
 /** Confirmed Batch API base (no trailing slash). */
 export const DEFAULT_BATCH_API_BASE = "https://async.api.zenrows.com/v1";
 /** Env var to override the Batch API base (local/staging testing). */
@@ -116,6 +117,17 @@ function problemToError(status, body, method, path) {
             suggested_commands: ["zenrows batch estimate jobs.jsonl"],
         });
     }
+    if (status === 401 && serverCode === "subscription_inactive") {
+        // The key is fine; the subscription needs a billing action. Same
+        // AUTH_INVALID code as any 401 so callers that branch on it keep
+        // working; only the guidance changes, since re-entering the key won't help.
+        return new ToolkitError({
+            code: "AUTH_INVALID",
+            message: "Zenrows refused the Batch API request because the subscription is not active.",
+            likely_cause: `${cause.replace(/\.?$/, ".")} The API key is valid, but the subscription is past due, unpaid, incomplete or paused.`,
+            next_action: `Fix the subscription in Billing (${BILLING_URL}), or in Azure Marketplace if you subscribed there, then retry.`,
+        });
+    }
     if (status === 401) {
         return new ToolkitError({
             code: "AUTH_INVALID",

package/dist/src/core/config.js

@@ -14,7 +14,7 @@ export const CONFIG_VERSION = "0.1.0";
  * entrypoint) so core modules — telemetry, signup provenance — can read it
  * without an import cycle. `VERSION` in `cli/index.ts` re-exports this.
  */
-export const CLI_VERSION = "1.5.5"; // x-release-please-version
+export const CLI_VERSION = "1.6.0"; // x-release-please-version
 /** The User-Agent on every request to a Zenrows API. The gateway reads the version from it. */
 export const CLI_USER_AGENT = `zenrows-cli/${CLI_VERSION}`;
 /** Env var to override the Fetch and Extract API base (local/staging testing). */

package/dist/src/core/crawl-api.js

@@ -10,7 +10,7 @@
  * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable
  * for tests.
  *
- * `output_format` takes `html` only (absent = URLs only). Response fields we do
+ * `output_format` takes `html` or `markdown` (absent = URLs only). Response fields we do
  * not model pass through untouched.
  */
 import { Readable } from "node:stream";
@@ -297,6 +297,8 @@ export function createCrawl(params, opts) {
         body.exclude_patterns = params.exclude_patterns;
     if (params.output_format !== undefined)
         body.output_format = params.output_format;
+    if (params.discovery?.length)
+        body.discovery = params.discovery;
     return crawlJson("POST", "/crawls", {
         ...opts,
         body,
@@ -319,7 +321,7 @@ export function listCrawls(opts) {
 export function stopCrawl(id, opts) {
     return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id, retryStop: true });
 }
-/** The page of one kept URL (HTML for `output_format: html`). */
+/** The page of one kept URL, in the crawl's `output_format`. */
 export async function getCrawlContent(id, contentId, opts) {
     const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/contents/${encodeURIComponent(contentId)}`, {
         ...opts,
@@ -390,9 +392,9 @@ export async function listAllResults(id, opts) {
         const rows = page.results ?? [];
         results.push(...rows);
         if (!rows.length || !page.next_cursor || page.next_cursor === cursor) {
-            // Every page was read. An ended crawl keeps no new URLs, even while pages in flight after
-            // a stop keep next_cursor set, so only a running crawl's results are partial.
-            return { crawl: page, results, partial: !isTerminal(page.status) };
+            // Every page was read. The API keeps next_cursor set while URLs can still come: on a
+            // running crawl, and for up to 10 minutes after a stop, from the pages still in flight.
+            return { crawl: page, results, partial: page.next_cursor != null };
         }
         cursor = page.next_cursor;
     }

package/dist/src/core/open-url.js

@@ -12,6 +12,7 @@ export const DASHBOARD_URL = "https://app.zenrows.com/dashboard";
 /** Lands on Billing with the credit-pack purchase modal already open (app PR #1972). */
 export const BILLING_TOPUP_URL = "https://app.zenrows.com/billing?topup=open";
 export const PLANS_URL = "https://app.zenrows.com/plans";
+export const BILLING_URL = "https://app.zenrows.com/billing";
 export const DOCS_URL = "https://docs.zenrows.com";
 /** Append query params to a URL (used to attach anonymous `utm_*` attribution). */
 export function withParams(url, params) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenrows/cli",
-  "version": "1.5.5",
+  "version": "1.6.0",
   "description": "Open-source CLI, MCP, skills, templates, workflows, recipes, and evals layer for giving AI agents reliable access to protected web data through Zenrows cloud infrastructure.",
   "type": "module",
   "license": "MIT",

package/skills/crawl/SKILL.md

@@ -11,8 +11,8 @@ Give Crawl one start URL; it follows the links on that page (and on the
 pages behind it, up to the depth you set) and returns the URLs it keeps. Use it
 when you know where the items live (a category page, a blog index, a docs
 section) but not the item URLs themselves. Once you have the URLs, read pages
-with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML
-with `--output-format html`.
+with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page itself
+with `--output-format html` or `--output-format markdown`.
 
 Crawl stays on the start URL's registrable domain; subdomains count. An account without Crawl access gets
 403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account");
@@ -50,16 +50,18 @@ zenrows crawl results <id> --out urls.jsonl  # same, as JSONL
   `partial: true`. Read again once the crawl ends. `--limit` sets the page size
   of each request.
 
-## Page HTML
+## Page content
 
 ```
 zenrows crawl create <url> --depth 1 --output-format html --max-pages 20 --follow
+zenrows crawl create <url> --depth 1 --output-format markdown --max-pages 20 --follow
 zenrows crawl results <id> --json        # fetched results carry content_url
-zenrows crawl content <id> <content_id>  # one page's HTML (or pass the content_url)
-zenrows crawl download <id> --out pages.jsonl  # every URL + HTML, NDJSON
+zenrows crawl content <id> <content_id>  # one page (or pass the content_url)
+zenrows crawl download <id> --out pages.jsonl  # every URL + page, NDJSON
 ```
 
-`--output-format html` uses up the page budget: each kept page is one more
+`--output-format html` stores each page as fetched; `--output-format markdown`
+stores it converted to Markdown. Either uses up the page budget: each kept page is one more
 fetch, and `--max-pages` counts it. Raise `--max-pages` to keep the same number
 of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only
 `fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). `download`