@zenrows/cli 1.6.0 → 1.6.2

package/README.md

@@ -187,7 +187,10 @@ to open. `--max-items` and `--max-pages` cap the crawl (API default 100 each),
 and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default
 1000). `--output-format html` uses up the page budget: each kept page is one
 more fetch, and `--max-pages` counts it. `--include-patterns` /
-`--exclude-patterns` are repeatable substring filters on the URL. `get` and
+`--exclude-patterns` are repeatable substring filters on the URL. A crawl whose
+pages held nothing new to follow ends `completed` with no results and exits 0.
+When patterns were set, it prints a note (under `--json`, in `note`) to check
+them against the URLs the start page links to. `get` and
 `list` read one page and take `--cursor` and `--limit`. `results` follows every
 page (`--limit` is the page size); on a running crawl it returns the URLs kept
 so far, with `partial: true`. `download` reports the crawl's `status` (`null`
@@ -203,7 +206,12 @@ run out (default 600). When the time runs out, the command exits 0 and prints
 the crawl with status `running` and the hint `zenrows crawl wait <crawl-id>`
 (under `--json`, in `note`); the crawl keeps running. Ctrl-C stops the wait, not the crawl, and exits 130;
 resume with `zenrows crawl wait <crawl-id>`. A crawl that ends `failed` exits 1
-with `CRAWL_FAILED`.
+with `CRAWL_FAILED`. A crawl that ends `stopped` because the account ran out of
+credits or the API key hit a credit cap also exits 1, with the code a refused
+create gets: `CRAWL_INSUFFICIENT_CREDITS` or `CRAWL_KEY_CAP_REACHED`, and the
+API's `insufficient_credits` or `api_key_cap_reached` as `server_code`. Older
+deployments end such a crawl `failed`; it gets the same codes. Its fetched pages
+stay readable.
 
 Errors carry the API's code as `server_code`, `crawl_id` on any call about one
 crawl, and `retry_after` on a 429. When the account has reached its limit of

package/dist/src/cli/commands/crawl.js

@@ -15,7 +15,8 @@ import { assertDomainAllowed, assertWithinLimits, loadPolicy } from "../../core/
 import { newRunId, writeRun } from "../../core/artifacts.js";
 import { contentIdOf, createCrawl, downloadCrawl, getCrawl, getCrawlContent, isTerminal, listAllResults, listCrawls, stopCrawl, waitForCrawl, } from "../../core/crawl-api.js";
 import { asNumber, asString, parse } from "../command.js";
-import { ToolkitError } from "../../core/errors.js";
+import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.js";
+import { readAccount } from "../../core/agent-account.js";
 import { printError, writeOut, writeOutStream } from "../output.js";
 /** The API's default for `max_pages` (and `max_items`) when the flag is unset. */
 const DEFAULT_MAX_PAGES = 100;
@@ -61,7 +62,8 @@ export const crawl = {
         "                                    a crawl still running then is printed, not stopped",
         "  --json                            print structured output: create/get/wait/stop print",
         "                                    {ok, crawl_id, status, crawl}; get adds results and next_cursor;",
-        "                                    a wait that runs out adds note",
+        "                                    a wait that runs out, or a completed crawl with patterns",
+        "                                    and 0 results, adds note",
     ].join("\n"),
     async run(argv, ctx) {
         const [sub, ...rest] = argv;
@@ -415,7 +417,8 @@ async function stopCmd(rest, ctx) {
     assertUsable("crawl");
     const apiKey = requireApiKey();
     const s = await stopCrawl(id, { apiKey });
-    const headline = s.status === "stopped" ? `Stopped crawl ${id}` : `Nothing to stop: crawl ${id} had already ended`;
+    // A crawl stopped on credits carries `error`: it had already ended, this call did not stop it.
+    const headline = s.status === "stopped" && !s.error ? `Stopped crawl ${id}` : `Nothing to stop: crawl ${id} had already ended`;
     return printCrawl(s, ctx.json || values.json === true, headline);
 }
 /**
@@ -437,37 +440,67 @@ async function waitUntilEnd(id, opts) {
     }
 }
 /**
- * The error for a crawl that ended `failed`, or null for any other state. The
- * crawl's own `error.code` picks the next action.
+ * The error for a crawl that ended `failed`, or `stopped` with an `error` (the
+ * account ran out of credits or the API key hit a credit cap), or null for any
+ * other state. A crawl the credits ended gets the code and advice a create
+ * refused with 402 gets (`CRAWL_INSUFFICIENT_CREDITS`, `CRAWL_KEY_CAP_REACHED`),
+ * whether the API reports it `stopped` or, as older deployments do, `failed`.
+ * Any other `error.code` picks the next action of a `CRAWL_FAILED`.
  */
 export function crawlFailure(c) {
-    if (c.status !== "failed")
+    const creditStop = c.status === "stopped" && !!c.error;
+    if (c.status !== "failed" && !creditStop)
         return null;
     const code = c.error?.code;
+    const ids = { server_code: code, crawl_id: c.crawl_id };
+    const where = `crawl ${c.crawl_id}`;
+    const detail = c.error?.detail || undefined;
+    const getCmd = `zenrows crawl get ${c.crawl_id}`;
+    if (code && isKeyCapReached(code)) {
+        const cap = keyCapReached(where, { status: null, detail }).toJSON();
+        return new ToolkitError({ ...cap, ...ids, code: "CRAWL_KEY_CAP_REACHED", suggested_commands: [...(cap.suggested_commands ?? []), getCmd] });
+    }
+    if (code === "insufficient_credits") {
+        const acct = readAccount();
+        const quota = quotaExhausted(where, acct?.unclaimed ? acct.claimUrl : undefined, { status: null, detail }).toJSON();
+        return new ToolkitError({ ...quota, ...ids, code: "CRAWL_INSUFFICIENT_CREDITS", suggested_commands: [...(quota.suggested_commands ?? []), getCmd] });
+    }
     const why = [code, c.error?.detail].filter(Boolean).join(": ");
     const next = {
-        insufficient_credits: "The account ran out of credits. Check `zenrows usage`, add credits, then start a new crawl.",
-        seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch <url>`.",
-        no_items_found: "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise or drop --depth, or check the start page with `zenrows fetch <url>`.",
+        seed_unreachable: "The start URL could not be fetched or returned an empty page (the site may be blocking it). Check it with `zenrows fetch <url>`.",
+        no_items_found: "The crawl read the start page but found no data it could use, so it had nothing to follow. Start from a listing or product page.",
     };
     return new ToolkitError({
         code: "CRAWL_FAILED",
-        message: `Crawl ${c.crawl_id} failed.`,
+        message: creditStop ? `Crawl ${c.crawl_id} stopped before it finished.` : `Crawl ${c.crawl_id} failed.`,
         likely_cause: why || "The crawl ended in status failed without a reason.",
         next_action: (code && next[code]) || "Read the error, fix the cause, then start a new crawl.",
-        suggested_commands: [`zenrows crawl get ${c.crawl_id}`],
-        server_code: code,
-        crawl_id: c.crawl_id,
+        suggested_commands: [getCmd],
+        ...ids,
     });
 }
+/**
+ * Set only when a crawl with patterns completed and kept nothing. The API ends a
+ * crawl whose pages held nothing new to follow `completed`, and does not count
+ * the links the patterns rejected, so the patterns are the first thing to check.
+ */
+function noMatchNote(c) {
+    const crawl = c;
+    const patterns = [...(crawl.include_patterns ?? []), ...(crawl.exclude_patterns ?? [])];
+    if (crawl.status !== "completed" || crawl.coverage?.items_found !== 0 || !patterns.length)
+        return undefined;
+    return "The crawl completed with no results. Check --include-patterns / --exclude-patterns against the URLs the start page links to.";
+}
 /**
  * Print a crawl (or a stop answer), structured under --json, and return the exit
- * code: 1 when it ended `failed`, else 0. `stopped` is a deliberate outcome, so
- * it exits 0 but prints as a warning. `extra` carries the results a `get` read,
+ * code: 1 when it ended `failed`, or `stopped` because credits ran out or the
+ * API key hit a credit cap, else 0. A `stopped` crawl the caller stopped is a
+ * deliberate outcome, so it exits 0 but prints as a warning. `extra` carries the results a `get` read,
  * or the note of a wait that ran out.
  */
 function printCrawl(c, json, headline, extra = {}) {
     const err = crawlFailure(c);
+    const note = extra.note ?? noMatchNote(c);
     const crawlFields = { ...c };
     // A wait polls with one result to stay cheap; that page is not "the results".
     delete crawlFields.results;
@@ -479,6 +512,7 @@ function printCrawl(c, json, headline, extra = {}) {
             status: c.status,
             crawl: crawlFields,
             ...extra,
+            note,
             ...(err ? { error: err.toJSON() } : {}),
         }, null, 2));
         return err ? 1 : 0;
@@ -496,6 +530,8 @@ function printCrawl(c, json, headline, extra = {}) {
     }
     if (extra.note)
         log.dim(`  next: zenrows crawl wait ${c.crawl_id}`);
+    else if (note)
+        log.warn(`  ${note}`);
     if (err)
         printError(err, false);
     return err ? 1 : 0;

package/dist/src/core/config.js

@@ -14,7 +14,7 @@ export const CONFIG_VERSION = "0.1.0";
  * entrypoint) so core modules — telemetry, signup provenance — can read it
  * without an import cycle. `VERSION` in `cli/index.ts` re-exports this.
  */
-export const CLI_VERSION = "1.6.0"; // x-release-please-version
+export const CLI_VERSION = "1.6.2"; // x-release-please-version
 /** The User-Agent on every request to a Zenrows API. The gateway reads the version from it. */
 export const CLI_USER_AGENT = `zenrows-cli/${CLI_VERSION}`;
 /** Env var to override the Fetch and Extract API base (local/staging testing). */

package/dist/src/core/errors.js

@@ -45,7 +45,9 @@ export class ToolkitError extends Error {
  * (so `claimUrl` is surfaced); for a real logged-in account there is nothing to
  * claim — the step is to add credits / upgrade in the dashboard. `status`/`detail`
  * let the caller record the actual upstream response (402 usage-limit, 429
- * quota, …) instead of a hardcoded status.
+ * quota, …) instead of a hardcoded status. `status: null` is for a crawl the
+ * credits stopped: its status call succeeded, so there is no HTTP error to cite,
+ * only `url` (the crawl).
  */
 export function quotaExhausted(url, claimUrl, opts = {}) {
     // Say that the allowance comes back. Without it this reads as a permanent paywall,
@@ -58,10 +60,11 @@ export function quotaExhausted(url, claimUrl, opts = {}) {
         ? `You are on the Zenrows Free plan. Claim your account to keep your usage and add credits: ${claimUrl}. ${renewLine}`
         : `You are out of Zenrows credits. ${renewLine} To carry on now, add a credit pack (${BILLING_TOPUP_URL} opens the purchase directly) or upgrade your plan: ${PLANS_URL}`;
     const detail = opts.detail ? `${opts.detail.replace(/\.\s*$/, "")}. ` : "";
+    const where = opts.status === null ? `Stopped ${url}` : `HTTP ${opts.status ?? 429} for ${url}`;
     return new ToolkitError({
         code: "POLICY_MAX_CREDITS_EXCEEDED",
         message: "Zenrows request quota exhausted.",
-        likely_cause: `${detail}HTTP ${opts.status ?? 429} for ${url}`,
+        likely_cause: `${detail}${where}`,
         next_action: claimLine,
         suggested_commands: ["zenrows usage"],
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenrows/cli",
-  "version": "1.6.0",
+  "version": "1.6.2",
   "description": "Open-source CLI, MCP, skills, templates, workflows, recipes, and evals layer for giving AI agents reliable access to protected web data through Zenrows cloud infrastructure.",
   "type": "module",
   "license": "MIT",

package/skills/crawl/SKILL.md

@@ -39,6 +39,10 @@ zenrows crawl results <id> --out urls.jsonl  # same, as JSONL
 - `--include-patterns` / `--exclude-patterns` are substring matches on the
   normalized URL and repeat:
   `--include-patterns /product/ --exclude-patterns ?add-to-cart`.
+- A crawl whose pages held nothing new to follow (no link, only links to other
+  sites, or only links the patterns reject) ends `completed` with no results
+  and exits 0. When patterns were set, it prints a note (under `--json`, in
+  `note`) to check them against the URLs the start page links to.
 - `create` returns at once. `--follow` (or `zenrows crawl wait <id>`) polls
   until the crawl ends or `--timeout` seconds run out (default 600). When the
   time runs out, the command exits 0 and prints the crawl with status
@@ -77,8 +81,18 @@ zenrows crawl stop <id>                      # stop it; kept URLs stay readable
 ```
 
 A crawl ends `completed` (nothing left, or a cap hit: `stop_reason`
-`max_items` / `max_pages`), `stopped` (you stopped it) or `failed` (`error.code`
-and `error.detail` say why). Only `failed` exits non-zero (`CRAWL_FAILED`).
+`max_items` / `max_pages`), `stopped` or `failed` (`error.code` and
+`error.detail` say why). `stopped` with `stop_reason` `user` means you stopped
+it. `stopped` with no `stop_reason` and `error.code` `insufficient_credits` or
+`api_key_cap_reached` means the account ran out of credits or the API key hit a
+credit cap; the pages fetched before that stay readable. `failed`, and a
+`stopped` crawl with an `error`, exit non-zero: `CRAWL_INSUFFICIENT_CREDITS` or
+`CRAWL_KEY_CAP_REACHED` when credits ended it (the same codes a refused
+`create` gets, also when an older deployment reports it `failed`), else
+`CRAWL_FAILED`. `server_code` carries the API's `error.code`:
+`seed_unreachable` when the start URL could not be fetched or returned an empty
+page, `no_items_found` when the crawl read the start page but found no data it
+could use.
 `stop` on a crawl that already ended prints the status it ended with.
 
 With `--json`, `create`, `get`, `wait` and `stop` print
@@ -112,15 +126,15 @@ The "Retry?" column is for errors that remain after these retries.
 | --- | --- | --- |
 | `CRAWL_NOT_ENABLED` | 403 `REQS008`: Crawl is not enabled for this account | no |
 | `AUTH_INVALID` | 401: the API key was rejected | no, log in again |
-| `CRAWL_INSUFFICIENT_CREDITS` | 402: the account is out of credits (claim the account or add credits) | after credits renew or are added |
-| `CRAWL_KEY_CAP_REACHED` | 402 `AUTH014` or `api_key_cap_reached`: this API key reached its credit cap | after the cap resets or is raised |
+| `CRAWL_INSUFFICIENT_CREDITS` | 402 on `create`, or a crawl that ended on `insufficient_credits`: the account is out of credits (claim the account or add credits) | after credits renew or are added |
+| `CRAWL_KEY_CAP_REACHED` | 402 `AUTH014` or `api_key_cap_reached` on `create`, or a crawl that ended on `api_key_cap_reached`: this API key reached its credit cap | after the cap resets or is raised |
 | `CRAWL_INVALID_REQUEST` | 400 or 422: fix the request. For `idempotency_key_reused`, send a new key or no key | no, not as is |
 | `CRAWL_REQUEST_IN_FLIGHT` | 409: the same Idempotency-Key is still in flight | yes, once the first request ends |
 | `CRAWL_NOT_FOUND` / `CRAWL_CONTENT_NOT_FOUND` | 404: wrong id, or the page has no content | no |
 | `CRAWL_TOO_MANY_CRAWLS` | 429 `too_many_crawls` on `create`: the account's active-job limit | yes, after `retry_after` |
 | `CRAWL_RATE_LIMITED` | any other 429: requests are being throttled | yes, after `retry_after` |
 | `CRAWL_BUSY` | 503 `crawl_busy` on `stop`: the stop was not saved yet and the crawl still runs | yes, run the stop again after `retry_after` |
-| `CRAWL_FAILED` | the crawl ended `failed`, or another error | 5xx: yes, with backoff |
+| `CRAWL_FAILED` | the crawl ended `failed`, or `stopped` with an error other than credits, or another error | 5xx: yes, with backoff |
 | `BACKEND_UNAVAILABLE` | the API could not be reached | yes |
 
 Rules: start small (`--depth 1`, low caps) and check the results before a

package/skills/trace-debug/SKILL.md

@@ -49,12 +49,16 @@ zenrows trace export <run-id>     # JSON for sharing
   enabled for this account. Do not retry; ask Zenrows support for access.
 - `CRAWL_INSUFFICIENT_CREDITS` / `CRAWL_KEY_CAP_REACHED` → Crawl's 402: the
   account is out of credits, or this key reached its credit cap (`AUTH014`).
+  The same codes come back for a crawl that credits ended (`stopped`, or
+  `failed` on older deployments; `server_code` `insufficient_credits` /
+  `api_key_cap_reached`); its fetched pages stay readable.
   Same advice as `POLICY_MAX_CREDITS_EXCEEDED` / `KEY_CREDIT_CAP_REACHED`.
 - `CRAWL_TOO_MANY_CRAWLS` → 429 `too_many_crawls`: the account has reached
   its limit of active jobs (3 by default), shared with its Batch jobs.
   Nothing was created; retry after `retry_after` seconds (see
   `zenrows crawl list`).
-- `CRAWL_FAILED` → the crawl ended `failed`; `error.code` in
+- `CRAWL_FAILED` → the crawl ended `failed`, or `stopped` with an error
+  other than credits; `error.code` in
   `zenrows crawl get <id> --json` says why, with `error.detail`. See
   [[crawl]].
 - `PARAM_CONFLICT_AUTO_MANUAL` → drop the managed flags or add `--manual`.