@zenrows/cli 1.4.1 → 1.5.0

package/dist/src/core/crawl-api.js

@@ -0,0 +1,418 @@
+/**
+ * Client for the Zenrows Crawl API.
+ *
+ * Crawl lives on the same host as Fetch and Extract (`https://api.zenrows.com/v1`)
+ * under `/crawls`. Auth is the `X-API-Key` header, never the `apikey` query
+ * param: Crawl answers any query parameter it does not know with 400. Bodies are
+ * JSON; errors come back as `application/problem+json` (RFC 9457) and we branch
+ * on `status` + `code` only. A crawl is an async job: create it, then poll it
+ * until its status leaves `running`. Calls retry transient failures the way the
+ * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable
+ * for tests.
+ *
+ * `output_format` takes `html` only (absent = URLs only). Response fields we do
+ * not model pass through untouched.
+ */
+import { Readable } from "node:stream";
+import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.js";
+import { readAccount } from "./agent-account.js";
+import { registerSecret } from "./logger.js";
+import { CLI_USER_AGENT, loadConfig } from "./config.js";
+import { agentClientHeader } from "./agent-client.js";
+/** Production Crawl API base (no trailing slash): the Fetch/Extract host. */
+export const DEFAULT_CRAWL_API_BASE = "https://api.zenrows.com/v1";
+/** Env var to override the Crawl API base (local/staging testing). */
+export const CRAWL_API_BASE_ENV = "ZENROWS_CRAWL_API_BASE";
+/**
+ * Resolve the Crawl API base, trimmed of trailing `/`. `ZENROWS_CRAWL_API_BASE`
+ * wins; otherwise Crawl follows the Fetch/Extract `apiBase` (config or
+ * `ZENROWS_API_BASE`), since both live on the same API host.
+ */
+export function crawlBase() {
+    const env = process.env[CRAWL_API_BASE_ENV];
+    const base = env && env.trim() ? env.trim() : loadConfig().apiBase || DEFAULT_CRAWL_API_BASE;
+    return base.replace(/\/+$/, "");
+}
+/** The API's "Crawl is not enabled for this account" code (403). */
+export const CRAWL_NOT_ENABLED_CODE = "REQS008";
+/** A crawl stops progressing once its status is anything but `running`. */
+export function isTerminal(status) {
+    return !!status && status !== "running";
+}
+const MAX_RETRIES = 3;
+/** The longest a Retry-After may make one retry wait, so a large server value cannot hang a command. */
+const MAX_RETRY_AFTER_SECONDS = 30;
+const READ_RETRY_STATUSES = new Set([429, 502, 503, 504]);
+// 429 on create is the active-job limit: a retry only waits for a slot that may never free.
+const CREATE_RETRY_STATUSES = new Set([502, 503, 504]);
+// Any 503 on stop, crawl_busy included: crawl_busy means the stop was not saved and the crawl still runs.
+const STOP_RETRY_STATUSES = new Set([429, 502, 503, 504]);
+/**
+ * The statuses to retry, or undefined when the request must run once: a GET is
+ * safe to repeat, a create only with an Idempotency-Key. Stop retries on its own
+ * statuses and wait rule (see `retryStop`).
+ */
+function retryStatuses(method, headers) {
+    if (method === "GET")
+        return READ_RETRY_STATUSES;
+    return headers["Idempotency-Key"] ? CREATE_RETRY_STATUSES : undefined;
+}
+/** 250 ms x 2^attempt, capped at 10 s, +/-20% jitter. */
+function backoffMs(attempt) {
+    return Math.min(250 * 2 ** attempt, 10_000) * (0.8 + Math.random() * 0.4);
+}
+/** `Retry-After` in whole seconds, as milliseconds, at most `MAX_RETRY_AFTER_SECONDS`. */
+function retryAfterMs(value) {
+    return value && /^\d+$/.test(value.trim()) ? Math.min(Number(value), MAX_RETRY_AFTER_SECONDS) * 1000 : undefined;
+}
+const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** One HTTP attempt with its own timeout; the body is read inside it. */
+async function attempt(url, init, opts) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000);
+    try {
+        const res = await (opts.fetchImpl ?? fetch)(url, { ...init, signal: controller.signal });
+        if (opts.sink && res.ok) {
+            await opts.sink(res.body ?? Readable.from([]));
+            return { status: res.status, headers: res.headers, text: "" };
+        }
+        return { status: res.status, headers: res.headers, text: await res.text() };
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+/**
+ * Perform a Crawl API request and return the raw body. Sets `X-API-Key`, adds a
+ * JSON content-type when a body is present, retries transient failures as
+ * `retryStatuses` allows, and on a non-2xx parses the problem+json body into a
+ * normalized ToolkitError.
+ */
+async function crawlFetch(method, path, opts) {
+    registerSecret(opts.apiKey);
+    const url = new URL(crawlBase() + path);
+    for (const [k, v] of Object.entries(opts.query ?? {})) {
+        if (v !== undefined && v !== null)
+            url.searchParams.set(k, String(v));
+    }
+    const headers = {
+        "X-API-Key": opts.apiKey,
+        Accept: opts.accept ?? "application/json",
+        "User-Agent": CLI_USER_AGENT,
+        ...agentClientHeader(),
+        ...(opts.headers ?? {}),
+    };
+    if (opts.body !== undefined)
+        headers["Content-Type"] = "application/json";
+    const init = { method, headers, body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined };
+    const retryOn = retryStatuses(method, headers);
+    const sleep = opts.sleepImpl ?? defaultSleep;
+    for (let n = 0;; n++) {
+        const canRetry = retryOn !== undefined && n < MAX_RETRIES;
+        let res;
+        try {
+            res = await attempt(url.toString(), init, opts);
+        }
+        catch (err) {
+            if (err instanceof ToolkitError)
+                throw err;
+            if (canRetry) {
+                await sleep(backoffMs(n));
+                continue;
+            }
+            throw new ToolkitError({
+                code: "BACKEND_UNAVAILABLE",
+                message: "Could not reach the Zenrows Crawl API.",
+                likely_cause: err instanceof Error ? err.message : String(err),
+                next_action: "Check connectivity and retry. Override the host with ZENROWS_CRAWL_API_BASE if you are testing against staging.",
+                suggested_commands: ["zenrows status"],
+                crawl_id: opts.crawlId,
+            });
+        }
+        const retryAfter = res.headers.get("retry-after");
+        if (canRetry && retryOn.has(res.status)) {
+            await sleep(retryAfterMs(retryAfter) ?? backoffMs(n));
+            continue;
+        }
+        if (opts.retryStop && n < MAX_RETRIES && STOP_RETRY_STATUSES.has(res.status)) {
+            await sleep(retryAfterMs(retryAfter) ?? 1000);
+            continue;
+        }
+        if (res.status < 200 || res.status >= 300) {
+            throw problemToError(res.status, res.text, `${method} ${path}`, retryAfter, opts.crawlId);
+        }
+        return res;
+    }
+}
+/** `crawlFetch` + JSON parse. */
+async function crawlJson(method, path, opts) {
+    const { text } = await crawlFetch(method, path, opts);
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new ToolkitError({
+            code: "CRAWL_FAILED",
+            message: "The Crawl API response was not valid JSON.",
+            likely_cause: text.slice(0, 240) || "Empty response body.",
+            next_action: "Retry, or check the crawl with `zenrows crawl get <id>`.",
+            crawl_id: opts.crawlId,
+        });
+    }
+}
+/**
+ * Map a problem+json body + HTTP status to a normalized ToolkitError that
+ * carries the API's `code` as `server_code`, and `crawl_id` when known.
+ */
+function problemToError(status, body, request, retryAfter, crawlId) {
+    let problem = {};
+    try {
+        problem = JSON.parse(body);
+    }
+    catch {
+        // non-JSON error body — fall through with an empty problem
+    }
+    const serverCode = problem.code || undefined;
+    const throttled = status === 429 || (status === 503 && serverCode === "crawl_busy");
+    const seconds = throttled && retryAfter && /^\d+$/.test(retryAfter.trim()) ? Number(retryAfter) : undefined;
+    const err = classifyProblem(status, problem, body, request, serverCode, seconds);
+    return new ToolkitError({ ...err.toJSON(), server_code: serverCode, crawl_id: crawlId, retry_after: seconds });
+}
+function classifyProblem(status, problem, body, request, serverCode, retryAfter) {
+    const detail = problem.detail || problem.title || body.slice(0, 240) || `HTTP ${status}`;
+    // The API's detail often ends in a period already; add one only when it does not.
+    const raw = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${request}: ${detail}`;
+    const cause = /[.!?]$/.test(raw) ? raw : `${raw}.`;
+    if (status === 403 && serverCode === CRAWL_NOT_ENABLED_CODE) {
+        return new ToolkitError({
+            code: "CRAWL_NOT_ENABLED",
+            message: "Crawl is not enabled for this account.",
+            likely_cause: `${cause} Crawl is not enabled for this account yet.`,
+            next_action: "Ask Zenrows support to enable Crawl for this account. Meanwhile fetch known URLs with `zenrows fetch`.",
+            suggested_commands: ["zenrows fetch <url>"],
+        });
+    }
+    if (status === 401) {
+        return new ToolkitError({
+            code: "AUTH_INVALID",
+            message: "Zenrows rejected the API key for the Crawl API.",
+            likely_cause: cause,
+            next_action: "Re-check your key and log in again.",
+            suggested_commands: ["zenrows login --api-key <your-key>"],
+        });
+    }
+    if (status === 404 && serverCode === "content_not_found") {
+        return new ToolkitError({
+            code: "CRAWL_CONTENT_NOT_FOUND",
+            message: "Crawl content not found.",
+            likely_cause: `${cause} The page was not fetched (yet), its fetch failed, or the crawl ran without --output-format html.`,
+            next_action: "Read the result's content_status with `zenrows crawl results <id>`; only `fetched` results have content.",
+            suggested_commands: ["zenrows crawl list"],
+        });
+    }
+    if (status === 404) {
+        return new ToolkitError({
+            code: "CRAWL_NOT_FOUND",
+            message: "Crawl not found.",
+            likely_cause: `${cause} The id may be wrong or the crawl is not owned by this account.`,
+            next_action: "Check the crawl id, or list your crawls with `zenrows crawl list`.",
+            suggested_commands: ["zenrows crawl list"],
+        });
+    }
+    if (status === 429 && serverCode === "too_many_crawls") {
+        return new ToolkitError({
+            code: "CRAWL_TOO_MANY_CRAWLS",
+            message: "The account has reached its limit of active jobs.",
+            likely_cause: `${cause} The account has reached its limit of active jobs (3 by default), shared with its Batch jobs.`,
+            next_action: `Nothing was created. ${retryAfter !== undefined ? `Retry after ${retryAfter} seconds` : "Retry later"}, or stop one of your crawls (\`zenrows crawl stop <id>\`) or Batch jobs first.`,
+            suggested_commands: ["zenrows crawl list"],
+        });
+    }
+    if (status === 429) {
+        // The Crawl API sends 429 only on create, as too_many_crawls. Any other 429
+        // comes from something in front of it and says nothing about active jobs.
+        return new ToolkitError({
+            code: "CRAWL_RATE_LIMITED",
+            message: "The Crawl API is rate limiting requests.",
+            likely_cause: cause,
+            next_action: retryAfter !== undefined ? `Retry after ${retryAfter} seconds.` : "Retry later.",
+        });
+    }
+    if (status === 402) {
+        // Same advice as the shared credit errors, under Crawl's own codes.
+        const detail = problem.detail || problem.title || undefined;
+        if (isKeyCapReached(serverCode)) {
+            return new ToolkitError({ ...keyCapReached(request, { status, detail }).toJSON(), code: "CRAWL_KEY_CAP_REACHED" });
+        }
+        const acct = readAccount();
+        const quota = quotaExhausted(request, acct?.unclaimed ? acct.claimUrl : undefined, { status, detail });
+        return new ToolkitError({ ...quota.toJSON(), code: "CRAWL_INSUFFICIENT_CREDITS" });
+    }
+    if (status === 409) {
+        return new ToolkitError({
+            code: "CRAWL_REQUEST_IN_FLIGHT",
+            message: "A request with the same Idempotency-Key is still in flight.",
+            likely_cause: cause,
+            next_action: "Retry the same request once the first one has finished.",
+        });
+    }
+    if (status === 503 && serverCode === "crawl_busy") {
+        return new ToolkitError({
+            code: "CRAWL_BUSY",
+            message: "The stop was not saved yet; the crawl is still running.",
+            likely_cause: cause,
+            next_action: `Run the stop again${retryAfter !== undefined ? ` after ${retryAfter} seconds` : ""}. Repeating a stop is safe.`,
+            suggested_commands: ["zenrows crawl stop <id>"],
+        });
+    }
+    if (status === 400 || status === 422) {
+        return new ToolkitError({
+            code: "CRAWL_INVALID_REQUEST",
+            message: `Crawl rejected the request (HTTP ${status}).`,
+            likely_cause: cause,
+            next_action: serverCode === "idempotency_key_reused"
+                ? "This Idempotency-Key was already used for a different request. Send a new key, or no key. Do not retry as is."
+                : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.",
+        });
+    }
+    return new ToolkitError({
+        code: "CRAWL_FAILED",
+        message: `Crawl request failed (HTTP ${status}).`,
+        likely_cause: cause,
+        next_action: status >= 500 ? "Retry with a short backoff." : "Fix the reported problem and retry.",
+    });
+}
+/** Start a crawl (202). Sends only the fields set in `params`. */
+export function createCrawl(params, opts) {
+    const body = { url: params.url };
+    if (params.depth !== undefined)
+        body.depth = params.depth;
+    if (params.max_items !== undefined)
+        body.max_items = params.max_items;
+    if (params.max_pages !== undefined)
+        body.max_pages = params.max_pages;
+    if (params.include_patterns?.length)
+        body.include_patterns = params.include_patterns;
+    if (params.exclude_patterns?.length)
+        body.exclude_patterns = params.exclude_patterns;
+    if (params.output_format !== undefined)
+        body.output_format = params.output_format;
+    return crawlJson("POST", "/crawls", {
+        ...opts,
+        body,
+        headers: opts.idempotencyKey ? { "Idempotency-Key": opts.idempotencyKey } : undefined,
+    });
+}
+/** Read a crawl and one page of its results. */
+export function getCrawl(id, opts) {
+    return crawlJson("GET", `/crawls/${encodeURIComponent(id)}`, {
+        ...opts,
+        crawlId: id,
+        query: { cursor: opts.cursor, limit: opts.limit },
+    });
+}
+/** One page of the account's crawls, newest first. */
+export function listCrawls(opts) {
+    return crawlJson("GET", "/crawls", { ...opts, query: { cursor: opts.cursor, limit: opts.limit } });
+}
+/** Stop a running crawl. Idempotent: an ended crawl answers as it ended. */
+export function stopCrawl(id, opts) {
+    return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id, retryStop: true });
+}
+/** The page of one kept URL (HTML for `output_format: html`). */
+export async function getCrawlContent(id, contentId, opts) {
+    const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/contents/${encodeURIComponent(contentId)}`, {
+        ...opts,
+        crawlId: id,
+        accept: "text/html, application/json;q=0.9, application/problem+json;q=0.8",
+    });
+    return { contentType: res.headers.get("content-type") ?? "", body: res.text };
+}
+/**
+ * Every result in one NDJSON file (`{url, content_status?, content?}` per line),
+ * streamed into `write` because page HTML makes it large. `count` is its
+ * non-blank lines. `status` is the `X-Crawl-Status` header: `running` means the
+ * file is partial.
+ */
+export async function downloadCrawl(id, write, opts) {
+    let count = 0;
+    const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/download`, {
+        ...opts,
+        crawlId: id,
+        accept: "application/x-ndjson, application/problem+json;q=0.8",
+        timeoutMs: opts.timeoutMs ?? 300_000,
+        sink: (body) => {
+            count = 0;
+            return write(countLines(body, () => count++));
+        },
+    });
+    return { status: res.headers.get("x-crawl-status") ?? undefined, count };
+}
+/** Pass `chunks` through, calling `onLine` for each line that is not blank. */
+async function* countLines(chunks, onLine) {
+    let blank = true;
+    for await (const chunk of chunks) {
+        for (const b of chunk) {
+            if (b === 0x0a) {
+                if (!blank)
+                    onLine();
+                blank = true;
+            }
+            else if (b !== 0x20 && b !== 0x09 && b !== 0x0d) {
+                blank = false;
+            }
+        }
+        yield chunk;
+    }
+    if (!blank)
+        onLine();
+}
+/**
+ * The content id of a fetched result: the last segment of its `content_url`.
+ * Undefined when the result has no content yet.
+ */
+export function contentIdOf(result) {
+    if (!result.content_url)
+        return undefined;
+    const last = result.content_url.split("?")[0].replace(/\/+$/, "").split("/").pop();
+    return last ? decodeURIComponent(last) : undefined;
+}
+/**
+ * Read every kept result, following `next_cursor`. While a crawl runs its
+ * `next_cursor` is never null, so the read also ends at the first empty page;
+ * `partial` is true when the crawl was still running or a cursor remains.
+ */
+export async function listAllResults(id, opts) {
+    const results = [];
+    let cursor;
+    for (;;) {
+        const page = await getCrawl(id, { ...opts, cursor, limit: opts.pageSize });
+        const rows = page.results ?? [];
+        results.push(...rows);
+        if (!rows.length || !page.next_cursor || page.next_cursor === cursor) {
+            // Every page was read. An ended crawl keeps no new URLs, even while pages in flight after
+            // a stop keep next_cursor set, so only a running crawl's results are partial.
+            return { crawl: page, results, partial: !isTerminal(page.status) };
+        }
+        cursor = page.next_cursor;
+    }
+}
+/**
+ * Poll `GET /crawls/{id}?limit=1` until the status leaves `running` or
+ * `timeoutMs` (default 600 s) runs out, backing off 2s → ×1.5 → capped at 15s.
+ * A timeout is not an error: it returns the crawl, still `running`.
+ */
+export async function waitForCrawl(id, opts) {
+    const sleep = opts.sleepImpl ?? defaultSleep;
+    const deadline = Date.now() + (opts.timeoutMs ?? 600_000);
+    let delay = 2000;
+    for (;;) {
+        const crawl = await getCrawl(id, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, sleepImpl: opts.sleepImpl, limit: 1 });
+        if (isTerminal(crawl.status))
+            return crawl;
+        if (Date.now() + delay > deadline)
+            return crawl;
+        await sleep(delay);
+        delay = Math.min(delay * 1.5, 15_000);
+    }
+}

package/dist/src/core/errors.js

@@ -11,6 +11,9 @@ export class ToolkitError extends Error {
     likely_cause;
     next_action;
     suggested_commands;
+    server_code;
+    crawl_id;
+    retry_after;
     constructor(shape) {
         super(shape.message);
         this.name = "ToolkitError";
@@ -18,6 +21,9 @@ export class ToolkitError extends Error {
         this.likely_cause = shape.likely_cause;
         this.next_action = shape.next_action;
         this.suggested_commands = shape.suggested_commands ?? [];
+        this.server_code = shape.server_code;
+        this.crawl_id = shape.crawl_id;
+        this.retry_after = shape.retry_after;
     }
     toJSON() {
         return {
@@ -26,6 +32,9 @@ export class ToolkitError extends Error {
             likely_cause: this.likely_cause,
             next_action: this.next_action,
             suggested_commands: this.suggested_commands,
+            ...(this.server_code ? { server_code: this.server_code } : {}),
+            ...(this.crawl_id ? { crawl_id: this.crawl_id } : {}),
+            ...(this.retry_after !== undefined ? { retry_after: this.retry_after } : {}),
         };
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenrows/cli",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "description": "Open-source CLI, MCP, skills, templates, workflows, recipes, and evals layer for giving AI agents reliable access to protected web data through Zenrows cloud infrastructure.",
   "type": "module",
   "license": "MIT",
@@ -48,6 +48,7 @@
     "zenrows": "node bin/zenrows.ts",
     "dev": "node bin/zenrows.ts",
     "test": "npm run build && node --test --import ./dist/tests/setup.js dist/tests/*.test.js",
+    "test:e2e": "npm run build && node --test dist/tests/e2e/*.test.js",
     "typecheck": "tsc --noEmit",
     "build": "rm -rf dist && tsc && chmod +x dist/bin/zenrows.js",
     "prepack": "npm run build",

package/registry/capabilities.json

@@ -28,6 +28,15 @@
       "requiresAuth": true,
       "notes": "Open beta. Cloud subcommands (create/status/results/cancel/wait/retry-failed) call the Batch API; `zenrows batch estimate` works locally with no API key."
     },
+    "crawl": {
+      "key": "crawl",
+      "label": "Crawl",
+      "status": "available",
+      "command": "zenrows crawl",
+      "backend": "Crawl — https://api.zenrows.com/v1/crawls (X-API-Key header; same host as Fetch/Extract)",
+      "requiresAuth": true,
+      "notes": "Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. Follows the links on each page from one start URL, up to depth when set (optional; without it max_pages / max_items stop the crawl, both default 100), on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (create/get/list/results/content/download/stop/wait). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds. Any other 429 is CRAWL_RATE_LIMITED (requests are being throttled); retry after retry_after seconds."
+    },
     "browser": {
       "key": "browser",
       "label": "Browser Sessions",

package/registry/skills.json

@@ -44,6 +44,17 @@
       "path": "skills/batch-jobs",
       "tags": ["batch", "scale", "jsonl"]
     },
+    {
+      "name": "crawl",
+      "type": "skill",
+      "description": "Collect the URLs behind one start page with Crawl (new): create, wait, read results and page HTML, stop.",
+      "status": "available",
+      "requires_backend_capabilities": ["crawl"],
+      "requires_auth": true,
+      "version": "0.1.0",
+      "path": "skills/crawl",
+      "tags": ["crawl", "links", "urls"]
+    },
     {
       "name": "interact-browser",
       "type": "skill",

package/skills/crawl/SKILL.md

@@ -0,0 +1,126 @@
+---
+name: crawl
+description: Collect the URLs behind one start page with Crawl (new): create a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs.
+version: 0.1.0
+requires_backend_capabilities: [crawl]
+---
+
+# Crawl (new)
+
+Give Crawl one start URL; it follows the links on that page (and on the
+pages behind it, up to the depth you set) and returns the URLs it keeps. Use it
+when you know where the items live (a category page, a blog index, a docs
+section) but not the item URLs themselves. Once you have the URLs, read pages
+with `zenrows fetch` / `zenrows extract`, or ask Crawl for each page's HTML
+with `--output-format html`.
+
+Crawl stays on the start URL's registrable domain; subdomains count. An account without Crawl access gets
+403 `REQS008` → `CRAWL_NOT_ENABLED` ("Crawl is not enabled for this account");
+do not retry it.
+
+## Create, wait, read
+
+```
+zenrows crawl create https://example.com/products/ --depth 1 \
+  --max-items 20 --max-pages 25 --include-patterns /product/ --follow --json
+zenrows crawl results <id> --json            # every kept URL
+zenrows crawl results <id> --out urls.jsonl  # same, as JSONL
+```
+
+- `--depth N` (1 to 100000) is optional: link hops from the start URL. `1` =
+  the start page's links; `2` = also the links on each of those pages. Leave
+  it out for no depth limit: the crawl follows links until `--max-items` or
+  `--max-pages` stops it, or nothing is left to open. A crawl created without
+  a depth has no `depth` field in its output.
+- `--max-items` (kept URLs) and `--max-pages` (fetched pages) cap the crawl;
+  both default to 100. `--max-pages` bounds the cost: each page is one fetch.
+  The local policy `max_pages_per_run` caps `--max-pages` (default 1000);
+  above it the CLI refuses with `POLICY_LIMIT_EXCEEDED` before any request.
+- `--include-patterns` / `--exclude-patterns` are substring matches on the
+  normalized URL and repeat:
+  `--include-patterns /product/ --exclude-patterns ?add-to-cart`.
+- `create` returns at once. `--follow` (or `zenrows crawl wait <id>`) polls
+  until the crawl ends or `--timeout` seconds run out (default 600). When the
+  time runs out, the command exits 0 and prints the crawl with status
+  `running` and the hint `zenrows crawl wait <id>` (under `--json`, in
+  `note`). The crawl keeps running;
+  it is not stopped. Ctrl-C also stops the wait, not the crawl, and exits 130;
+  resume with `zenrows crawl wait <id>`.
+- `results` on a running crawl returns the URLs kept so far, with
+  `partial: true`. Read again once the crawl ends. `--limit` sets the page size
+  of each request.
+
+## Page HTML
+
+```
+zenrows crawl create <url> --depth 1 --output-format html --max-pages 20 --follow
+zenrows crawl results <id> --json        # fetched results carry content_url
+zenrows crawl content <id> <content_id>  # one page's HTML (or pass the content_url)
+zenrows crawl download <id> --out pages.jsonl  # every URL + HTML, NDJSON
+```
+
+`--output-format html` uses up the page budget: each kept page is one more
+fetch, and `--max-pages` counts it. Raise `--max-pages` to keep the same number
+of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only
+`fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). `download`
+reports the crawl's `status`, and `partial: true` while it runs.
+
+## While it runs, and after
+
+```
+zenrows crawl get <id>                       # status, coverage, stop_reason / error, one page of results
+zenrows crawl get <id> --cursor <c> --json   # the next page; keep polling with next_cursor
+zenrows crawl list                           # your crawls, newest first (--cursor, --limit)
+zenrows crawl stop <id>                      # stop it; kept URLs stay readable
+```
+
+A crawl ends `completed` (nothing left, or a cap hit: `stop_reason`
+`max_items` / `max_pages`), `stopped` (you stopped it) or `failed` (`error.code`
+and `error.detail` say why). Only `failed` exits non-zero (`CRAWL_FAILED`).
+`stop` on a crawl that already ended prints the status it ended with.
+
+With `--json`, `create`, `get`, `wait` and `stop` print
+`{ok, crawl_id, status, crawl}`; read `coverage`, `stop_reason` and `error`
+under `crawl`. `get` adds `results` and `next_cursor`. `download` prints
+`status: null` when the API sends no status.
+
+## Limits
+
+When the account has reached its limit of active jobs (3 by default), shared
+with its Batch jobs, a create gets 429 `too_many_crawls` →
+`CRAWL_TOO_MANY_CRAWLS`; nothing was created. Retry after `retry_after`
+seconds (about 30), or stop one of your crawls or Batch jobs first. Any other
+429 is `CRAWL_RATE_LIMITED`: requests are being throttled; retry after
+`retry_after` seconds.
+
+## Errors
+
+Every Crawl error carries the API's code as `server_code` (when the API sent
+one), and `crawl_id` on any call about one crawl.
+
+The CLI retries transient failures, as the Crawl SDKs do: up to 3 retries,
+waiting the `Retry-After` seconds (at most 30) or 250 ms x 2^n (+/-20% jitter,
+at most 10 s). Reads retry on 429, 502, 503, 504 and network errors (a timeout
+counts). `create` retries only with `--idempotency-key`, and only on 502, 503,
+504 and network errors, never on 429. `stop` retries on 429, 502, 503 (including
+`crawl_busy`) and 504, waiting the `Retry-After` seconds (at most 30) or 1 s.
+The "Retry?" column is for errors that remain after these retries.
+
+| Code | Meaning | Retry? |
+| --- | --- | --- |
+| `CRAWL_NOT_ENABLED` | 403 `REQS008`: Crawl is not enabled for this account | no |
+| `AUTH_INVALID` | 401: the API key was rejected | no, log in again |
+| `CRAWL_INSUFFICIENT_CREDITS` | 402: the account is out of credits (claim the account or add credits) | after credits renew or are added |
+| `CRAWL_KEY_CAP_REACHED` | 402 `AUTH014` or `api_key_cap_reached`: this API key reached its credit cap | after the cap resets or is raised |
+| `CRAWL_INVALID_REQUEST` | 400 or 422: fix the request. For `idempotency_key_reused`, send a new key or no key | no, not as is |
+| `CRAWL_REQUEST_IN_FLIGHT` | 409: the same Idempotency-Key is still in flight | yes, once the first request ends |
+| `CRAWL_NOT_FOUND` / `CRAWL_CONTENT_NOT_FOUND` | 404: wrong id, or the page has no content | no |
+| `CRAWL_TOO_MANY_CRAWLS` | 429 `too_many_crawls` on `create`: the account's active-job limit | yes, after `retry_after` |
+| `CRAWL_RATE_LIMITED` | any other 429: requests are being throttled | yes, after `retry_after` |
+| `CRAWL_BUSY` | 503 `crawl_busy` on `stop`: the stop was not saved yet and the crawl still runs | yes, run the stop again after `retry_after` |
+| `CRAWL_FAILED` | the crawl ended `failed`, or another error | 5xx: yes, with backoff |
+| `BACKEND_UNAVAILABLE` | the API could not be reached | yes |
+
+Rules: start small (`--depth 1`, low caps) and check the results before a
+deep crawl. Mind the cost ([[cost-control]]). Read errors with
+[[trace-debug]].

package/skills/trace-debug/SKILL.md

@@ -20,9 +20,10 @@ zenrows trace export <run-id>     # JSON for sharing
 
 ## Failure → action map
 - `FETCH_FAILED` / empty content → first retry in auto mode (`mode=auto`), which
-  escalates for you and bills only for what succeeds. For slow pages add
-  `--wait-for <selector>`. Only when auto mode has failed on its own, take
-  manual control with `--manual --js-render`, then `--premium-proxy`. Each step
+  escalates for you and bills only for what succeeds. Only when auto mode has
+  failed on its own, take manual control with `--manual --js-render`, then
+  `--premium-proxy`. For slow pages, add `--wait-for <selector>` to a
+  `--manual --js-render` run; auto mode may ignore it. Each step
   multiplies the cost of the request, and both together are the most expensive
   configuration available ([[cost-control]]).
 - `REQUEST_TIMEOUT` → the CLI stopped waiting; the API was reached. Raise
@@ -44,6 +45,18 @@ zenrows trace export <run-id>     # JSON for sharing
   auto-created Free plan.
 - `BATCH_FAILED` → the batch run ended `failed`; `failure_reason` and
   `failure_detail` in `zenrows batch status <id> --json` say why.
+- `CRAWL_NOT_ENABLED` → the API answered 403 `REQS008`: Crawl is not
+  enabled for this account. Do not retry; ask Zenrows support for access.
+- `CRAWL_INSUFFICIENT_CREDITS` / `CRAWL_KEY_CAP_REACHED` → Crawl's 402: the
+  account is out of credits, or this key reached its credit cap (`AUTH014`).
+  Same advice as `POLICY_MAX_CREDITS_EXCEEDED` / `KEY_CREDIT_CAP_REACHED`.
+- `CRAWL_TOO_MANY_CRAWLS` → 429 `too_many_crawls`: the account has reached
+  its limit of active jobs (3 by default), shared with its Batch jobs.
+  Nothing was created; retry after `retry_after` seconds (see
+  `zenrows crawl list`).
+- `CRAWL_FAILED` → the crawl ended `failed`; `error.code` in
+  `zenrows crawl get <id> --json` says why, with `error.detail`. See
+  [[crawl]].
 - `PARAM_CONFLICT_AUTO_MANUAL` → drop the managed flags or add `--manual`.
 - `CAPABILITY_UNAVAILABLE` → the primitive is not available on this account (e.g. beta/invite-only); use the local-spec path where offered.