@zenrows/cli 1.4.2 → 1.5.1

package/dist/src/core/capabilities.js

@@ -24,8 +24,8 @@ export function getCapability(key) {
 const USABLE = new Set([
     "available",
     "available-but-needs-confirmation",
-    // Open beta: any key can call; product-specific limits (e.g. Extract domain
-    // coverage, Batch 403) are handled by the adapters / API errors, not here.
+    // Beta: any key can call; product-specific limits are handled by the
+    // adapters / API errors, not here.
     "beta",
 ]);
 export function isUsable(key) {

package/dist/src/core/config.js

@@ -14,7 +14,7 @@ export const CONFIG_VERSION = "0.1.0";
  * entrypoint) so core modules — telemetry, signup provenance — can read it
  * without an import cycle. `VERSION` in `cli/index.ts` re-exports this.
  */
-export const CLI_VERSION = "1.4.2";
+export const CLI_VERSION = "1.5.1";
 /** The User-Agent on every request to a Zenrows API. The gateway reads the version from it. */
 export const CLI_USER_AGENT = `zenrows-cli/${CLI_VERSION}`;
 /** Env var to override the Fetch and Extract API base (local/staging testing). */

package/dist/src/core/crawl-api.js

@@ -0,0 +1,418 @@
+/**
+ * Client for the Zenrows Crawl API.
+ *
+ * Crawl lives on the same host as Fetch and Extract (`https://api.zenrows.com/v1`)
+ * under `/crawls`. Auth is the `X-API-Key` header, never the `apikey` query
+ * param: Crawl answers any query parameter it does not know with 400. Bodies are
+ * JSON; errors come back as `application/problem+json` (RFC 9457) and we branch
+ * on `status` + `code` only. A crawl is an async job: create it, then poll it
+ * until its status leaves `running`. Calls retry transient failures the way the
+ * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable
+ * for tests.
+ *
+ * `output_format` takes `html` only (absent = URLs only). Response fields we do
+ * not model pass through untouched.
+ */
+import { Readable } from "node:stream";
+import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.js";
+import { readAccount } from "./agent-account.js";
+import { registerSecret } from "./logger.js";
+import { CLI_USER_AGENT, loadConfig } from "./config.js";
+import { agentClientHeader } from "./agent-client.js";
+/** Production Crawl API base (no trailing slash): the Fetch/Extract host. */
+export const DEFAULT_CRAWL_API_BASE = "https://api.zenrows.com/v1";
+/** Env var to override the Crawl API base (local/staging testing). */
+export const CRAWL_API_BASE_ENV = "ZENROWS_CRAWL_API_BASE";
+/**
+ * Resolve the Crawl API base, trimmed of trailing `/`. `ZENROWS_CRAWL_API_BASE`
+ * wins; otherwise Crawl follows the Fetch/Extract `apiBase` (config or
+ * `ZENROWS_API_BASE`), since both live on the same API host.
+ */
+export function crawlBase() {
+    const env = process.env[CRAWL_API_BASE_ENV];
+    const base = env && env.trim() ? env.trim() : loadConfig().apiBase || DEFAULT_CRAWL_API_BASE;
+    return base.replace(/\/+$/, "");
+}
+/** The API's "Crawl is not enabled for this account" code (403). */
+export const CRAWL_NOT_ENABLED_CODE = "REQS008";
+/** A crawl stops progressing once its status is anything but `running`. */
+export function isTerminal(status) {
+    return !!status && status !== "running";
+}
+const MAX_RETRIES = 3;
+/** The longest a Retry-After may make one retry wait, so a large server value cannot hang a command. */
+const MAX_RETRY_AFTER_SECONDS = 30;
+const READ_RETRY_STATUSES = new Set([429, 502, 503, 504]);
+// 429 on create is the active-job limit: a retry only waits for a slot that may never free.
+const CREATE_RETRY_STATUSES = new Set([502, 503, 504]);
+// Any 503 on stop, crawl_busy included: crawl_busy means the stop was not saved and the crawl still runs.
+const STOP_RETRY_STATUSES = new Set([429, 502, 503, 504]);
+/**
+ * The statuses to retry, or undefined when the request must run once: a GET is
+ * safe to repeat, a create only with an Idempotency-Key. Stop retries on its own
+ * statuses and wait rule (see `retryStop`).
+ */
+function retryStatuses(method, headers) {
+    if (method === "GET")
+        return READ_RETRY_STATUSES;
+    return headers["Idempotency-Key"] ? CREATE_RETRY_STATUSES : undefined;
+}
+/** 250 ms x 2^attempt, capped at 10 s, +/-20% jitter. */
+function backoffMs(attempt) {
+    return Math.min(250 * 2 ** attempt, 10_000) * (0.8 + Math.random() * 0.4);
+}
+/** `Retry-After` in whole seconds, as milliseconds, at most `MAX_RETRY_AFTER_SECONDS`. */
+function retryAfterMs(value) {
+    return value && /^\d+$/.test(value.trim()) ? Math.min(Number(value), MAX_RETRY_AFTER_SECONDS) * 1000 : undefined;
+}
+const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** One HTTP attempt with its own timeout; the body is read inside it. */
+async function attempt(url, init, opts) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000);
+    try {
+        const res = await (opts.fetchImpl ?? fetch)(url, { ...init, signal: controller.signal });
+        if (opts.sink && res.ok) {
+            await opts.sink(res.body ?? Readable.from([]));
+            return { status: res.status, headers: res.headers, text: "" };
+        }
+        return { status: res.status, headers: res.headers, text: await res.text() };
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+/**
+ * Perform a Crawl API request and return the raw body. Sets `X-API-Key`, adds a
+ * JSON content-type when a body is present, retries transient failures as
+ * `retryStatuses` allows, and on a non-2xx parses the problem+json body into a
+ * normalized ToolkitError.
+ */
+async function crawlFetch(method, path, opts) {
+    registerSecret(opts.apiKey);
+    const url = new URL(crawlBase() + path);
+    for (const [k, v] of Object.entries(opts.query ?? {})) {
+        if (v !== undefined && v !== null)
+            url.searchParams.set(k, String(v));
+    }
+    const headers = {
+        "X-API-Key": opts.apiKey,
+        Accept: opts.accept ?? "application/json",
+        "User-Agent": CLI_USER_AGENT,
+        ...agentClientHeader(),
+        ...(opts.headers ?? {}),
+    };
+    if (opts.body !== undefined)
+        headers["Content-Type"] = "application/json";
+    const init = { method, headers, body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined };
+    const retryOn = retryStatuses(method, headers);
+    const sleep = opts.sleepImpl ?? defaultSleep;
+    for (let n = 0;; n++) {
+        const canRetry = retryOn !== undefined && n < MAX_RETRIES;
+        let res;
+        try {
+            res = await attempt(url.toString(), init, opts);
+        }
+        catch (err) {
+            if (err instanceof ToolkitError)
+                throw err;
+            if (canRetry) {
+                await sleep(backoffMs(n));
+                continue;
+            }
+            throw new ToolkitError({
+                code: "BACKEND_UNAVAILABLE",
+                message: "Could not reach the Zenrows Crawl API.",
+                likely_cause: err instanceof Error ? err.message : String(err),
+                next_action: "Check connectivity and retry. Override the host with ZENROWS_CRAWL_API_BASE if you are testing against staging.",
+                suggested_commands: ["zenrows status"],
+                crawl_id: opts.crawlId,
+            });
+        }
+        const retryAfter = res.headers.get("retry-after");
+        if (canRetry && retryOn.has(res.status)) {
+            await sleep(retryAfterMs(retryAfter) ?? backoffMs(n));
+            continue;
+        }
+        if (opts.retryStop && n < MAX_RETRIES && STOP_RETRY_STATUSES.has(res.status)) {
+            await sleep(retryAfterMs(retryAfter) ?? 1000);
+            continue;
+        }
+        if (res.status < 200 || res.status >= 300) {
+            throw problemToError(res.status, res.text, `${method} ${path}`, retryAfter, opts.crawlId);
+        }
+        return res;
+    }
+}
+/** `crawlFetch` + JSON parse. */
+async function crawlJson(method, path, opts) {
+    const { text } = await crawlFetch(method, path, opts);
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new ToolkitError({
+            code: "CRAWL_FAILED",
+            message: "The Crawl API response was not valid JSON.",
+            likely_cause: text.slice(0, 240) || "Empty response body.",
+            next_action: "Retry, or check the crawl with `zenrows crawl get <id>`.",
+            crawl_id: opts.crawlId,
+        });
+    }
+}
+/**
+ * Map a problem+json body + HTTP status to a normalized ToolkitError that
+ * carries the API's `code` as `server_code`, and `crawl_id` when known.
+ */
+function problemToError(status, body, request, retryAfter, crawlId) {
+    let problem = {};
+    try {
+        problem = JSON.parse(body);
+    }
+    catch {
+        // non-JSON error body — fall through with an empty problem
+    }
+    const serverCode = problem.code || undefined;
+    const throttled = status === 429 || (status === 503 && serverCode === "crawl_busy");
+    const seconds = throttled && retryAfter && /^\d+$/.test(retryAfter.trim()) ? Number(retryAfter) : undefined;
+    const err = classifyProblem(status, problem, body, request, serverCode, seconds);
+    return new ToolkitError({ ...err.toJSON(), server_code: serverCode, crawl_id: crawlId, retry_after: seconds });
+}
+function classifyProblem(status, problem, body, request, serverCode, retryAfter) {
+    const detail = problem.detail || problem.title || body.slice(0, 240) || `HTTP ${status}`;
+    // The API's detail often ends in a period already; add one only when it does not.
+    const raw = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${request}: ${detail}`;
+    const cause = /[.!?]$/.test(raw) ? raw : `${raw}.`;
+    if (status === 403 && serverCode === CRAWL_NOT_ENABLED_CODE) {
+        return new ToolkitError({
+            code: "CRAWL_NOT_ENABLED",
+            message: "Crawl is not enabled for this account.",
+            likely_cause: `${cause} Crawl is not enabled for this account yet.`,
+            next_action: "Ask Zenrows support to enable Crawl for this account. Meanwhile fetch known URLs with `zenrows fetch`.",
+            suggested_commands: ["zenrows fetch <url>"],
+        });
+    }
+    if (status === 401) {
+        return new ToolkitError({
+            code: "AUTH_INVALID",
+            message: "Zenrows rejected the API key for the Crawl API.",
+            likely_cause: cause,
+            next_action: "Re-check your key and log in again.",
+            suggested_commands: ["zenrows login --api-key <your-key>"],
+        });
+    }
+    if (status === 404 && serverCode === "content_not_found") {
+        return new ToolkitError({
+            code: "CRAWL_CONTENT_NOT_FOUND",
+            message: "Crawl content not found.",
+            likely_cause: `${cause} The page was not fetched (yet), its fetch failed, or the crawl ran without --output-format html.`,
+            next_action: "Read the result's content_status with `zenrows crawl results <id>`; only `fetched` results have content.",
+            suggested_commands: ["zenrows crawl list"],
+        });
+    }
+    if (status === 404) {
+        return new ToolkitError({
+            code: "CRAWL_NOT_FOUND",
+            message: "Crawl not found.",
+            likely_cause: `${cause} The id may be wrong or the crawl is not owned by this account.`,
+            next_action: "Check the crawl id, or list your crawls with `zenrows crawl list`.",
+            suggested_commands: ["zenrows crawl list"],
+        });
+    }
+    if (status === 429 && serverCode === "too_many_crawls") {
+        return new ToolkitError({
+            code: "CRAWL_TOO_MANY_CRAWLS",
+            message: "The account has reached its limit of active jobs.",
+            likely_cause: `${cause} The account has reached its limit of active jobs (3 by default), shared with its Batch jobs.`,
+            next_action: `Nothing was created. ${retryAfter !== undefined ? `Retry after ${retryAfter} seconds` : "Retry later"}, or stop one of your crawls (\`zenrows crawl stop <id>\`) or Batch jobs first.`,
+            suggested_commands: ["zenrows crawl list"],
+        });
+    }
+    if (status === 429) {
+        // The Crawl API sends 429 only on create, as too_many_crawls. Any other 429
+        // comes from something in front of it and says nothing about active jobs.
+        return new ToolkitError({
+            code: "CRAWL_RATE_LIMITED",
+            message: "The Crawl API is rate limiting requests.",
+            likely_cause: cause,
+            next_action: retryAfter !== undefined ? `Retry after ${retryAfter} seconds.` : "Retry later.",
+        });
+    }
+    if (status === 402) {
+        // Same advice as the shared credit errors, under Crawl's own codes.
+        const detail = problem.detail || problem.title || undefined;
+        if (isKeyCapReached(serverCode)) {
+            return new ToolkitError({ ...keyCapReached(request, { status, detail }).toJSON(), code: "CRAWL_KEY_CAP_REACHED" });
+        }
+        const acct = readAccount();
+        const quota = quotaExhausted(request, acct?.unclaimed ? acct.claimUrl : undefined, { status, detail });
+        return new ToolkitError({ ...quota.toJSON(), code: "CRAWL_INSUFFICIENT_CREDITS" });
+    }
+    if (status === 409) {
+        return new ToolkitError({
+            code: "CRAWL_REQUEST_IN_FLIGHT",
+            message: "A request with the same Idempotency-Key is still in flight.",
+            likely_cause: cause,
+            next_action: "Retry the same request once the first one has finished.",
+        });
+    }
+    if (status === 503 && serverCode === "crawl_busy") {
+        return new ToolkitError({
+            code: "CRAWL_BUSY",
+            message: "The stop was not saved yet; the crawl is still running.",
+            likely_cause: cause,
+            next_action: `Run the stop again${retryAfter !== undefined ? ` after ${retryAfter} seconds` : ""}. Repeating a stop is safe.`,
+            suggested_commands: ["zenrows crawl stop <id>"],
+        });
+    }
+    if (status === 400 || status === 422) {
+        return new ToolkitError({
+            code: "CRAWL_INVALID_REQUEST",
+            message: `Crawl rejected the request (HTTP ${status}).`,
+            likely_cause: cause,
+            next_action: serverCode === "idempotency_key_reused"
+                ? "This Idempotency-Key was already used for a different request. Send a new key, or no key. Do not retry as is."
+                : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.",
+        });
+    }
+    return new ToolkitError({
+        code: "CRAWL_FAILED",
+        message: `Crawl request failed (HTTP ${status}).`,
+        likely_cause: cause,
+        next_action: status >= 500 ? "Retry with a short backoff." : "Fix the reported problem and retry.",
+    });
+}
+/** Start a crawl (202). Sends only the fields set in `params`. */
+export function createCrawl(params, opts) {
+    const body = { url: params.url };
+    if (params.depth !== undefined)
+        body.depth = params.depth;
+    if (params.max_items !== undefined)
+        body.max_items = params.max_items;
+    if (params.max_pages !== undefined)
+        body.max_pages = params.max_pages;
+    if (params.include_patterns?.length)
+        body.include_patterns = params.include_patterns;
+    if (params.exclude_patterns?.length)
+        body.exclude_patterns = params.exclude_patterns;
+    if (params.output_format !== undefined)
+        body.output_format = params.output_format;
+    return crawlJson("POST", "/crawls", {
+        ...opts,
+        body,
+        headers: opts.idempotencyKey ? { "Idempotency-Key": opts.idempotencyKey } : undefined,
+    });
+}
+/** Read a crawl and one page of its results. */
+export function getCrawl(id, opts) {
+    return crawlJson("GET", `/crawls/${encodeURIComponent(id)}`, {
+        ...opts,
+        crawlId: id,
+        query: { cursor: opts.cursor, limit: opts.limit },
+    });
+}
+/** One page of the account's crawls, newest first. */
+export function listCrawls(opts) {
+    return crawlJson("GET", "/crawls", { ...opts, query: { cursor: opts.cursor, limit: opts.limit } });
+}
+/** Stop a running crawl. Idempotent: an ended crawl answers as it ended. */
+export function stopCrawl(id, opts) {
+    return crawlJson("POST", `/crawls/${encodeURIComponent(id)}/stop`, { ...opts, crawlId: id, retryStop: true });
+}
+/** The page of one kept URL (HTML for `output_format: html`). */
+export async function getCrawlContent(id, contentId, opts) {
+    const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/contents/${encodeURIComponent(contentId)}`, {
+        ...opts,
+        crawlId: id,
+        accept: "text/html, application/json;q=0.9, application/problem+json;q=0.8",
+    });
+    return { contentType: res.headers.get("content-type") ?? "", body: res.text };
+}
+/**
+ * Every result in one NDJSON file (`{url, content_status?, content?}` per line),
+ * streamed into `write` because page HTML makes it large. `count` is its
+ * non-blank lines. `status` is the `X-Crawl-Status` header: `running` means the
+ * file is partial.
+ */
+export async function downloadCrawl(id, write, opts) {
+    let count = 0;
+    const res = await crawlFetch("GET", `/crawls/${encodeURIComponent(id)}/download`, {
+        ...opts,
+        crawlId: id,
+        accept: "application/x-ndjson, application/problem+json;q=0.8",
+        timeoutMs: opts.timeoutMs ?? 300_000,
+        sink: (body) => {
+            count = 0;
+            return write(countLines(body, () => count++));
+        },
+    });
+    return { status: res.headers.get("x-crawl-status") ?? undefined, count };
+}
+/** Pass `chunks` through, calling `onLine` for each line that is not blank. */
+async function* countLines(chunks, onLine) {
+    let blank = true;
+    for await (const chunk of chunks) {
+        for (const b of chunk) {
+            if (b === 0x0a) {
+                if (!blank)
+                    onLine();
+                blank = true;
+            }
+            else if (b !== 0x20 && b !== 0x09 && b !== 0x0d) {
+                blank = false;
+            }
+        }
+        yield chunk;
+    }
+    if (!blank)
+        onLine();
+}
+/**
+ * The content id of a fetched result: the last segment of its `content_url`.
+ * Undefined when the result has no content yet.
+ */
+export function contentIdOf(result) {
+    if (!result.content_url)
+        return undefined;
+    const last = result.content_url.split("?")[0].replace(/\/+$/, "").split("/").pop();
+    return last ? decodeURIComponent(last) : undefined;
+}
+/**
+ * Read every kept result, following `next_cursor`. While a crawl runs its
+ * `next_cursor` is never null, so the read also ends at the first empty page;
+ * `partial` is true when the crawl was still running or a cursor remains.
+ */
+export async function listAllResults(id, opts) {
+    const results = [];
+    let cursor;
+    for (;;) {
+        const page = await getCrawl(id, { ...opts, cursor, limit: opts.pageSize });
+        const rows = page.results ?? [];
+        results.push(...rows);
+        if (!rows.length || !page.next_cursor || page.next_cursor === cursor) {
+            // Every page was read. An ended crawl keeps no new URLs, even while pages in flight after
+            // a stop keep next_cursor set, so only a running crawl's results are partial.
+            return { crawl: page, results, partial: !isTerminal(page.status) };
+        }
+        cursor = page.next_cursor;
+    }
+}
+/**
+ * Poll `GET /crawls/{id}?limit=1` until the status leaves `running` or
+ * `timeoutMs` (default 600 s) runs out, backing off 2s → ×1.5 → capped at 15s.
+ * A timeout is not an error: it returns the crawl, still `running`.
+ */
+export async function waitForCrawl(id, opts) {
+    const sleep = opts.sleepImpl ?? defaultSleep;
+    const deadline = Date.now() + (opts.timeoutMs ?? 600_000);
+    let delay = 2000;
+    for (;;) {
+        const crawl = await getCrawl(id, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, sleepImpl: opts.sleepImpl, limit: 1 });
+        if (isTerminal(crawl.status))
+            return crawl;
+        if (Date.now() + delay > deadline)
+            return crawl;
+        await sleep(delay);
+        delay = Math.min(delay * 1.5, 15_000);
+    }
+}

package/dist/src/core/errors.js

@@ -11,6 +11,9 @@ export class ToolkitError extends Error {
     likely_cause;
     next_action;
     suggested_commands;
+    server_code;
+    crawl_id;
+    retry_after;
     constructor(shape) {
         super(shape.message);
         this.name = "ToolkitError";
@@ -18,6 +21,9 @@ export class ToolkitError extends Error {
         this.likely_cause = shape.likely_cause;
         this.next_action = shape.next_action;
         this.suggested_commands = shape.suggested_commands ?? [];
+        this.server_code = shape.server_code;
+        this.crawl_id = shape.crawl_id;
+        this.retry_after = shape.retry_after;
     }
     toJSON() {
         return {
@@ -26,6 +32,9 @@ export class ToolkitError extends Error {
             likely_cause: this.likely_cause,
             next_action: this.next_action,
             suggested_commands: this.suggested_commands,
+            ...(this.server_code ? { server_code: this.server_code } : {}),
+            ...(this.crawl_id ? { crawl_id: this.crawl_id } : {}),
+            ...(this.retry_after !== undefined ? { retry_after: this.retry_after } : {}),
         };
     }
 }

package/dist/src/core/http.js

@@ -106,10 +106,29 @@ export async function scrape(apiBase, apiKey, params, opts = {}) {
             code: "EXTRACT_DOMAIN_NOT_ENABLED",
             message: "Extract has not been prepared for this domain yet.",
             likely_cause: zrErrorDetail(body) ?? "This domain is not prepared for Extract.",
-            next_action: "Retry with --autoparse for general-purpose extraction on any site.",
+            next_action: "Retry with --autoparse for general-purpose extraction on any site, or get the domain prepared for Extract (POST /v1/extract/prepared-domains).",
             suggested_commands: [`zenrows extract ${params.url} --autoparse`],
         });
     }
+    // 422 no_data_extracted on an Extract request: the page was processed and had
+    // nothing to extract. It is an answer, not a failure to reach the target, so
+    // return it for the extract adapter to report as an empty result instead of
+    // advising escalation (which would repeat a slow, deterministic request).
+    if (res.status === 422 && params.extract !== undefined && zrErrorCode(body) === "NO_DATA_EXTRACTED") {
+        return { ...result, noDataExtracted: true };
+    }
+    // 422 blocked_document on an Extract request: the fetched page was a bot wall,
+    // an error page or an empty shell, so Extract had nothing real to read. Not
+    // charged. The fix is a stronger fetch of the same URL, not another extractor.
+    if (res.status === 422 && params.extract !== undefined && zrErrorCode(body) === "BLOCKED_DOCUMENT") {
+        throw new ToolkitError({
+            code: "EXTRACT_FAILED",
+            message: "Extract could not read this page: the site answered with a block or error page (HTTP 422 blocked_document, not charged).",
+            likely_cause: zrErrorDetail(body) ?? "The fetched page was a bot challenge, an error page or a page that had not finished loading.",
+            next_action: "Retry with a stronger fetch: --manual --js-render --premium-proxy. Check the URL if the site may be returning an error page.",
+            suggested_commands: [`zenrows extract ${params.url} --manual --js-render --premium-proxy`],
+        });
+    }
     if (res.status === 401 || res.status === 403) {
         // Distinguish Zenrows auth errors from target-site 403s. Zenrows auth
         // failures surface AUTH00x codes in the body.
@@ -143,15 +162,15 @@ export async function scrape(apiBase, apiKey, params, opts = {}) {
         }
     }
     if (res.status === 402 && isZenrowsErrorEnvelope(body)) {
-        // AUTH010 on extract=auto means the target domain is not in the Extract
-        // beta — distinct from credit exhaustion (AUTH004). Let the extract
+        // AUTH010 on extract=auto means the target domain is not enabled for
+        // Extract, distinct from credit exhaustion (AUTH004). Let the extract
         // adapter fall back to autoparse.
         if (params.extract !== undefined && zrErrorCode(body) === "AUTH010") {
             throw new ToolkitError({
                 code: "EXTRACT_DOMAIN_NOT_ENABLED",
                 message: "Extract is not enabled for this domain yet.",
-                likely_cause: zrErrorDetail(body) ?? "This domain is not part of the Extract open beta.",
-                next_action: "Retry with --autoparse for general-purpose extraction on any site, or contact Zenrows support to enable this domain for Extract.",
+                likely_cause: zrErrorDetail(body) ?? "This domain is not enabled for Extract.",
+                next_action: "Retry with --autoparse for general-purpose extraction on any site, or get the domain prepared for Extract (POST /v1/extract/prepared-domains).",
                 suggested_commands: [`zenrows extract ${params.url} --autoparse`],
             });
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenrows/cli",
-  "version": "1.4.2",
+  "version": "1.5.1",
   "description": "Open-source CLI, MCP, skills, templates, workflows, recipes, and evals layer for giving AI agents reliable access to protected web data through Zenrows cloud infrastructure.",
   "type": "module",
   "license": "MIT",
@@ -48,6 +48,7 @@
     "zenrows": "node bin/zenrows.ts",
     "dev": "node bin/zenrows.ts",
     "test": "npm run build && node --test --import ./dist/tests/setup.js dist/tests/*.test.js",
+    "test:e2e": "npm run build && node --test dist/tests/e2e/*.test.js",
     "typecheck": "tsc --noEmit",
     "build": "rm -rf dist && tsc && chmod +x dist/bin/zenrows.js",
     "prepack": "npm run build",

package/recipes/batch-estimate/RECIPE.md

@@ -26,8 +26,8 @@ Swap `jobs.jsonl` for your own spec. Per-line keys (`js_render`,
 `premium_proxy`, `proxy_country`, `mode`, `autoparse`, `external_id`) tune each
 URL; cost multipliers are basic 1x / JS 5x / proxy 10x / both 25x.
 
-Once the spec is clean, submit it with beta access:
+Once the spec is clean, submit it:
 ```
 zenrows batch create jobs.jsonl --wait
 ```
-See [[batch-jobs]] for the full cloud flow (beta).
+See [[batch-jobs]] for the full cloud flow.

package/registry/capabilities.json

@@ -13,20 +13,29 @@
     "extract": {
       "key": "extract",
       "label": "Extract (extract=auto / Autoparse / CSS / Markdown)",
-      "status": "beta",
+      "status": "available",
       "command": "zenrows extract",
       "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, autoparse, css_extractor, outputs, response_type)",
       "requiresAuth": true,
-      "notes": "Open beta. Default is extract=auto (domain-gated); CLI falls back to autoparse on AUTH010. Autoparse / CSS / outputs / markdown work on any domain."
+      "notes": "Default is extract=auto, which works on any prepared domain with any key; CLI falls back to autoparse on REQS007 (domain not prepared yet) or AUTH010. Autoparse / CSS / outputs / markdown work on any domain."
     },
     "batch": {
       "key": "batch",
-      "label": "Batch (beta)",
-      "status": "beta",
+      "label": "Batch",
+      "status": "available",
       "command": "zenrows batch",
       "backend": "Batch — https://async.api.zenrows.com/v1 (X-API-Key header; separate host from the Fetch/Extract /v1/)",
       "requiresAuth": true,
-      "notes": "Open beta. Cloud subcommands (create/status/results/cancel/wait/retry-failed) call the Batch API; `zenrows batch estimate` works locally with no API key."
+      "notes": "Cloud subcommands (create/status/results/cancel/wait/retry-failed) call the Batch API with your API key; `zenrows batch estimate` works locally with no API key."
+    },
+    "crawl": {
+      "key": "crawl",
+      "label": "Crawl",
+      "status": "available",
+      "command": "zenrows crawl",
+      "backend": "Crawl — https://api.zenrows.com/v1/crawls (X-API-Key header; same host as Fetch/Extract)",
+      "requiresAuth": true,
+      "notes": "Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. Follows the links on each page from one start URL, up to depth when set (optional; without it max_pages / max_items stop the crawl, both default 100), on the start URL's registrable domain (subdomains count), and returns the URLs it keeps, optionally with each page's HTML (create/get/list/results/content/download/stop/wait). An account without Crawl access gets 403 REQS008 (CRAWL_NOT_ENABLED). When the account has reached its limit of active jobs (3 by default), shared with its Batch jobs, a create gets 429 too_many_crawls (CRAWL_TOO_MANY_CRAWLS); retry after retry_after seconds. Any other 429 is CRAWL_RATE_LIMITED (requests are being throttled); retry after retry_after seconds."
     },
     "browser": {
       "key": "browser",

package/registry/skills.json

@@ -26,7 +26,7 @@
       "name": "extract",
       "type": "skill",
       "description": "Turn protected pages into structured data with Autoparse / CSS / Markdown.",
-      "status": "beta",
+      "status": "available",
       "requires_backend_capabilities": ["extract"],
       "requires_auth": true,
       "version": "0.1.0",
@@ -36,14 +36,25 @@
     {
       "name": "batch-jobs",
       "type": "skill",
-      "description": "Scale protected fetch/extract over many URLs with Batch (beta): submit/track/collect jobs with beta access; validate + estimate specs locally with no key.",
-      "status": "beta",
+      "description": "Scale protected fetch/extract over many URLs with Batch: submit/track/collect jobs with an API key; validate + estimate specs locally with no key.",
+      "status": "available",
       "requires_backend_capabilities": ["batch"],
       "requires_auth": true,
       "version": "0.1.0",
       "path": "skills/batch-jobs",
       "tags": ["batch", "scale", "jsonl"]
     },
+    {
+      "name": "crawl",
+      "type": "skill",
+      "description": "Collect the URLs behind one start page with Crawl (new): create, wait, read results and page HTML, stop.",
+      "status": "available",
+      "requires_backend_capabilities": ["crawl"],
+      "requires_auth": true,
+      "version": "0.1.0",
+      "path": "skills/crawl",
+      "tags": ["crawl", "links", "urls"]
+    },
     {
       "name": "interact-browser",
       "type": "skill",

package/registry/templates.json

@@ -25,8 +25,8 @@
     {
       "name": "batch-jsonl-pipeline",
       "type": "template",
-      "description": "JSONL job-spec scaffold for high-scale workloads on Batch: submit/track/collect with beta access, validate + estimate locally with no key (beta).",
-      "status": "beta",
+      "description": "JSONL job-spec scaffold for high-scale workloads on Batch: submit/track/collect with an API key, validate + estimate locally with no key.",
+      "status": "available",
       "requires_backend_capabilities": [],
       "requires_auth": false,
       "version": "0.1.0",