@zenrows/cli 1.7.0 → 1.9.0

package/dist/src/core/config.js

@@ -14,7 +14,7 @@ export const CONFIG_VERSION = "0.1.0";
  * entrypoint) so core modules — telemetry, signup provenance — can read it
  * without an import cycle. `VERSION` in `cli/index.ts` re-exports this.
  */
-export const CLI_VERSION = "1.7.0"; // x-release-please-version
+export const CLI_VERSION = "1.9.0"; // x-release-please-version
 /** The User-Agent on every request to a Zenrows API. The gateway reads the version from it. */
 export const CLI_USER_AGENT = `zenrows-cli/${CLI_VERSION}`;
 /** Env var to override the Fetch and Extract API base (local/staging testing). */

package/dist/src/core/errors.js

@@ -13,6 +13,8 @@ export class ToolkitError extends Error {
     suggested_commands;
     server_code;
     crawl_id;
+    job_id;
+    idempotency_key;
     retry_after;
     constructor(shape) {
         super(shape.message);
@@ -23,6 +25,8 @@ export class ToolkitError extends Error {
         this.suggested_commands = shape.suggested_commands ?? [];
         this.server_code = shape.server_code;
         this.crawl_id = shape.crawl_id;
+        this.job_id = shape.job_id;
+        this.idempotency_key = shape.idempotency_key;
         this.retry_after = shape.retry_after;
     }
     toJSON() {
@@ -34,6 +38,8 @@ export class ToolkitError extends Error {
             suggested_commands: this.suggested_commands,
             ...(this.server_code ? { server_code: this.server_code } : {}),
             ...(this.crawl_id ? { crawl_id: this.crawl_id } : {}),
+            ...(this.job_id ? { job_id: this.job_id } : {}),
+            ...(this.idempotency_key ? { idempotency_key: this.idempotency_key } : {}),
             ...(this.retry_after !== undefined ? { retry_after: this.retry_after } : {}),
         };
     }

package/dist/src/core/http.js

@@ -12,6 +12,7 @@ import { ENV_KEY, resolveApiKey } from "./auth.js";
 import { registerSecret } from "./logger.js";
 import { CLI_USER_AGENT } from "./config.js";
 import { agentClientHeader } from "./agent-client.js";
+import { normalizeDomain, PAGE_NOT_RECOGNISED_STEPS, prepareCommand } from "./prepared-domains-api.js";
 /**
  * Client-side timeout for a Fetch and Extract request, deliberately ABOVE the
  * gateway's own 90s request budget.
@@ -99,17 +100,10 @@ export async function scrape(apiBase, apiKey, params, opts = {}) {
         concurrencyRemaining: numOrNull(res.headers.get("concurrency-remaining")),
         redactedUrl: redacted,
     };
-    // REQS007 on extract=auto: the domain hasn't been prepared for Extract yet.
-    // Same recovery as AUTH010 (the extract adapter falls back to autoparse), so it
-    // maps to the same code. Checked before the 403 auth branch below.
+    // REQS007 on extract=auto: Extract has not prepared the domain yet. Reported,
+    // never retried: the fix is to prepare the domain, which the error names.
     if (res.status === 403 && params.extract !== undefined && zrErrorCode(body) === "REQS007") {
-        throw new ToolkitError({
-            code: "EXTRACT_DOMAIN_NOT_ENABLED",
-            message: "Extract has not been prepared for this domain yet.",
-            likely_cause: zrErrorDetail(body) ?? "This domain is not prepared for Extract.",
-            next_action: "Retry with --autoparse for general-purpose extraction on any site, or get the domain prepared for Extract (POST /v1/extract/prepared-domains).",
-            suggested_commands: [`zenrows extract ${params.url} --autoparse`],
-        });
+        throw extractDomainNotPrepared(params.url, body);
     }
     // 422 no_data_extracted on an Extract request: the page was processed and had
     // nothing to extract. It is an answer, not a failure to reach the target, so
@@ -163,17 +157,10 @@ export async function scrape(apiBase, apiKey, params, opts = {}) {
         }
     }
     if (res.status === 402 && isZenrowsErrorEnvelope(body)) {
-        // AUTH010 on extract=auto means the target domain is not enabled for
-        // Extract, distinct from credit exhaustion (AUTH004). Let the extract
-        // adapter fall back to autoparse.
+        // AUTH010 on extract=auto is the legacy code for REQS007: the domain is
+        // not prepared for Extract. Distinct from credit exhaustion (AUTH004).
         if (params.extract !== undefined && zrErrorCode(body) === "AUTH010") {
-            throw new ToolkitError({
-                code: "EXTRACT_DOMAIN_NOT_ENABLED",
-                message: "Extract is not enabled for this domain yet.",
-                likely_cause: zrErrorDetail(body) ?? "This domain is not enabled for Extract.",
-                next_action: "Retry with --autoparse for general-purpose extraction on any site, or get the domain prepared for Extract (POST /v1/extract/prepared-domains).",
-                suggested_commands: [`zenrows extract ${params.url} --autoparse`],
-            });
+            throw extractDomainNotPrepared(params.url, body);
         }
         // AUTH014: this key hit one of its credit caps; the account still has credits.
         if (isKeyCapReached(zrErrorCode(body) ?? undefined)) {
@@ -261,6 +248,77 @@ export async function scrape(apiBase, apiKey, params, opts = {}) {
     }
     return result;
 }
+const REFUSAL_REASONS = ["domain_not_prepared", "page_not_recognised", "domain_not_enabled"];
+/**
+ * Which REQS007 this is. Only domain_not_prepared is fixed by preparing the
+ * domain: on page_not_recognised the domain is already prepared, and
+ * resubmitting it returns the existing ready record, so pointing there would
+ * loop.
+ *
+ * The gateway says it in the problem's `reason` field, read first. The sentence
+ * match below is only the fallback for a gateway deployed before that field
+ * existed, and reads the titles that gateway sends; remove it once every gateway
+ * sends `reason`.
+ */
+export function refusalReason(body) {
+    let j;
+    try {
+        j = JSON.parse(body);
+    }
+    catch {
+        return "domain_not_prepared";
+    }
+    if (typeof j.reason === "string" && REFUSAL_REASONS.includes(j.reason))
+        return j.reason;
+    const title = typeof j.title === "string" ? j.title : "";
+    const detail = typeof j.detail === "string" ? j.detail : "";
+    if (/did not recogni[sz]e this page/i.test(title) || /^Extract serves /.test(detail))
+        return "page_not_recognised";
+    if (/^Extract is not enabled/i.test(title))
+        return "domain_not_enabled";
+    return "domain_not_prepared";
+}
+/**
+ * REQS007 (or the legacy AUTH010) on extract=auto. Each reason names its own
+ * next step, because agents copy it: the exact prepare command for a domain not
+ * prepared yet, the gateway's advice in the gateway's order for a page it did not
+ * recognise. None offers Autoparse.
+ */
+export function extractDomainNotPrepared(url, body) {
+    const domain = normalizeDomain(url);
+    const detail = zrErrorDetail(body);
+    const css = `zenrows extract ${url} --css '{"title":"h1"}'`;
+    const reason = refusalReason(body);
+    if (reason === "page_not_recognised") {
+        return new ToolkitError({
+            code: "EXTRACT_PAGE_NOT_RECOGNISED",
+            message: `Extract serves ${domain} but did not recognise this page as one it can extract (it may be a page with nothing to extract, such as a home page). ${PAGE_NOT_RECOGNISED_STEPS} The request was not charged.`,
+            likely_cause: detail ?? `Extract serves ${domain} but did not recognise this page.`,
+            next_action: PAGE_NOT_RECOGNISED_STEPS,
+            suggested_commands: [css],
+            server_code: "REQS007",
+        });
+    }
+    if (reason === "domain_not_enabled") {
+        return new ToolkitError({
+            code: "EXTRACT_DOMAIN_NOT_ENABLED",
+            message: `Extract is not enabled for ${domain}, and preparing it does not change this.`,
+            likely_cause: detail ?? `Extract is not enabled for ${domain}.`,
+            next_action: "Extract this page with --css and explicit selectors, or contact Zenrows support to have the site enabled.",
+            suggested_commands: [css],
+            server_code: "REQS007",
+        });
+    }
+    const prepare = prepareCommand(url);
+    return new ToolkitError({
+        code: "EXTRACT_DOMAIN_NOT_PREPARED",
+        message: `Extract hasn't prepared ${domain} yet. Prepare it with \`${prepare}\` (a few example URLs, a few minutes, no credits), then retry; meanwhile use --css.`,
+        likely_cause: detail ?? `Extract has not prepared ${domain} yet.`,
+        next_action: `Run \`${prepare} --wait\` (up to 5 --url pages, one per page kind you want), then retry \`zenrows extract ${url}\`. Without --wait, check progress about every 30 seconds with \`zenrows extract prepare status ${domain}\`. Meanwhile extract with --css and explicit selectors.`,
+        suggested_commands: [`${prepare} --wait`, `zenrows extract prepare status ${domain}`, css],
+        server_code: "REQS007",
+    });
+}
 /**
  * The CLI gave up waiting before the API answered. This is NOT unreachability:
  * the connection was established and the gateway was still working, so telling

package/dist/src/core/policy.js

@@ -1,9 +1,16 @@
 import { ToolkitError } from "./errors.js";
 import { findWorkspace, readJson, workspacePaths, writeJson } from "./workspace.js";
+/**
+ * Safe defaults. The per-run caps let a default run submit up to 10,000 URLs
+ * in any mode: an Adaptive Stealth Mode task is estimated at its 25-credit
+ * upper bound, so 10,000 of them estimate at 250,000 credits. Larger runs
+ * (the Batch API takes up to 100,000 tasks per job) need the caps raised
+ * explicitly with `zenrows policy set`.
+ */
 export function defaultPolicy() {
     return {
-        max_credits_per_run: 10000,
-        max_pages_per_run: 1000,
+        max_credits_per_run: 250_000,
+        max_pages_per_run: 10_000,
         max_concurrency: 20,
         allow_browser: true,
         allow_experimental: false,
@@ -94,8 +101,8 @@ export function assertWithinLimits(requested, policy, context = "run") {
             code: "POLICY_LIMIT_EXCEEDED",
             message: `This ${context} submits ${requested.pages} page(s), over the policy cap of ${policy.max_pages_per_run}.`,
             likely_cause: "policy.max_pages_per_run bounds how many URLs a single run may submit.",
-            next_action: `Split the job into runs of ≤${policy.max_pages_per_run} page(s), or raise max_pages_per_run in policy.json.`,
-            suggested_commands: ["zenrows policy show"],
+            next_action: `Split the job into runs of at most ${policy.max_pages_per_run} page(s), or raise the cap: \`zenrows policy set max_pages_per_run ${requested.pages}\` (default ${defaultPolicy().max_pages_per_run}).`,
+            suggested_commands: [`zenrows policy set max_pages_per_run ${requested.pages}`, "zenrows policy show"],
         });
     }
     if (requested.credits !== undefined && requested.credits > policy.max_credits_per_run) {
@@ -103,8 +110,8 @@ export function assertWithinLimits(requested, policy, context = "run") {
             code: "POLICY_LIMIT_EXCEEDED",
             message: `This ${context} is estimated at ${requested.credits} credit(s), over the policy cap of ${policy.max_credits_per_run}.`,
             likely_cause: "policy.max_credits_per_run bounds the estimated credit cost of a single run.",
-            next_action: "Reduce the job size or per-task cost (e.g. drop premium_proxy/js_render), or raise max_credits_per_run in policy.json.",
-            suggested_commands: ["zenrows policy show"],
+            next_action: `Reduce the job size or per-task cost (e.g. drop premium_proxy/js_render), or raise the cap: \`zenrows policy set max_credits_per_run ${requested.credits}\` (default ${defaultPolicy().max_credits_per_run}). The estimate is an upper bound: an Adaptive Stealth Mode task counts 25 credits, and is billed 1 to 25.`,
+            suggested_commands: [`zenrows policy set max_credits_per_run ${requested.credits}`, "zenrows policy show"],
         });
     }
 }

package/dist/src/core/prepared-domains-api.js

@@ -0,0 +1,289 @@
+/**
+ * Client for Extract's prepared-domains API.
+ *
+ * `extract=auto` answers REQS007 on a domain Extract has not prepared yet. Preparing
+ * it is a call any key can make: `POST {apiBase}/extract/prepared-domains` with the
+ * domain and 1 to 5 example URLs answers 202 at once; the preparation itself takes a
+ * few minutes and costs no credits. `GET .../{domain}` reports progress and
+ * `GET .../allowance` the account's monthly allowance of new domains. Auth is the
+ * `X-API-Key` header; errors are problem+json and we branch on `status` + `code`.
+ * `fetchImpl` / `sleepImpl` / `now` are injectable for tests.
+ */
+import { ToolkitError } from "./errors.js";
+import { registerSecret } from "./logger.js";
+import { CLI_USER_AGENT, loadConfig } from "./config.js";
+import { agentClientHeader } from "./agent-client.js";
+export const MAX_EXAMPLE_URLS = 5;
+/** How often `--wait` polls, and for how long by default. */
+export const DEFAULT_POLL_INTERVAL_MS = 30_000;
+export const DEFAULT_WAIT_TIMEOUT_MS = 15 * 60_000;
+/**
+ * The domain as the API keys it: host only, no scheme, path or leading `www.`.
+ * Accepts a URL.
+ *
+ * Deliberately not the registrable domain: Extract prepares and serves each host
+ * on its own. A prepared `es.wikipedia.org` does not cover `pt.wikipedia.org`
+ * (REQS007 names `pt.wikipedia.org`), and `wikipedia.org` has no record of its
+ * own. The API itself folds `www.` into the bare host, so dropping it here
+ * matches what it stores.
+ */
+export function normalizeDomain(input) {
+    const trimmed = input.trim();
+    let host = trimmed;
+    try {
+        host = new URL(/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`).hostname;
+    }
+    catch {
+        // Left as given: the API answers invalid_argument with the reason.
+    }
+    return host.toLowerCase().replace(/^www\./, "");
+}
+/** The exact command an agent runs to prepare the domain of `url`. */
+export function prepareCommand(url) {
+    return `zenrows extract prepare ${normalizeDomain(url)} --url ${url}`;
+}
+function base(opts) {
+    return (opts.apiBase ?? loadConfig().apiBase).replace(/\/+$/, "");
+}
+async function request(method, path, domain, opts) {
+    registerSecret(opts.apiKey);
+    const headers = {
+        "X-API-Key": opts.apiKey,
+        Accept: "application/json",
+        "User-Agent": CLI_USER_AGENT,
+        ...agentClientHeader(),
+    };
+    if (opts.body !== undefined)
+        headers["Content-Type"] = "application/json";
+    let res;
+    try {
+        res = await (opts.fetchImpl ?? fetch)(`${base(opts)}/extract/prepared-domains${path}`, {
+            method,
+            headers,
+            body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined,
+        });
+    }
+    catch (err) {
+        throw new ToolkitError({
+            code: "BACKEND_UNAVAILABLE",
+            message: "Could not reach the Zenrows API.",
+            likely_cause: err instanceof Error ? err.message : String(err),
+            next_action: "Check connectivity and retry. Verify the api base in `zenrows config show`.",
+            suggested_commands: ["zenrows status", "zenrows config show"],
+        });
+    }
+    const text = await res.text();
+    if (res.status < 200 || res.status >= 300)
+        throw prepareError(res.status, text, domain);
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new ToolkitError({
+            code: "EXTRACT_PREPARE_FAILED",
+            message: "The prepared-domains API returned a non-JSON answer.",
+            likely_cause: text.slice(0, 200),
+            next_action: "Retry in a minute.",
+        });
+    }
+}
+/** Map a non-2xx prepared-domains answer to an error that names the next step. */
+export function prepareError(status, body, domain) {
+    let p = {};
+    try {
+        p = JSON.parse(body) ?? {};
+    }
+    catch {
+        // not problem+json
+    }
+    const code = typeof p.code === "string" ? p.code.toLowerCase() : undefined;
+    const detail = typeof p.detail === "string" ? p.detail : body.slice(0, 200) || `HTTP ${status}`;
+    const cause = code ? `(${code}) ${detail}` : detail;
+    if (code === "invalid_argument" || status === 400) {
+        return new ToolkitError({
+            code: "EXTRACT_PREPARE_INVALID",
+            message: `The request to prepare ${domain} was rejected.`,
+            likely_cause: p.reason ? `${cause} [reason: ${String(p.reason)}]` : cause,
+            next_action: `Pass the domain without scheme or path, and 1 to ${MAX_EXAMPLE_URLS} full page URLs from that site with --url.`,
+            suggested_commands: [`zenrows extract prepare ${domain} --url https://${domain}/<page>`],
+            server_code: code,
+        });
+    }
+    if (code === "not_found" || status === 404) {
+        return new ToolkitError({
+            code: "EXTRACT_PREPARE_NOT_FOUND",
+            message: `This account has not submitted ${domain} for preparation.`,
+            likely_cause: cause,
+            next_action: `Prepare it first: zenrows extract prepare ${domain} --url <page url> (1 to ${MAX_EXAMPLE_URLS} --url).`,
+            suggested_commands: [`zenrows extract prepare ${domain} --url https://${domain}/<page>`],
+            server_code: code,
+        });
+    }
+    if (code === "in_progress") {
+        return new ToolkitError({
+            code: "EXTRACT_PREPARE_IN_PROGRESS",
+            message: `${domain} is already being prepared.`,
+            likely_cause: cause,
+            next_action: "Do not resubmit. Retry the extract in a few minutes.",
+            suggested_commands: [`zenrows extract prepare status ${domain}`],
+            server_code: code,
+        });
+    }
+    if (code === "idempotency_conflict") {
+        return new ToolkitError({
+            code: "EXTRACT_PREPARE_INVALID",
+            message: "This idempotency key was already used with a different request.",
+            likely_cause: cause,
+            next_action: "Retry without the key, or with a new one.",
+            server_code: code,
+        });
+    }
+    if (code === "out_of_allowance" || status === 429) {
+        const used = p.used !== undefined && p.limit !== undefined ? ` (${String(p.used)} of ${String(p.limit)} used)` : "";
+        return new ToolkitError({
+            code: "EXTRACT_PREPARE_OUT_OF_ALLOWANCE",
+            message: `This account has used its monthly allowance of new domains to prepare${used}.`,
+            likely_cause: cause,
+            next_action: "Do not retry this month. Meanwhile extract this domain with --css and explicit selectors.",
+            suggested_commands: [`zenrows extract <url> --css '{"title":"h1"}'`],
+            server_code: code,
+        });
+    }
+    if (code === "overloaded" || code === "disabled" || status === 503) {
+        return new ToolkitError({
+            code: "EXTRACT_PREPARE_UNAVAILABLE",
+            message: "Domain preparation is unavailable right now.",
+            likely_cause: cause,
+            next_action: "Retry in a few minutes; do not loop. Meanwhile extract with --css and explicit selectors.",
+            server_code: code,
+        });
+    }
+    if (status === 401 || status === 403) {
+        return new ToolkitError({
+            code: "AUTH_INVALID",
+            message: "Zenrows rejected the API key.",
+            likely_cause: cause,
+            next_action: "Re-check your key and log in again.",
+            suggested_commands: ["zenrows login --api-key <your-key>"],
+            server_code: code,
+        });
+    }
+    return new ToolkitError({
+        code: "EXTRACT_PREPARE_FAILED",
+        message: `Preparing ${domain} failed with HTTP ${status}.`,
+        likely_cause: cause,
+        next_action: "Retry in a few minutes.",
+        server_code: code,
+    });
+}
+export function submitPreparedDomain(domain, exampleUrls, opts) {
+    return request("POST", "", domain, { ...opts, body: { domain, example_urls: exampleUrls } });
+}
+export function getPreparedDomain(domain, opts) {
+    return request("GET", `/${encodeURIComponent(domain)}`, domain, opts);
+}
+export function getAllowance(opts) {
+    return request("GET", "/allowance", "", opts);
+}
+/**
+ * Submit a domain. A 409 in_progress (already being prepared) answers with the
+ * current status when this account can read it, so the caller just follows it.
+ */
+export async function prepareDomain(domain, exampleUrls, opts) {
+    try {
+        return await submitPreparedDomain(domain, exampleUrls, opts);
+    }
+    catch (err) {
+        if (!(err instanceof ToolkitError) || err.code !== "EXTRACT_PREPARE_IN_PROGRESS")
+            throw err;
+        try {
+            return { ...(await getPreparedDomain(domain, opts)), already_in_progress: true };
+        }
+        catch {
+            throw err;
+        }
+    }
+}
+export function isTerminal(record) {
+    return record.status === "ready" || record.status === "failed";
+}
+/** Poll until ready or failed. Throws EXTRACT_PREPARE_TIMEOUT when the wait runs out. */
+export async function waitForPreparedDomain(domain, opts) {
+    const sleep = opts.sleepImpl ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    const now = opts.now ?? Date.now;
+    const interval = opts.intervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+    const deadline = now() + (opts.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS);
+    for (;;) {
+        const record = await getPreparedDomain(domain, opts);
+        opts.onPoll?.(record);
+        if (isTerminal(record))
+            return record;
+        if (now() + interval > deadline) {
+            throw new ToolkitError({
+                code: "EXTRACT_PREPARE_TIMEOUT",
+                message: `${domain} is still being prepared (state: ${record.state ?? record.status}).`,
+                likely_cause: "Preparation usually takes a few minutes; this one is taking longer than the wait.",
+                next_action: "Check again later; it keeps running on the server.",
+                suggested_commands: [`zenrows extract prepare status ${domain}`],
+            });
+        }
+        await sleep(interval);
+    }
+}
+/** Whether a failed preparation used an allowance slot, from the record's failure.slot_consumed. */
+function slotSentence(slotConsumed) {
+    if (slotConsumed === true)
+        return " This attempt used an allowance slot.";
+    if (slotConsumed === false)
+        return " This attempt did not use an allowance slot.";
+    return "";
+}
+/** What to do after a failed preparation, by failure.code (docs.zenrows.com extract/endpoints). */
+export function failureNextStep(domain, failure) {
+    let step;
+    switch (failure?.code) {
+        case "interrupted":
+        case "preparation_exhausted":
+            step = `Run the same zenrows extract prepare ${domain} command again.`;
+            break;
+        case "discovery_failed":
+            step = `Run zenrows extract prepare ${domain} again with 1 to ${MAX_EXAMPLE_URLS} --url page URLs.`;
+            break;
+        case "unpreparable":
+            step = `Run zenrows extract prepare ${domain} again with different --url pages from the same site; the same URLs fail again.`;
+            break;
+        case "preparation_failed":
+            step = "Try other --url pages, or contact Zenrows support.";
+            break;
+        default:
+            step = "Contact Zenrows support before preparing this domain again. Meanwhile extract with --css.";
+    }
+    return step + slotSentence(failure?.slot_consumed);
+}
+/**
+ * What to do when a page on a prepared domain answers REQS007 with reason
+ * page_not_recognised, in the gateway's own order. On a prepared domain Extract
+ * learns a new kind of page on its first request, so that answer means this page
+ * is not one Extract extracts (a page with nothing to extract, such as a home page) or a
+ * transient failure kept it from trying, never that the domain needs preparing
+ * again: resubmitting returns the existing ready record, so pointing there would
+ * loop.
+ */
+export const PAGE_NOT_RECOGNISED_STEPS = "Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use --css with explicit selectors. Preparing the domain again does not change this.";
+/** Said once a domain is ready, so a user who then meets EXTRACT_PAGE_NOT_RECOGNISED does not resubmit. */
+const IF_PAGE_NOT_RECOGNISED = `If the extract then fails with EXTRACT_PAGE_NOT_RECOGNISED, Extract did not recognise that page. ${PAGE_NOT_RECOGNISED_STEPS}`;
+/**
+ * The next step for a status record. `alreadyReady` is set when a submit answered
+ * ready at once: the domain was prepared before this call.
+ */
+export function statusNextStep(record, opts = {}) {
+    if (record.status === "ready") {
+        const extract = opts.exampleUrl ? ` (\`zenrows extract ${opts.exampleUrl}\`)` : "";
+        if (opts.alreadyReady)
+            return `${record.domain} is already prepared, so nothing new was submitted: run the extract${extract}. ${IF_PAGE_NOT_RECOGNISED}`;
+        return `${record.domain} is ready: run the extract again${extract}. ${IF_PAGE_NOT_RECOGNISED}`;
+    }
+    if (record.status === "failed")
+        return failureNextStep(record.domain, record.failure);
+    return `Preparing; it takes a few minutes. Check about every 30 seconds with: zenrows extract prepare status ${record.domain}`;
+}

package/evals/extract-smoke/README.md

@@ -2,7 +2,7 @@
 
 Reproducible smoke test for Extract (`extract=auto`).
 
-- **Target:** `https://www.owler.com/company/meltwater` (enabled domain, real company page)
+- **Target:** `https://www.owler.com/company/meltwater` (prepared domain, real company page)
 - **Config:** `extract=auto`
 - **Success criteria:** HTTP 2xx and a non-empty body
 - **Cost:** ~1× basic request

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenrows/cli",
-  "version": "1.7.0",
+  "version": "1.9.0",
   "description": "Open-source CLI, MCP, skills, templates, workflows, recipes, and evals layer for giving AI agents reliable access to protected web data through Zenrows cloud infrastructure.",
   "type": "module",
   "license": "MIT",

package/recipes/batch-estimate/RECIPE.md

@@ -23,11 +23,14 @@ zenrows batch estimate jobs.jsonl
 ```
 
 Swap `jobs.jsonl` for your own spec. Per-line keys (`js_render`,
-`premium_proxy`, `proxy_country`, `mode`, `autoparse`, `external_id`) tune each
-URL; cost multipliers are basic 1x / JS 5x / proxy 10x / both 25x.
+`premium_proxy`, `proxy_country`, `mode`, `autoparse`, `external_id`,
+`metadata`, `method`, `body`) tune each URL; cost multipliers are basic 1x / JS
+5x / proxy 10x / both 25x, and Adaptive Stealth Mode costs 1x to 25x. Pass
+`create`'s job-level flags (`--premium-proxy`, `--js-render`, `--manual`) to
+`estimate` to price the same request.
 
 Once the spec is clean, submit it:
 ```
-zenrows batch create jobs.jsonl --wait
+zenrows batch create jobs.jsonl --follow
 ```
 See [[batch-jobs]] for the full cloud flow.

package/registry/capabilities.json

@@ -12,12 +12,12 @@
     },
     "extract": {
       "key": "extract",
-      "label": "Extract (extract=auto / Autoparse / CSS / Markdown)",
+      "label": "Extract (extract=auto / CSS / Markdown)",
       "status": "available",
       "command": "zenrows extract",
-      "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, autoparse, css_extractor, outputs, response_type)",
+      "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, css_extractor, outputs, response_type; autoparse deprecated)",
       "requiresAuth": true,
-      "notes": "Default is extract=auto, which works on any prepared domain with any key; CLI falls back to autoparse on REQS007 (domain not prepared yet) or AUTH010. Autoparse / CSS / outputs / markdown work on any domain."
+      "notes": "Default is extract=auto, which works on any prepared domain with any key. A REQS007 (legacy AUTH010) is reported, never retried, with the next step for its reason: on a domain not prepared yet (EXTRACT_DOMAIN_NOT_PREPARED), prepare it with `zenrows extract prepare <domain> --url <page> [--wait]` (POST /v1/extract/prepared-domains: 1 to 5 example URLs, a few minutes, no credits) and check it with `zenrows extract prepare status <domain>`; on a page Extract did not recognise on a prepared domain (EXTRACT_PAGE_NOT_RECOGNISED), try a product, article or listing page, retry it once in a few minutes, or use --css, because preparing again does not change it. CSS / outputs / markdown work on any domain; autoparse is deprecated and runs only with --autoparse."
     },
     "batch": {
       "key": "batch",

package/registry/skills.json

@@ -25,13 +25,13 @@
     {
       "name": "extract",
       "type": "skill",
-      "description": "Turn protected pages into structured data with Autoparse / CSS / Markdown.",
+      "description": "Turn protected pages into structured data with Extract / CSS / Markdown (Autoparse is deprecated).",
       "status": "available",
       "requires_backend_capabilities": ["extract"],
       "requires_auth": true,
       "version": "0.1.0",
       "path": "skills/extract",
-      "tags": ["extract", "autoparse", "structured-data"]
+      "tags": ["extract", "structured-data"]
     },
     {
       "name": "batch-jobs",

package/registry/templates.json

@@ -14,7 +14,7 @@
     {
       "name": "extract-pricing-pages",
       "type": "template",
-      "description": "Extract structured pricing fields from product pages (Autoparse / CSS).",
+      "description": "Extract structured pricing fields from product pages (extract=auto / CSS).",
       "status": "available",
       "requires_backend_capabilities": ["extract"],
       "requires_auth": true,