@zenrows/cli 1.8.0 → 1.10.0

package/dist/src/core/signup-challenge.js

@@ -0,0 +1,114 @@
+/**
+ * The signup challenge: before it creates an account, the signup endpoint may
+ * answer 428 with one short natural-language question for the agent running
+ * the CLI to answer itself. The CLI never tries to solve it.
+ *
+ * Each CLI run is its own process, so the pending question is kept in
+ * `.zenrows/signup-challenge.json` (no secret: the id, the question, when it
+ * expires, and the signup URL that issued it, which is where the answer goes).
+ */
+import { existsSync, unlinkSync } from "node:fs";
+import { join } from "node:path";
+import { ToolkitError } from "./errors.js";
+import { findWorkspace, readJson, workspacePaths, writeJson } from "./workspace.js";
+/** The command an agent runs to answer the pending question. */
+export const ANSWER_COMMAND = 'zenrows signup --answer "<your answer>"';
+export function challengePath(projectRoot) {
+    const ws = projectRoot ? workspacePaths(projectRoot) : (findWorkspace() ?? workspacePaths());
+    return join(ws.dir, "signup-challenge.json");
+}
+export function readChallenge(projectRoot) {
+    try {
+        const c = readJson(challengePath(projectRoot));
+        if (!c || typeof c.id !== "string" || typeof c.prompt !== "string" || typeof c.signupUrl !== "string")
+            return null;
+        return c;
+    }
+    catch {
+        return null;
+    }
+}
+export function writeChallenge(c, projectRoot) {
+    writeJson(challengePath(projectRoot), c);
+}
+export function clearChallenge(projectRoot) {
+    const file = challengePath(projectRoot);
+    if (!existsSync(file))
+        return;
+    try {
+        unlinkSync(file);
+    }
+    catch {
+        // best effort
+    }
+}
+/** True when the stored question can no longer be answered. */
+export function challengeExpired(c, now = Date.now()) {
+    const t = Date.parse(c.expiresAt);
+    return Number.isFinite(t) && t <= now;
+}
+function parseBody(body) {
+    try {
+        const json = JSON.parse(body);
+        return json && typeof json === "object" ? json : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Parse a 428 `challenge_required` body into a pending challenge, or null when
+ * the body is not one (then the caller reports a generic failure).
+ */
+export function parseChallenge(body, signupUrl, now = Date.now()) {
+    const json = parseBody(body);
+    if (!json || json.error !== "challenge_required")
+        return null;
+    const ch = json.challenge;
+    if (!ch || typeof ch.id !== "string" || !ch.id || typeof ch.prompt !== "string" || !ch.prompt.trim())
+        return null;
+    const ttl = typeof ch.expires_in === "number" && ch.expires_in > 0 ? ch.expires_in : 600;
+    return { id: ch.id, prompt: ch.prompt.trim(), expiresAt: new Date(now + ttl * 1000).toISOString(), signupUrl };
+}
+/** True when a 428 says the previous answer was wrong or came too late. */
+export function wasWrongAnswer(body) {
+    const msg = parseBody(body)?.message;
+    return typeof msg === "string" && /^wrong or expired/i.test(msg.trim());
+}
+/** True when a 429 is the per-network lock after too many questions or wrong answers. */
+export function isChallengeLocked(body) {
+    return parseBody(body)?.error === "challenge_locked";
+}
+/** The error an agent sees while a question is waiting for its answer. */
+export function challengeError(c, opts = {}) {
+    const lead = opts.wrong ? "That answer was wrong or came too late, so here is a new question. " : "";
+    return new ToolkitError({
+        code: "SIGNUP_CHALLENGE",
+        message: `${lead}Zenrows needs one quick answer before it creates an account: ${c.prompt}`,
+        likely_cause: "No API key is configured, and new accounts created by agents answer one short question first.",
+        next_action: `Answer the question yourself, then run: ${ANSWER_COMMAND}. When it succeeds, re-run your original command.`,
+        suggested_commands: [ANSWER_COMMAND, "zenrows login --api-key <your-key>"],
+        challenge: { id: c.id, question: c.prompt, expires_at: c.expiresAt },
+    });
+}
+export function challengeLockedError(retryAfter) {
+    return new ToolkitError({
+        code: "SIGNUP_CHALLENGE_LOCKED",
+        message: "Too many signup questions or wrong answers from this network, so Zenrows paused account creation here.",
+        likely_cause: "Too many signup questions asked, or wrong answers given, from this network within an hour.",
+        next_action: "Try again in an hour by re-running your original command, which asks a new question. If you already have a Zenrows API key, use it now: zenrows login --api-key <key> (or set ZENROWS_API_KEY).",
+        suggested_commands: ["zenrows login --api-key <your-key>"],
+        ...(retryAfter !== undefined ? { retry_after: retryAfter } : {}),
+    });
+}
+export function noChallengeError(expired) {
+    return new ToolkitError({
+        code: "SIGNUP_CHALLENGE_MISSING",
+        message: expired ? "The signup question expired before it was answered." : "There is no signup question waiting for an answer.",
+        likely_cause: expired
+            ? "Questions are valid for a few minutes."
+            : "No signup was attempted from this directory, or the question was already answered.",
+        next_action: "Re-run your original command (for example zenrows fetch <url>) to get a new question, then answer it with: " + ANSWER_COMMAND,
+        suggested_commands: ["zenrows fetch <url>"],
+    });
+}

package/dist/src/core/workspace.js

@@ -70,6 +70,7 @@ export function ensureGitignore(paths) {
         `${WORKSPACE_DIRNAME}/secrets.json`,
         `${WORKSPACE_DIRNAME}/config.json`,
         `${WORKSPACE_DIRNAME}/account.json`,
+        `${WORKSPACE_DIRNAME}/signup-challenge.json`,
         `${WORKSPACE_DIRNAME}/runs/`,
         `${WORKSPACE_DIRNAME}/traces/`,
         `${WORKSPACE_DIRNAME}/logs/`,

package/evals/extract-smoke/README.md

@@ -2,7 +2,7 @@
 
 Reproducible smoke test for Extract (`extract=auto`).
 
-- **Target:** `https://www.owler.com/company/meltwater` (enabled domain, real company page)
+- **Target:** `https://www.owler.com/company/meltwater` (prepared domain, real company page)
 - **Config:** `extract=auto`
 - **Success criteria:** HTTP 2xx and a non-empty body
 - **Cost:** ~1× basic request

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenrows/cli",
-  "version": "1.8.0",
+  "version": "1.10.0",
   "description": "Open-source CLI, MCP, skills, templates, workflows, recipes, and evals layer for giving AI agents reliable access to protected web data through Zenrows cloud infrastructure.",
   "type": "module",
   "license": "MIT",

package/registry/capabilities.json

@@ -12,12 +12,12 @@
     },
     "extract": {
       "key": "extract",
-      "label": "Extract (extract=auto / Autoparse / CSS / Markdown)",
+      "label": "Extract (extract=auto / CSS / Markdown)",
       "status": "available",
       "command": "zenrows extract",
-      "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, autoparse, css_extractor, outputs, response_type)",
+      "backend": "Extract — GET https://api.zenrows.com/v1/ (extract=auto, css_extractor, outputs, response_type; autoparse deprecated)",
       "requiresAuth": true,
-      "notes": "Default is extract=auto, which works on any prepared domain with any key; CLI falls back to autoparse on REQS007 (domain not prepared yet) or AUTH010. Autoparse / CSS / outputs / markdown work on any domain."
+      "notes": "Default is extract=auto, which works on any prepared domain with any key. A REQS007 (legacy AUTH010) is reported, never retried, with the next step for its reason: on a domain not prepared yet (EXTRACT_DOMAIN_NOT_PREPARED), prepare it with `zenrows extract prepare <domain> --url <page> [--wait]` (POST /v1/extract/prepared-domains: 1 to 5 example URLs, a few minutes, no credits) and check it with `zenrows extract prepare status <domain>`; on a page Extract did not recognise on a prepared domain (EXTRACT_PAGE_NOT_RECOGNISED), try a product, article or listing page, retry it once in a few minutes, or use --css, because preparing again does not change it. CSS / outputs / markdown work on any domain; autoparse is deprecated and runs only with --autoparse."
     },
     "batch": {
       "key": "batch",

package/registry/skills.json

@@ -25,13 +25,13 @@
     {
       "name": "extract",
       "type": "skill",
-      "description": "Turn protected pages into structured data with Autoparse / CSS / Markdown.",
+      "description": "Turn protected pages into structured data with Extract / CSS / Markdown (Autoparse is deprecated).",
       "status": "available",
       "requires_backend_capabilities": ["extract"],
       "requires_auth": true,
       "version": "0.1.0",
       "path": "skills/extract",
-      "tags": ["extract", "autoparse", "structured-data"]
+      "tags": ["extract", "structured-data"]
     },
     {
       "name": "batch-jobs",

package/registry/templates.json

@@ -14,7 +14,7 @@
     {
       "name": "extract-pricing-pages",
       "type": "template",
-      "description": "Extract structured pricing fields from product pages (Autoparse / CSS).",
+      "description": "Extract structured pricing fields from product pages (extract=auto / CSS).",
       "status": "available",
       "requires_backend_capabilities": ["extract"],
       "requires_auth": true,

package/skills/extract/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: extract
-description: Turn protected pages into structured data with Extract / Autoparse / CSS / Markdown.
+description: Turn protected pages into structured data with Extract / CSS / Markdown (Autoparse is deprecated).
 version: 0.1.0
 requires_backend_capabilities: [extract]
 ---
@@ -12,8 +12,8 @@ access + extraction**, not generic LLM parsing.
 
 ## Methods
 ```
-zenrows extract <url>                              # extract=auto (default); falls back to Autoparse if the domain is not prepared yet
-zenrows extract <url> --autoparse                  # general-purpose Autoparse (any domain)
+zenrows extract <url>                              # extract=auto (default), any prepared domain
+zenrows extract <url> --autoparse                  # deprecated general-purpose Autoparse (any domain)
 zenrows extract <url> --css '{"title":"h1","price":".price"}'   # selector map
 zenrows extract <url> --outputs emails,links       # built-in output filters → JSON
 zenrows extract <url> --output markdown            # Markdown conversion
@@ -21,14 +21,58 @@ zenrows extract <url> --validate                   # fail if not valid JSON
 ```
 
 ## When to use which
-- **Extract** (`extract=auto`, default): richest fields; works on any domain Zenrows has prepared.
-- **Autoparse**: any website; also the automatic fallback when the domain is not prepared for Extract yet.
+- **Extract** (`extract=auto`, default): richest fields; works on any prepared
+  domain (you can prepare one yourself, see below).
+- **Autoparse** (deprecated): only with `--autoparse`. Never automatic; do not
+  build new work on it.
 - **CSS**: you know the exact fields/selectors and want determinism.
 - **Markdown/plaintext**: feed clean content to an LLM yourself.
 
+## REQS007: prepare the domain, then extract again
+
+`extract=auto` works on any prepared domain, and you can prepare one yourself.
+When Extract refuses a page with `403 REQS007`, `zenrows extract` says which
+of three cases it is and prints the next step. It never retries with Autoparse.
+
+| Error code | Meaning | Next step |
+| --- | --- | --- |
+| `EXTRACT_DOMAIN_NOT_PREPARED` | The domain is not prepared yet (also the legacy `402 AUTH010`). | Prepare it (below), then extract again. |
+| `EXTRACT_PAGE_NOT_RECOGNISED` | The domain is prepared, but Extract did not recognise this page as one it can extract (a page with nothing to extract, such as a home page, or a transient failure). | Try a product, article or listing page on the same site instead. If this page should work, retry it once in a few minutes; do not retry in a loop. Otherwise use `--css`. Preparing the domain again does not change this. |
+| `EXTRACT_DOMAIN_NOT_ENABLED` | Extract is not enabled for the site. | Use `--css`, or contact Zenrows support to have the site enabled. Preparing does not change this. |
+
+Preparing a domain:
+
+```
+zenrows extract https://example.com/p/1             # 1. fails: EXTRACT_DOMAIN_NOT_PREPARED
+zenrows extract prepare example.com \
+  --url https://example.com/p/1 --url https://example.com/list --wait
+                                                    # 2. prepare it and wait until ready (a few minutes)
+zenrows extract https://example.com/p/1             # 3. extract again
+```
+
+- `--url`: 1 to 5 real pages you want to extract, one per page kind (for
+  example a product page and a listing page).
+- Without `--wait` the command returns at once; then check about every 30
+  seconds with `zenrows extract prepare status example.com` until it says
+  `ready`. `--wait` polls at the same pace.
+- The domain is the host: `shop.example.com` and `example.com` are prepared
+  separately, and a leading `www.` is dropped.
+- Preparing takes a few minutes and costs no credits. Each new domain uses one
+  slot of the account's monthly allowance; `prepare status` shows `used` /
+  `limit` / `remaining`.
+- `EXTRACT_PREPARE_OUT_OF_ALLOWANCE`: no slots left this month; use `--css`.
+  `EXTRACT_PREPARE_UNAVAILABLE`: retry in a few minutes; do not loop.
+  A `failed` status prints the next step for its failure code and says whether
+  the attempt used an allowance slot.
+- A domain that is already prepared answers `ready` at once
+  (`already_prepared`) and uses no slot: run the extract. If it then fails with
+  `EXTRACT_PAGE_NOT_RECOGNISED`, follow that row of the table, not `prepare`.
+
 ## Rules
 - Validate on a single page before scaling across many URLs.
-- If Autoparse misses fields, switch to `--css` with explicit selectors. Auto
+- Check `empty` in `--json` output: true means every field came back null or
+  empty, so the result is not data.
+- If a field is missing, switch to `--css` with explicit selectors. Auto
   mode already renders JS when the target needs it, so reach for
   `--manual --js-render` only after auto mode has returned an incomplete page:
   it costs several times a basic call ([[cost-control]]) and it makes the

package/skills/zenrows/SKILL.md

@@ -20,6 +20,13 @@ the toolkit automatically creates a **free, unclaimed Zenrows Free plan account*
 key locally — no human step required. To opt out, pass `--no-signup` or set
 `auto_signup: false` in `.zenrows/policy.json`, then use `zenrows login --api-key <key>`.
 
+Before it creates the account, Zenrows may ask you one short question (error code
+`SIGNUP_CHALLENGE`, with the question in the message and in `error.challenge.question`).
+Answer it yourself, do not ask the human, then run `zenrows signup --answer "<your answer>"`
+and re-run your original command. A wrong answer returns a new question the same way.
+After too many questions or wrong answers from one network (`SIGNUP_CHALLENGE_LOCKED`), wait an hour or use
+`zenrows login --api-key <key>`.
+
 When the Free plan nears its limit (or the user asks), tell the human they can **claim** the
 account to keep their usage and history: run `zenrows account claim` (or open the claim URL
 shown on quota errors). It prints a valid link, requesting a fresh one when the stored link is
@@ -34,7 +41,7 @@ If the user has a known URL and wants page content:
   → Use Protected Fetch.            (zenrows fetch <url>)
 
 If the user has a known URL and wants structured data:
-  → Use Extract.                    (zenrows extract <url> | --autoparse | --css)
+  → Use Extract.                    (zenrows extract <url> | --css)
 
 If the user has a start page (listing, index, section) but not the item URLs:
   → Use Crawl.                      (zenrows crawl create <url> --depth 1 --follow,
@@ -75,13 +82,16 @@ Run `zenrows status` for the live capability matrix. As of this toolkit:
 | Primitive | Command | Status |
 | --- | --- | --- |
 | Protected Fetch | `zenrows fetch` | **available**: `GET https://api.zenrows.com/v1/` |
-| Extract (`extract=auto`, Autoparse, CSS, Markdown) | `zenrows extract` | **available**: `GET https://api.zenrows.com/v1/?extract=auto` |
+| Extract (`extract=auto`, CSS, Markdown; Autoparse deprecated) | `zenrows extract` | **available**: `GET https://api.zenrows.com/v1/?extract=auto` |
 | Batch | `zenrows batch` | **available**: `POST https://async.api.zenrows.com/v1/jobs` |
 | Crawl (new) | `zenrows crawl` | **available**: `POST https://api.zenrows.com/v1/crawls` |
 | Browser | `zenrows browser` | **available**: `POST https://mcp.zenrows.com/browser/sessions` |
 | MCP | `zenrows mcp` | **available**: `https://mcp.zenrows.com/mcp` |
 
-- Extract: `extract=auto` falls back to Autoparse on a domain not prepared yet.
+- Extract: `extract=auto` works on any prepared domain. On one not prepared yet
+  (`EXTRACT_DOMAIN_NOT_PREPARED`), run `zenrows extract prepare <domain> --url
+  <page> --wait`, then extract again. `EXTRACT_PAGE_NOT_RECOGNISED` means the
+  domain is prepared but not this page: do not prepare again; see [[extract]].
 - Batch: the reliable path at scale, at Fetch's prices (create, list, status, wait, results, content, pause, resume, rerun, delete). Estimate specs locally with no key.
 - Crawl: collects the URLs behind one start page, optionally with each page's HTML.
 - Browser: escalation-only, for pages Fetch can't handle; same backend as the MCP `browser_*` tools.

package/templates/extract-pricing-pages/README.md

@@ -3,13 +3,11 @@
 Extract structured pricing fields from product pages with Zenrows Extract.
 
 ```bash
-# Default: extract=auto on an enabled domain (site-tailored JSON)
+# Default: extract=auto on a prepared domain (site-tailored JSON).
+# Not prepared yet (EXTRACT_DOMAIN_NOT_PREPARED)? zenrows extract prepare <domain> --url <product-url> --wait
 zenrows extract https://www.owler.com/company/meltwater
 
-# Autoparse — general-purpose JSON on any domain
-zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse
-
-# Or a precise CSS selector map
+# A precise CSS selector map, on any domain
 zenrows extract <product-url> --css '{"title":"h1","price":".price"}' --validate
 ```
 

package/workflows/competitor-intelligence/WORKFLOW.md

@@ -9,7 +9,7 @@ status: available
 
 A higher-level process built on Zenrows **Fetch and Extract**: retrieve
 protected pages with `zenrows fetch`, then turn them into structured data with
-`zenrows extract` (the same API's Autoparse / CSS / Markdown — not a separate
+`zenrows extract` (the same API's `extract=auto` / CSS / Markdown — not a separate
 product). Scheduling and scaling use the tools you already have (cron/CI, a
 simple per-URL loop).
 
@@ -20,7 +20,7 @@ simple per-URL loop).
    ```
    zenrows fetch https://competitor.example/products
    ```
-2. **Validate extraction** on one page (extract=auto falls back to autoparse on a domain not prepared yet).
+2. **Validate extraction** on one page (on a domain not prepared yet, `EXTRACT_DOMAIN_NOT_PREPARED`, run `zenrows extract prepare <domain> --url <page> --wait` first).
    ```
    zenrows extract https://competitor.example/p/123 --validate
    ```