@zenrows/cli 1.3.1 → 1.4.1

package/agent-plugin/skills/extract/SKILL.md

@@ -17,12 +17,18 @@ Use **`extract`** when you need structured JSON fields rather than a full page:
 extract({ url })                                    # mode=auto (default)
 extract({ url, mode: "autoparse" })                 # general Autoparse (any domain)
 extract({ url, mode: "css", css_extractor: '{"title":"h1","price":".price"}' })
-extract({ url, js_render: true, premium_proxy: true })  # stealth flags when needed
+extract({ url, proxy_country: "US" })               # geo-targeted
 ```
 
 `mode=auto` (`extract=auto`) is **open beta**: richest on enabled domains,
-currently free; billing may apply later. If the domain is not enabled (AUTH010),
-the tool retries once with Autoparse by default (`fallback_autoparse`).
+currently free; billing may apply later. If the domain is not enabled (AUTH010)
+or not prepared yet (REQS007), the tool retries once with Autoparse by default
+(`fallback_autoparse`).
+
+Every mode uses Adaptive Stealth Mode by default: pass just the URL, even for
+protected or dynamic sites. Set `js_render` or `premium_proxy` only to force a
+fixed configuration; that turns Adaptive Stealth Mode off. Check `empty` in the
+result: true means no field came back with a value.
 
 ## Fallback: `scrape` extraction params
 
@@ -40,7 +46,7 @@ scrape({ url, outputs: "emails,links" })            # or "*" for all built-ins
 ## When to use which
 
 - **`mode=auto`**: site-tailored fields on beta-enabled domains.
-- **`mode=autoparse`**: any website; also the automatic AUTH010 fallback.
+- **`mode=autoparse`**: any website; also the automatic AUTH010 / REQS007 fallback.
 - **`mode=css`**: known fields/selectors; deterministic.
 - **Markdown via `scrape`**: feed clean content to an LLM yourself.
 

package/dist/src/adapters/batch.js

@@ -69,6 +69,25 @@ export function estimateCredits(jobs) {
     });
     return { credits: perJob.reduce((a, b) => a + b, 0), perJob };
 }
+/**
+ * Adaptive Stealth Mode (mode=auto) by default, like `fetch` and `extract`: returns
+ * the jobs with `mode: "auto"` on every line that doesn't force `js_render` /
+ * `premium_proxy` or set `mode` itself, and none at all when the job-level params
+ * force either flag. Applied per task rather than job-level because the API rejects
+ * a task that combines a job-level mode=auto with its own js_render or
+ * premium_proxy (REQS004).
+ */
+export function withAdaptiveStealth(jobs, jobParams = {}) {
+    const forced = (p) => isTruthyFlag(p.js_render) || isTruthyFlag(p.premium_proxy);
+    if (forced(jobParams) || jobParams.mode != null)
+        return jobs;
+    return jobs.map((job) => {
+        const own = { ...(job.zenrows_params ?? {}), ...job };
+        if (forced(own) || own.mode != null)
+            return job;
+        return { ...job, mode: "auto" };
+    });
+}
 /**
  * Map a validated JSONL spec + job-level flags to the Batch API request body.
  * The flat per-line options (`js_render`, `premium_proxy`, `proxy_country`,
@@ -82,7 +101,8 @@ export function estimateCredits(jobs) {
  * `protected-fetch.ts::validateAutoManual`.
  */
 export function toJobBody(jobs, jobParams = {}) {
-    assertProxyCountryPremium(jobParams, "job-level params");
+    // No separate job-level check: a job-level proxy_country is valid for each task
+    // in Adaptive Stealth Mode, and the per-task check below sees job ⊕ task params.
     const tasks = jobs.map((job, idx) => {
         const { url, external_id, metadata, zenrows_params } = job;
         const flat = {};

package/dist/src/cli/commands/batch.js

@@ -9,7 +9,8 @@
 import { statSync } from "node:fs";
 import { join } from "node:path";
 import { log, ANSI, c } from "../../core/logger.js";
-import { estimateCredits, toJobBody, validateJsonl } from "../../adapters/batch.js";
+import { estimateCredits, toJobBody, validateJsonl, withAdaptiveStealth } from "../../adapters/batch.js";
+import { loadConfig } from "../../core/config.js";
 import { requireApiKey } from "../../core/auth.js";
 import { ensureApiKey } from "../../core/ensure-key.js";
 import { assertUsable } from "../../core/capabilities.js";
@@ -25,12 +26,15 @@ export const batch = {
     usage: "zenrows batch <estimate|create <file.jsonl>|status <id>|results <id>|cancel <id>|wait <id>|retry-failed <id>>",
     help: [
         "Local (no key):",
-        "  estimate <file.jsonl>            validate the spec + estimate credits",
+        "  estimate <file.jsonl> [--manual] validate the spec + estimate credits (upper bound)",
         "Cloud (needs a key + Batch beta access):",
         "  create <file.jsonl> [flags]      validate, then submit the job",
-        "    --js-render                    job-level: render JavaScript",
-        "    --premium-proxy                job-level: use residential IPs",
-        "    --proxy-country <cc>           job-level: geo-target (needs --premium-proxy)",
+        "    (default)                      Adaptive Stealth Mode (mode=auto) on every task that",
+        "                                   doesn't force js_render / premium_proxy itself",
+        "    --manual                       no Adaptive Stealth Mode; plain requests unless flags below",
+        "    --js-render                    job-level: force JavaScript rendering (turns auto off)",
+        "    --premium-proxy                job-level: force residential IPs (turns auto off)",
+        "    --proxy-country <cc>           job-level: geo-target (auto mode, or with --premium-proxy)",
         "    --output <fmt>                 job-level response_type (markdown|plaintext|pdf|html)",
         "    --follow                       poll until the run finishes (alias: --wait)",
         "    --no-signup                    do not auto-create a Free plan account if no key exists",
@@ -76,20 +80,24 @@ export const batch = {
         }
     },
 };
+/** Jobs as they will be sent: Adaptive Stealth Mode applied unless manual or disabled in config. */
+function effectiveJobs(jobs, jobParams, manual) {
+    return manual || loadConfig().defaultMode !== "auto" ? jobs : withAdaptiveStealth(jobs, jobParams);
+}
 function estimateCmd(rest, ctx) {
-    const { positionals } = parse(rest, {});
+    const { values, positionals } = parse(rest, { manual: { type: "boolean" } });
     const file = positionals[0];
     if (!file)
         throw needFile();
     const v = validateJsonl(file);
-    const est = estimateCredits(v.jobs);
+    const est = estimateCredits(effectiveJobs(v.jobs, {}, values.manual === true));
     if (ctx.json) {
         log.out(JSON.stringify({ ok: v.errors.length === 0, ...v, estimatedCredits: est.credits }, null, 2));
     }
     else {
         log.info(c(ANSI.bold, `Job spec: ${file}`));
         log.info(`valid jobs: ${v.validJobs}/${v.totalLines}`);
-        log.info(`estimated credits (1x basic / 5x js / 10x proxy / 25x both): ${est.credits}`);
+        log.info(`estimated credits, upper bound (1x basic / 5x js / 10x proxy / 25x both or auto): ${est.credits}`);
         if (v.errors.length) {
             log.warn(`${v.errors.length} invalid line(s):`);
             v.errors.slice(0, 10).forEach((e) => log.dim(`  line ${e.line}: ${e.reason}`));
@@ -105,6 +113,7 @@ async function createCmd(rest, ctx) {
         "js-render": { type: "boolean" },
         "premium-proxy": { type: "boolean" },
         "proxy-country": { type: "string" },
+        manual: { type: "boolean" },
         output: { type: "string" },
         follow: { type: "boolean" },
         wait: { type: "boolean" }, // back-compat alias for --follow
@@ -136,11 +145,6 @@ async function createCmd(rest, ctx) {
     //  1. every task URL must pass the allow/deny domain policy,
     //  2. the run must fit the batch-only page/credit caps (a batch is the one
     //     primitive that fans out into many requests, so the caps bind here).
-    const policy = loadPolicy();
-    for (const job of v.jobs)
-        assertDomainAllowed(job.url, policy);
-    const est = estimateCredits(v.jobs);
-    assertWithinLimits({ pages: v.validJobs, credits: est.credits }, policy, "batch");
     const jobParams = {};
     if (values["js-render"] === true)
         jobParams.js_render = true;
@@ -152,8 +156,14 @@ async function createCmd(rest, ctx) {
     const responseType = normalizeOutput(asString(values.output));
     if (responseType)
         jobParams.response_type = responseType;
+    const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true);
+    const policy = loadPolicy();
+    for (const job of jobs)
+        assertDomainAllowed(job.url, policy);
+    const est = estimateCredits(jobs);
+    assertWithinLimits({ pages: v.validJobs, credits: est.credits }, policy, "batch");
     // toJobBody validates proxy_country/premium_proxy BEFORE any HTTP call.
-    const body = toJobBody(v.jobs, jobParams);
+    const body = toJobBody(jobs, jobParams);
     const apiKey = await ensureApiKey(values["no-signup"] ? { ...policy, auto_signup: false } : policy, {
         onProvision: (a) => {
             log.info("No API key found — created a Zenrows Free plan account for you.");

package/dist/src/cli/commands/extract.js

@@ -2,7 +2,7 @@
  * `zenrows extract <url> [flags]` → Extract (structured extraction on `/v1/`).
  *
  * Default: `extract=auto` (site-tailored Extract, open beta). If the domain is
- * not enabled (AUTH010), falls back once to Autoparse. Opt into Autoparse
+ * not enabled or prepared (AUTH010 / REQS007), falls back once to Autoparse. Opt into Autoparse
  * directly with `--autoparse`. Other methods: `--css`, `--outputs`, `--output`.
  */
 import { ensureApiKey } from "../../core/ensure-key.js";
@@ -23,7 +23,7 @@ export const extract = {
     usage: "zenrows extract <url> [--autoparse | --css <json> | --outputs <filters> | --output md|text] [flags]",
     help: [
         "Methods:",
-        "  (default)              extract=auto — site-tailored Extract (open beta); falls back to Autoparse if the domain is not enabled",
+        "  (default)              extract=auto — site-tailored Extract (open beta); falls back to Autoparse if the domain is not enabled or prepared",
         "  --autoparse            general-purpose Autoparse (any domain, no Extract fallback)",
         "  --css <json>           CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'",
         "  --outputs <list>       built-in output filters → JSON: emails, phone_numbers, headings,",
@@ -119,7 +119,7 @@ export const extract = {
             const safeParams = { ...outcome.params };
             delete safeParams.apikey;
             if (outcome.fellBackToAutoparse) {
-                log.info("Extract not enabled for this domain — fell back to Autoparse.");
+                log.info("Extract not available for this domain yet — fell back to Autoparse.");
             }
             const runDir = writeRun({
                 runId,

package/dist/src/core/config.js

@@ -2,6 +2,9 @@
  * Toolkit config (`.zenrows/config.json`). Non-secret, safe defaults.
  */
 import { randomUUID } from "node:crypto";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
 import { findWorkspace, readJson, workspacePaths, writeJson } from "./workspace.js";
 /** Confirmed Zenrows Fetch and Extract API base. */
 export const DEFAULT_API_BASE = "https://api.zenrows.com/v1/";
@@ -11,7 +14,7 @@ export const CONFIG_VERSION = "0.1.0";
  * entrypoint) so core modules — telemetry, signup provenance — can read it
  * without an import cycle. `VERSION` in `cli/index.ts` re-exports this.
  */
-export const CLI_VERSION = "1.3.1";
+export const CLI_VERSION = "1.4.1";
 /** The User-Agent on every request to a Zenrows API. The gateway reads the version from it. */
 export const CLI_USER_AGENT = `zenrows-cli/${CLI_VERSION}`;
 /** Env var to override the Fetch and Extract API base (local/staging testing). */
@@ -63,21 +66,57 @@ export function saveConfig(config, projectRoot) {
     const ws = projectRoot ? workspacePaths(projectRoot) : (findWorkspace() ?? workspacePaths());
     writeJson(ws.config, config);
 }
+/** Env var to relocate the per-machine config directory (tests, sandboxes). */
+export const CONFIG_HOME_ENV = "ZENROWS_CONFIG_HOME";
+/** Per-machine config directory, shared by every workspace on this machine. */
+export function machineConfigDir() {
+    const override = process.env[CONFIG_HOME_ENV];
+    if (override && override.trim())
+        return override.trim();
+    if (process.platform === "win32" && process.env.APPDATA)
+        return join(process.env.APPDATA, "zenrows");
+    const xdg = process.env.XDG_CONFIG_HOME;
+    return join(xdg && xdg.trim() ? xdg.trim() : join(homedir(), ".config"), "zenrows");
+}
+function readMachineTelemetryId(file) {
+    try {
+        const id = readFileSync(file, "utf8").trim();
+        return id || undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
 /**
  * Return the stable anonymous agent id, generating + persisting one on first
  * use. Sent as the `X-ZR-Agent-Id` header on signup so the backend can correlate
  * the (anonymous) device with the account it later merges on claim. Contains no
  * PII — a random uuid only.
+ *
+ * Stored per machine, not per workspace: a per-workspace id made every new
+ * project folder look like a new device, so the backend could not tell a
+ * rate-limited retry from the machine that had just signed up. An id already in
+ * the workspace is adopted on first run so existing devices keep their identity.
+ * If the machine directory is not writable, falls back to the workspace config.
  */
 export function getOrCreateTelemetryId(projectRoot) {
+    const file = join(machineConfigDir(), "telemetry-id");
+    const machineId = readMachineTelemetryId(file);
+    if (machineId)
+        return machineId;
     const ws = projectRoot ? workspacePaths(projectRoot) : (findWorkspace() ?? workspacePaths());
     // Merge onto the raw stored config (not loadConfig) so we never persist the
     // ZENROWS_API_BASE env override back into config.json.
     const stored = readJson(ws.config) ?? {};
-    const existing = stored.telemetryId;
-    if (existing && existing.trim())
-        return existing;
-    const id = randomUUID();
-    saveConfig({ ...defaultConfig(), ...stored, telemetryId: id }, projectRoot);
+    const workspaceId = stored.telemetryId?.trim() ? stored.telemetryId.trim() : undefined;
+    const id = workspaceId ?? randomUUID();
+    try {
+        mkdirSync(dirname(file), { recursive: true });
+        writeFileSync(file, `${id}\n`, { mode: 0o600 });
+    }
+    catch {
+        if (!workspaceId)
+            saveConfig({ ...defaultConfig(), ...stored, telemetryId: id }, projectRoot);
+    }
     return id;
 }

package/dist/src/core/http.js

@@ -98,6 +98,18 @@ export async function scrape(apiBase, apiKey, params, opts = {}) {
         concurrencyRemaining: numOrNull(res.headers.get("concurrency-remaining")),
         redactedUrl: redacted,
     };
+    // REQS007 on extract=auto: the domain hasn't been prepared for Extract yet.
+    // Same recovery as AUTH010 (the extract adapter falls back to autoparse), so it
+    // maps to the same code. Checked before the 403 auth branch below.
+    if (res.status === 403 && params.extract !== undefined && zrErrorCode(body) === "REQS007") {
+        throw new ToolkitError({
+            code: "EXTRACT_DOMAIN_NOT_ENABLED",
+            message: "Extract has not been prepared for this domain yet.",
+            likely_cause: zrErrorDetail(body) ?? "This domain is not prepared for Extract.",
+            next_action: "Retry with --autoparse for general-purpose extraction on any site.",
+            suggested_commands: [`zenrows extract ${params.url} --autoparse`],
+        });
+    }
     if (res.status === 401 || res.status === 403) {
         // Distinguish Zenrows auth errors from target-site 403s. Zenrows auth
         // failures surface AUTH00x codes in the body.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenrows/cli",
-  "version": "1.3.1",
+  "version": "1.4.1",
   "description": "Open-source CLI, MCP, skills, templates, workflows, recipes, and evals layer for giving AI agents reliable access to protected web data through Zenrows cloud infrastructure.",
   "type": "module",
   "license": "MIT",

package/skills/batch-jobs/SKILL.md

@@ -35,14 +35,15 @@ structured data instead of raw HTML.
 {"url": "https://example.com/products", "extract": "auto", "external_id": "p1"}
 ```
 
-An Extract task costs the same as a regular one (1 credit at base tier), so
-`batch estimate` prices it correctly. The result carries two keys: `html` (the
+An Extract task costs the same as a regular one, so `batch estimate` prices it
+correctly. The result carries two keys: `html` (the
 raw page) and `parsed` (the structured data). Validate on a single URL with
 `zenrows extract <url>` before running the full batch.
 
 ## Cloud (needs a key + beta access)
 ```
-zenrows batch create jobs.jsonl [--js-render] [--premium-proxy] [--proxy-country cc] [--wait]
+zenrows batch create jobs.jsonl [--proxy-country cc] [--wait]
+zenrows batch create jobs.jsonl --manual [--js-render] [--premium-proxy]
 zenrows batch status <id>             # run status + stats
 zenrows batch results <id> [--status successful|failed|all] [--out results.jsonl]
 zenrows batch wait <id> [--timeout ms]
@@ -50,9 +51,17 @@ zenrows batch cancel <id>             # stop an in-flight run
 zenrows batch retry-failed <id>       # rerun only the failed tasks (new run)
 ```
 
-`create` validates the spec first, then submits. `--proxy-country` needs
-`--premium-proxy` (or `mode=auto`) and is rejected before any request. Job-level
-flags apply to every task; per-task keys in the JSONL override them.
+Every task runs in Adaptive Stealth Mode (`mode=auto`) by default, like `fetch`
+and `extract`: Zenrows enables JS rendering and premium proxies only on the pages
+that need them, and bills only the configuration that succeeds. A task that sets
+`js_render`, `premium_proxy` or `mode` itself keeps its own settings, and
+`--js-render` / `--premium-proxy` force that configuration on every task instead.
+`--manual` turns Adaptive Stealth Mode off. `batch estimate` counts auto tasks at
+the 25x upper bound, so it's the most the run can cost, not the expected cost.
+
+`create` validates the spec first, then submits. Outside Adaptive Stealth Mode,
+`--proxy-country` needs `--premium-proxy` and is rejected before any request.
+Job-level flags apply to every task; per-task keys in the JSONL override them.
 
 A run ends in one of `completed`, `failed`, `stopped`, or `deleted`. `status`,
 `wait`, and `create --wait` exit non-zero (`ok: false` under `--json`) only for

package/templates/batch-jsonl-pipeline/README.md

@@ -29,7 +29,9 @@ zenrows batch retry-failed <job-id>                     # rerun only the failed
 zenrows batch cancel <job-id>                           # stop an in-flight run
 ```
 
-Job-level scrape options come from flags (`--js-render`, `--premium-proxy`,
-`--proxy-country`, `--output`); per-line keys override them for that URL.
+Tasks use Adaptive Stealth Mode (`mode=auto`) by default. Job-level scrape
+options come from flags (`--js-render`, `--premium-proxy`, `--proxy-country`,
+`--output`, or `--manual` to turn Adaptive Stealth Mode off); per-line keys
+override them for that URL.
 Without beta access yet, validate/estimate locally or fan out with
 `zenrows fetch` per URL on a small sample first.