@zenrows/cli 1.7.0 → 1.9.0

package/dist/src/cli/commands/crawl.js

@@ -34,7 +34,7 @@ export const crawl = {
         "                                    leave it out for no depth limit (the caps below stop the crawl)",
         "    --max-items <n>                 stop after keeping n URLs (API default 100)",
         "    --max-pages <n>                 stop after fetching n pages (API default 100; bounds cost);",
-        "                                    the local policy max_pages_per_run caps it (default 1000)",
+        "                                    the local policy max_pages_per_run caps it (default 10000)",
         "    --include-patterns <p>          keep only URLs containing this substring (repeatable)",
         "    --exclude-patterns <p>          drop URLs containing this substring (repeatable)",
         "    --output-format <html|markdown> also fetch each kept URL's page, as HTML or Markdown",

package/dist/src/cli/commands/extract-prepare.js

@@ -0,0 +1,168 @@
+/**
+ * `zenrows extract prepare <domain> --url <u> [--url <u>...] [--wait]` and
+ * `zenrows extract prepare status <domain>`.
+ *
+ * The way out of REQS007: prepare the domain so `extract=auto` works on it.
+ * Submitting returns at once; `--wait` polls until the preparation is ready or
+ * failed (a few minutes). Costs no credits; each new domain uses one slot of the
+ * account's monthly allowance.
+ */
+import { ensureApiKey } from "../../core/ensure-key.js";
+import { loadConfig } from "../../core/config.js";
+import { loadPolicy } from "../../core/policy.js";
+import { log } from "../../core/logger.js";
+import { ToolkitError } from "../../core/errors.js";
+import { DEFAULT_WAIT_TIMEOUT_MS, MAX_EXAMPLE_URLS, failureNextStep, getAllowance, getPreparedDomain, normalizeDomain, prepareDomain, statusNextStep, waitForPreparedDomain, } from "../../core/prepared-domains-api.js";
+import { parse, asString } from "../command.js";
+import { printError } from "../output.js";
+export const PREPARE_USAGE = "zenrows extract prepare <domain> --url <page-url> [--url <page-url>...] [--wait [--timeout <seconds>]]\n" +
+    "       zenrows extract prepare status <domain>";
+export const PREPARE_HELP = [
+    "Prepare a domain so `zenrows extract <url>` (extract=auto) works on it. Use it when an",
+    "extract fails with EXTRACT_DOMAIN_NOT_PREPARED (REQS007: the domain is not prepared yet).",
+    "Not for EXTRACT_PAGE_NOT_RECOGNISED: that domain is already prepared, and preparing it",
+    "again does not change that answer.",
+    "",
+    `  --url <page-url>       1 to ${MAX_EXAMPLE_URLS} real pages you want to extract, one per page kind (repeat --url)`,
+    "  --wait                 poll every 30 seconds until the domain is ready or failed; off by default",
+    `  --timeout <seconds>    give up waiting after this long (default ${DEFAULT_WAIT_TIMEOUT_MS / 1000})`,
+    "  status <domain>        show progress and the account's monthly allowance",
+    "  --json                 structured result",
+    "",
+    "Preparing takes a few minutes and costs no credits. Each new domain uses one slot of the",
+    "account's monthly allowance; a domain that is already prepared answers ready at once and",
+    "uses none. The domain is the host (shop.example.com and example.com are separate); a",
+    "leading www. is dropped.",
+].join("\n");
+export async function runPrepare(argv, ctx, deps = {}) {
+    const isStatus = argv[0] === "status";
+    const { values, positionals } = parse(isStatus ? argv.slice(1) : argv, {
+        url: { type: "string", multiple: true },
+        wait: { type: "boolean" },
+        timeout: { type: "string" },
+        json: { type: "boolean" },
+        help: { type: "boolean" },
+    });
+    const json = ctx.json || values.json === true;
+    try {
+        const raw = positionals[0];
+        if (!raw) {
+            throw new ToolkitError({
+                code: "INVALID_USAGE",
+                message: "Missing domain.",
+                likely_cause: "No <domain> was given.",
+                next_action: `Usage: ${PREPARE_USAGE}`,
+                suggested_commands: ["zenrows extract prepare example.com --url https://example.com/products/1"],
+            });
+        }
+        const domain = normalizeDomain(raw);
+        const urls = isStatus ? [] : validateUrls(values.url, domain);
+        const timeoutMs = parseTimeout(asString(values.timeout));
+        const policy = loadPolicy();
+        const apiKey = deps.apiKey ??
+            // Preparing never creates a Free plan account: it needs a key the user already has.
+            (await ensureApiKey({ ...policy, auto_signup: false }));
+        const opts = { apiKey, apiBase: loadConfig().apiBase, fetchImpl: deps.fetchImpl };
+        if (isStatus) {
+            const [record, allowance] = await Promise.all([getPreparedDomain(domain, opts), getAllowance(opts).catch(() => undefined)]);
+            report(record, allowance, json);
+            return 0;
+        }
+        log.step(`Prepare ${domain} for Extract (${urls.length} example URL${urls.length === 1 ? "" : "s"})…`);
+        let record = await prepareDomain(domain, urls, opts);
+        if (record.already_in_progress)
+            log.info(`${domain} was already being prepared; following that preparation.`);
+        // A submit that answers ready at once found the domain already prepared.
+        const alreadyReady = record.status === "ready" && !record.already_in_progress;
+        if (values.wait === true && record.status !== "ready" && record.status !== "failed") {
+            record = await waitForPreparedDomain(domain, {
+                ...opts,
+                timeoutMs,
+                intervalMs: deps.intervalMs,
+                sleepImpl: deps.sleepImpl,
+                now: deps.now,
+                onPoll: (r) => log.dim(`  ${r.domain}: ${r.status} (${r.state ?? r.status})`),
+            });
+        }
+        if (record.status === "failed") {
+            throw new ToolkitError({
+                code: "EXTRACT_PREPARE_FAILED",
+                message: `Preparing ${domain} failed${record.failure?.code ? ` (${record.failure.code})` : ""}.`,
+                likely_cause: record.failure?.detail ?? "The preparation did not finish.",
+                next_action: failureNextStep(domain, record.failure),
+                server_code: record.failure?.code,
+            });
+        }
+        report(record, undefined, json, { exampleUrl: urls[0], alreadyReady });
+        return 0;
+    }
+    catch (err) {
+        printError(err, json);
+        return 1;
+    }
+}
+function validateUrls(raw, domain) {
+    const urls = (Array.isArray(raw) ? raw : raw === undefined ? [] : [raw]).map(String);
+    if (urls.length < 1 || urls.length > MAX_EXAMPLE_URLS) {
+        throw new ToolkitError({
+            code: "INVALID_USAGE",
+            message: `Pass 1 to ${MAX_EXAMPLE_URLS} example page URLs with --url (got ${urls.length}).`,
+            likely_cause: "Preparation learns the domain from real pages you want to extract.",
+            next_action: "Add one --url per page kind, e.g. a product page and a listing page.",
+            suggested_commands: [`zenrows extract prepare ${domain} --url https://${domain}/<page>`],
+        });
+    }
+    for (const u of urls) {
+        let parsed;
+        try {
+            parsed = new URL(u);
+        }
+        catch {
+            // reported below
+        }
+        if (!parsed || (parsed.protocol !== "http:" && parsed.protocol !== "https:")) {
+            throw new ToolkitError({
+                code: "INVALID_USAGE",
+                message: `--url ${u} is not a full http(s) URL.`,
+                likely_cause: "Each --url must be a page URL including its scheme.",
+                next_action: `Use full URLs, e.g. --url https://${domain}/<page>.`,
+            });
+        }
+    }
+    return urls;
+}
+function parseTimeout(v) {
+    if (v === undefined)
+        return DEFAULT_WAIT_TIMEOUT_MS;
+    const n = Number(v);
+    if (!Number.isFinite(n) || n <= 0) {
+        throw new ToolkitError({
+            code: "INVALID_USAGE",
+            message: `--timeout must be a positive number of seconds (got ${v}).`,
+            likely_cause: "The value is not a number.",
+            next_action: "Pass seconds, e.g. --timeout 600.",
+        });
+    }
+    return n * 1000;
+}
+function report(record, allowance, json, opts = {}) {
+    const next = statusNextStep(record, opts);
+    if (json) {
+        log.out(JSON.stringify({ ok: true, ...record, ...(opts.alreadyReady ? { already_prepared: true } : {}), ...(allowance ? { allowance } : {}), next }, null, 2));
+        return;
+    }
+    const line = `${record.domain}: ${record.status}${record.state && record.state !== record.status ? ` (${record.state})` : ""}`;
+    if (record.status === "ready")
+        log.success(line);
+    else if (record.status === "failed")
+        log.warn(line);
+    else
+        log.info(line);
+    if (record.failure)
+        log.dim(`  failure: ${record.failure.code}${record.failure.detail ? ` — ${record.failure.detail}` : ""}`);
+    for (const role of record.roles ?? [])
+        log.dim(`  ${role.role}: ${role.status}${role.url ? ` ${role.url}` : ""}`);
+    if (allowance)
+        log.dim(`  allowance: ${allowance.used} of ${allowance.limit} used, ${allowance.remaining} left${allowance.month ? ` (${allowance.month})` : ""}`);
+    log.out(next);
+}

package/dist/src/cli/commands/extract.js

@@ -1,9 +1,12 @@
 /**
  * `zenrows extract <url> [flags]` → Extract (structured extraction on `/v1/`).
  *
- * Default: `extract=auto` (site-tailored Extract). If the domain is
- * not enabled or prepared (AUTH010 / REQS007), falls back once to Autoparse. Opt into Autoparse
- * directly with `--autoparse`. Other methods: `--css`, `--outputs`, `--output`.
+ * Default: `extract=auto` (site-tailored Extract). On a REQS007 / AUTH010 the
+ * error names the next step for its reason (prepare the domain, or try another
+ * page); nothing is retried. `--autoparse` runs the deprecated Autoparse
+ * explicitly. Other
+ * methods: `--css`, `--outputs`, `--output`. `zenrows extract prepare` prepares a
+ * domain (see extract-prepare.ts).
  */
 import { ensureApiKey } from "../../core/ensure-key.js";
 import { maybeNudgeClaim } from "../../core/nudge.js";
@@ -18,14 +21,17 @@ import { formatRequestCost } from "../../core/http.js";
 import { parse, asString, asNumber } from "../command.js";
 import { printError, writeOut } from "../output.js";
 import { refreshClaimAdvice } from "../../core/claim-link.js";
+import { PREPARE_HELP, PREPARE_USAGE, runPrepare } from "./extract-prepare.js";
 export const extract = {
     name: "extract",
     summary: "Turn a protected page into structured data (Extract / Autoparse / CSS / Markdown).",
-    usage: "zenrows extract <url> [--autoparse | --css <json> | --outputs <filters> | --output md|text] [flags]",
+    usage: "zenrows extract <url> [--autoparse | --css <json> | --outputs <filters> | --output md|text] [flags]\n       " +
+        PREPARE_USAGE,
     help: [
         "Methods:",
-        "  (default)              extract=auto: site-tailored Extract; falls back to Autoparse if the domain is not enabled or prepared",
-        "  --autoparse            general-purpose Autoparse (any domain, no Extract fallback)",
+        "  (default)              extract=auto: site-tailored Extract on any prepared domain",
+        "                         (EXTRACT_DOMAIN_NOT_PREPARED = not prepared yet: run zenrows extract prepare, below)",
+        "  --autoparse            deprecated general-purpose Autoparse (any domain)",
         "  --css <json>           CSS selector map, e.g. '{\"title\":\"h1\",\"price\":\".price\"}'",
         "  --outputs <list>       built-in output filters → JSON: emails, phone_numbers, headings,",
         "                         images, audios, videos, links, menus, hashtags, metadata, tables,",
@@ -40,8 +46,13 @@ export const extract = {
         "",
         "Cost (credits per request): 1x normal · 5x --js-render · 10x --premium-proxy · 25x both.",
         "The exact charge is reported after every request (costCredits / X-Request-Credits).",
+        "",
+        "Prepare a domain:",
+        PREPARE_HELP.split("\n").map((l) => (l ? "  " + l : l)).join("\n"),
     ].join("\n"),
     async run(argv, ctx) {
+        if (argv[0] === "prepare")
+            return runPrepare(argv.slice(1), ctx);
         const { values, positionals } = parse(argv, {
             autoparse: { type: "boolean" },
             css: { type: "string" },
@@ -114,14 +125,14 @@ export const extract = {
         const runId = newRunId();
         const startedAt = new Date().toISOString();
         log.step(`Extract ${url} (method=${opts.method ?? "extract"})…`);
+        if (opts.method === "autoparse") {
+            log.warn("Autoparse is deprecated. Prefer the default extract=auto on a prepared domain, or --css.");
+        }
         try {
             const outcome = await runExtract(opts, config, policy, apiKey);
             const finishedAt = new Date().toISOString();
             const safeParams = { ...outcome.params };
             delete safeParams.apikey;
-            if (outcome.fellBackToAutoparse) {
-                log.info("Extract not available for this domain yet — fell back to Autoparse.");
-            }
             const runDir = writeRun({
                 runId,
                 command: "zenrows extract",
@@ -133,7 +144,6 @@ export const extract = {
                 request: {
                     ...safeParams,
                     method: outcome.method,
-                    fellBackToAutoparse: outcome.fellBackToAutoparse ?? false,
                 },
                 result: { httpStatus: outcome.result.status, bytes: outcome.result.body.length, parsed: outcome.data !== undefined },
                 costUsd: outcome.result.costUsd,
@@ -145,8 +155,7 @@ export const extract = {
                 writeOut(values.out, payload);
                 log.success(`Wrote output → ${values.out}`);
             }
-            const via = outcome.fellBackToAutoparse ? `${outcome.method} (fallback)` : outcome.method;
-            log.success(`Extracted via ${via} · ${outcome.result.body.length} bytes · ${formatRequestCost(outcome.result.costUsd, outcome.result.costCredits)} · run ${runId}`);
+            log.success(`Extracted via ${outcome.method} · ${outcome.result.body.length} bytes · ${formatRequestCost(outcome.result.costUsd, outcome.result.costCredits)} · run ${runId}`);
             if (runDir)
                 log.dim(`  artifact: ${runDir}`);
             if (outcome.noDataExtracted) {
@@ -160,7 +169,6 @@ export const extract = {
                     ok: true,
                     runId,
                     method: outcome.method,
-                    fellBackToAutoparse: outcome.fellBackToAutoparse ?? false,
                     empty: outcome.empty ?? false,
                     noDataExtracted: outcome.noDataExtracted ?? false,
                     httpStatus: outcome.result.status,