@zenrows/cli 1.7.0 → 1.9.0

package/dist/src/adapters/batch.js

@@ -3,14 +3,28 @@
  *
  * Status: `available`. The cloud subcommands call the Batch API with an API key
  * (see `core/batch-api.ts`); a 403 maps to BATCH_ACCESS_DENIED. This adapter owns
- * the local, no-network pieces: validating a JSONL job spec, estimating credit
- * cost, and mapping the validated spec to the Batch API request body.
+ * the local, no-network pieces: validating a JSONL job spec against the API's
+ * limits, estimating credit cost, and mapping the validated spec to the Batch
+ * API request body.
  */
 import { existsSync, readFileSync } from "node:fs";
 import { ToolkitError } from "../core/errors.js";
-/** Per-task keys handled explicitly; everything else on a line is a scrape param. */
-const TASK_RESERVED = new Set(["url", "external_id", "metadata", "zenrows_params"]);
-/** Parse + validate a JSONL job spec locally. */
+/** Per-task keys sent at task level; everything else on a line is a scrape param. */
+const TASK_RESERVED = new Set(["url", "external_id", "metadata", "zenrows_params", "method", "body"]);
+/** The API's limits on a task and on job-level fields. */
+export const LIMITS = {
+    urlLength: 2048,
+    externalIdLength: 128,
+    metadataEntries: 20,
+    metadataKeyLength: 64,
+    metadataValueLength: 1024,
+    metadataTotalBytes: 4 * 1024,
+    bodyBytes: 16 * 1024,
+    nameLength: 100,
+};
+/** `external_id` and metadata keys: letters, digits, `.`, `_` and `-`. */
+const SAFE_ID = /^[A-Za-z0-9._-]+$/;
+/** Parse + validate a JSONL job spec locally, against the API's per-task limits. */
 export function validateJsonl(file) {
     if (!existsSync(file)) {
         throw new ToolkitError({
@@ -28,45 +42,171 @@ export function validateJsonl(file) {
         if (!trimmed)
             return;
         result.totalLines++;
+        let obj;
         try {
-            const obj = JSON.parse(trimmed);
-            if (!obj.url || typeof obj.url !== "string") {
-                result.errors.push({ line: idx + 1, reason: "missing string `url`" });
-                return;
-            }
-            try {
-                new URL(obj.url);
-            }
-            catch {
-                result.errors.push({ line: idx + 1, reason: `invalid url: ${obj.url}` });
-                return;
-            }
-            result.validJobs++;
-            result.jobs.push(obj);
+            obj = JSON.parse(trimmed);
         }
         catch {
             result.errors.push({ line: idx + 1, reason: "invalid JSON" });
+            return;
+        }
+        const reason = taskError(obj);
+        if (reason) {
+            result.errors.push({ line: idx + 1, reason });
+            return;
         }
+        result.validJobs++;
+        result.jobs.push(obj);
     });
     return result;
 }
+/** Why one JSONL line would be rejected by the API, or null when it is valid. */
+function taskError(obj) {
+    if (typeof obj !== "object" || obj === null || Array.isArray(obj))
+        return "a line must be a JSON object";
+    const job = obj;
+    if (!job.url || typeof job.url !== "string")
+        return "missing string `url`";
+    let u;
+    try {
+        u = new URL(job.url);
+    }
+    catch {
+        return `invalid url: ${job.url}`;
+    }
+    if (u.protocol !== "http:" && u.protocol !== "https:")
+        return `url must be http or https: ${job.url}`;
+    // The API counts these limits in UTF-8 bytes, not characters.
+    if (Buffer.byteLength(job.url) > LIMITS.urlLength)
+        return `url is longer than ${LIMITS.urlLength} bytes`;
+    if (job.external_id !== undefined && job.external_id !== null) {
+        const e = externalIdError(job.external_id);
+        if (e)
+            return e;
+    }
+    const meta = metadataError(job.metadata);
+    if (meta)
+        return meta;
+    if (job.zenrows_params !== undefined && (typeof job.zenrows_params !== "object" || job.zenrows_params === null || Array.isArray(job.zenrows_params))) {
+        return "`zenrows_params` must be an object";
+    }
+    return methodBodyError(job.method, job.body);
+}
+/** Why an `external_id` is invalid, or null. Numbers are sent as their string form. */
+export function externalIdError(v) {
+    if (typeof v !== "string" && typeof v !== "number")
+        return "`external_id` must be a string";
+    const s = String(v);
+    if (s.length === 0 || s.length > LIMITS.externalIdLength)
+        return `\`external_id\` must be 1 to ${LIMITS.externalIdLength} characters`;
+    if (!SAFE_ID.test(s))
+        return "`external_id` may only use letters, digits, `.`, `_` and `-`";
+    return null;
+}
 /**
- * Rough local credit estimate using documented multipliers
- * (basic 1x, js 5x, proxies 10x, both 25x). Counts in abstract "credits".
+ * Why `metadata` is invalid, or null: the API takes at most 20 string-to-string
+ * entries, keys of 1 to 64 safe characters, values up to 1024 bytes, 4 KiB
+ * in total. Any other shape fails the whole submit.
  */
-export function estimateCredits(jobs) {
+export function metadataError(metadata) {
+    if (metadata === undefined || metadata === null)
+        return null;
+    if (typeof metadata !== "object" || Array.isArray(metadata))
+        return "`metadata` must be an object of string values";
+    const entries = Object.entries(metadata);
+    if (entries.length > LIMITS.metadataEntries)
+        return `\`metadata\` has more than ${LIMITS.metadataEntries} entries`;
+    let total = 0;
+    for (const [k, v] of entries) {
+        if (k.length === 0 || k.length > LIMITS.metadataKeyLength || !SAFE_ID.test(k)) {
+            return `\`metadata\` key "${k}" must be 1 to ${LIMITS.metadataKeyLength} letters, digits, \`.\`, \`_\` or \`-\``;
+        }
+        if (typeof v !== "string")
+            return `\`metadata.${k}\` must be a string`;
+        if (Buffer.byteLength(v) > LIMITS.metadataValueLength)
+            return `\`metadata.${k}\` is longer than ${LIMITS.metadataValueLength} bytes`;
+        total += Buffer.byteLength(k) + Buffer.byteLength(v);
+    }
+    if (total > LIMITS.metadataTotalBytes)
+        return "`metadata` is larger than 4 KiB";
+    return null;
+}
+/** Why `method` / `body` are invalid, or null. Only GET and POST exist; a body needs POST. */
+function methodBodyError(method, body) {
+    if (method !== undefined && method !== null) {
+        if (typeof method !== "string" || !/^(get|post)$/i.test(method))
+            return "`method` must be GET or POST";
+    }
+    if (body === undefined || body === null)
+        return null;
+    if (!isPost(method))
+        return "`body` needs `\"method\": \"POST\"`";
+    const bytes = typeof body === "string" ? Buffer.byteLength(body) : Buffer.byteLength(JSON.stringify(body));
+    if (bytes > LIMITS.bodyBytes)
+        return `\`body\` is larger than ${LIMITS.bodyBytes / 1024} KiB`;
+    return null;
+}
+function isPost(method) {
+    return typeof method === "string" && method.toUpperCase() === "POST";
+}
+/**
+ * Scrape params the API reads as booleans. It accepts JSON `true`/`false` or the
+ * strings "true"/"false" in any case, and rejects anything else with a 400.
+ */
+const BOOLEAN_PARAMS = new Set(["js_render", "premium_proxy", "original_status", "json_response", "autoparse"]);
+/** JSON `true` and "true" in any case become "true"; likewise "false". Any other value is unchanged. */
+function canonicalBool(v) {
+    if (v === true)
+        return "true";
+    if (v === false)
+        return "false";
+    if (typeof v === "string") {
+        const s = v.trim().toLowerCase();
+        if (s === "true" || s === "false")
+            return s;
+    }
+    return v;
+}
+/** A task's own params: its nested `zenrows_params`, then its flat keys (later wins). */
+function taskParamsOf(job) {
+    const flat = {};
+    for (const [k, v] of Object.entries(job)) {
+        if (!TASK_RESERVED.has(k))
+            flat[k] = v;
+    }
+    return { ...(job.zenrows_params ?? {}), ...flat };
+}
+/**
+ * The params a task actually runs with: the job-level params and the task's
+ * own, each canonicalized exactly as submitted, merged the way the API merges
+ * them (the task wins on a shared key). A task key set to null is not
+ * submitted, so the job's value applies.
+ */
+export function effectiveParams(job, jobParams = {}) {
+    return { ...normalizeParams(jobParams), ...normalizeParams(taskParamsOf(job)) };
+}
+/**
+ * Local credit estimate from the rate card: basic 1, js_render 5,
+ * premium_proxy 10, both 25. An Adaptive Stealth Mode task (`mode=auto`)
+ * costs 1 to 25 depending on what each page needs, so it counts 1 in `min`
+ * and 25 in `credits` (the upper bound). Each task is priced on the params
+ * it is submitted with, job-level flags included.
+ */
+export function estimateCredits(jobs, jobParams = {}) {
+    let min = 0;
     const perJob = jobs.map((j) => {
-        const js = j.js_render || j.mode === "auto";
-        const proxy = j.premium_proxy || j.mode === "auto";
-        if (js && proxy)
+        const p = effectiveParams(j, jobParams);
+        if (p.mode === "auto") {
+            min += 1;
             return 25;
-        if (proxy)
-            return 10;
-        if (js)
-            return 5;
-        return 1;
+        }
+        const js = p.js_render === "true";
+        const proxy = p.premium_proxy === "true";
+        const cost = js && proxy ? 25 : proxy ? 10 : js ? 5 : 1;
+        min += cost;
+        return cost;
     });
-    return { credits: perJob.reduce((a, b) => a + b, 0), perJob };
+    return { credits: perJob.reduce((a, b) => a + b, 0), min, perJob };
 }
 /**
  * Adaptive Stealth Mode (mode=auto) by default, like `fetch` and `extract`: returns
@@ -74,14 +214,17 @@ export function estimateCredits(jobs) {
  * `premium_proxy` or set `mode` itself, and none at all when the job-level params
  * force either flag. Applied per task rather than job-level because the API rejects
  * a task that combines a job-level mode=auto with its own js_render or
- * premium_proxy (REQS004).
+ * premium_proxy (REQS004). A POST task is left alone: Adaptive Stealth Mode may
+ * turn on JavaScript rendering, which a POST cannot use.
  */
 export function withAdaptiveStealth(jobs, jobParams = {}) {
     const forced = (p) => isTruthyFlag(p.js_render) || isTruthyFlag(p.premium_proxy);
     if (forced(jobParams) || jobParams.mode != null)
         return jobs;
     return jobs.map((job) => {
-        const own = { ...(job.zenrows_params ?? {}), ...job };
+        if (isPost(job.method))
+            return job;
+        const own = taskParamsOf(job);
         if (forced(own) || own.mode != null)
             return job;
         return { ...job, mode: "auto" };
@@ -92,38 +235,45 @@ export function withAdaptiveStealth(jobs, jobParams = {}) {
  * The flat per-line options (`js_render`, `premium_proxy`, `proxy_country`,
  * `autoparse`, `mode`, …) plus any nested `zenrows_params` become that task's
  * `zenrows_params`; `jobParams` (from CLI flags) becomes the job-level
- * `zenrows_params`. `url`/`external_id`/`metadata` map straight through.
+ * `zenrows_params`. `url`, `external_id`, `metadata`, `method` and `body` are
+ * task fields and map straight through.
  *
- * The API rejects `proxy_country` without `premium_proxy` (unless mode=auto),
- * so we surface PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM here — before submitting —
- * for the effective (job + task) params, mirroring the manual-mode guard in
- * `protected-fetch.ts::validateAutoManual`.
+ * Checked here, before submitting, on the effective (job + task) params:
+ * boolean params hold true or false; `proxy_country` needs `premium_proxy`
+ * unless mode=auto (PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM, mirroring
+ * `protected-fetch.ts::validateAutoManual`); a POST task does not turn on
+ * `js_render`, `js_instructions` or `json_response`, which the API rejects.
  */
-export function toJobBody(jobs, jobParams = {}) {
-    // No separate job-level check: a job-level proxy_country is valid for each task
-    // in Adaptive Stealth Mode, and the per-task check below sees job ⊕ task params.
+export function toJobBody(jobs, jobParams = {}, fields = {}) {
     const tasks = jobs.map((job, idx) => {
-        const { url, external_id, metadata, zenrows_params } = job;
-        const flat = {};
-        for (const [k, v] of Object.entries(job)) {
-            if (!TASK_RESERVED.has(k))
-                flat[k] = v;
-        }
-        const taskParams = { ...(zenrows_params ?? {}), ...flat };
-        // Validate on the effective (job ⊕ task) params so a job-level premium_proxy
-        // covers a per-task proxy_country, matching how the API merges them.
-        assertProxyCountryPremium({ ...jobParams, ...taskParams }, `line ${idx + 1} (${url})`);
+        const { url, external_id, metadata } = job;
+        const where = `line ${idx + 1} (${url})`;
+        const effective = effectiveParams(job, jobParams);
+        assertBooleanParams(effective, where);
+        assertProxyCountryPremium(effective, where);
         const task = { url };
-        if (external_id !== undefined)
+        if (external_id !== undefined && external_id !== null)
             task.external_id = String(external_id);
-        if (metadata !== undefined)
+        if (metadata !== undefined && metadata !== null)
             task.metadata = metadata;
-        const zp = normalizeParams(taskParams);
+        if (isPost(job.method)) {
+            assertPostParams(effective, where);
+            task.method = "POST";
+            if (job.body !== undefined && job.body !== null)
+                task.body = job.body;
+        }
+        const zp = normalizeParams(taskParamsOf(job));
         if (Object.keys(zp).length)
             task.zenrows_params = zp;
         return task;
     });
     const body = { type: "regular", status: "closed", tasks };
+    if (fields.name)
+        body.name = fields.name;
+    if (fields.external_id)
+        body.external_id = fields.external_id;
+    if (fields.metadata && Object.keys(fields.metadata).length)
+        body.metadata = fields.metadata;
     const jobZp = normalizeParams(jobParams);
     if (Object.keys(jobZp).length)
         body.zenrows_params = jobZp;
@@ -131,14 +281,16 @@ export function toJobBody(jobs, jobParams = {}) {
 }
 /**
  * Coerce a flat options object into the string-valued map the Batch API expects
- * (its docs pass `zenrows_params` values as strings, e.g. `"true"`, `"us"`).
- * Booleans/numbers are stringified; objects are JSON-encoded; nullish is dropped.
+ * (`zenrows_params` values are strings, e.g. `"true"`, `"us"`). Boolean params
+ * are canonicalized to "true"/"false"; other booleans/numbers are stringified;
+ * objects are JSON-encoded; nullish is dropped.
  */
 function normalizeParams(obj) {
     const out = {};
-    for (const [k, v] of Object.entries(obj)) {
-        if (v === undefined || v === null)
+    for (const [k, raw] of Object.entries(obj)) {
+        if (raw === undefined || raw === null)
             continue;
+        const v = BOOLEAN_PARAMS.has(k) ? canonicalBool(raw) : raw;
         if (typeof v === "string")
             out[k] = v;
         else if (typeof v === "boolean" || typeof v === "number")
@@ -153,7 +305,33 @@ function isAutoMode(v) {
     return v === "auto" || v === true;
 }
 function isTruthyFlag(v) {
-    return v === true || v === "true";
+    return canonicalBool(v) === "true";
+}
+/** Throw before submitting if a boolean param holds a value the API rejects. */
+function assertBooleanParams(params, where) {
+    for (const k of BOOLEAN_PARAMS) {
+        const v = params[k];
+        if (v !== undefined && v !== "true" && v !== "false") {
+            throw new ToolkitError({
+                code: "INVALID_USAGE",
+                message: `${k} must be true or false (got ${JSON.stringify(v)}) in ${where}.`,
+                likely_cause: `The Batch API accepts only true or false for ${k}; any other value fails the whole job.`,
+                next_action: `Set ${k} to true or false on that line, then retry.`,
+            });
+        }
+    }
+}
+/** Throw if a POST task would turn on a param the API refuses with POST (method_param_conflict). */
+function assertPostParams(params, where) {
+    const conflict = params.js_render === "true" ? "js_render" : params.json_response === "true" ? "json_response" : params.js_instructions ? "js_instructions" : undefined;
+    if (!conflict)
+        return;
+    throw new ToolkitError({
+        code: "INVALID_USAGE",
+        message: `A POST task cannot use ${conflict}: ${where}.`,
+        likely_cause: "POST tasks run without a browser, so the Batch API rejects js_render, js_instructions and json_response on them (method_param_conflict).",
+        next_action: "Drop that param from the line (or the job-level --js-render flag), or send the task as GET.",
+    });
 }
 /** Throw if proxy_country is set without premium_proxy and outside auto mode. */
 function assertProxyCountryPremium(params, where) {

package/dist/src/adapters/extract.js

@@ -1,17 +1,26 @@
 import { ToolkitError } from "../core/errors.js";
+import { log } from "../core/logger.js";
 import { runFetch } from "./protected-fetch.js";
-/** True when a parsed extraction result carries no usable structured data. */
-function isEmptyData(data) {
+/**
+ * True when a parsed extraction result carries no usable structured data: null,
+ * a blank string, or an array / object whose every value is itself empty. So
+ * `{"title": null, "items": [], "author": {"name": ""}}` is empty. Numbers and
+ * booleans are values, so `0` and `false` count as data.
+ */
+export function isEmptyData(data) {
     if (data === null || data === undefined)
         return true;
-    if (Array.isArray(data))
-        return data.length === 0;
-    if (typeof data === "object")
-        return Object.keys(data).length === 0;
     if (typeof data === "string")
         return data.trim() === "";
+    if (Array.isArray(data))
+        return data.every(isEmptyData);
+    if (typeof data === "object")
+        return Object.values(data).every(isEmptyData);
     return false;
 }
+/** The warning a caller that still passes `fallbackAutoparse: true` gets. */
+export const FALLBACK_AUTOPARSE_DEPRECATION = "fallbackAutoparse is deprecated and has no effect: Extract no longer falls back to Autoparse. " +
+    "On REQS007, prepare the domain with `zenrows extract prepare <domain> --url <page>` instead.";
 export async function runExtract(opts, config, policy, apiKey) {
     const method = opts.method ?? (opts.outputs ? "outputs" : opts.cssExtractor ? "css" : "extract");
     if (method === "css" && !opts.cssExtractor) {
@@ -30,18 +39,8 @@ export async function runExtract(opts, config, policy, apiKey) {
             next_action: `Pass one or more filters, e.g. --outputs emails,links (or --outputs '*').`,
         });
     }
-    if (method === "extract" && opts.fallbackAutoparse !== false) {
-        try {
-            return await runExtractOnce({ ...opts, method: "extract" }, config, policy, apiKey);
-        }
-        catch (err) {
-            if (err instanceof ToolkitError && err.code === "EXTRACT_DOMAIN_NOT_ENABLED") {
-                const outcome = await runExtractOnce({ ...opts, method: "autoparse" }, config, policy, apiKey);
-                return { ...outcome, fellBackToAutoparse: true };
-            }
-            throw err;
-        }
-    }
+    if (opts.fallbackAutoparse === true)
+        log.warn(FALLBACK_AUTOPARSE_DEPRECATION);
     return runExtractOnce({ ...opts, method }, config, policy, apiKey);
 }
 async function runExtractOnce(opts, config, policy, apiKey) {
@@ -101,7 +100,7 @@ function parseExtractBody(method, body, opts) {
         }
     }
     const empty = isEmptyData(data);
-    // ACT-1514: an empty result must not read as a silent success. Under
+    // An empty result must not read as a silent success. Under
     // --validate it's an error; otherwise the command surfaces a warning.
     if (opts.validate && empty) {
         throw new ToolkitError({