@zenrows/cli 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -9
- package/agent-plugin/skills/batch-jobs/SKILL.md +16 -3
- package/dist/src/adapters/batch.js +237 -59
- package/dist/src/cli/commands/batch.js +665 -126
- package/dist/src/cli/commands/crawl.js +1 -1
- package/dist/src/core/batch-api.js +398 -145
- package/dist/src/core/config.js +1 -1
- package/dist/src/core/errors.js +6 -0
- package/dist/src/core/policy.js +13 -6
- package/package.json +1 -1
- package/recipes/batch-estimate/RECIPE.md +6 -3
- package/skills/batch-jobs/SKILL.md +160 -43
- package/skills/crawl/SKILL.md +1 -1
- package/skills/trace-debug/SKILL.md +17 -2
- package/skills/zenrows/SKILL.md +3 -2
- package/templates/batch-jsonl-pipeline/README.md +10 -5
package/README.md
CHANGED
|
@@ -128,20 +128,75 @@ zenrows extract <url> --output markdown
|
|
|
128
128
|
|
|
129
129
|
Zenrows **Batch** (`https://async.api.zenrows.com/v1`) fans a
|
|
130
130
|
protected fetch/extract out over many URLs. The cloud subcommands work
|
|
131
|
-
with any active API key and
|
|
132
|
-
|
|
131
|
+
with any active API key and cost the same credits as Fetch: Batch is the
|
|
132
|
+
reliable path at scale, at Fetch's prices. Local spec validation + credit
|
|
133
|
+
estimation always work (no key needed).
|
|
133
134
|
|
|
134
135
|
```bash
|
|
135
|
-
zenrows batch estimate jobs.jsonl
|
|
136
|
-
zenrows batch create jobs.jsonl --
|
|
137
|
-
zenrows batch
|
|
138
|
-
zenrows batch
|
|
139
|
-
zenrows batch
|
|
136
|
+
zenrows batch estimate jobs.jsonl # validate the spec + estimate credits (local)
|
|
137
|
+
zenrows batch create jobs.jsonl --follow # submit a job and wait for it to finish
|
|
138
|
+
zenrows batch list # your jobs, newest first
|
|
139
|
+
zenrows batch status <job-id> # run status, progress stats, pause state
|
|
140
|
+
zenrows batch wait <job-id> --timeout 30m # wait for the run to end
|
|
141
|
+
zenrows batch results <job-id> --out results.jsonl # collect results (paginated)
|
|
142
|
+
zenrows batch content <job-id> <task-id> # one task's result body
|
|
143
|
+
zenrows batch pause <job-id> # pause the run (resume <job-id> continues it)
|
|
144
|
+
zenrows batch cancel <job-id> # stop an in-flight run
|
|
145
|
+
zenrows batch retry-failed <job-id> --include-pending # rerun every task that did not succeed
|
|
146
|
+
zenrows batch rerun <job-id> # rerun failed + pending tasks (--all: every task)
|
|
147
|
+
zenrows batch delete <job-id> --yes # delete the job, its runs and results
|
|
140
148
|
```
|
|
141
149
|
|
|
142
150
|
`jobs.jsonl` is one JSON object per line, each with a `url` (plus optional
|
|
143
151
|
per-line overrides like `js_render`, `premium_proxy`, `proxy_country`,
|
|
144
|
-
`extract
|
|
152
|
+
`extract`, nested `zenrows_params`, an `external_id`, string `metadata`, and
|
|
153
|
+
`"method": "POST"` with a `body` of up to 16 KiB). `estimate` and `create`
|
|
154
|
+
check every line against the API's limits before anything is sent: `url` up
|
|
155
|
+
to 2048 bytes, `external_id` up to 128 letters, digits, `.`, `_` or `-`,
|
|
156
|
+
and `metadata` up to 20 string entries (4 KiB). Validate and estimate locally
|
|
157
|
+
first, then submit with `create`. `estimate` takes `create`'s
|
|
158
|
+
`--manual` / `--js-render` / `--premium-proxy` / `--proxy-country` flags and
|
|
159
|
+
prices the same request, as a range (Adaptive Stealth Mode costs 1 to 25).
|
|
160
|
+
|
|
161
|
+
`create` also takes `--name`, `--external-id` and `--metadata key=value`
|
|
162
|
+
(repeatable) for the job itself. It sends an `Idempotency-Key` (generated, or
|
|
163
|
+
`--idempotency-key`) and retries network errors and 502/503/504 under it; an
|
|
164
|
+
outcome that stays unknown prints the command that repeats the request with
|
|
165
|
+
that key, which cannot create a second job.
|
|
166
|
+
|
|
167
|
+
The local policy caps a run at 10,000 URLs and 250,000 estimated credits by
|
|
168
|
+
default (an Adaptive Stealth Mode task counts 25), so a default run of up to
|
|
169
|
+
10,000 URLs passes in any mode. Raise them for larger jobs (the API takes up to
|
|
170
|
+
100,000 tasks per job): `zenrows policy set max_pages_per_run 100000` and
|
|
171
|
+
`zenrows policy set max_credits_per_run 2500000`. A workspace created by an
|
|
172
|
+
earlier version keeps its caps (1,000 and 10,000) in `policy.json`.
|
|
173
|
+
|
|
174
|
+
`wait`, `create --follow` and `rerun --follow` poll until the run ends, the run
|
|
175
|
+
is paused, or `--timeout` runs out (default 10 minutes; `30s`, `5m`, `1h`; a
|
|
176
|
+
bare number is milliseconds, as before). When the time runs out, or the run is
|
|
177
|
+
paused, the command exits 0 and prints the run with a `note` and the next
|
|
178
|
+
command; the job keeps running. Ctrl-C stops the wait, not the job, and exits
|
|
179
|
+
130. A run that ends `failed` exits 1: out of credits is
|
|
180
|
+
`POLICY_MAX_CREDITS_EXCEEDED`, an inactive subscription `AUTH_INVALID`, a key
|
|
181
|
+
cap `KEY_CREDIT_CAP_REACHED` (the same codes a refused request gets), anything
|
|
182
|
+
else `BATCH_FAILED`, each with the run's `failure_reason` as `server_code` and
|
|
183
|
+
the `job_id`. Then run the unfinished tasks with
|
|
184
|
+
`zenrows batch rerun <job-id> --status failed,pending,processing`. With no
|
|
185
|
+
flag, `rerun` reruns the `failed` and `pending` tasks, so successful tasks are
|
|
186
|
+
never billed again by surprise; `--status <list>` picks the statuses and
|
|
187
|
+
`--all` reruns every task (not with `--status`). A partial rerun works for 10
|
|
188
|
+
days after the run started; `--all` always does.
|
|
189
|
+
|
|
190
|
+
With `--json`, `create`, `status`, `wait`, `cancel`, `pause`, `resume` and
|
|
191
|
+
`rerun` print `{ok, job_id, run_id, status, stats, job}` (`jobId` is kept for
|
|
192
|
+
older scripts), plus `paused`, `ingest_status`, `accepted_tasks`, `rerun_of`,
|
|
193
|
+
`retried_tasks` and `inherited_tasks` when the API sends them. Results are
|
|
194
|
+
kept 14 days by default and then read `expired`; a `result_url` is valid 2
|
|
195
|
+
hours. Lifecycle errors get their own codes (`BATCH_RUN_IN_PROGRESS`,
|
|
196
|
+
`BATCH_NO_MATCHING_TASKS`, `BATCH_RERUN_WINDOW_EXPIRED`,
|
|
197
|
+
`BATCH_REQUEST_IN_FLIGHT`, `BATCH_RESULT_EXPIRED`, ...) and carry the API's code
|
|
198
|
+
as `server_code`. Schedules, webhooks, ZIP exports, CSV inputs and open jobs
|
|
199
|
+
are available through the Batch API and the SDKs, not this CLI.
|
|
145
200
|
|
|
146
201
|
**Extract in Batch.** Set `"extract": "auto"` on a line to run that URL through
|
|
147
202
|
Extract — structured data instead of raw HTML:
|
|
@@ -186,7 +241,7 @@ zenrows crawl stop <crawl-id> # stop a running crawl
|
|
|
186
241
|
follows links until `--max-items` or `--max-pages` stops it, or nothing is left
|
|
187
242
|
to open. `--max-items` and `--max-pages` cap the crawl (API default 100 each),
|
|
188
243
|
and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default
|
|
189
|
-
|
|
244
|
+
10000). `--output-format html` uses up the page budget: each kept page is one
|
|
190
245
|
more fetch, and `--max-pages` counts it. `--include-patterns` /
|
|
191
246
|
`--exclude-patterns` are repeatable substring filters on the URL. A crawl whose
|
|
192
247
|
pages held nothing new to follow ends `completed` with no results and exits 0.
|
|
@@ -7,7 +7,8 @@ version: 0.1.0
|
|
|
7
7
|
# Batch Jobs (MCP)
|
|
8
8
|
|
|
9
9
|
Process large workloads asynchronously. Batch is where Zenrows' high-scale
|
|
10
|
-
anti-bot advantage becomes obvious
|
|
10
|
+
anti-bot advantage becomes obvious: thousands to 100,000 URLs per job, at
|
|
11
|
+
Fetch's prices.
|
|
11
12
|
|
|
12
13
|
> Cloud Batch lives on `async.api.zenrows.com/v1`. A 403 maps to
|
|
13
14
|
> `BATCH_ACCESS_DENIED`. Do not fake a successful
|
|
@@ -33,10 +34,22 @@ Optional if exposed by the server: `batch_wait` to block until terminal status.
|
|
|
33
34
|
`autoparse`, `extract`, `external_id`, `metadata`). `extract: "auto"`
|
|
34
35
|
returns structured data instead of raw HTML, at the same credit cost as a
|
|
35
36
|
regular task; the result carries `html` and `parsed` keys.
|
|
36
|
-
3. Poll `batch_status` until terminal (`completed
|
|
37
|
-
|
|
37
|
+
3. Poll `batch_status` until terminal (`completed`, `stopped`, `failed` or
|
|
38
|
+
`deleted`). A paused run (`pause_state: paused`) is not terminal and picks
|
|
39
|
+
up no new task until resumed: report it instead of polling on.
|
|
40
|
+
4. Collect with `batch_results`. Results are kept 14 days by default (then a
|
|
41
|
+
row reads `expired`); a `result_url` is valid 2 hours.
|
|
38
42
|
5. `batch_cancel` if the user aborts or the run should stop early.
|
|
39
43
|
|
|
44
|
+
A run ends `failed` when an account problem stops it: `failure_reason`
|
|
45
|
+
`insufficient_credits` (add credits), `subscription_inactive` (fix it in
|
|
46
|
+
Billing) or `api_key_cap_reached` (the key's own cap; the account still has
|
|
47
|
+
credits). The tasks not yet run stay `pending`: once fixed, rerun them with
|
|
48
|
+
status `failed,pending,processing` rather than resubmitting the job.
|
|
49
|
+
|
|
50
|
+
Schedules, webhooks, ZIP exports and CSV inputs are available through the
|
|
51
|
+
Batch API and the SDKs.
|
|
52
|
+
|
|
40
53
|
## Rules
|
|
41
54
|
|
|
42
55
|
- **Never batch before validating** on a single page ([[extract]] /
|
|
@@ -3,14 +3,28 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Status: `available`. The cloud subcommands call the Batch API with an API key
|
|
5
5
|
* (see `core/batch-api.ts`); a 403 maps to BATCH_ACCESS_DENIED. This adapter owns
|
|
6
|
-
* the local, no-network pieces: validating a JSONL job spec
|
|
7
|
-
* cost, and mapping the validated spec to the Batch
|
|
6
|
+
* the local, no-network pieces: validating a JSONL job spec against the API's
|
|
7
|
+
* limits, estimating credit cost, and mapping the validated spec to the Batch
|
|
8
|
+
* API request body.
|
|
8
9
|
*/
|
|
9
10
|
import { existsSync, readFileSync } from "node:fs";
|
|
10
11
|
import { ToolkitError } from "../core/errors.js";
|
|
11
|
-
/** Per-task keys
|
|
12
|
-
const TASK_RESERVED = new Set(["url", "external_id", "metadata", "zenrows_params"]);
|
|
13
|
-
/**
|
|
12
|
+
/** Per-task keys sent at task level; everything else on a line is a scrape param. */
|
|
13
|
+
const TASK_RESERVED = new Set(["url", "external_id", "metadata", "zenrows_params", "method", "body"]);
|
|
14
|
+
/** The API's limits on a task and on job-level fields. */
|
|
15
|
+
export const LIMITS = {
|
|
16
|
+
urlLength: 2048,
|
|
17
|
+
externalIdLength: 128,
|
|
18
|
+
metadataEntries: 20,
|
|
19
|
+
metadataKeyLength: 64,
|
|
20
|
+
metadataValueLength: 1024,
|
|
21
|
+
metadataTotalBytes: 4 * 1024,
|
|
22
|
+
bodyBytes: 16 * 1024,
|
|
23
|
+
nameLength: 100,
|
|
24
|
+
};
|
|
25
|
+
/** `external_id` and metadata keys: letters, digits, `.`, `_` and `-`. */
|
|
26
|
+
const SAFE_ID = /^[A-Za-z0-9._-]+$/;
|
|
27
|
+
/** Parse + validate a JSONL job spec locally, against the API's per-task limits. */
|
|
14
28
|
export function validateJsonl(file) {
|
|
15
29
|
if (!existsSync(file)) {
|
|
16
30
|
throw new ToolkitError({
|
|
@@ -28,45 +42,171 @@ export function validateJsonl(file) {
|
|
|
28
42
|
if (!trimmed)
|
|
29
43
|
return;
|
|
30
44
|
result.totalLines++;
|
|
45
|
+
let obj;
|
|
31
46
|
try {
|
|
32
|
-
|
|
33
|
-
if (!obj.url || typeof obj.url !== "string") {
|
|
34
|
-
result.errors.push({ line: idx + 1, reason: "missing string `url`" });
|
|
35
|
-
return;
|
|
36
|
-
}
|
|
37
|
-
try {
|
|
38
|
-
new URL(obj.url);
|
|
39
|
-
}
|
|
40
|
-
catch {
|
|
41
|
-
result.errors.push({ line: idx + 1, reason: `invalid url: ${obj.url}` });
|
|
42
|
-
return;
|
|
43
|
-
}
|
|
44
|
-
result.validJobs++;
|
|
45
|
-
result.jobs.push(obj);
|
|
47
|
+
obj = JSON.parse(trimmed);
|
|
46
48
|
}
|
|
47
49
|
catch {
|
|
48
50
|
result.errors.push({ line: idx + 1, reason: "invalid JSON" });
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const reason = taskError(obj);
|
|
54
|
+
if (reason) {
|
|
55
|
+
result.errors.push({ line: idx + 1, reason });
|
|
56
|
+
return;
|
|
49
57
|
}
|
|
58
|
+
result.validJobs++;
|
|
59
|
+
result.jobs.push(obj);
|
|
50
60
|
});
|
|
51
61
|
return result;
|
|
52
62
|
}
|
|
63
|
+
/** Why one JSONL line would be rejected by the API, or null when it is valid. */
|
|
64
|
+
function taskError(obj) {
|
|
65
|
+
if (typeof obj !== "object" || obj === null || Array.isArray(obj))
|
|
66
|
+
return "a line must be a JSON object";
|
|
67
|
+
const job = obj;
|
|
68
|
+
if (!job.url || typeof job.url !== "string")
|
|
69
|
+
return "missing string `url`";
|
|
70
|
+
let u;
|
|
71
|
+
try {
|
|
72
|
+
u = new URL(job.url);
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return `invalid url: ${job.url}`;
|
|
76
|
+
}
|
|
77
|
+
if (u.protocol !== "http:" && u.protocol !== "https:")
|
|
78
|
+
return `url must be http or https: ${job.url}`;
|
|
79
|
+
// The API counts these limits in UTF-8 bytes, not characters.
|
|
80
|
+
if (Buffer.byteLength(job.url) > LIMITS.urlLength)
|
|
81
|
+
return `url is longer than ${LIMITS.urlLength} bytes`;
|
|
82
|
+
if (job.external_id !== undefined && job.external_id !== null) {
|
|
83
|
+
const e = externalIdError(job.external_id);
|
|
84
|
+
if (e)
|
|
85
|
+
return e;
|
|
86
|
+
}
|
|
87
|
+
const meta = metadataError(job.metadata);
|
|
88
|
+
if (meta)
|
|
89
|
+
return meta;
|
|
90
|
+
if (job.zenrows_params !== undefined && (typeof job.zenrows_params !== "object" || job.zenrows_params === null || Array.isArray(job.zenrows_params))) {
|
|
91
|
+
return "`zenrows_params` must be an object";
|
|
92
|
+
}
|
|
93
|
+
return methodBodyError(job.method, job.body);
|
|
94
|
+
}
|
|
95
|
+
/** Why an `external_id` is invalid, or null. Numbers are sent as their string form. */
|
|
96
|
+
export function externalIdError(v) {
|
|
97
|
+
if (typeof v !== "string" && typeof v !== "number")
|
|
98
|
+
return "`external_id` must be a string";
|
|
99
|
+
const s = String(v);
|
|
100
|
+
if (s.length === 0 || s.length > LIMITS.externalIdLength)
|
|
101
|
+
return `\`external_id\` must be 1 to ${LIMITS.externalIdLength} characters`;
|
|
102
|
+
if (!SAFE_ID.test(s))
|
|
103
|
+
return "`external_id` may only use letters, digits, `.`, `_` and `-`";
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
53
106
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
107
|
+
* Why `metadata` is invalid, or null: the API takes at most 20 string-to-string
|
|
108
|
+
* entries, keys of 1 to 64 safe characters, values up to 1024 bytes, 4 KiB
|
|
109
|
+
* in total. Any other shape fails the whole submit.
|
|
56
110
|
*/
|
|
57
|
-
export function
|
|
111
|
+
export function metadataError(metadata) {
|
|
112
|
+
if (metadata === undefined || metadata === null)
|
|
113
|
+
return null;
|
|
114
|
+
if (typeof metadata !== "object" || Array.isArray(metadata))
|
|
115
|
+
return "`metadata` must be an object of string values";
|
|
116
|
+
const entries = Object.entries(metadata);
|
|
117
|
+
if (entries.length > LIMITS.metadataEntries)
|
|
118
|
+
return `\`metadata\` has more than ${LIMITS.metadataEntries} entries`;
|
|
119
|
+
let total = 0;
|
|
120
|
+
for (const [k, v] of entries) {
|
|
121
|
+
if (k.length === 0 || k.length > LIMITS.metadataKeyLength || !SAFE_ID.test(k)) {
|
|
122
|
+
return `\`metadata\` key "${k}" must be 1 to ${LIMITS.metadataKeyLength} letters, digits, \`.\`, \`_\` or \`-\``;
|
|
123
|
+
}
|
|
124
|
+
if (typeof v !== "string")
|
|
125
|
+
return `\`metadata.${k}\` must be a string`;
|
|
126
|
+
if (Buffer.byteLength(v) > LIMITS.metadataValueLength)
|
|
127
|
+
return `\`metadata.${k}\` is longer than ${LIMITS.metadataValueLength} bytes`;
|
|
128
|
+
total += Buffer.byteLength(k) + Buffer.byteLength(v);
|
|
129
|
+
}
|
|
130
|
+
if (total > LIMITS.metadataTotalBytes)
|
|
131
|
+
return "`metadata` is larger than 4 KiB";
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
/** Why `method` / `body` are invalid, or null. Only GET and POST exist; a body needs POST. */
|
|
135
|
+
function methodBodyError(method, body) {
|
|
136
|
+
if (method !== undefined && method !== null) {
|
|
137
|
+
if (typeof method !== "string" || !/^(get|post)$/i.test(method))
|
|
138
|
+
return "`method` must be GET or POST";
|
|
139
|
+
}
|
|
140
|
+
if (body === undefined || body === null)
|
|
141
|
+
return null;
|
|
142
|
+
if (!isPost(method))
|
|
143
|
+
return "`body` needs `\"method\": \"POST\"`";
|
|
144
|
+
const bytes = typeof body === "string" ? Buffer.byteLength(body) : Buffer.byteLength(JSON.stringify(body));
|
|
145
|
+
if (bytes > LIMITS.bodyBytes)
|
|
146
|
+
return `\`body\` is larger than ${LIMITS.bodyBytes / 1024} KiB`;
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
function isPost(method) {
|
|
150
|
+
return typeof method === "string" && method.toUpperCase() === "POST";
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Scrape params the API reads as booleans. It accepts JSON `true`/`false` or the
|
|
154
|
+
* strings "true"/"false" in any case, and rejects anything else with a 400.
|
|
155
|
+
*/
|
|
156
|
+
const BOOLEAN_PARAMS = new Set(["js_render", "premium_proxy", "original_status", "json_response", "autoparse"]);
|
|
157
|
+
/** JSON `true` and "true" in any case become "true"; likewise "false". Any other value is unchanged. */
|
|
158
|
+
function canonicalBool(v) {
|
|
159
|
+
if (v === true)
|
|
160
|
+
return "true";
|
|
161
|
+
if (v === false)
|
|
162
|
+
return "false";
|
|
163
|
+
if (typeof v === "string") {
|
|
164
|
+
const s = v.trim().toLowerCase();
|
|
165
|
+
if (s === "true" || s === "false")
|
|
166
|
+
return s;
|
|
167
|
+
}
|
|
168
|
+
return v;
|
|
169
|
+
}
|
|
170
|
+
/** A task's own params: its nested `zenrows_params`, then its flat keys (later wins). */
|
|
171
|
+
function taskParamsOf(job) {
|
|
172
|
+
const flat = {};
|
|
173
|
+
for (const [k, v] of Object.entries(job)) {
|
|
174
|
+
if (!TASK_RESERVED.has(k))
|
|
175
|
+
flat[k] = v;
|
|
176
|
+
}
|
|
177
|
+
return { ...(job.zenrows_params ?? {}), ...flat };
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* The params a task actually runs with: the job-level params and the task's
|
|
181
|
+
* own, each canonicalized exactly as submitted, merged the way the API merges
|
|
182
|
+
* them (the task wins on a shared key). A task key set to null is not
|
|
183
|
+
* submitted, so the job's value applies.
|
|
184
|
+
*/
|
|
185
|
+
export function effectiveParams(job, jobParams = {}) {
|
|
186
|
+
return { ...normalizeParams(jobParams), ...normalizeParams(taskParamsOf(job)) };
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Local credit estimate from the rate card: basic 1, js_render 5,
|
|
190
|
+
* premium_proxy 10, both 25. An Adaptive Stealth Mode task (`mode=auto`)
|
|
191
|
+
* costs 1 to 25 depending on what each page needs, so it counts 1 in `min`
|
|
192
|
+
* and 25 in `credits` (the upper bound). Each task is priced on the params
|
|
193
|
+
* it is submitted with, job-level flags included.
|
|
194
|
+
*/
|
|
195
|
+
export function estimateCredits(jobs, jobParams = {}) {
|
|
196
|
+
let min = 0;
|
|
58
197
|
const perJob = jobs.map((j) => {
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
198
|
+
const p = effectiveParams(j, jobParams);
|
|
199
|
+
if (p.mode === "auto") {
|
|
200
|
+
min += 1;
|
|
62
201
|
return 25;
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
202
|
+
}
|
|
203
|
+
const js = p.js_render === "true";
|
|
204
|
+
const proxy = p.premium_proxy === "true";
|
|
205
|
+
const cost = js && proxy ? 25 : proxy ? 10 : js ? 5 : 1;
|
|
206
|
+
min += cost;
|
|
207
|
+
return cost;
|
|
68
208
|
});
|
|
69
|
-
return { credits: perJob.reduce((a, b) => a + b, 0), perJob };
|
|
209
|
+
return { credits: perJob.reduce((a, b) => a + b, 0), min, perJob };
|
|
70
210
|
}
|
|
71
211
|
/**
|
|
72
212
|
* Adaptive Stealth Mode (mode=auto) by default, like `fetch` and `extract`: returns
|
|
@@ -74,14 +214,17 @@ export function estimateCredits(jobs) {
|
|
|
74
214
|
* `premium_proxy` or set `mode` itself, and none at all when the job-level params
|
|
75
215
|
* force either flag. Applied per task rather than job-level because the API rejects
|
|
76
216
|
* a task that combines a job-level mode=auto with its own js_render or
|
|
77
|
-
* premium_proxy (REQS004).
|
|
217
|
+
* premium_proxy (REQS004). A POST task is left alone: Adaptive Stealth Mode may
|
|
218
|
+
* turn on JavaScript rendering, which a POST cannot use.
|
|
78
219
|
*/
|
|
79
220
|
export function withAdaptiveStealth(jobs, jobParams = {}) {
|
|
80
221
|
const forced = (p) => isTruthyFlag(p.js_render) || isTruthyFlag(p.premium_proxy);
|
|
81
222
|
if (forced(jobParams) || jobParams.mode != null)
|
|
82
223
|
return jobs;
|
|
83
224
|
return jobs.map((job) => {
|
|
84
|
-
|
|
225
|
+
if (isPost(job.method))
|
|
226
|
+
return job;
|
|
227
|
+
const own = taskParamsOf(job);
|
|
85
228
|
if (forced(own) || own.mode != null)
|
|
86
229
|
return job;
|
|
87
230
|
return { ...job, mode: "auto" };
|
|
@@ -92,38 +235,45 @@ export function withAdaptiveStealth(jobs, jobParams = {}) {
|
|
|
92
235
|
* The flat per-line options (`js_render`, `premium_proxy`, `proxy_country`,
|
|
93
236
|
* `autoparse`, `mode`, …) plus any nested `zenrows_params` become that task's
|
|
94
237
|
* `zenrows_params`; `jobParams` (from CLI flags) becomes the job-level
|
|
95
|
-
* `zenrows_params`. `url
|
|
238
|
+
* `zenrows_params`. `url`, `external_id`, `metadata`, `method` and `body` are
|
|
239
|
+
* task fields and map straight through.
|
|
96
240
|
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
* `protected-fetch.ts::validateAutoManual
|
|
241
|
+
* Checked here, before submitting, on the effective (job + task) params:
|
|
242
|
+
* boolean params hold true or false; `proxy_country` needs `premium_proxy`
|
|
243
|
+
* unless mode=auto (PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM, mirroring
|
|
244
|
+
* `protected-fetch.ts::validateAutoManual`); a POST task does not turn on
|
|
245
|
+
* `js_render`, `js_instructions` or `json_response`, which the API rejects.
|
|
101
246
|
*/
|
|
102
|
-
export function toJobBody(jobs, jobParams = {}) {
|
|
103
|
-
// No separate job-level check: a job-level proxy_country is valid for each task
|
|
104
|
-
// in Adaptive Stealth Mode, and the per-task check below sees job ⊕ task params.
|
|
247
|
+
export function toJobBody(jobs, jobParams = {}, fields = {}) {
|
|
105
248
|
const tasks = jobs.map((job, idx) => {
|
|
106
|
-
const { url, external_id, metadata
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
}
|
|
112
|
-
const taskParams = { ...(zenrows_params ?? {}), ...flat };
|
|
113
|
-
// Validate on the effective (job ⊕ task) params so a job-level premium_proxy
|
|
114
|
-
// covers a per-task proxy_country, matching how the API merges them.
|
|
115
|
-
assertProxyCountryPremium({ ...jobParams, ...taskParams }, `line ${idx + 1} (${url})`);
|
|
249
|
+
const { url, external_id, metadata } = job;
|
|
250
|
+
const where = `line ${idx + 1} (${url})`;
|
|
251
|
+
const effective = effectiveParams(job, jobParams);
|
|
252
|
+
assertBooleanParams(effective, where);
|
|
253
|
+
assertProxyCountryPremium(effective, where);
|
|
116
254
|
const task = { url };
|
|
117
|
-
if (external_id !== undefined)
|
|
255
|
+
if (external_id !== undefined && external_id !== null)
|
|
118
256
|
task.external_id = String(external_id);
|
|
119
|
-
if (metadata !== undefined)
|
|
257
|
+
if (metadata !== undefined && metadata !== null)
|
|
120
258
|
task.metadata = metadata;
|
|
121
|
-
|
|
259
|
+
if (isPost(job.method)) {
|
|
260
|
+
assertPostParams(effective, where);
|
|
261
|
+
task.method = "POST";
|
|
262
|
+
if (job.body !== undefined && job.body !== null)
|
|
263
|
+
task.body = job.body;
|
|
264
|
+
}
|
|
265
|
+
const zp = normalizeParams(taskParamsOf(job));
|
|
122
266
|
if (Object.keys(zp).length)
|
|
123
267
|
task.zenrows_params = zp;
|
|
124
268
|
return task;
|
|
125
269
|
});
|
|
126
270
|
const body = { type: "regular", status: "closed", tasks };
|
|
271
|
+
if (fields.name)
|
|
272
|
+
body.name = fields.name;
|
|
273
|
+
if (fields.external_id)
|
|
274
|
+
body.external_id = fields.external_id;
|
|
275
|
+
if (fields.metadata && Object.keys(fields.metadata).length)
|
|
276
|
+
body.metadata = fields.metadata;
|
|
127
277
|
const jobZp = normalizeParams(jobParams);
|
|
128
278
|
if (Object.keys(jobZp).length)
|
|
129
279
|
body.zenrows_params = jobZp;
|
|
@@ -131,14 +281,16 @@ export function toJobBody(jobs, jobParams = {}) {
|
|
|
131
281
|
}
|
|
132
282
|
/**
|
|
133
283
|
* Coerce a flat options object into the string-valued map the Batch API expects
|
|
134
|
-
* (
|
|
135
|
-
*
|
|
284
|
+
* (`zenrows_params` values are strings, e.g. `"true"`, `"us"`). Boolean params
|
|
285
|
+
* are canonicalized to "true"/"false"; other booleans/numbers are stringified;
|
|
286
|
+
* objects are JSON-encoded; nullish is dropped.
|
|
136
287
|
*/
|
|
137
288
|
function normalizeParams(obj) {
|
|
138
289
|
const out = {};
|
|
139
|
-
for (const [k,
|
|
140
|
-
if (
|
|
290
|
+
for (const [k, raw] of Object.entries(obj)) {
|
|
291
|
+
if (raw === undefined || raw === null)
|
|
141
292
|
continue;
|
|
293
|
+
const v = BOOLEAN_PARAMS.has(k) ? canonicalBool(raw) : raw;
|
|
142
294
|
if (typeof v === "string")
|
|
143
295
|
out[k] = v;
|
|
144
296
|
else if (typeof v === "boolean" || typeof v === "number")
|
|
@@ -153,7 +305,33 @@ function isAutoMode(v) {
|
|
|
153
305
|
return v === "auto" || v === true;
|
|
154
306
|
}
|
|
155
307
|
function isTruthyFlag(v) {
|
|
156
|
-
return v ===
|
|
308
|
+
return canonicalBool(v) === "true";
|
|
309
|
+
}
|
|
310
|
+
/** Throw before submitting if a boolean param holds a value the API rejects. */
|
|
311
|
+
function assertBooleanParams(params, where) {
|
|
312
|
+
for (const k of BOOLEAN_PARAMS) {
|
|
313
|
+
const v = params[k];
|
|
314
|
+
if (v !== undefined && v !== "true" && v !== "false") {
|
|
315
|
+
throw new ToolkitError({
|
|
316
|
+
code: "INVALID_USAGE",
|
|
317
|
+
message: `${k} must be true or false (got ${JSON.stringify(v)}) in ${where}.`,
|
|
318
|
+
likely_cause: `The Batch API accepts only true or false for ${k}; any other value fails the whole job.`,
|
|
319
|
+
next_action: `Set ${k} to true or false on that line, then retry.`,
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
/** Throw if a POST task would turn on a param the API refuses with POST (method_param_conflict). */
|
|
325
|
+
function assertPostParams(params, where) {
|
|
326
|
+
const conflict = params.js_render === "true" ? "js_render" : params.json_response === "true" ? "json_response" : params.js_instructions ? "js_instructions" : undefined;
|
|
327
|
+
if (!conflict)
|
|
328
|
+
return;
|
|
329
|
+
throw new ToolkitError({
|
|
330
|
+
code: "INVALID_USAGE",
|
|
331
|
+
message: `A POST task cannot use ${conflict}: ${where}.`,
|
|
332
|
+
likely_cause: "POST tasks run without a browser, so the Batch API rejects js_render, js_instructions and json_response on them (method_param_conflict).",
|
|
333
|
+
next_action: "Drop that param from the line (or the job-level --js-render flag), or send the task as GET.",
|
|
334
|
+
});
|
|
157
335
|
}
|
|
158
336
|
/** Throw if proxy_country is set without premium_proxy and outside auto mode. */
|
|
159
337
|
function assertProxyCountryPremium(params, where) {
|