@zenrows/cli 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -12
- package/agent-plugin/README.md +1 -1
- package/agent-plugin/skills/batch-jobs/SKILL.md +16 -3
- package/agent-plugin/skills/cost-control/SKILL.md +2 -2
- package/agent-plugin/skills/extract/SKILL.md +49 -9
- package/agent-plugin/skills/protected-fetch/SKILL.md +1 -1
- package/agent-plugin/skills/zenrows/SKILL.md +3 -2
- package/dist/src/adapters/batch.js +237 -59
- package/dist/src/adapters/extract.js +18 -19
- package/dist/src/cli/commands/batch.js +665 -126
- package/dist/src/cli/commands/crawl.js +1 -1
- package/dist/src/cli/commands/extract-prepare.js +168 -0
- package/dist/src/cli/commands/extract.js +21 -13
- package/dist/src/core/batch-api.js +398 -145
- package/dist/src/core/config.js +1 -1
- package/dist/src/core/errors.js +6 -0
- package/dist/src/core/http.js +78 -20
- package/dist/src/core/policy.js +13 -6
- package/dist/src/core/prepared-domains-api.js +289 -0
- package/evals/extract-smoke/README.md +1 -1
- package/package.json +1 -1
- package/recipes/batch-estimate/RECIPE.md +6 -3
- package/registry/capabilities.json +3 -3
- package/registry/skills.json +2 -2
- package/registry/templates.json +1 -1
- package/skills/batch-jobs/SKILL.md +160 -43
- package/skills/crawl/SKILL.md +1 -1
- package/skills/extract/SKILL.md +50 -6
- package/skills/trace-debug/SKILL.md +17 -2
- package/skills/zenrows/SKILL.md +9 -5
- package/templates/batch-jsonl-pipeline/README.md +10 -5
- package/templates/extract-pricing-pages/README.md +3 -5
- package/workflows/competitor-intelligence/WORKFLOW.md +2 -2
|
@@ -1,52 +1,89 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `zenrows batch`: Batch API (status: available).
|
|
3
3
|
*
|
|
4
|
-
* Local (no key, always works): `estimate
|
|
5
|
-
*
|
|
6
|
-
* `
|
|
7
|
-
* BATCH_ACCESS_DENIED.
|
|
4
|
+
* Local (no key, always works): `estimate` validates a JSONL spec and prices
|
|
5
|
+
* it. Cloud (needs a key): `create`, `list`, `status`, `wait`, `results`,
|
|
6
|
+
* `content`, `cancel`, `pause`, `resume`, `rerun`, `retry-failed`, `delete`.
|
|
7
|
+
* A 403 maps to BATCH_ACCESS_DENIED. Schedules, webhooks, exports, CSV inputs,
|
|
8
|
+
* open jobs, runs and task history are API/SDK-only.
|
|
8
9
|
*/
|
|
9
|
-
import { statSync } from "node:fs";
|
|
10
10
|
import { join } from "node:path";
|
|
11
11
|
import { log, ANSI, c } from "../../core/logger.js";
|
|
12
|
-
import { estimateCredits, toJobBody, validateJsonl, withAdaptiveStealth } from "../../adapters/batch.js";
|
|
12
|
+
import { estimateCredits, externalIdError, LIMITS, metadataError, toJobBody, validateJsonl, withAdaptiveStealth, } from "../../adapters/batch.js";
|
|
13
13
|
import { loadConfig } from "../../core/config.js";
|
|
14
14
|
import { requireApiKey } from "../../core/auth.js";
|
|
15
15
|
import { ensureApiKey } from "../../core/ensure-key.js";
|
|
16
16
|
import { assertUsable } from "../../core/capabilities.js";
|
|
17
17
|
import { assertDomainAllowed, assertWithinLimits, loadPolicy } from "../../core/policy.js";
|
|
18
18
|
import { newRunId, writeRun } from "../../core/artifacts.js";
|
|
19
|
-
import { createJob, downloadResults, getJob, listResults, rerunJob, stopJob, waitForJob } from "../../core/batch-api.js";
|
|
19
|
+
import { RERUN_STATUSES, assertIdempotencyKey, createJob, deleteJob, downloadResults, getJob, getRun, getTaskContent, isPaused, isTerminal, listJobs, listResults, newIdempotencyKey, pauseJob, rerunJob, resumeJob, stopJob, subscriptionInactive, waitForJob, } from "../../core/batch-api.js";
|
|
20
|
+
import { readAccount } from "../../core/agent-account.js";
|
|
20
21
|
import { asNumber, asString, parse } from "../command.js";
|
|
21
|
-
import { ToolkitError, isKeyCapReached, keyCapReached } from "../../core/errors.js";
|
|
22
|
+
import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.js";
|
|
22
23
|
import { printError, writeOut } from "../output.js";
|
|
23
|
-
import { refreshClaimAdvice } from "../../core/claim-link.js";
|
|
24
|
+
import { refreshClaimAdvice, storedClaimHint } from "../../core/claim-link.js";
|
|
25
|
+
/** What `rerun` selects with no --status: the unfinished tasks, never the successful ones. */
|
|
26
|
+
const DEFAULT_RERUN = "failed,pending";
|
|
27
|
+
/** The task statuses that recover every task a run did not finish successfully. */
|
|
28
|
+
const UNFINISHED = "failed,pending,processing";
|
|
24
29
|
export const batch = {
|
|
25
30
|
name: "batch",
|
|
26
31
|
summary: "Run JSONL batch jobs on Zenrows Batch.",
|
|
27
|
-
usage: "zenrows batch <estimate
|
|
32
|
+
usage: "zenrows batch <estimate <file.jsonl>|create <file.jsonl>|list|status <id>|wait <id>|results <id>|content <id> <task_id>|cancel <id>|pause <id>|resume <id>|rerun <id>|retry-failed <id>|delete <id>>",
|
|
28
33
|
help: [
|
|
29
34
|
"Local (no key):",
|
|
30
|
-
" estimate <file.jsonl> [
|
|
35
|
+
" estimate <file.jsonl> [flags] validate the spec and price it (credits: min and upper bound);",
|
|
36
|
+
" takes the same --manual / --js-render / --premium-proxy /",
|
|
37
|
+
" --proxy-country flags as create, so both price the same request",
|
|
31
38
|
"Cloud (needs a key):",
|
|
32
39
|
" create <file.jsonl> [flags] validate, then submit the job",
|
|
33
|
-
" (default) Adaptive Stealth Mode (mode=auto) on every task that",
|
|
34
|
-
" doesn't force js_render / premium_proxy itself",
|
|
40
|
+
" (default) Adaptive Stealth Mode (mode=auto) on every GET task that",
|
|
41
|
+
" doesn't force js_render / premium_proxy or set mode itself",
|
|
35
42
|
" --manual no Adaptive Stealth Mode; plain requests unless flags below",
|
|
36
43
|
" --js-render job-level: force JavaScript rendering (turns auto off)",
|
|
37
44
|
" --premium-proxy job-level: force residential IPs (turns auto off)",
|
|
38
45
|
" --proxy-country <cc> job-level: geo-target (auto mode, or with --premium-proxy)",
|
|
39
46
|
" --output <fmt> job-level response_type (markdown|plaintext|pdf|html)",
|
|
40
|
-
" --
|
|
47
|
+
" --name <name> job name (up to 100 characters)",
|
|
48
|
+
" --external-id <id> job-level external id (letters, digits, . _ -; up to 128)",
|
|
49
|
+
" --metadata <key=value> job-level metadata entry (repeatable, up to 20)",
|
|
50
|
+
" --idempotency-key <key> so a repeated create starts no second job (one is generated",
|
|
51
|
+
" otherwise; the API keeps it 24 hours)",
|
|
52
|
+
" --follow wait until the run ends (alias: --wait; Ctrl-C stops the",
|
|
53
|
+
" wait, not the job)",
|
|
54
|
+
" --timeout <duration> with --follow: stop waiting after this long (10s, 5m, 1h;",
|
|
55
|
+
" a bare number is milliseconds; default 10m)",
|
|
41
56
|
" --no-signup do not auto-create a Free plan account if no key exists",
|
|
42
|
-
" status
|
|
43
|
-
"
|
|
57
|
+
" list [--status open|closed] [--type regular|scheduled] [--limit n] [--cursor c]",
|
|
58
|
+
" your jobs, newest first (limit up to 100)",
|
|
59
|
+
" status <id> show the job's latest run: status, stats, pause state",
|
|
60
|
+
" wait <id> [--timeout <duration>] poll until the run ends, is paused, or the time runs out",
|
|
61
|
+
" (default 10m); a run still going then is printed, not stopped",
|
|
62
|
+
" results <id> [--status s] list results (successful|failed|processing|all); partial",
|
|
63
|
+
" while the run is in progress",
|
|
64
|
+
" --run <run_id> read an earlier run instead of the latest",
|
|
44
65
|
" --out <file> write results as JSONL instead of printing",
|
|
45
66
|
" --download <dir> fetch each result body into <dir> (+ _manifest.jsonl)",
|
|
46
|
-
"
|
|
47
|
-
"
|
|
48
|
-
"
|
|
49
|
-
"
|
|
67
|
+
" content <id> <task_id> [--run <run_id>] [--out <file>]",
|
|
68
|
+
" print or save one task's result body",
|
|
69
|
+
" cancel <id> stop the in-flight run (an ended run answers as it ended)",
|
|
70
|
+
" pause <id> / resume <id> pause the in-flight run, or resume it",
|
|
71
|
+
" rerun <id> start a new run of the failed and pending tasks",
|
|
72
|
+
" (successful tasks carry over and are not billed again)",
|
|
73
|
+
" --status <list> rerun only tasks with these statuses",
|
|
74
|
+
" (pending,processing,failed,successful)",
|
|
75
|
+
" --all rerun every task (a full rerun; not with --status)",
|
|
76
|
+
" rerun also takes --idempotency-key, --follow and --timeout",
|
|
77
|
+
" retry-failed <id> rerun only the failed tasks (rerun --status failed); takes",
|
|
78
|
+
" --idempotency-key, --follow and --timeout like rerun",
|
|
79
|
+
" --include-pending also the tasks a stop or a credit failure left unfinished",
|
|
80
|
+
" (rerun --status failed,pending,processing)",
|
|
81
|
+
" delete <id> --yes delete the job, its runs and stored results; frees its slot",
|
|
82
|
+
" --json print structured output: create/status/wait/cancel/pause/",
|
|
83
|
+
" resume/rerun print {ok, job_id, run_id, status, stats, job};",
|
|
84
|
+
" a wait that runs out or meets a paused run adds note",
|
|
85
|
+
"Schedules, webhooks, exports, CSV inputs and open jobs are available through the Batch API",
|
|
86
|
+
"and the SDKs: https://docs.zenrows.com",
|
|
50
87
|
].join("\n"),
|
|
51
88
|
async run(argv, ctx) {
|
|
52
89
|
const [sub, ...rest] = argv;
|
|
@@ -56,51 +93,100 @@ export const batch = {
|
|
|
56
93
|
return estimateCmd(rest, ctx);
|
|
57
94
|
case "create":
|
|
58
95
|
return await createCmd(rest, ctx);
|
|
96
|
+
case "list":
|
|
97
|
+
return await listCmd(rest, ctx);
|
|
59
98
|
case "status":
|
|
60
99
|
return await statusCmd(rest, ctx);
|
|
100
|
+
case "wait":
|
|
101
|
+
return await waitCmd(rest, ctx);
|
|
61
102
|
case "results":
|
|
62
103
|
return await resultsCmd(rest, ctx);
|
|
104
|
+
case "content":
|
|
105
|
+
return await contentCmd(rest, ctx);
|
|
63
106
|
case "cancel":
|
|
107
|
+
case "stop":
|
|
64
108
|
return await cancelCmd(rest, ctx);
|
|
65
|
-
case "
|
|
66
|
-
return await
|
|
109
|
+
case "pause":
|
|
110
|
+
return await pauseCmd(rest, ctx, "pause");
|
|
111
|
+
case "resume":
|
|
112
|
+
return await pauseCmd(rest, ctx, "resume");
|
|
113
|
+
case "rerun":
|
|
114
|
+
return await rerunCmd(rest, ctx, "rerun");
|
|
67
115
|
case "retry-failed":
|
|
68
|
-
return await
|
|
116
|
+
return await rerunCmd(rest, ctx, "retry-failed");
|
|
117
|
+
case "delete":
|
|
118
|
+
return await deleteCmd(rest, ctx);
|
|
69
119
|
default:
|
|
70
120
|
throw new ToolkitError({
|
|
71
121
|
code: "INVALID_USAGE",
|
|
72
122
|
message: `Unknown batch subcommand: ${sub ?? "(none)"}`,
|
|
73
123
|
likely_cause: "Subcommand not recognized.",
|
|
74
|
-
next_action: "Use estimate | create <file.jsonl> | status <id> | results <id> | cancel <id> |
|
|
124
|
+
next_action: "Use estimate | create <file.jsonl> | list | status <id> | wait <id> | results <id> | content <id> <task_id> | cancel <id> | pause <id> | resume <id> | rerun <id> | retry-failed <id> | delete <id>.",
|
|
125
|
+
suggested_commands: ["zenrows batch --help"],
|
|
75
126
|
});
|
|
76
127
|
}
|
|
77
128
|
}
|
|
78
129
|
catch (caught) {
|
|
79
130
|
// A quota error's claim advice is built from local state; check it is still current.
|
|
80
|
-
const err = await
|
|
131
|
+
const err = await refreshQuotaAdvice(caught);
|
|
81
132
|
printError(err, ctx.json);
|
|
82
133
|
return 1;
|
|
83
134
|
}
|
|
84
135
|
},
|
|
85
136
|
};
|
|
137
|
+
/** Flags that shape the request each task sends; shared by `estimate` and `create`. */
|
|
138
|
+
const PARAM_FLAGS = {
|
|
139
|
+
"js-render": { type: "boolean" },
|
|
140
|
+
"premium-proxy": { type: "boolean" },
|
|
141
|
+
"proxy-country": { type: "string" },
|
|
142
|
+
manual: { type: "boolean" },
|
|
143
|
+
output: { type: "string" },
|
|
144
|
+
};
|
|
145
|
+
/** Job-level `zenrows_params` from the CLI flags. */
|
|
146
|
+
function jobParamsOf(values) {
|
|
147
|
+
const jobParams = {};
|
|
148
|
+
if (values["js-render"] === true)
|
|
149
|
+
jobParams.js_render = true;
|
|
150
|
+
if (values["premium-proxy"] === true)
|
|
151
|
+
jobParams.premium_proxy = true;
|
|
152
|
+
const proxyCountry = asString(values["proxy-country"]);
|
|
153
|
+
if (proxyCountry)
|
|
154
|
+
jobParams.proxy_country = proxyCountry;
|
|
155
|
+
const responseType = normalizeOutput(asString(values.output));
|
|
156
|
+
if (responseType)
|
|
157
|
+
jobParams.response_type = responseType;
|
|
158
|
+
return jobParams;
|
|
159
|
+
}
|
|
86
160
|
/** Jobs as they will be sent: Adaptive Stealth Mode applied unless manual or disabled in config. */
|
|
87
161
|
function effectiveJobs(jobs, jobParams, manual) {
|
|
88
162
|
return manual || loadConfig().defaultMode !== "auto" ? jobs : withAdaptiveStealth(jobs, jobParams);
|
|
89
163
|
}
|
|
90
164
|
function estimateCmd(rest, ctx) {
|
|
91
|
-
const { values, positionals } = parse(rest, {
|
|
165
|
+
const { values, positionals } = parse(rest, { ...PARAM_FLAGS, json: { type: "boolean" } });
|
|
166
|
+
const json = ctx.json || values.json === true;
|
|
92
167
|
const file = positionals[0];
|
|
93
168
|
if (!file)
|
|
94
169
|
throw needFile();
|
|
95
170
|
const v = validateJsonl(file);
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
|
|
171
|
+
const jobParams = jobParamsOf(values);
|
|
172
|
+
const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true);
|
|
173
|
+
// Price exactly what create would submit, and catch what it would refuse.
|
|
174
|
+
if (!v.errors.length)
|
|
175
|
+
toJobBody(jobs, jobParams);
|
|
176
|
+
const est = estimateCredits(jobs, jobParams);
|
|
177
|
+
if (json) {
|
|
178
|
+
log.out(JSON.stringify({
|
|
179
|
+
ok: v.errors.length === 0,
|
|
180
|
+
...v,
|
|
181
|
+
estimatedCredits: est.credits,
|
|
182
|
+
estimated_credits: est.credits,
|
|
183
|
+
estimated_credits_min: est.min,
|
|
184
|
+
}, null, 2));
|
|
99
185
|
}
|
|
100
186
|
else {
|
|
101
187
|
log.info(c(ANSI.bold, `Job spec: ${file}`));
|
|
102
188
|
log.info(`valid jobs: ${v.validJobs}/${v.totalLines}`);
|
|
103
|
-
log.info(`estimated credits
|
|
189
|
+
log.info(`estimated credits: ${est.min} to ${est.credits} (1 basic / 5 js / 10 proxy / 25 both; auto costs 1 to 25)`);
|
|
104
190
|
if (v.errors.length) {
|
|
105
191
|
log.warn(`${v.errors.length} invalid line(s):`);
|
|
106
192
|
v.errors.slice(0, 10).forEach((e) => log.dim(` line ${e.line}: ${e.reason}`));
|
|
@@ -111,15 +197,51 @@ function estimateCmd(rest, ctx) {
|
|
|
111
197
|
}
|
|
112
198
|
return v.errors.length ? 1 : 0;
|
|
113
199
|
}
|
|
200
|
+
/** `--name`, `--external-id` and `--metadata key=value`, checked against the API's limits. */
|
|
201
|
+
function jobFieldsOf(values) {
|
|
202
|
+
const fields = {};
|
|
203
|
+
const name = asString(values.name);
|
|
204
|
+
if (name !== undefined) {
|
|
205
|
+
// The API counts the name in characters (code points), not UTF-16 units.
|
|
206
|
+
if (!name.trim() || [...name].length > LIMITS.nameLength) {
|
|
207
|
+
throw usageError(`Invalid --name: it must be 1 to ${LIMITS.nameLength} characters.`, "Pass a shorter name.");
|
|
208
|
+
}
|
|
209
|
+
fields.name = name;
|
|
210
|
+
}
|
|
211
|
+
const externalId = asString(values["external-id"]);
|
|
212
|
+
if (externalId !== undefined) {
|
|
213
|
+
const e = externalIdError(externalId);
|
|
214
|
+
if (e)
|
|
215
|
+
throw usageError(`Invalid --external-id: ${e}.`, "Pass e.g. --external-id nightly-2026-10-16.");
|
|
216
|
+
fields.external_id = externalId;
|
|
217
|
+
}
|
|
218
|
+
const pairs = Array.isArray(values.metadata) ? values.metadata : [];
|
|
219
|
+
if (pairs.length) {
|
|
220
|
+
const metadata = {};
|
|
221
|
+
for (const p of pairs) {
|
|
222
|
+
const s = String(p);
|
|
223
|
+
const eq = s.indexOf("=");
|
|
224
|
+
if (eq <= 0)
|
|
225
|
+
throw usageError(`Invalid --metadata '${s}': use key=value.`, "Pass e.g. --metadata source=catalog.");
|
|
226
|
+
metadata[s.slice(0, eq)] = s.slice(eq + 1);
|
|
227
|
+
}
|
|
228
|
+
const e = metadataError(metadata);
|
|
229
|
+
if (e)
|
|
230
|
+
throw usageError(`Invalid --metadata: ${e}.`, "Fix the entry, then retry.");
|
|
231
|
+
fields.metadata = metadata;
|
|
232
|
+
}
|
|
233
|
+
return fields;
|
|
234
|
+
}
|
|
114
235
|
async function createCmd(rest, ctx) {
|
|
115
236
|
const { values, positionals } = parse(rest, {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
"
|
|
119
|
-
|
|
120
|
-
|
|
237
|
+
...PARAM_FLAGS,
|
|
238
|
+
name: { type: "string" },
|
|
239
|
+
"external-id": { type: "string" },
|
|
240
|
+
metadata: { type: "string", multiple: true },
|
|
241
|
+
"idempotency-key": { type: "string" },
|
|
121
242
|
follow: { type: "boolean" },
|
|
122
243
|
wait: { type: "boolean" }, // back-compat alias for --follow
|
|
244
|
+
timeout: { type: "string" },
|
|
123
245
|
"no-signup": { type: "boolean" },
|
|
124
246
|
json: { type: "boolean" },
|
|
125
247
|
});
|
|
@@ -130,55 +252,57 @@ async function createCmd(rest, ctx) {
|
|
|
130
252
|
const file = positionals[0];
|
|
131
253
|
if (!file)
|
|
132
254
|
throw needFile();
|
|
133
|
-
|
|
255
|
+
const waitMs = timeoutFlag(values.timeout);
|
|
256
|
+
const fields = jobFieldsOf(values);
|
|
257
|
+
const givenKey = asString(values["idempotency-key"]);
|
|
258
|
+
if (givenKey !== undefined)
|
|
259
|
+
assertIdempotencyKey(givenKey);
|
|
260
|
+
// Capability gate first: a disabled backend never gets a spec read.
|
|
134
261
|
assertUsable("batch");
|
|
135
262
|
const v = validateJsonl(file);
|
|
136
263
|
if (v.errors.length) {
|
|
137
|
-
log.warn(`${v.errors.length} invalid line(s)
|
|
264
|
+
log.warn(`${v.errors.length} invalid line(s), fix them before submitting:`);
|
|
138
265
|
v.errors.slice(0, 10).forEach((e) => log.dim(` line ${e.line}: ${e.reason}`));
|
|
139
266
|
throw new ToolkitError({
|
|
140
267
|
code: "INVALID_USAGE",
|
|
141
268
|
message: `Job spec has ${v.errors.length} invalid line(s).`,
|
|
142
|
-
likely_cause:
|
|
269
|
+
likely_cause: `One or more JSONL lines are not valid JSON or break the API's task limits (first: line ${v.errors[0].line}: ${v.errors[0].reason}).`,
|
|
143
270
|
next_action: "Fix the flagged lines (see `zenrows batch estimate`), then retry.",
|
|
144
271
|
suggested_commands: [`zenrows batch estimate ${file}`],
|
|
145
272
|
});
|
|
146
273
|
}
|
|
147
|
-
// Pre-flight governance
|
|
274
|
+
// Pre-flight governance, all local, before any network call:
|
|
148
275
|
// 1. every task URL must pass the allow/deny domain policy,
|
|
149
|
-
// 2. the run must fit the
|
|
150
|
-
|
|
151
|
-
const jobParams = {};
|
|
152
|
-
if (values["js-render"] === true)
|
|
153
|
-
jobParams.js_render = true;
|
|
154
|
-
if (values["premium-proxy"] === true)
|
|
155
|
-
jobParams.premium_proxy = true;
|
|
156
|
-
const proxyCountry = asString(values["proxy-country"]);
|
|
157
|
-
if (proxyCountry)
|
|
158
|
-
jobParams.proxy_country = proxyCountry;
|
|
159
|
-
const responseType = normalizeOutput(asString(values.output));
|
|
160
|
-
if (responseType)
|
|
161
|
-
jobParams.response_type = responseType;
|
|
276
|
+
// 2. the run must fit the page/credit caps, priced on the params as submitted.
|
|
277
|
+
const jobParams = jobParamsOf(values);
|
|
162
278
|
const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true);
|
|
163
279
|
const policy = loadPolicy();
|
|
164
280
|
for (const job of jobs)
|
|
165
281
|
assertDomainAllowed(job.url, policy);
|
|
166
|
-
const est = estimateCredits(jobs);
|
|
282
|
+
const est = estimateCredits(jobs, jobParams);
|
|
167
283
|
assertWithinLimits({ pages: v.validJobs, credits: est.credits }, policy, "batch");
|
|
168
|
-
// toJobBody validates proxy_country/premium_proxy BEFORE any HTTP call.
|
|
169
|
-
const body = toJobBody(jobs, jobParams);
|
|
284
|
+
// toJobBody validates params (proxy_country/premium_proxy, booleans, POST) BEFORE any HTTP call.
|
|
285
|
+
const body = toJobBody(jobs, jobParams, fields);
|
|
170
286
|
const apiKey = await ensureApiKey(values["no-signup"] ? { ...policy, auto_signup: false } : policy, {
|
|
171
287
|
onProvision: (a) => {
|
|
172
|
-
log.info("No API key found
|
|
288
|
+
log.info("No API key found: created a Zenrows Free plan account for you.");
|
|
173
289
|
log.dim(`Claim it anytime (keeps your usage): ${a.claimUrl}`);
|
|
174
290
|
},
|
|
175
291
|
});
|
|
292
|
+
const idempotencyKey = givenKey ?? newIdempotencyKey();
|
|
176
293
|
const runId = newRunId();
|
|
177
294
|
const startedAt = new Date().toISOString();
|
|
178
|
-
|
|
295
|
+
const request = { file, tasks: body.tasks.length, estimatedCredits: est.credits, jobParams, ...fields };
|
|
296
|
+
log.step(`Submitting batch job (${body.tasks.length} tasks, ${est.min} to ${est.credits} credits)…`);
|
|
179
297
|
try {
|
|
180
|
-
|
|
181
|
-
|
|
298
|
+
let job;
|
|
299
|
+
try {
|
|
300
|
+
job = await createJob(body, { apiKey, idempotencyKey });
|
|
301
|
+
}
|
|
302
|
+
catch (err) {
|
|
303
|
+
throw withRepeatCommand(err, "create", rest);
|
|
304
|
+
}
|
|
305
|
+
const finished = follow ? await followJob(job.job_id, apiKey, waitMs, job) : job;
|
|
182
306
|
const runError = runFailure(finished);
|
|
183
307
|
const runDir = writeRun({
|
|
184
308
|
runId,
|
|
@@ -187,11 +311,19 @@ async function createCmd(rest, ctx) {
|
|
|
187
311
|
startedAt,
|
|
188
312
|
finishedAt: new Date().toISOString(),
|
|
189
313
|
status: runError ? "error" : "ok",
|
|
190
|
-
request
|
|
191
|
-
result: {
|
|
314
|
+
request,
|
|
315
|
+
result: {
|
|
316
|
+
jobId: job.job_id,
|
|
317
|
+
job_id: job.job_id,
|
|
318
|
+
run_id: finished.latest_run?.run_id,
|
|
319
|
+
status: finished.latest_run?.status ?? "unknown",
|
|
320
|
+
...failureFields(finished),
|
|
321
|
+
},
|
|
192
322
|
...(runError ? { error: runError.toJSON() } : {}),
|
|
193
323
|
});
|
|
194
|
-
const code =
|
|
324
|
+
const code = follow
|
|
325
|
+
? printJob(finished, json, waitHeadline(finished), { note: waitNote(finished) })
|
|
326
|
+
: printJob(finished, json, `Submitted job ${job.job_id}`);
|
|
195
327
|
if (runDir && !json)
|
|
196
328
|
log.dim(` artifact: ${runDir}`);
|
|
197
329
|
return code;
|
|
@@ -204,171 +336,542 @@ async function createCmd(rest, ctx) {
|
|
|
204
336
|
startedAt,
|
|
205
337
|
finishedAt: new Date().toISOString(),
|
|
206
338
|
status: "error",
|
|
207
|
-
request
|
|
339
|
+
request,
|
|
208
340
|
error: err instanceof ToolkitError ? err.toJSON() : { message: String(err) },
|
|
209
341
|
});
|
|
210
342
|
throw err;
|
|
211
343
|
}
|
|
212
344
|
}
|
|
345
|
+
/**
|
|
346
|
+
* A write whose outcome is unknown carries its Idempotency-Key: suggest the
|
|
347
|
+
* exact command that repeats it under that key, which cannot create a duplicate.
|
|
348
|
+
*/
|
|
349
|
+
function withRepeatCommand(err, sub, rest) {
|
|
350
|
+
if (!(err instanceof ToolkitError) || !err.idempotency_key)
|
|
351
|
+
return err;
|
|
352
|
+
const args = [];
|
|
353
|
+
for (let i = 0; i < rest.length; i++) {
|
|
354
|
+
const a = rest[i];
|
|
355
|
+
if (a === "--idempotency-key") {
|
|
356
|
+
i++;
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
if (a.startsWith("--idempotency-key="))
|
|
360
|
+
continue;
|
|
361
|
+
args.push(shellQuote(a));
|
|
362
|
+
}
|
|
363
|
+
const repeat = ["zenrows", "batch", sub, ...args, "--idempotency-key", shellQuote(err.idempotency_key)].join(" ");
|
|
364
|
+
return new ToolkitError({ ...err.toJSON(), suggested_commands: [repeat, "zenrows batch list"] });
|
|
365
|
+
}
|
|
366
|
+
/** `s` as one POSIX shell word: bare when it is plainly safe, else single-quoted. */
|
|
367
|
+
export function shellQuote(s) {
|
|
368
|
+
return /^[A-Za-z0-9._\-/:=,@%+]+$/.test(s) ? s : `'${s.replace(/'/g, "'\\''")}'`;
|
|
369
|
+
}
|
|
370
|
+
async function listCmd(rest, ctx) {
|
|
371
|
+
const { values } = parse(rest, {
|
|
372
|
+
status: { type: "string" },
|
|
373
|
+
type: { type: "string" },
|
|
374
|
+
limit: { type: "string" },
|
|
375
|
+
cursor: { type: "string" },
|
|
376
|
+
json: { type: "boolean" },
|
|
377
|
+
});
|
|
378
|
+
const json = ctx.json || values.json === true;
|
|
379
|
+
const status = oneOf("status", asString(values.status), ["open", "closed"]);
|
|
380
|
+
const type = oneOf("type", asString(values.type), ["regular", "scheduled"]);
|
|
381
|
+
const limit = pageLimit(values.limit, 100);
|
|
382
|
+
const cursor = asString(values.cursor);
|
|
383
|
+
assertUsable("batch");
|
|
384
|
+
const apiKey = requireApiKey();
|
|
385
|
+
const page = await listJobs({ apiKey, limit, cursor, status, type });
|
|
386
|
+
if (json) {
|
|
387
|
+
log.out(JSON.stringify({ ok: true, count: page.jobs.length, jobs: page.jobs, next_cursor: page.next_cursor }, null, 2));
|
|
388
|
+
return 0;
|
|
389
|
+
}
|
|
390
|
+
if (!page.jobs.length)
|
|
391
|
+
log.info("No jobs.");
|
|
392
|
+
for (const j of page.jobs) {
|
|
393
|
+
const run = j.latest_run;
|
|
394
|
+
const runState = run ? `${run.status}${isPaused(run) ? " (paused)" : ""}` : "no run yet";
|
|
395
|
+
const progress = run?.stats ? `${run.stats.completed}/${run.stats.total}` : "";
|
|
396
|
+
const created = typeof j.created_at === "string" ? j.created_at : "";
|
|
397
|
+
const name = typeof j.name === "string" ? j.name : "";
|
|
398
|
+
log.out(`${j.job_id} ${String(j.type ?? "")} ${String(j.status ?? "")} ${runState} ${progress} ${created} ${name}`.trimEnd());
|
|
399
|
+
}
|
|
400
|
+
if (page.next_cursor)
|
|
401
|
+
log.dim(` more: zenrows batch list --cursor ${page.next_cursor}`);
|
|
402
|
+
return 0;
|
|
403
|
+
}
|
|
213
404
|
async function statusCmd(rest, ctx) {
|
|
214
|
-
const { positionals } = parse(rest, {});
|
|
215
|
-
const id = requireId(positionals[0]);
|
|
405
|
+
const { values, positionals } = parse(rest, { json: { type: "boolean" } });
|
|
406
|
+
const id = requireId(positionals[0], "status");
|
|
216
407
|
assertUsable("batch");
|
|
217
408
|
const apiKey = requireApiKey();
|
|
218
409
|
const job = await getJob(id, { apiKey });
|
|
219
|
-
return printJob(job, ctx.json, `Job ${id}
|
|
410
|
+
return printJob(job, ctx.json || values.json === true, `Job ${id}`, { note: isPaused(job.latest_run) ? pausedNote(id) : undefined });
|
|
411
|
+
}
|
|
412
|
+
async function waitCmd(rest, ctx) {
|
|
413
|
+
const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } });
|
|
414
|
+
const json = ctx.json || values.json === true;
|
|
415
|
+
const id = requireId(positionals[0], "wait");
|
|
416
|
+
const waitMs = timeoutFlag(values.timeout);
|
|
417
|
+
assertUsable("batch");
|
|
418
|
+
const apiKey = requireApiKey();
|
|
419
|
+
const job = await followJob(id, apiKey, waitMs);
|
|
420
|
+
return printJob(job, json, waitHeadline(job), { note: waitNote(job) });
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* Wait for a job's run to end. Ctrl-C ends the wait, not the job: say so, give
|
|
424
|
+
* the command that resumes the wait, and exit 130 as shells do on SIGINT. A
|
|
425
|
+
* submit's own answer is printed instead of a read when the wait ends before
|
|
426
|
+
* the first poll, so its `accepted_tasks` survive.
|
|
427
|
+
*/
|
|
428
|
+
async function followJob(id, apiKey, waitMs, created) {
|
|
429
|
+
const onSigint = () => {
|
|
430
|
+
log.warn(`Stopped waiting. Job ${id} is still running; it was not stopped.`);
|
|
431
|
+
log.dim(` resume: zenrows batch wait ${id}`);
|
|
432
|
+
process.exit(130);
|
|
433
|
+
};
|
|
434
|
+
process.once("SIGINT", onSigint);
|
|
435
|
+
try {
|
|
436
|
+
const job = await waitForJob(id, { apiKey, waitMs });
|
|
437
|
+
// Keep the submit/rerun answer's own fields (accepted_tasks, rerun_of, ...).
|
|
438
|
+
return created ? { ...pickAnswerFields(created), ...job } : job;
|
|
439
|
+
}
|
|
440
|
+
finally {
|
|
441
|
+
process.off("SIGINT", onSigint);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
/** The fields only a submit or rerun answer carries. */
|
|
445
|
+
function pickAnswerFields(job) {
|
|
446
|
+
const out = {};
|
|
447
|
+
for (const k of ["accepted_tasks", "rerun_of", "retried_tasks", "inherited_tasks"]) {
|
|
448
|
+
if (job[k] !== undefined)
|
|
449
|
+
out[k] = job[k];
|
|
450
|
+
}
|
|
451
|
+
return out;
|
|
452
|
+
}
|
|
453
|
+
function waitHeadline(job) {
|
|
454
|
+
if (isTerminal(job.latest_run))
|
|
455
|
+
return `Job ${job.job_id} finished`;
|
|
456
|
+
if (isPaused(job.latest_run))
|
|
457
|
+
return `Stopped waiting: job ${job.job_id} is paused`;
|
|
458
|
+
return `Stopped waiting: job ${job.job_id} is still running`;
|
|
459
|
+
}
|
|
460
|
+
/** Set when a wait ended without a terminal run: it ran out, or the run is paused. */
|
|
461
|
+
function waitNote(job) {
|
|
462
|
+
if (isTerminal(job.latest_run))
|
|
463
|
+
return undefined;
|
|
464
|
+
if (isPaused(job.latest_run))
|
|
465
|
+
return pausedNote(job.job_id);
|
|
466
|
+
return `The wait ran out and the run is still in progress. Run zenrows batch wait ${job.job_id}.`;
|
|
467
|
+
}
|
|
468
|
+
function pausedNote(id) {
|
|
469
|
+
return `The run is paused: it picks up no new task until it is resumed. Run zenrows batch resume ${id}.`;
|
|
220
470
|
}
|
|
221
471
|
async function resultsCmd(rest, ctx) {
|
|
222
472
|
const { values, positionals } = parse(rest, {
|
|
223
473
|
status: { type: "string" },
|
|
474
|
+
run: { type: "string" },
|
|
224
475
|
out: { type: "string" },
|
|
225
476
|
download: { type: "string" },
|
|
226
477
|
json: { type: "boolean" },
|
|
227
478
|
});
|
|
228
479
|
const json = ctx.json || values.json === true;
|
|
229
|
-
const id = requireId(positionals[0]);
|
|
480
|
+
const id = requireId(positionals[0], "results");
|
|
230
481
|
const status = normalizeResultStatus(asString(values.status));
|
|
482
|
+
const runId = asString(values.run);
|
|
231
483
|
assertUsable("batch");
|
|
232
484
|
const apiKey = requireApiKey();
|
|
233
|
-
const rows = await listResults(id, { apiKey, status });
|
|
485
|
+
const rows = await listResults(id, { apiKey, status, runId });
|
|
486
|
+
const state = await runState(id, runId, apiKey);
|
|
487
|
+
const note = state.unknown
|
|
488
|
+
? `The run's state could not be read, so these results may be partial. Check it with zenrows batch status ${id}, then read them again.`
|
|
489
|
+
: state.partial
|
|
490
|
+
? partialNote(state.status, id)
|
|
491
|
+
: undefined;
|
|
492
|
+
const meta = { jobId: id, job_id: id, ...(runId ? { run_id: runId } : {}), ...(state.status ? { status: state.status } : {}) };
|
|
234
493
|
const downloadDir = asString(values.download);
|
|
235
494
|
if (downloadDir) {
|
|
236
495
|
const out = await downloadResults(rows, downloadDir);
|
|
496
|
+
const expired = out.skipped.filter((s) => s.reason.startsWith("result expired")).length;
|
|
237
497
|
if (json) {
|
|
238
498
|
log.out(JSON.stringify({
|
|
239
499
|
ok: out.failed.length === 0,
|
|
240
|
-
|
|
500
|
+
...meta,
|
|
241
501
|
dir: out.dir,
|
|
242
502
|
downloaded: out.downloaded.length,
|
|
243
503
|
failed: out.failed.length,
|
|
244
504
|
skipped: out.skipped.length,
|
|
505
|
+
expired,
|
|
506
|
+
partial: state.partial,
|
|
507
|
+
...(note ? { note } : {}),
|
|
245
508
|
results: out,
|
|
246
509
|
}, null, 2));
|
|
247
510
|
}
|
|
248
511
|
else {
|
|
249
512
|
log.success(`Downloaded ${out.downloaded.length}/${rows.length} result(s) → ${out.dir}`);
|
|
250
513
|
log.dim(` index: ${join(out.dir, "_manifest.jsonl")}`);
|
|
251
|
-
if (out.skipped.length)
|
|
252
|
-
log.dim(` skipped ${out.skipped.length}
|
|
514
|
+
if (out.skipped.length) {
|
|
515
|
+
log.dim(` skipped ${out.skipped.length} with no body (failed, expired or unfinished tasks${expired ? `; ${expired} expired` : ""})`);
|
|
516
|
+
}
|
|
517
|
+
if (expired)
|
|
518
|
+
log.dim(` expired results are gone after 14 days by default; rerun the job in full: zenrows batch rerun ${id} --all`);
|
|
253
519
|
if (out.failed.length) {
|
|
254
520
|
log.warn(` ${out.failed.length} download(s) failed:`);
|
|
255
521
|
out.failed.slice(0, 10).forEach((f) => log.dim(` ${f.external_id ?? f.task_id}: ${f.reason}`));
|
|
256
522
|
}
|
|
523
|
+
if (note)
|
|
524
|
+
log.warn(` ${note}`);
|
|
257
525
|
}
|
|
258
526
|
return out.failed.length ? 1 : 0;
|
|
259
527
|
}
|
|
260
528
|
const jsonl = rows.map((r) => JSON.stringify(r)).join("\n");
|
|
261
529
|
const outFile = asString(values.out);
|
|
262
530
|
if (outFile) {
|
|
263
|
-
if (statSync(outFile, { throwIfNoEntry: false })?.isDirectory()) {
|
|
264
|
-
throw new ToolkitError({
|
|
265
|
-
code: "INVALID_USAGE",
|
|
266
|
-
message: `--out expects a file, but '${outFile}' is a directory.`,
|
|
267
|
-
likely_cause: "--out writes the results listing to a single JSONL file; you passed a directory.",
|
|
268
|
-
next_action: "Use a file path (e.g. --out results.jsonl), or download each result body into a directory with --download <dir>.",
|
|
269
|
-
suggested_commands: [`zenrows batch results ${id} --download ${outFile}`],
|
|
270
|
-
});
|
|
271
|
-
}
|
|
272
531
|
writeOut(outFile, jsonl + (jsonl ? "\n" : ""));
|
|
273
|
-
|
|
532
|
+
if (json) {
|
|
533
|
+
log.out(JSON.stringify({ ok: true, ...meta, count: rows.length, file: outFile, partial: state.partial, ...(note ? { note } : {}) }, null, 2));
|
|
534
|
+
}
|
|
535
|
+
else {
|
|
536
|
+
log.success(`Wrote ${rows.length} result(s) → ${outFile}`);
|
|
537
|
+
}
|
|
274
538
|
}
|
|
275
539
|
else if (json) {
|
|
276
|
-
log.out(JSON.stringify({ ok: true,
|
|
540
|
+
log.out(JSON.stringify({ ok: true, ...meta, count: rows.length, results: rows, partial: state.partial, ...(note ? { note } : {}) }, null, 2));
|
|
277
541
|
}
|
|
278
542
|
else {
|
|
279
|
-
log.info(`${rows.length} result(s) for job ${id}:`);
|
|
543
|
+
log.info(`${rows.length} result(s) for job ${id}${state.status ? ` (status: ${state.status})` : ""}:`);
|
|
280
544
|
log.out(jsonl);
|
|
281
545
|
}
|
|
546
|
+
if (note && !json)
|
|
547
|
+
log.warn(` ${note}`);
|
|
282
548
|
return 0;
|
|
283
549
|
}
|
|
550
|
+
/**
|
|
551
|
+
* Whether the run whose results were read can still change: in progress, or
|
|
552
|
+
* still receiving its tasks. An error here never fails the listing.
|
|
553
|
+
*/
|
|
554
|
+
async function runState(id, runId, apiKey) {
|
|
555
|
+
try {
|
|
556
|
+
const run = runId ? await getRun(id, runId, { apiKey }) : (await getJob(id, { apiKey })).latest_run;
|
|
557
|
+
if (!run)
|
|
558
|
+
return { partial: false };
|
|
559
|
+
return { status: run.status, partial: !isTerminal(run) || run.ingest_status === "pending" };
|
|
560
|
+
}
|
|
561
|
+
catch {
|
|
562
|
+
// Unknown is not complete: report the rows as possibly partial.
|
|
563
|
+
return { partial: true, unknown: true };
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
function partialNote(status, id) {
|
|
567
|
+
return status && isTerminal({ status })
|
|
568
|
+
? `The run is still receiving its tasks: these results are partial. Read them again once zenrows batch status ${id} shows ingest_status done.`
|
|
569
|
+
: `The run is still in progress: these results are partial. Wait with zenrows batch wait ${id}, then read them again.`;
|
|
570
|
+
}
|
|
571
|
+
async function contentCmd(rest, ctx) {
|
|
572
|
+
const { values, positionals } = parse(rest, { run: { type: "string" }, out: { type: "string" }, json: { type: "boolean" } });
|
|
573
|
+
const json = ctx.json || values.json === true;
|
|
574
|
+
const id = requireId(positionals[0], "content");
|
|
575
|
+
const taskId = positionals[1];
|
|
576
|
+
if (!taskId) {
|
|
577
|
+
throw new ToolkitError({
|
|
578
|
+
code: "INVALID_USAGE",
|
|
579
|
+
message: "Missing task id.",
|
|
580
|
+
likely_cause: "No <task_id> positional was provided.",
|
|
581
|
+
next_action: `Usage: zenrows batch content ${id} <task_id>. Each result row carries its task_id.`,
|
|
582
|
+
suggested_commands: [`zenrows batch results ${id}`],
|
|
583
|
+
});
|
|
584
|
+
}
|
|
585
|
+
const runId = asString(values.run);
|
|
586
|
+
assertUsable("batch");
|
|
587
|
+
const apiKey = requireApiKey();
|
|
588
|
+
const { contentType, body } = await getTaskContent(id, taskId, { apiKey, runId });
|
|
589
|
+
const ids = { job_id: id, task_id: taskId, ...(runId ? { run_id: runId } : {}) };
|
|
590
|
+
const outFile = asString(values.out);
|
|
591
|
+
if (outFile) {
|
|
592
|
+
writeOut(outFile, body);
|
|
593
|
+
if (json) {
|
|
594
|
+
log.out(JSON.stringify({ ok: true, ...ids, content_type: contentType, bytes: body.length, file: outFile }, null, 2));
|
|
595
|
+
}
|
|
596
|
+
else {
|
|
597
|
+
log.success(`Wrote ${body.length} bytes (${contentType || "unknown type"}) → ${outFile}`);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
else if (json) {
|
|
601
|
+
log.out(JSON.stringify({ ok: true, ...ids, content_type: contentType, bytes: body.length, content: body.toString("utf8") }, null, 2));
|
|
602
|
+
}
|
|
603
|
+
else {
|
|
604
|
+
process.stdout.write(body);
|
|
605
|
+
if (body.length && body[body.length - 1] !== 0x0a && isText(contentType))
|
|
606
|
+
process.stdout.write("\n");
|
|
607
|
+
}
|
|
608
|
+
return 0;
|
|
609
|
+
}
|
|
610
|
+
function isText(contentType) {
|
|
611
|
+
return /^(text\/|application\/(json|xml))/i.test(contentType);
|
|
612
|
+
}
|
|
284
613
|
async function cancelCmd(rest, ctx) {
|
|
285
|
-
const { positionals } = parse(rest, {});
|
|
286
|
-
const
|
|
614
|
+
const { values, positionals } = parse(rest, { json: { type: "boolean" } });
|
|
615
|
+
const json = ctx.json || values.json === true;
|
|
616
|
+
const id = requireId(positionals[0], "cancel");
|
|
287
617
|
assertUsable("batch");
|
|
288
618
|
const apiKey = requireApiKey();
|
|
289
|
-
|
|
290
|
-
|
|
619
|
+
let run;
|
|
620
|
+
try {
|
|
621
|
+
run = await stopJob(id, { apiKey });
|
|
622
|
+
}
|
|
623
|
+
catch (err) {
|
|
624
|
+
// An ended run cannot be stopped: answer with the job as it ended, like crawl's stop.
|
|
625
|
+
if (err instanceof ToolkitError && err.server_code === "run_not_stoppable") {
|
|
626
|
+
const job = await getJob(id, { apiKey });
|
|
627
|
+
return printJob(job, json, `Nothing to stop: job ${id} had already ended`);
|
|
628
|
+
}
|
|
629
|
+
throw err;
|
|
630
|
+
}
|
|
631
|
+
return printJob(asJob(id, run), json, `Stopped job ${id}`);
|
|
291
632
|
}
|
|
292
|
-
async function
|
|
293
|
-
const { values, positionals } = parse(rest, {
|
|
633
|
+
async function pauseCmd(rest, ctx, op) {
|
|
634
|
+
const { values, positionals } = parse(rest, { json: { type: "boolean" } });
|
|
635
|
+
const json = ctx.json || values.json === true;
|
|
636
|
+
const id = requireId(positionals[0], op);
|
|
637
|
+
assertUsable("batch");
|
|
638
|
+
const apiKey = requireApiKey();
|
|
639
|
+
const run = op === "pause" ? await pauseJob(id, { apiKey }) : await resumeJob(id, { apiKey });
|
|
640
|
+
const job = asJob(id, run);
|
|
641
|
+
return op === "pause"
|
|
642
|
+
? printJob(job, json, `Paused job ${id}`, { note: isPaused(run) ? pausedNote(id) : undefined })
|
|
643
|
+
: printJob(job, json, `Resumed job ${id}`);
|
|
644
|
+
}
|
|
645
|
+
/** stop, pause and resume answer with the run, not the job. */
|
|
646
|
+
function asJob(id, run) {
|
|
647
|
+
return { job_id: run.job_id ?? id, latest_run: run };
|
|
648
|
+
}
|
|
649
|
+
async function rerunCmd(rest, ctx, sub) {
|
|
650
|
+
const { values, positionals } = parse(rest, {
|
|
651
|
+
...(sub === "rerun" ? { status: { type: "string" }, all: { type: "boolean" } } : { "include-pending": { type: "boolean" } }),
|
|
652
|
+
"idempotency-key": { type: "string" },
|
|
653
|
+
follow: { type: "boolean" },
|
|
654
|
+
timeout: { type: "string" },
|
|
655
|
+
json: { type: "boolean" },
|
|
656
|
+
});
|
|
294
657
|
const json = ctx.json || values.json === true;
|
|
295
|
-
const id = requireId(positionals[0]);
|
|
658
|
+
const id = requireId(positionals[0], sub);
|
|
659
|
+
if (values.all === true && values.status !== undefined) {
|
|
660
|
+
throw usageError("--all and --status cannot be combined.", "Pass --all to rerun every task, or --status <list> to pick statuses, not both.", "--all reruns every task; --status reruns only some.");
|
|
661
|
+
}
|
|
662
|
+
const status = sub === "rerun"
|
|
663
|
+
? values.all === true
|
|
664
|
+
? undefined
|
|
665
|
+
: (rerunStatus(asString(values.status)) ?? DEFAULT_RERUN)
|
|
666
|
+
: values["include-pending"] === true
|
|
667
|
+
? UNFINISHED
|
|
668
|
+
: "failed";
|
|
669
|
+
const waitMs = timeoutFlag(values.timeout);
|
|
670
|
+
const givenKey = asString(values["idempotency-key"]);
|
|
671
|
+
if (givenKey !== undefined)
|
|
672
|
+
assertIdempotencyKey(givenKey);
|
|
296
673
|
assertUsable("batch");
|
|
297
674
|
const apiKey = requireApiKey();
|
|
298
|
-
|
|
299
|
-
|
|
675
|
+
let job;
|
|
676
|
+
try {
|
|
677
|
+
job = await rerunJob(id, { apiKey, status, idempotencyKey: givenKey ?? newIdempotencyKey() });
|
|
678
|
+
}
|
|
679
|
+
catch (err) {
|
|
680
|
+
throw withRepeatCommand(err, sub, rest);
|
|
681
|
+
}
|
|
682
|
+
const what = status ? `tasks with status ${status}` : "every task";
|
|
683
|
+
if (values.follow === true) {
|
|
684
|
+
const finished = await followJob(job.job_id ?? id, apiKey, waitMs, job);
|
|
685
|
+
return printJob(finished, json, waitHeadline(finished), { note: waitNote(finished) });
|
|
686
|
+
}
|
|
687
|
+
return printJob(job, json, `Started a new run of job ${id} (${what})`);
|
|
300
688
|
}
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
689
|
+
/** `--status` for a rerun: a comma-separated list of task statuses; undefined when unset. */
|
|
690
|
+
function rerunStatus(v) {
|
|
691
|
+
if (v === undefined)
|
|
692
|
+
return undefined;
|
|
693
|
+
const parts = [...new Set(v.split(",").map((s) => s.trim().toLowerCase()).filter(Boolean))];
|
|
694
|
+
const bad = parts.find((p) => !RERUN_STATUSES.includes(p));
|
|
695
|
+
if (!parts.length || bad !== undefined) {
|
|
696
|
+
throw usageError(`Invalid --status '${v}'.`, `Pass a comma-separated list of ${RERUN_STATUSES.join(", ")} (e.g. --status ${UNFINISHED}), leave it out to rerun failed and pending tasks, or pass --all to rerun every task.`, `A rerun selects tasks of the previous run by status: ${RERUN_STATUSES.join(", ")}.`);
|
|
697
|
+
}
|
|
698
|
+
return parts.join(",");
|
|
699
|
+
}
|
|
700
|
+
async function deleteCmd(rest, ctx) {
|
|
701
|
+
const { values, positionals } = parse(rest, { json: { type: "boolean" } });
|
|
702
|
+
const json = ctx.json || values.json === true;
|
|
703
|
+
const id = requireId(positionals[0], "delete");
|
|
704
|
+
if (!ctx.yes) {
|
|
705
|
+
throw new ToolkitError({
|
|
706
|
+
code: "INVALID_USAGE",
|
|
707
|
+
message: `Deleting job ${id} needs --yes.`,
|
|
708
|
+
likely_cause: "Delete stops the job's run and removes the job, its runs and its stored results for good.",
|
|
709
|
+
next_action: "Re-run with --yes to confirm.",
|
|
710
|
+
suggested_commands: [`zenrows batch delete ${id} --yes`],
|
|
711
|
+
});
|
|
712
|
+
}
|
|
304
713
|
assertUsable("batch");
|
|
305
714
|
const apiKey = requireApiKey();
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
715
|
+
await deleteJob(id, { apiKey });
|
|
716
|
+
if (json)
|
|
717
|
+
log.out(JSON.stringify({ ok: true, job_id: id, deleted: true }, null, 2));
|
|
718
|
+
else
|
|
719
|
+
log.success(`Deleted job ${id}. Its stored results are removed in the background.`);
|
|
720
|
+
return 0;
|
|
310
721
|
}
|
|
311
722
|
/** The run's failure_reason / failure_detail, when the API reports them. */
|
|
312
723
|
function failureFields(job) {
|
|
313
|
-
const run = job.latest_run
|
|
724
|
+
const run = job.latest_run;
|
|
314
725
|
const out = {};
|
|
315
|
-
if (typeof run
|
|
726
|
+
if (typeof run?.failure_reason === "string" && run.failure_reason)
|
|
316
727
|
out.failure_reason = run.failure_reason;
|
|
317
|
-
if (typeof run
|
|
728
|
+
if (typeof run?.failure_detail === "string" && run.failure_detail)
|
|
318
729
|
out.failure_detail = run.failure_detail;
|
|
319
730
|
return out;
|
|
320
731
|
}
|
|
321
732
|
/**
|
|
322
|
-
* The error for a run that ended `failed`, or null for any other state. A run
|
|
323
|
-
*
|
|
324
|
-
*
|
|
733
|
+
* The error for a run that ended `failed`, or null for any other state. A run
|
|
734
|
+
* the account's credits, its subscription or the key's credit cap stopped gets
|
|
735
|
+
* the code and advice the matching HTTP refusal gets (POLICY_MAX_CREDITS_EXCEEDED,
|
|
736
|
+
* AUTH_INVALID, KEY_CREDIT_CAP_REACHED); any other reason is BATCH_FAILED. All
|
|
737
|
+
* carry `server_code` (the failure_reason) and `job_id`, and say how to run the
|
|
738
|
+
* tasks the failure left unfinished.
|
|
325
739
|
*/
|
|
326
740
|
export function runFailure(job) {
|
|
327
741
|
if (job.latest_run?.status !== "failed")
|
|
328
742
|
return null;
|
|
329
743
|
const { failure_reason, failure_detail } = failureFields(job);
|
|
330
|
-
const
|
|
744
|
+
const id = job.job_id;
|
|
745
|
+
const where = `batch job ${id}`;
|
|
746
|
+
const rerun = `zenrows batch rerun ${id} --status ${UNFINISHED}`;
|
|
747
|
+
const ids = { ...(failure_reason ? { server_code: failure_reason } : {}), job_id: id };
|
|
748
|
+
const detail = failure_detail ? `${failure_detail.replace(/[.\s]+$/, "")}. ` : "";
|
|
331
749
|
if (isKeyCapReached(failure_reason)) {
|
|
332
|
-
|
|
750
|
+
const cap = keyCapReached(where, { status: null, detail: failure_detail }).toJSON();
|
|
751
|
+
return new ToolkitError({
|
|
752
|
+
...cap,
|
|
753
|
+
...ids,
|
|
754
|
+
next_action: thenRerun(cap.next_action, rerun),
|
|
755
|
+
suggested_commands: [...(cap.suggested_commands ?? []), rerun],
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
if (failure_reason === "insufficient_credits") {
|
|
759
|
+
const acct = readAccount();
|
|
760
|
+
const quota = quotaExhausted(where, storedClaimHint(acct), { detail: failure_detail }).toJSON();
|
|
761
|
+
return new ToolkitError({
|
|
762
|
+
...quota,
|
|
763
|
+
...ids,
|
|
764
|
+
likely_cause: `${detail}Stopped ${where}: the account ran out of credits.`,
|
|
765
|
+
next_action: thenRerun(quota.next_action, rerun),
|
|
766
|
+
suggested_commands: [...(quota.suggested_commands ?? []), rerun],
|
|
767
|
+
});
|
|
768
|
+
}
|
|
769
|
+
if (failure_reason === "subscription_inactive") {
|
|
770
|
+
const sub = subscriptionInactive(`${detail}Stopped ${where}`).toJSON();
|
|
771
|
+
return new ToolkitError({
|
|
772
|
+
...sub,
|
|
773
|
+
...ids,
|
|
774
|
+
next_action: thenRerun(sub.next_action, rerun),
|
|
775
|
+
suggested_commands: [rerun],
|
|
776
|
+
});
|
|
333
777
|
}
|
|
334
778
|
const why = [failure_reason, failure_detail].filter(Boolean).join(": ");
|
|
335
779
|
return new ToolkitError({
|
|
336
780
|
code: "BATCH_FAILED",
|
|
337
|
-
message: `Batch job ${
|
|
781
|
+
message: `Batch job ${id} stopped before it finished.`,
|
|
338
782
|
likely_cause: why || "The run ended in status failed without a reason.",
|
|
339
|
-
next_action:
|
|
340
|
-
suggested_commands: [`zenrows batch
|
|
783
|
+
next_action: `Check the run with \`zenrows batch status ${id}\`. Once the cause is fixed, run the unfinished tasks: \`${rerun}\`.`,
|
|
784
|
+
suggested_commands: [`zenrows batch status ${id}`, rerun],
|
|
785
|
+
...ids,
|
|
341
786
|
});
|
|
342
787
|
}
|
|
788
|
+
/** Where batch advice continues after a quota error's own `next_action`. */
|
|
789
|
+
const ADVICE_SUFFIX = / (An earlier attempt of this request|Then run the unfinished tasks)/;
|
|
343
790
|
/**
|
|
344
|
-
*
|
|
345
|
-
*
|
|
346
|
-
*
|
|
347
|
-
*
|
|
791
|
+
* `refreshClaimAdvice` for a batch error: it rewrites a quota error's claim
|
|
792
|
+
* advice from the current claim state, which would drop what batch adds after
|
|
793
|
+
* that advice (the safe-repeat steps of an unknown submit, or the rerun of a
|
|
794
|
+
* failed run's unfinished tasks). Refresh the quota part and keep the rest.
|
|
348
795
|
*/
|
|
349
|
-
function
|
|
350
|
-
|
|
351
|
-
|
|
796
|
+
export async function refreshQuotaAdvice(err, opts = {}) {
|
|
797
|
+
if (!(err instanceof ToolkitError))
|
|
798
|
+
return refreshClaimAdvice(err, opts);
|
|
799
|
+
const m = ADVICE_SUFFIX.exec(err.next_action);
|
|
800
|
+
if (!m)
|
|
801
|
+
return refreshClaimAdvice(err, opts);
|
|
802
|
+
const head = err.next_action.slice(0, m.index);
|
|
803
|
+
const fresh = await refreshClaimAdvice(new ToolkitError({ ...err.toJSON(), next_action: head }), opts);
|
|
804
|
+
if (!(fresh instanceof ToolkitError) || fresh.next_action === head)
|
|
805
|
+
return err;
|
|
806
|
+
const next = `${fresh.next_action.replace(/[.\s]*$/, ".")}${err.next_action.slice(m.index)}`;
|
|
807
|
+
return new ToolkitError({ ...err.toJSON(), next_action: next });
|
|
808
|
+
}
|
|
809
|
+
/** `next` followed by the command that runs the tasks a failure left unfinished. */
|
|
810
|
+
function thenRerun(next, rerun) {
|
|
811
|
+
return `${next}${/[.!?]$/.test(next) ? "" : "."} Then run the unfinished tasks: \`${rerun}\`.`;
|
|
812
|
+
}
|
|
813
|
+
/**
|
|
814
|
+
* Print a job's latest run, structured under --json, and return the exit code:
|
|
815
|
+
* 1 when the run ended `failed`, else 0. `stopped` and `deleted` are deliberate
|
|
816
|
+
* outcomes (someone cancelled or removed the run), so they exit 0 but print as
|
|
817
|
+
* a warning. `extra.note` says why a wait ended without a terminal run.
|
|
818
|
+
*/
|
|
819
|
+
function printJob(job, json, headline, extra = {}) {
|
|
820
|
+
const run = job.latest_run;
|
|
821
|
+
const stats = run?.stats;
|
|
352
822
|
const failure = failureFields(job);
|
|
353
823
|
const err = runFailure(job);
|
|
824
|
+
const paused = isPaused(run);
|
|
825
|
+
const answer = pickAnswerFields(job);
|
|
826
|
+
const ingest = run?.ingest_status;
|
|
354
827
|
if (json) {
|
|
355
|
-
log.out(JSON.stringify({
|
|
828
|
+
log.out(JSON.stringify({
|
|
829
|
+
ok: !err,
|
|
830
|
+
job_id: job.job_id,
|
|
831
|
+
jobId: job.job_id,
|
|
832
|
+
...(run?.run_id ? { run_id: run.run_id } : {}),
|
|
833
|
+
status: run?.status ?? null,
|
|
834
|
+
...(paused ? { paused: true } : {}),
|
|
835
|
+
stats,
|
|
836
|
+
...failure,
|
|
837
|
+
...(ingest ? { ingest_status: ingest } : {}),
|
|
838
|
+
...answer,
|
|
839
|
+
job,
|
|
840
|
+
...(extra.note ? { note: extra.note } : {}),
|
|
841
|
+
...(err ? { error: err.toJSON() } : {}),
|
|
842
|
+
}, null, 2));
|
|
356
843
|
return err ? 1 : 0;
|
|
357
844
|
}
|
|
358
|
-
const line = `${headline} · status: ${run
|
|
845
|
+
const line = `${headline} · status: ${run?.status ?? "no run yet"}${paused ? " (paused)" : ""}`;
|
|
359
846
|
if (err)
|
|
360
847
|
log.error(line);
|
|
361
|
-
else if (run
|
|
848
|
+
else if (run?.status === "stopped" || run?.status === "deleted" || paused || extra.note)
|
|
362
849
|
log.warn(line);
|
|
363
850
|
else
|
|
364
851
|
log.success(line);
|
|
852
|
+
if (run?.run_id)
|
|
853
|
+
log.info(` run: ${run.run_id}`);
|
|
365
854
|
if (stats) {
|
|
366
855
|
log.info(` ${stats.completed}/${stats.total} completed · ${stats.successful} successful · ${stats.failed} failed`);
|
|
367
856
|
}
|
|
857
|
+
if (answer.accepted_tasks !== undefined)
|
|
858
|
+
log.info(` accepted_tasks: ${answer.accepted_tasks}`);
|
|
859
|
+
if (answer.rerun_of) {
|
|
860
|
+
log.info(` rerun of ${answer.rerun_of}: ${answer.retried_tasks ?? 0} task(s) retried, ${answer.inherited_tasks ?? 0} carried over`);
|
|
861
|
+
}
|
|
862
|
+
if (ingest === "pending")
|
|
863
|
+
log.dim(" ingest_status: pending (the API is still receiving the tasks; results stay partial until done)");
|
|
368
864
|
if (failure.failure_reason)
|
|
369
865
|
log.info(` failure_reason: ${failure.failure_reason}`);
|
|
370
866
|
if (failure.failure_detail)
|
|
371
867
|
log.info(` failure_detail: ${failure.failure_detail}`);
|
|
868
|
+
if (!err && run?.status === "stopped" && stats && stats.completed < stats.total) {
|
|
869
|
+
log.dim(` ${stats.total - stats.completed} task(s) did not finish: zenrows batch rerun ${job.job_id} --status ${UNFINISHED}`);
|
|
870
|
+
}
|
|
871
|
+
if (extra.note) {
|
|
872
|
+
log.dim(` ${extra.note}`);
|
|
873
|
+
log.dim(` next: zenrows batch ${paused ? "resume" : "wait"} ${job.job_id}`);
|
|
874
|
+
}
|
|
372
875
|
if (err)
|
|
373
876
|
printError(err, false);
|
|
374
877
|
return err ? 1 : 0;
|
|
@@ -377,13 +880,13 @@ function normalizeResultStatus(v) {
|
|
|
377
880
|
if (!v)
|
|
378
881
|
return undefined;
|
|
379
882
|
const s = v.toLowerCase();
|
|
380
|
-
if (s === "successful" || s === "failed" || s === "all")
|
|
883
|
+
if (s === "successful" || s === "failed" || s === "processing" || s === "all")
|
|
381
884
|
return s;
|
|
382
885
|
throw new ToolkitError({
|
|
383
886
|
code: "INVALID_USAGE",
|
|
384
887
|
message: `Invalid --status: ${v}`,
|
|
385
|
-
likely_cause: "Only successful | failed | all are supported.",
|
|
386
|
-
next_action: "Use --status successful | failed | all (default: all).",
|
|
888
|
+
likely_cause: "Only successful | failed | processing | all are supported.",
|
|
889
|
+
next_action: "Use --status successful | failed | processing | all (default: all). processing lists the tasks not finished yet; expired rows come back with successful.",
|
|
387
890
|
});
|
|
388
891
|
}
|
|
389
892
|
function normalizeOutput(v) {
|
|
@@ -412,15 +915,51 @@ function normalizeOutput(v) {
|
|
|
412
915
|
const mapped = map[key];
|
|
413
916
|
return mapped ? mapped : undefined; // html → undefined (no response_type)
|
|
414
917
|
}
|
|
415
|
-
|
|
918
|
+
/**
|
|
919
|
+
* `--timeout` as a duration, returned in milliseconds; undefined when unset.
|
|
920
|
+
* Takes `500ms`, `30s`, `5m` or `1h`; a bare number keeps its earlier meaning,
|
|
921
|
+
* milliseconds.
|
|
922
|
+
*/
|
|
923
|
+
export function timeoutFlag(v) {
|
|
924
|
+
if (v === undefined)
|
|
925
|
+
return undefined;
|
|
926
|
+
const m = typeof v === "string" ? /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h)?\s*$/i.exec(v) : null;
|
|
927
|
+
const unit = { ms: 1, s: 1000, m: 60_000, h: 3_600_000 }[(m?.[2] ?? "ms").toLowerCase()];
|
|
928
|
+
const ms = m ? Number(m[1]) * unit : NaN;
|
|
929
|
+
if (!Number.isFinite(ms) || ms <= 0) {
|
|
930
|
+
throw usageError(`Invalid --timeout value '${String(v)}'.`, "Pass a duration such as --timeout 30s, 5m or 1h (a bare number is milliseconds).", "--timeout takes a positive duration.");
|
|
931
|
+
}
|
|
932
|
+
return Math.round(ms);
|
|
933
|
+
}
|
|
934
|
+
/** `--limit` for a paged read: a whole number from 1 to `max`. */
|
|
935
|
+
function pageLimit(v, max) {
|
|
936
|
+
if (v === undefined)
|
|
937
|
+
return undefined;
|
|
938
|
+
const n = asNumber(v);
|
|
939
|
+
if (n === undefined || !Number.isInteger(n) || n < 1 || n > max) {
|
|
940
|
+
throw usageError(`Invalid --limit value '${String(v)}'.`, `Pass e.g. --limit ${Math.min(max, 50)}.`, `--limit takes a whole number from 1 to ${max}.`);
|
|
941
|
+
}
|
|
942
|
+
return n;
|
|
943
|
+
}
|
|
944
|
+
function oneOf(flag, v, allowed) {
|
|
945
|
+
if (v === undefined)
|
|
946
|
+
return undefined;
|
|
947
|
+
if (allowed.includes(v))
|
|
948
|
+
return v;
|
|
949
|
+
throw usageError(`Invalid --${flag} '${v}'.`, `Pass --${flag} ${allowed.join(" or ")}, or leave it out.`);
|
|
950
|
+
}
|
|
951
|
+
function usageError(message, next_action, likely_cause = "A flag value is outside what the Batch API accepts.") {
|
|
952
|
+
return new ToolkitError({ code: "INVALID_USAGE", message, likely_cause, next_action });
|
|
953
|
+
}
|
|
954
|
+
function requireId(id, sub) {
|
|
416
955
|
if (id)
|
|
417
956
|
return id;
|
|
418
957
|
throw new ToolkitError({
|
|
419
958
|
code: "INVALID_USAGE",
|
|
420
959
|
message: "Missing job id.",
|
|
421
960
|
likely_cause: "No <id> positional was provided.",
|
|
422
|
-
next_action:
|
|
423
|
-
suggested_commands: ["zenrows batch
|
|
961
|
+
next_action: `Usage: zenrows batch ${sub} <id>. List your jobs with \`zenrows batch list\`.`,
|
|
962
|
+
suggested_commands: ["zenrows batch list"],
|
|
424
963
|
});
|
|
425
964
|
}
|
|
426
965
|
function needFile() {
|