@zenrows/cli 1.6.2 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -11
- package/agent-plugin/skills/batch-jobs/SKILL.md +16 -3
- package/agent-plugin/skills/zenrows/SKILL.md +2 -1
- package/dist/src/adapters/batch.js +237 -59
- package/dist/src/cli/commands/account.js +37 -27
- package/dist/src/cli/commands/batch.js +667 -125
- package/dist/src/cli/commands/browser.js +4 -1
- package/dist/src/cli/commands/crawl.js +6 -3
- package/dist/src/cli/commands/extract.js +4 -1
- package/dist/src/cli/commands/fetch.js +4 -1
- package/dist/src/cli/commands/signup.js +3 -2
- package/dist/src/cli/index.js +2 -1
- package/dist/src/core/agent-account.js +111 -14
- package/dist/src/core/auth.js +4 -10
- package/dist/src/core/batch-api.js +400 -146
- package/dist/src/core/browser-api.js +2 -1
- package/dist/src/core/claim-link.js +237 -0
- package/dist/src/core/config.js +1 -1
- package/dist/src/core/crawl-api.js +2 -1
- package/dist/src/core/ensure-key.js +1 -0
- package/dist/src/core/errors.js +27 -11
- package/dist/src/core/http.js +4 -3
- package/dist/src/core/nudge.js +3 -1
- package/dist/src/core/policy.js +13 -6
- package/dist/src/core/provenance.js +10 -2
- package/dist/src/core/usage.js +2 -1
- package/dist/src/core/workspace.js +29 -1
- package/package.json +1 -1
- package/recipes/batch-estimate/RECIPE.md +6 -3
- package/skills/batch-jobs/SKILL.md +160 -43
- package/skills/crawl/SKILL.md +1 -1
- package/skills/trace-debug/SKILL.md +17 -2
- package/skills/zenrows/SKILL.md +6 -3
- package/templates/batch-jsonl-pipeline/README.md +10 -5
|
@@ -1,51 +1,89 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `zenrows batch`: Batch API (status: available).
|
|
3
3
|
*
|
|
4
|
-
* Local (no key, always works): `estimate
|
|
5
|
-
*
|
|
6
|
-
* `
|
|
7
|
-
* BATCH_ACCESS_DENIED.
|
|
4
|
+
* Local (no key, always works): `estimate` validates a JSONL spec and prices
|
|
5
|
+
* it. Cloud (needs a key): `create`, `list`, `status`, `wait`, `results`,
|
|
6
|
+
* `content`, `cancel`, `pause`, `resume`, `rerun`, `retry-failed`, `delete`.
|
|
7
|
+
* A 403 maps to BATCH_ACCESS_DENIED. Schedules, webhooks, exports, CSV inputs,
|
|
8
|
+
* open jobs, runs and task history are API/SDK-only.
|
|
8
9
|
*/
|
|
9
|
-
import { statSync } from "node:fs";
|
|
10
10
|
import { join } from "node:path";
|
|
11
11
|
import { log, ANSI, c } from "../../core/logger.js";
|
|
12
|
-
import { estimateCredits, toJobBody, validateJsonl, withAdaptiveStealth } from "../../adapters/batch.js";
|
|
12
|
+
import { estimateCredits, externalIdError, LIMITS, metadataError, toJobBody, validateJsonl, withAdaptiveStealth, } from "../../adapters/batch.js";
|
|
13
13
|
import { loadConfig } from "../../core/config.js";
|
|
14
14
|
import { requireApiKey } from "../../core/auth.js";
|
|
15
15
|
import { ensureApiKey } from "../../core/ensure-key.js";
|
|
16
16
|
import { assertUsable } from "../../core/capabilities.js";
|
|
17
17
|
import { assertDomainAllowed, assertWithinLimits, loadPolicy } from "../../core/policy.js";
|
|
18
18
|
import { newRunId, writeRun } from "../../core/artifacts.js";
|
|
19
|
-
import { createJob, downloadResults, getJob, listResults, rerunJob, stopJob, waitForJob } from "../../core/batch-api.js";
|
|
19
|
+
import { RERUN_STATUSES, assertIdempotencyKey, createJob, deleteJob, downloadResults, getJob, getRun, getTaskContent, isPaused, isTerminal, listJobs, listResults, newIdempotencyKey, pauseJob, rerunJob, resumeJob, stopJob, subscriptionInactive, waitForJob, } from "../../core/batch-api.js";
|
|
20
|
+
import { readAccount } from "../../core/agent-account.js";
|
|
20
21
|
import { asNumber, asString, parse } from "../command.js";
|
|
21
|
-
import { ToolkitError, isKeyCapReached, keyCapReached } from "../../core/errors.js";
|
|
22
|
+
import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.js";
|
|
22
23
|
import { printError, writeOut } from "../output.js";
|
|
24
|
+
import { refreshClaimAdvice, storedClaimHint } from "../../core/claim-link.js";
|
|
25
|
+
/** What `rerun` selects with no --status: the unfinished tasks, never the successful ones. */
|
|
26
|
+
const DEFAULT_RERUN = "failed,pending";
|
|
27
|
+
/** The task statuses that recover every task a run did not finish successfully. */
|
|
28
|
+
const UNFINISHED = "failed,pending,processing";
|
|
23
29
|
export const batch = {
|
|
24
30
|
name: "batch",
|
|
25
31
|
summary: "Run JSONL batch jobs on Zenrows Batch.",
|
|
26
|
-
usage: "zenrows batch <estimate
|
|
32
|
+
usage: "zenrows batch <estimate <file.jsonl>|create <file.jsonl>|list|status <id>|wait <id>|results <id>|content <id> <task_id>|cancel <id>|pause <id>|resume <id>|rerun <id>|retry-failed <id>|delete <id>>",
|
|
27
33
|
help: [
|
|
28
34
|
"Local (no key):",
|
|
29
|
-
" estimate <file.jsonl> [
|
|
35
|
+
" estimate <file.jsonl> [flags] validate the spec and price it (credits: min and upper bound);",
|
|
36
|
+
" takes the same --manual / --js-render / --premium-proxy /",
|
|
37
|
+
" --proxy-country flags as create, so both price the same request",
|
|
30
38
|
"Cloud (needs a key):",
|
|
31
39
|
" create <file.jsonl> [flags] validate, then submit the job",
|
|
32
|
-
" (default) Adaptive Stealth Mode (mode=auto) on every task that",
|
|
33
|
-
" doesn't force js_render / premium_proxy itself",
|
|
40
|
+
" (default) Adaptive Stealth Mode (mode=auto) on every GET task that",
|
|
41
|
+
" doesn't force js_render / premium_proxy or set mode itself",
|
|
34
42
|
" --manual no Adaptive Stealth Mode; plain requests unless flags below",
|
|
35
43
|
" --js-render job-level: force JavaScript rendering (turns auto off)",
|
|
36
44
|
" --premium-proxy job-level: force residential IPs (turns auto off)",
|
|
37
45
|
" --proxy-country <cc> job-level: geo-target (auto mode, or with --premium-proxy)",
|
|
38
46
|
" --output <fmt> job-level response_type (markdown|plaintext|pdf|html)",
|
|
39
|
-
" --
|
|
47
|
+
" --name <name> job name (up to 100 characters)",
|
|
48
|
+
" --external-id <id> job-level external id (letters, digits, . _ -; up to 128)",
|
|
49
|
+
" --metadata <key=value> job-level metadata entry (repeatable, up to 20)",
|
|
50
|
+
" --idempotency-key <key> so a repeated create starts no second job (one is generated",
|
|
51
|
+
" otherwise; the API keeps it 24 hours)",
|
|
52
|
+
" --follow wait until the run ends (alias: --wait; Ctrl-C stops the",
|
|
53
|
+
" wait, not the job)",
|
|
54
|
+
" --timeout <duration> with --follow: stop waiting after this long (10s, 5m, 1h;",
|
|
55
|
+
" a bare number is milliseconds; default 10m)",
|
|
40
56
|
" --no-signup do not auto-create a Free plan account if no key exists",
|
|
41
|
-
" status
|
|
42
|
-
"
|
|
57
|
+
" list [--status open|closed] [--type regular|scheduled] [--limit n] [--cursor c]",
|
|
58
|
+
" your jobs, newest first (limit up to 100)",
|
|
59
|
+
" status <id> show the job's latest run: status, stats, pause state",
|
|
60
|
+
" wait <id> [--timeout <duration>] poll until the run ends, is paused, or the time runs out",
|
|
61
|
+
" (default 10m); a run still going then is printed, not stopped",
|
|
62
|
+
" results <id> [--status s] list results (successful|failed|processing|all); partial",
|
|
63
|
+
" while the run is in progress",
|
|
64
|
+
" --run <run_id> read an earlier run instead of the latest",
|
|
43
65
|
" --out <file> write results as JSONL instead of printing",
|
|
44
66
|
" --download <dir> fetch each result body into <dir> (+ _manifest.jsonl)",
|
|
45
|
-
"
|
|
46
|
-
"
|
|
47
|
-
"
|
|
48
|
-
"
|
|
67
|
+
" content <id> <task_id> [--run <run_id>] [--out <file>]",
|
|
68
|
+
" print or save one task's result body",
|
|
69
|
+
" cancel <id> stop the in-flight run (an ended run answers as it ended)",
|
|
70
|
+
" pause <id> / resume <id> pause the in-flight run, or resume it",
|
|
71
|
+
" rerun <id> start a new run of the failed and pending tasks",
|
|
72
|
+
" (successful tasks carry over and are not billed again)",
|
|
73
|
+
" --status <list> rerun only tasks with these statuses",
|
|
74
|
+
" (pending,processing,failed,successful)",
|
|
75
|
+
" --all rerun every task (a full rerun; not with --status)",
|
|
76
|
+
" rerun also takes --idempotency-key, --follow and --timeout",
|
|
77
|
+
" retry-failed <id> rerun only the failed tasks (rerun --status failed); takes",
|
|
78
|
+
" --idempotency-key, --follow and --timeout like rerun",
|
|
79
|
+
" --include-pending also the tasks a stop or a credit failure left unfinished",
|
|
80
|
+
" (rerun --status failed,pending,processing)",
|
|
81
|
+
" delete <id> --yes delete the job, its runs and stored results; frees its slot",
|
|
82
|
+
" --json print structured output: create/status/wait/cancel/pause/",
|
|
83
|
+
" resume/rerun print {ok, job_id, run_id, status, stats, job};",
|
|
84
|
+
" a wait that runs out or meets a paused run adds note",
|
|
85
|
+
"Schedules, webhooks, exports, CSV inputs and open jobs are available through the Batch API",
|
|
86
|
+
"and the SDKs: https://docs.zenrows.com",
|
|
49
87
|
].join("\n"),
|
|
50
88
|
async run(argv, ctx) {
|
|
51
89
|
const [sub, ...rest] = argv;
|
|
@@ -55,49 +93,100 @@ export const batch = {
|
|
|
55
93
|
return estimateCmd(rest, ctx);
|
|
56
94
|
case "create":
|
|
57
95
|
return await createCmd(rest, ctx);
|
|
96
|
+
case "list":
|
|
97
|
+
return await listCmd(rest, ctx);
|
|
58
98
|
case "status":
|
|
59
99
|
return await statusCmd(rest, ctx);
|
|
100
|
+
case "wait":
|
|
101
|
+
return await waitCmd(rest, ctx);
|
|
60
102
|
case "results":
|
|
61
103
|
return await resultsCmd(rest, ctx);
|
|
104
|
+
case "content":
|
|
105
|
+
return await contentCmd(rest, ctx);
|
|
62
106
|
case "cancel":
|
|
107
|
+
case "stop":
|
|
63
108
|
return await cancelCmd(rest, ctx);
|
|
64
|
-
case "
|
|
65
|
-
return await
|
|
109
|
+
case "pause":
|
|
110
|
+
return await pauseCmd(rest, ctx, "pause");
|
|
111
|
+
case "resume":
|
|
112
|
+
return await pauseCmd(rest, ctx, "resume");
|
|
113
|
+
case "rerun":
|
|
114
|
+
return await rerunCmd(rest, ctx, "rerun");
|
|
66
115
|
case "retry-failed":
|
|
67
|
-
return await
|
|
116
|
+
return await rerunCmd(rest, ctx, "retry-failed");
|
|
117
|
+
case "delete":
|
|
118
|
+
return await deleteCmd(rest, ctx);
|
|
68
119
|
default:
|
|
69
120
|
throw new ToolkitError({
|
|
70
121
|
code: "INVALID_USAGE",
|
|
71
122
|
message: `Unknown batch subcommand: ${sub ?? "(none)"}`,
|
|
72
123
|
likely_cause: "Subcommand not recognized.",
|
|
73
|
-
next_action: "Use estimate | create <file.jsonl> | status <id> | results <id> | cancel <id> |
|
|
124
|
+
next_action: "Use estimate | create <file.jsonl> | list | status <id> | wait <id> | results <id> | content <id> <task_id> | cancel <id> | pause <id> | resume <id> | rerun <id> | retry-failed <id> | delete <id>.",
|
|
125
|
+
suggested_commands: ["zenrows batch --help"],
|
|
74
126
|
});
|
|
75
127
|
}
|
|
76
128
|
}
|
|
77
|
-
catch (
|
|
129
|
+
catch (caught) {
|
|
130
|
+
// A quota error's claim advice is built from local state; check it is still current.
|
|
131
|
+
const err = await refreshQuotaAdvice(caught);
|
|
78
132
|
printError(err, ctx.json);
|
|
79
133
|
return 1;
|
|
80
134
|
}
|
|
81
135
|
},
|
|
82
136
|
};
|
|
137
|
+
/** Flags that shape the request each task sends; shared by `estimate` and `create`. */
|
|
138
|
+
const PARAM_FLAGS = {
|
|
139
|
+
"js-render": { type: "boolean" },
|
|
140
|
+
"premium-proxy": { type: "boolean" },
|
|
141
|
+
"proxy-country": { type: "string" },
|
|
142
|
+
manual: { type: "boolean" },
|
|
143
|
+
output: { type: "string" },
|
|
144
|
+
};
|
|
145
|
+
/** Job-level `zenrows_params` from the CLI flags. */
|
|
146
|
+
function jobParamsOf(values) {
|
|
147
|
+
const jobParams = {};
|
|
148
|
+
if (values["js-render"] === true)
|
|
149
|
+
jobParams.js_render = true;
|
|
150
|
+
if (values["premium-proxy"] === true)
|
|
151
|
+
jobParams.premium_proxy = true;
|
|
152
|
+
const proxyCountry = asString(values["proxy-country"]);
|
|
153
|
+
if (proxyCountry)
|
|
154
|
+
jobParams.proxy_country = proxyCountry;
|
|
155
|
+
const responseType = normalizeOutput(asString(values.output));
|
|
156
|
+
if (responseType)
|
|
157
|
+
jobParams.response_type = responseType;
|
|
158
|
+
return jobParams;
|
|
159
|
+
}
|
|
83
160
|
/** Jobs as they will be sent: Adaptive Stealth Mode applied unless manual or disabled in config. */
|
|
84
161
|
function effectiveJobs(jobs, jobParams, manual) {
|
|
85
162
|
return manual || loadConfig().defaultMode !== "auto" ? jobs : withAdaptiveStealth(jobs, jobParams);
|
|
86
163
|
}
|
|
87
164
|
function estimateCmd(rest, ctx) {
|
|
88
|
-
const { values, positionals } = parse(rest, {
|
|
165
|
+
const { values, positionals } = parse(rest, { ...PARAM_FLAGS, json: { type: "boolean" } });
|
|
166
|
+
const json = ctx.json || values.json === true;
|
|
89
167
|
const file = positionals[0];
|
|
90
168
|
if (!file)
|
|
91
169
|
throw needFile();
|
|
92
170
|
const v = validateJsonl(file);
|
|
93
|
-
const
|
|
94
|
-
|
|
95
|
-
|
|
171
|
+
const jobParams = jobParamsOf(values);
|
|
172
|
+
const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true);
|
|
173
|
+
// Price exactly what create would submit, and catch what it would refuse.
|
|
174
|
+
if (!v.errors.length)
|
|
175
|
+
toJobBody(jobs, jobParams);
|
|
176
|
+
const est = estimateCredits(jobs, jobParams);
|
|
177
|
+
if (json) {
|
|
178
|
+
log.out(JSON.stringify({
|
|
179
|
+
ok: v.errors.length === 0,
|
|
180
|
+
...v,
|
|
181
|
+
estimatedCredits: est.credits,
|
|
182
|
+
estimated_credits: est.credits,
|
|
183
|
+
estimated_credits_min: est.min,
|
|
184
|
+
}, null, 2));
|
|
96
185
|
}
|
|
97
186
|
else {
|
|
98
187
|
log.info(c(ANSI.bold, `Job spec: ${file}`));
|
|
99
188
|
log.info(`valid jobs: ${v.validJobs}/${v.totalLines}`);
|
|
100
|
-
log.info(`estimated credits
|
|
189
|
+
log.info(`estimated credits: ${est.min} to ${est.credits} (1 basic / 5 js / 10 proxy / 25 both; auto costs 1 to 25)`);
|
|
101
190
|
if (v.errors.length) {
|
|
102
191
|
log.warn(`${v.errors.length} invalid line(s):`);
|
|
103
192
|
v.errors.slice(0, 10).forEach((e) => log.dim(` line ${e.line}: ${e.reason}`));
|
|
@@ -108,15 +197,51 @@ function estimateCmd(rest, ctx) {
|
|
|
108
197
|
}
|
|
109
198
|
return v.errors.length ? 1 : 0;
|
|
110
199
|
}
|
|
200
|
+
/** `--name`, `--external-id` and `--metadata key=value`, checked against the API's limits. */
|
|
201
|
+
function jobFieldsOf(values) {
|
|
202
|
+
const fields = {};
|
|
203
|
+
const name = asString(values.name);
|
|
204
|
+
if (name !== undefined) {
|
|
205
|
+
// The API counts the name in characters (code points), not UTF-16 units.
|
|
206
|
+
if (!name.trim() || [...name].length > LIMITS.nameLength) {
|
|
207
|
+
throw usageError(`Invalid --name: it must be 1 to ${LIMITS.nameLength} characters.`, "Pass a shorter name.");
|
|
208
|
+
}
|
|
209
|
+
fields.name = name;
|
|
210
|
+
}
|
|
211
|
+
const externalId = asString(values["external-id"]);
|
|
212
|
+
if (externalId !== undefined) {
|
|
213
|
+
const e = externalIdError(externalId);
|
|
214
|
+
if (e)
|
|
215
|
+
throw usageError(`Invalid --external-id: ${e}.`, "Pass e.g. --external-id nightly-2026-10-16.");
|
|
216
|
+
fields.external_id = externalId;
|
|
217
|
+
}
|
|
218
|
+
const pairs = Array.isArray(values.metadata) ? values.metadata : [];
|
|
219
|
+
if (pairs.length) {
|
|
220
|
+
const metadata = {};
|
|
221
|
+
for (const p of pairs) {
|
|
222
|
+
const s = String(p);
|
|
223
|
+
const eq = s.indexOf("=");
|
|
224
|
+
if (eq <= 0)
|
|
225
|
+
throw usageError(`Invalid --metadata '${s}': use key=value.`, "Pass e.g. --metadata source=catalog.");
|
|
226
|
+
metadata[s.slice(0, eq)] = s.slice(eq + 1);
|
|
227
|
+
}
|
|
228
|
+
const e = metadataError(metadata);
|
|
229
|
+
if (e)
|
|
230
|
+
throw usageError(`Invalid --metadata: ${e}.`, "Fix the entry, then retry.");
|
|
231
|
+
fields.metadata = metadata;
|
|
232
|
+
}
|
|
233
|
+
return fields;
|
|
234
|
+
}
|
|
111
235
|
async function createCmd(rest, ctx) {
|
|
112
236
|
const { values, positionals } = parse(rest, {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
"
|
|
116
|
-
|
|
117
|
-
|
|
237
|
+
...PARAM_FLAGS,
|
|
238
|
+
name: { type: "string" },
|
|
239
|
+
"external-id": { type: "string" },
|
|
240
|
+
metadata: { type: "string", multiple: true },
|
|
241
|
+
"idempotency-key": { type: "string" },
|
|
118
242
|
follow: { type: "boolean" },
|
|
119
243
|
wait: { type: "boolean" }, // back-compat alias for --follow
|
|
244
|
+
timeout: { type: "string" },
|
|
120
245
|
"no-signup": { type: "boolean" },
|
|
121
246
|
json: { type: "boolean" },
|
|
122
247
|
});
|
|
@@ -127,55 +252,57 @@ async function createCmd(rest, ctx) {
|
|
|
127
252
|
const file = positionals[0];
|
|
128
253
|
if (!file)
|
|
129
254
|
throw needFile();
|
|
130
|
-
|
|
255
|
+
const waitMs = timeoutFlag(values.timeout);
|
|
256
|
+
const fields = jobFieldsOf(values);
|
|
257
|
+
const givenKey = asString(values["idempotency-key"]);
|
|
258
|
+
if (givenKey !== undefined)
|
|
259
|
+
assertIdempotencyKey(givenKey);
|
|
260
|
+
// Capability gate first: a disabled backend never gets a spec read.
|
|
131
261
|
assertUsable("batch");
|
|
132
262
|
const v = validateJsonl(file);
|
|
133
263
|
if (v.errors.length) {
|
|
134
|
-
log.warn(`${v.errors.length} invalid line(s)
|
|
264
|
+
log.warn(`${v.errors.length} invalid line(s), fix them before submitting:`);
|
|
135
265
|
v.errors.slice(0, 10).forEach((e) => log.dim(` line ${e.line}: ${e.reason}`));
|
|
136
266
|
throw new ToolkitError({
|
|
137
267
|
code: "INVALID_USAGE",
|
|
138
268
|
message: `Job spec has ${v.errors.length} invalid line(s).`,
|
|
139
|
-
likely_cause:
|
|
269
|
+
likely_cause: `One or more JSONL lines are not valid JSON or break the API's task limits (first: line ${v.errors[0].line}: ${v.errors[0].reason}).`,
|
|
140
270
|
next_action: "Fix the flagged lines (see `zenrows batch estimate`), then retry.",
|
|
141
271
|
suggested_commands: [`zenrows batch estimate ${file}`],
|
|
142
272
|
});
|
|
143
273
|
}
|
|
144
|
-
// Pre-flight governance
|
|
274
|
+
// Pre-flight governance, all local, before any network call:
|
|
145
275
|
// 1. every task URL must pass the allow/deny domain policy,
|
|
146
|
-
// 2. the run must fit the
|
|
147
|
-
|
|
148
|
-
const jobParams = {};
|
|
149
|
-
if (values["js-render"] === true)
|
|
150
|
-
jobParams.js_render = true;
|
|
151
|
-
if (values["premium-proxy"] === true)
|
|
152
|
-
jobParams.premium_proxy = true;
|
|
153
|
-
const proxyCountry = asString(values["proxy-country"]);
|
|
154
|
-
if (proxyCountry)
|
|
155
|
-
jobParams.proxy_country = proxyCountry;
|
|
156
|
-
const responseType = normalizeOutput(asString(values.output));
|
|
157
|
-
if (responseType)
|
|
158
|
-
jobParams.response_type = responseType;
|
|
276
|
+
// 2. the run must fit the page/credit caps, priced on the params as submitted.
|
|
277
|
+
const jobParams = jobParamsOf(values);
|
|
159
278
|
const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true);
|
|
160
279
|
const policy = loadPolicy();
|
|
161
280
|
for (const job of jobs)
|
|
162
281
|
assertDomainAllowed(job.url, policy);
|
|
163
|
-
const est = estimateCredits(jobs);
|
|
282
|
+
const est = estimateCredits(jobs, jobParams);
|
|
164
283
|
assertWithinLimits({ pages: v.validJobs, credits: est.credits }, policy, "batch");
|
|
165
|
-
// toJobBody validates proxy_country/premium_proxy BEFORE any HTTP call.
|
|
166
|
-
const body = toJobBody(jobs, jobParams);
|
|
284
|
+
// toJobBody validates params (proxy_country/premium_proxy, booleans, POST) BEFORE any HTTP call.
|
|
285
|
+
const body = toJobBody(jobs, jobParams, fields);
|
|
167
286
|
const apiKey = await ensureApiKey(values["no-signup"] ? { ...policy, auto_signup: false } : policy, {
|
|
168
287
|
onProvision: (a) => {
|
|
169
|
-
log.info("No API key found
|
|
288
|
+
log.info("No API key found: created a Zenrows Free plan account for you.");
|
|
170
289
|
log.dim(`Claim it anytime (keeps your usage): ${a.claimUrl}`);
|
|
171
290
|
},
|
|
172
291
|
});
|
|
292
|
+
const idempotencyKey = givenKey ?? newIdempotencyKey();
|
|
173
293
|
const runId = newRunId();
|
|
174
294
|
const startedAt = new Date().toISOString();
|
|
175
|
-
|
|
295
|
+
const request = { file, tasks: body.tasks.length, estimatedCredits: est.credits, jobParams, ...fields };
|
|
296
|
+
log.step(`Submitting batch job (${body.tasks.length} tasks, ${est.min} to ${est.credits} credits)…`);
|
|
176
297
|
try {
|
|
177
|
-
|
|
178
|
-
|
|
298
|
+
let job;
|
|
299
|
+
try {
|
|
300
|
+
job = await createJob(body, { apiKey, idempotencyKey });
|
|
301
|
+
}
|
|
302
|
+
catch (err) {
|
|
303
|
+
throw withRepeatCommand(err, "create", rest);
|
|
304
|
+
}
|
|
305
|
+
const finished = follow ? await followJob(job.job_id, apiKey, waitMs, job) : job;
|
|
179
306
|
const runError = runFailure(finished);
|
|
180
307
|
const runDir = writeRun({
|
|
181
308
|
runId,
|
|
@@ -184,11 +311,19 @@ async function createCmd(rest, ctx) {
|
|
|
184
311
|
startedAt,
|
|
185
312
|
finishedAt: new Date().toISOString(),
|
|
186
313
|
status: runError ? "error" : "ok",
|
|
187
|
-
request
|
|
188
|
-
result: {
|
|
314
|
+
request,
|
|
315
|
+
result: {
|
|
316
|
+
jobId: job.job_id,
|
|
317
|
+
job_id: job.job_id,
|
|
318
|
+
run_id: finished.latest_run?.run_id,
|
|
319
|
+
status: finished.latest_run?.status ?? "unknown",
|
|
320
|
+
...failureFields(finished),
|
|
321
|
+
},
|
|
189
322
|
...(runError ? { error: runError.toJSON() } : {}),
|
|
190
323
|
});
|
|
191
|
-
const code =
|
|
324
|
+
const code = follow
|
|
325
|
+
? printJob(finished, json, waitHeadline(finished), { note: waitNote(finished) })
|
|
326
|
+
: printJob(finished, json, `Submitted job ${job.job_id}`);
|
|
192
327
|
if (runDir && !json)
|
|
193
328
|
log.dim(` artifact: ${runDir}`);
|
|
194
329
|
return code;
|
|
@@ -201,171 +336,542 @@ async function createCmd(rest, ctx) {
|
|
|
201
336
|
startedAt,
|
|
202
337
|
finishedAt: new Date().toISOString(),
|
|
203
338
|
status: "error",
|
|
204
|
-
request
|
|
339
|
+
request,
|
|
205
340
|
error: err instanceof ToolkitError ? err.toJSON() : { message: String(err) },
|
|
206
341
|
});
|
|
207
342
|
throw err;
|
|
208
343
|
}
|
|
209
344
|
}
|
|
345
|
+
/**
|
|
346
|
+
* A write whose outcome is unknown carries its Idempotency-Key: suggest the
|
|
347
|
+
* exact command that repeats it under that key, which cannot create a duplicate.
|
|
348
|
+
*/
|
|
349
|
+
function withRepeatCommand(err, sub, rest) {
|
|
350
|
+
if (!(err instanceof ToolkitError) || !err.idempotency_key)
|
|
351
|
+
return err;
|
|
352
|
+
const args = [];
|
|
353
|
+
for (let i = 0; i < rest.length; i++) {
|
|
354
|
+
const a = rest[i];
|
|
355
|
+
if (a === "--idempotency-key") {
|
|
356
|
+
i++;
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
if (a.startsWith("--idempotency-key="))
|
|
360
|
+
continue;
|
|
361
|
+
args.push(shellQuote(a));
|
|
362
|
+
}
|
|
363
|
+
const repeat = ["zenrows", "batch", sub, ...args, "--idempotency-key", shellQuote(err.idempotency_key)].join(" ");
|
|
364
|
+
return new ToolkitError({ ...err.toJSON(), suggested_commands: [repeat, "zenrows batch list"] });
|
|
365
|
+
}
|
|
366
|
+
/** `s` as one POSIX shell word: bare when it is plainly safe, else single-quoted. */
|
|
367
|
+
export function shellQuote(s) {
|
|
368
|
+
return /^[A-Za-z0-9._\-/:=,@%+]+$/.test(s) ? s : `'${s.replace(/'/g, "'\\''")}'`;
|
|
369
|
+
}
|
|
370
|
+
async function listCmd(rest, ctx) {
|
|
371
|
+
const { values } = parse(rest, {
|
|
372
|
+
status: { type: "string" },
|
|
373
|
+
type: { type: "string" },
|
|
374
|
+
limit: { type: "string" },
|
|
375
|
+
cursor: { type: "string" },
|
|
376
|
+
json: { type: "boolean" },
|
|
377
|
+
});
|
|
378
|
+
const json = ctx.json || values.json === true;
|
|
379
|
+
const status = oneOf("status", asString(values.status), ["open", "closed"]);
|
|
380
|
+
const type = oneOf("type", asString(values.type), ["regular", "scheduled"]);
|
|
381
|
+
const limit = pageLimit(values.limit, 100);
|
|
382
|
+
const cursor = asString(values.cursor);
|
|
383
|
+
assertUsable("batch");
|
|
384
|
+
const apiKey = requireApiKey();
|
|
385
|
+
const page = await listJobs({ apiKey, limit, cursor, status, type });
|
|
386
|
+
if (json) {
|
|
387
|
+
log.out(JSON.stringify({ ok: true, count: page.jobs.length, jobs: page.jobs, next_cursor: page.next_cursor }, null, 2));
|
|
388
|
+
return 0;
|
|
389
|
+
}
|
|
390
|
+
if (!page.jobs.length)
|
|
391
|
+
log.info("No jobs.");
|
|
392
|
+
for (const j of page.jobs) {
|
|
393
|
+
const run = j.latest_run;
|
|
394
|
+
const runState = run ? `${run.status}${isPaused(run) ? " (paused)" : ""}` : "no run yet";
|
|
395
|
+
const progress = run?.stats ? `${run.stats.completed}/${run.stats.total}` : "";
|
|
396
|
+
const created = typeof j.created_at === "string" ? j.created_at : "";
|
|
397
|
+
const name = typeof j.name === "string" ? j.name : "";
|
|
398
|
+
log.out(`${j.job_id} ${String(j.type ?? "")} ${String(j.status ?? "")} ${runState} ${progress} ${created} ${name}`.trimEnd());
|
|
399
|
+
}
|
|
400
|
+
if (page.next_cursor)
|
|
401
|
+
log.dim(` more: zenrows batch list --cursor ${page.next_cursor}`);
|
|
402
|
+
return 0;
|
|
403
|
+
}
|
|
210
404
|
async function statusCmd(rest, ctx) {
|
|
211
|
-
const { positionals } = parse(rest, {});
|
|
212
|
-
const id = requireId(positionals[0]);
|
|
405
|
+
const { values, positionals } = parse(rest, { json: { type: "boolean" } });
|
|
406
|
+
const id = requireId(positionals[0], "status");
|
|
213
407
|
assertUsable("batch");
|
|
214
408
|
const apiKey = requireApiKey();
|
|
215
409
|
const job = await getJob(id, { apiKey });
|
|
216
|
-
return printJob(job, ctx.json, `Job ${id}
|
|
410
|
+
return printJob(job, ctx.json || values.json === true, `Job ${id}`, { note: isPaused(job.latest_run) ? pausedNote(id) : undefined });
|
|
411
|
+
}
|
|
412
|
+
async function waitCmd(rest, ctx) {
|
|
413
|
+
const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } });
|
|
414
|
+
const json = ctx.json || values.json === true;
|
|
415
|
+
const id = requireId(positionals[0], "wait");
|
|
416
|
+
const waitMs = timeoutFlag(values.timeout);
|
|
417
|
+
assertUsable("batch");
|
|
418
|
+
const apiKey = requireApiKey();
|
|
419
|
+
const job = await followJob(id, apiKey, waitMs);
|
|
420
|
+
return printJob(job, json, waitHeadline(job), { note: waitNote(job) });
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* Wait for a job's run to end. Ctrl-C ends the wait, not the job: say so, give
|
|
424
|
+
* the command that resumes the wait, and exit 130 as shells do on SIGINT. A
|
|
425
|
+
* submit's own answer is printed instead of a read when the wait ends before
|
|
426
|
+
* the first poll, so its `accepted_tasks` survive.
|
|
427
|
+
*/
|
|
428
|
+
async function followJob(id, apiKey, waitMs, created) {
|
|
429
|
+
const onSigint = () => {
|
|
430
|
+
log.warn(`Stopped waiting. Job ${id} is still running; it was not stopped.`);
|
|
431
|
+
log.dim(` resume: zenrows batch wait ${id}`);
|
|
432
|
+
process.exit(130);
|
|
433
|
+
};
|
|
434
|
+
process.once("SIGINT", onSigint);
|
|
435
|
+
try {
|
|
436
|
+
const job = await waitForJob(id, { apiKey, waitMs });
|
|
437
|
+
// Keep the submit/rerun answer's own fields (accepted_tasks, rerun_of, ...).
|
|
438
|
+
return created ? { ...pickAnswerFields(created), ...job } : job;
|
|
439
|
+
}
|
|
440
|
+
finally {
|
|
441
|
+
process.off("SIGINT", onSigint);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
/** The fields only a submit or rerun answer carries. */
|
|
445
|
+
function pickAnswerFields(job) {
|
|
446
|
+
const out = {};
|
|
447
|
+
for (const k of ["accepted_tasks", "rerun_of", "retried_tasks", "inherited_tasks"]) {
|
|
448
|
+
if (job[k] !== undefined)
|
|
449
|
+
out[k] = job[k];
|
|
450
|
+
}
|
|
451
|
+
return out;
|
|
452
|
+
}
|
|
453
|
+
function waitHeadline(job) {
|
|
454
|
+
if (isTerminal(job.latest_run))
|
|
455
|
+
return `Job ${job.job_id} finished`;
|
|
456
|
+
if (isPaused(job.latest_run))
|
|
457
|
+
return `Stopped waiting: job ${job.job_id} is paused`;
|
|
458
|
+
return `Stopped waiting: job ${job.job_id} is still running`;
|
|
459
|
+
}
|
|
460
|
+
/** Set when a wait ended without a terminal run: it ran out, or the run is paused. */
|
|
461
|
+
function waitNote(job) {
|
|
462
|
+
if (isTerminal(job.latest_run))
|
|
463
|
+
return undefined;
|
|
464
|
+
if (isPaused(job.latest_run))
|
|
465
|
+
return pausedNote(job.job_id);
|
|
466
|
+
return `The wait ran out and the run is still in progress. Run zenrows batch wait ${job.job_id}.`;
|
|
467
|
+
}
|
|
468
|
+
function pausedNote(id) {
|
|
469
|
+
return `The run is paused: it picks up no new task until it is resumed. Run zenrows batch resume ${id}.`;
|
|
217
470
|
}
|
|
218
471
|
async function resultsCmd(rest, ctx) {
|
|
219
472
|
const { values, positionals } = parse(rest, {
|
|
220
473
|
status: { type: "string" },
|
|
474
|
+
run: { type: "string" },
|
|
221
475
|
out: { type: "string" },
|
|
222
476
|
download: { type: "string" },
|
|
223
477
|
json: { type: "boolean" },
|
|
224
478
|
});
|
|
225
479
|
const json = ctx.json || values.json === true;
|
|
226
|
-
const id = requireId(positionals[0]);
|
|
480
|
+
const id = requireId(positionals[0], "results");
|
|
227
481
|
const status = normalizeResultStatus(asString(values.status));
|
|
482
|
+
const runId = asString(values.run);
|
|
228
483
|
assertUsable("batch");
|
|
229
484
|
const apiKey = requireApiKey();
|
|
230
|
-
const rows = await listResults(id, { apiKey, status });
|
|
485
|
+
const rows = await listResults(id, { apiKey, status, runId });
|
|
486
|
+
const state = await runState(id, runId, apiKey);
|
|
487
|
+
const note = state.unknown
|
|
488
|
+
? `The run's state could not be read, so these results may be partial. Check it with zenrows batch status ${id}, then read them again.`
|
|
489
|
+
: state.partial
|
|
490
|
+
? partialNote(state.status, id)
|
|
491
|
+
: undefined;
|
|
492
|
+
const meta = { jobId: id, job_id: id, ...(runId ? { run_id: runId } : {}), ...(state.status ? { status: state.status } : {}) };
|
|
231
493
|
const downloadDir = asString(values.download);
|
|
232
494
|
if (downloadDir) {
|
|
233
495
|
const out = await downloadResults(rows, downloadDir);
|
|
496
|
+
const expired = out.skipped.filter((s) => s.reason.startsWith("result expired")).length;
|
|
234
497
|
if (json) {
|
|
235
498
|
log.out(JSON.stringify({
|
|
236
499
|
ok: out.failed.length === 0,
|
|
237
|
-
|
|
500
|
+
...meta,
|
|
238
501
|
dir: out.dir,
|
|
239
502
|
downloaded: out.downloaded.length,
|
|
240
503
|
failed: out.failed.length,
|
|
241
504
|
skipped: out.skipped.length,
|
|
505
|
+
expired,
|
|
506
|
+
partial: state.partial,
|
|
507
|
+
...(note ? { note } : {}),
|
|
242
508
|
results: out,
|
|
243
509
|
}, null, 2));
|
|
244
510
|
}
|
|
245
511
|
else {
|
|
246
512
|
log.success(`Downloaded ${out.downloaded.length}/${rows.length} result(s) → ${out.dir}`);
|
|
247
513
|
log.dim(` index: ${join(out.dir, "_manifest.jsonl")}`);
|
|
248
|
-
if (out.skipped.length)
|
|
249
|
-
log.dim(` skipped ${out.skipped.length}
|
|
514
|
+
if (out.skipped.length) {
|
|
515
|
+
log.dim(` skipped ${out.skipped.length} with no body (failed, expired or unfinished tasks${expired ? `; ${expired} expired` : ""})`);
|
|
516
|
+
}
|
|
517
|
+
if (expired)
|
|
518
|
+
log.dim(` expired results are gone after 14 days by default; rerun the job in full: zenrows batch rerun ${id} --all`);
|
|
250
519
|
if (out.failed.length) {
|
|
251
520
|
log.warn(` ${out.failed.length} download(s) failed:`);
|
|
252
521
|
out.failed.slice(0, 10).forEach((f) => log.dim(` ${f.external_id ?? f.task_id}: ${f.reason}`));
|
|
253
522
|
}
|
|
523
|
+
if (note)
|
|
524
|
+
log.warn(` ${note}`);
|
|
254
525
|
}
|
|
255
526
|
return out.failed.length ? 1 : 0;
|
|
256
527
|
}
|
|
257
528
|
const jsonl = rows.map((r) => JSON.stringify(r)).join("\n");
|
|
258
529
|
const outFile = asString(values.out);
|
|
259
530
|
if (outFile) {
|
|
260
|
-
if (statSync(outFile, { throwIfNoEntry: false })?.isDirectory()) {
|
|
261
|
-
throw new ToolkitError({
|
|
262
|
-
code: "INVALID_USAGE",
|
|
263
|
-
message: `--out expects a file, but '${outFile}' is a directory.`,
|
|
264
|
-
likely_cause: "--out writes the results listing to a single JSONL file; you passed a directory.",
|
|
265
|
-
next_action: "Use a file path (e.g. --out results.jsonl), or download each result body into a directory with --download <dir>.",
|
|
266
|
-
suggested_commands: [`zenrows batch results ${id} --download ${outFile}`],
|
|
267
|
-
});
|
|
268
|
-
}
|
|
269
531
|
writeOut(outFile, jsonl + (jsonl ? "\n" : ""));
|
|
270
|
-
|
|
532
|
+
if (json) {
|
|
533
|
+
log.out(JSON.stringify({ ok: true, ...meta, count: rows.length, file: outFile, partial: state.partial, ...(note ? { note } : {}) }, null, 2));
|
|
534
|
+
}
|
|
535
|
+
else {
|
|
536
|
+
log.success(`Wrote ${rows.length} result(s) → ${outFile}`);
|
|
537
|
+
}
|
|
271
538
|
}
|
|
272
539
|
else if (json) {
|
|
273
|
-
log.out(JSON.stringify({ ok: true,
|
|
540
|
+
log.out(JSON.stringify({ ok: true, ...meta, count: rows.length, results: rows, partial: state.partial, ...(note ? { note } : {}) }, null, 2));
|
|
274
541
|
}
|
|
275
542
|
else {
|
|
276
|
-
log.info(`${rows.length} result(s) for job ${id}:`);
|
|
543
|
+
log.info(`${rows.length} result(s) for job ${id}${state.status ? ` (status: ${state.status})` : ""}:`);
|
|
277
544
|
log.out(jsonl);
|
|
278
545
|
}
|
|
546
|
+
if (note && !json)
|
|
547
|
+
log.warn(` ${note}`);
|
|
279
548
|
return 0;
|
|
280
549
|
}
|
|
550
|
+
/**
|
|
551
|
+
* Whether the run whose results were read can still change: in progress, or
|
|
552
|
+
* still receiving its tasks. An error here never fails the listing.
|
|
553
|
+
*/
|
|
554
|
+
async function runState(id, runId, apiKey) {
|
|
555
|
+
try {
|
|
556
|
+
const run = runId ? await getRun(id, runId, { apiKey }) : (await getJob(id, { apiKey })).latest_run;
|
|
557
|
+
if (!run)
|
|
558
|
+
return { partial: false };
|
|
559
|
+
return { status: run.status, partial: !isTerminal(run) || run.ingest_status === "pending" };
|
|
560
|
+
}
|
|
561
|
+
catch {
|
|
562
|
+
// Unknown is not complete: report the rows as possibly partial.
|
|
563
|
+
return { partial: true, unknown: true };
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
function partialNote(status, id) {
|
|
567
|
+
return status && isTerminal({ status })
|
|
568
|
+
? `The run is still receiving its tasks: these results are partial. Read them again once zenrows batch status ${id} shows ingest_status done.`
|
|
569
|
+
: `The run is still in progress: these results are partial. Wait with zenrows batch wait ${id}, then read them again.`;
|
|
570
|
+
}
|
|
571
|
+
async function contentCmd(rest, ctx) {
|
|
572
|
+
const { values, positionals } = parse(rest, { run: { type: "string" }, out: { type: "string" }, json: { type: "boolean" } });
|
|
573
|
+
const json = ctx.json || values.json === true;
|
|
574
|
+
const id = requireId(positionals[0], "content");
|
|
575
|
+
const taskId = positionals[1];
|
|
576
|
+
if (!taskId) {
|
|
577
|
+
throw new ToolkitError({
|
|
578
|
+
code: "INVALID_USAGE",
|
|
579
|
+
message: "Missing task id.",
|
|
580
|
+
likely_cause: "No <task_id> positional was provided.",
|
|
581
|
+
next_action: `Usage: zenrows batch content ${id} <task_id>. Each result row carries its task_id.`,
|
|
582
|
+
suggested_commands: [`zenrows batch results ${id}`],
|
|
583
|
+
});
|
|
584
|
+
}
|
|
585
|
+
const runId = asString(values.run);
|
|
586
|
+
assertUsable("batch");
|
|
587
|
+
const apiKey = requireApiKey();
|
|
588
|
+
const { contentType, body } = await getTaskContent(id, taskId, { apiKey, runId });
|
|
589
|
+
const ids = { job_id: id, task_id: taskId, ...(runId ? { run_id: runId } : {}) };
|
|
590
|
+
const outFile = asString(values.out);
|
|
591
|
+
if (outFile) {
|
|
592
|
+
writeOut(outFile, body);
|
|
593
|
+
if (json) {
|
|
594
|
+
log.out(JSON.stringify({ ok: true, ...ids, content_type: contentType, bytes: body.length, file: outFile }, null, 2));
|
|
595
|
+
}
|
|
596
|
+
else {
|
|
597
|
+
log.success(`Wrote ${body.length} bytes (${contentType || "unknown type"}) → ${outFile}`);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
else if (json) {
|
|
601
|
+
log.out(JSON.stringify({ ok: true, ...ids, content_type: contentType, bytes: body.length, content: body.toString("utf8") }, null, 2));
|
|
602
|
+
}
|
|
603
|
+
else {
|
|
604
|
+
process.stdout.write(body);
|
|
605
|
+
if (body.length && body[body.length - 1] !== 0x0a && isText(contentType))
|
|
606
|
+
process.stdout.write("\n");
|
|
607
|
+
}
|
|
608
|
+
return 0;
|
|
609
|
+
}
|
|
610
|
+
function isText(contentType) {
|
|
611
|
+
return /^(text\/|application\/(json|xml))/i.test(contentType);
|
|
612
|
+
}
|
|
281
613
|
async function cancelCmd(rest, ctx) {
|
|
282
|
-
const { positionals } = parse(rest, {});
|
|
283
|
-
const
|
|
614
|
+
const { values, positionals } = parse(rest, { json: { type: "boolean" } });
|
|
615
|
+
const json = ctx.json || values.json === true;
|
|
616
|
+
const id = requireId(positionals[0], "cancel");
|
|
284
617
|
assertUsable("batch");
|
|
285
618
|
const apiKey = requireApiKey();
|
|
286
|
-
|
|
287
|
-
|
|
619
|
+
let run;
|
|
620
|
+
try {
|
|
621
|
+
run = await stopJob(id, { apiKey });
|
|
622
|
+
}
|
|
623
|
+
catch (err) {
|
|
624
|
+
// An ended run cannot be stopped: answer with the job as it ended, like crawl's stop.
|
|
625
|
+
if (err instanceof ToolkitError && err.server_code === "run_not_stoppable") {
|
|
626
|
+
const job = await getJob(id, { apiKey });
|
|
627
|
+
return printJob(job, json, `Nothing to stop: job ${id} had already ended`);
|
|
628
|
+
}
|
|
629
|
+
throw err;
|
|
630
|
+
}
|
|
631
|
+
return printJob(asJob(id, run), json, `Stopped job ${id}`);
|
|
288
632
|
}
|
|
289
|
-
async function
|
|
290
|
-
const { values, positionals } = parse(rest, {
|
|
633
|
+
async function pauseCmd(rest, ctx, op) {
|
|
634
|
+
const { values, positionals } = parse(rest, { json: { type: "boolean" } });
|
|
291
635
|
const json = ctx.json || values.json === true;
|
|
292
|
-
const id = requireId(positionals[0]);
|
|
636
|
+
const id = requireId(positionals[0], op);
|
|
293
637
|
assertUsable("batch");
|
|
294
638
|
const apiKey = requireApiKey();
|
|
295
|
-
const
|
|
296
|
-
|
|
639
|
+
const run = op === "pause" ? await pauseJob(id, { apiKey }) : await resumeJob(id, { apiKey });
|
|
640
|
+
const job = asJob(id, run);
|
|
641
|
+
return op === "pause"
|
|
642
|
+
? printJob(job, json, `Paused job ${id}`, { note: isPaused(run) ? pausedNote(id) : undefined })
|
|
643
|
+
: printJob(job, json, `Resumed job ${id}`);
|
|
644
|
+
}
|
|
645
|
+
/** stop, pause and resume answer with the run, not the job. */
|
|
646
|
+
function asJob(id, run) {
|
|
647
|
+
return { job_id: run.job_id ?? id, latest_run: run };
|
|
297
648
|
}
|
|
298
|
-
async function
|
|
299
|
-
const { positionals } = parse(rest, {
|
|
300
|
-
|
|
649
|
+
async function rerunCmd(rest, ctx, sub) {
|
|
650
|
+
const { values, positionals } = parse(rest, {
|
|
651
|
+
...(sub === "rerun" ? { status: { type: "string" }, all: { type: "boolean" } } : { "include-pending": { type: "boolean" } }),
|
|
652
|
+
"idempotency-key": { type: "string" },
|
|
653
|
+
follow: { type: "boolean" },
|
|
654
|
+
timeout: { type: "string" },
|
|
655
|
+
json: { type: "boolean" },
|
|
656
|
+
});
|
|
657
|
+
const json = ctx.json || values.json === true;
|
|
658
|
+
const id = requireId(positionals[0], sub);
|
|
659
|
+
if (values.all === true && values.status !== undefined) {
|
|
660
|
+
throw usageError("--all and --status cannot be combined.", "Pass --all to rerun every task, or --status <list> to pick statuses, not both.", "--all reruns every task; --status reruns only some.");
|
|
661
|
+
}
|
|
662
|
+
const status = sub === "rerun"
|
|
663
|
+
? values.all === true
|
|
664
|
+
? undefined
|
|
665
|
+
: (rerunStatus(asString(values.status)) ?? DEFAULT_RERUN)
|
|
666
|
+
: values["include-pending"] === true
|
|
667
|
+
? UNFINISHED
|
|
668
|
+
: "failed";
|
|
669
|
+
const waitMs = timeoutFlag(values.timeout);
|
|
670
|
+
const givenKey = asString(values["idempotency-key"]);
|
|
671
|
+
if (givenKey !== undefined)
|
|
672
|
+
assertIdempotencyKey(givenKey);
|
|
301
673
|
assertUsable("batch");
|
|
302
674
|
const apiKey = requireApiKey();
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
675
|
+
let job;
|
|
676
|
+
try {
|
|
677
|
+
job = await rerunJob(id, { apiKey, status, idempotencyKey: givenKey ?? newIdempotencyKey() });
|
|
678
|
+
}
|
|
679
|
+
catch (err) {
|
|
680
|
+
throw withRepeatCommand(err, sub, rest);
|
|
681
|
+
}
|
|
682
|
+
const what = status ? `tasks with status ${status}` : "every task";
|
|
683
|
+
if (values.follow === true) {
|
|
684
|
+
const finished = await followJob(job.job_id ?? id, apiKey, waitMs, job);
|
|
685
|
+
return printJob(finished, json, waitHeadline(finished), { note: waitNote(finished) });
|
|
686
|
+
}
|
|
687
|
+
return printJob(job, json, `Started a new run of job ${id} (${what})`);
|
|
688
|
+
}
|
|
689
|
+
/** `--status` for a rerun: a comma-separated list of task statuses; undefined when unset. */
|
|
690
|
+
function rerunStatus(v) {
|
|
691
|
+
if (v === undefined)
|
|
692
|
+
return undefined;
|
|
693
|
+
const parts = [...new Set(v.split(",").map((s) => s.trim().toLowerCase()).filter(Boolean))];
|
|
694
|
+
const bad = parts.find((p) => !RERUN_STATUSES.includes(p));
|
|
695
|
+
if (!parts.length || bad !== undefined) {
|
|
696
|
+
throw usageError(`Invalid --status '${v}'.`, `Pass a comma-separated list of ${RERUN_STATUSES.join(", ")} (e.g. --status ${UNFINISHED}), leave it out to rerun failed and pending tasks, or pass --all to rerun every task.`, `A rerun selects tasks of the previous run by status: ${RERUN_STATUSES.join(", ")}.`);
|
|
697
|
+
}
|
|
698
|
+
return parts.join(",");
|
|
699
|
+
}
|
|
700
|
+
async function deleteCmd(rest, ctx) {
|
|
701
|
+
const { values, positionals } = parse(rest, { json: { type: "boolean" } });
|
|
702
|
+
const json = ctx.json || values.json === true;
|
|
703
|
+
const id = requireId(positionals[0], "delete");
|
|
704
|
+
if (!ctx.yes) {
|
|
705
|
+
throw new ToolkitError({
|
|
706
|
+
code: "INVALID_USAGE",
|
|
707
|
+
message: `Deleting job ${id} needs --yes.`,
|
|
708
|
+
likely_cause: "Delete stops the job's run and removes the job, its runs and its stored results for good.",
|
|
709
|
+
next_action: "Re-run with --yes to confirm.",
|
|
710
|
+
suggested_commands: [`zenrows batch delete ${id} --yes`],
|
|
711
|
+
});
|
|
712
|
+
}
|
|
713
|
+
assertUsable("batch");
|
|
714
|
+
const apiKey = requireApiKey();
|
|
715
|
+
await deleteJob(id, { apiKey });
|
|
716
|
+
if (json)
|
|
717
|
+
log.out(JSON.stringify({ ok: true, job_id: id, deleted: true }, null, 2));
|
|
718
|
+
else
|
|
719
|
+
log.success(`Deleted job ${id}. Its stored results are removed in the background.`);
|
|
720
|
+
return 0;
|
|
307
721
|
}
|
|
308
722
|
/** The run's failure_reason / failure_detail, when the API reports them. */
|
|
309
723
|
function failureFields(job) {
|
|
310
|
-
const run = job.latest_run
|
|
724
|
+
const run = job.latest_run;
|
|
311
725
|
const out = {};
|
|
312
|
-
if (typeof run
|
|
726
|
+
if (typeof run?.failure_reason === "string" && run.failure_reason)
|
|
313
727
|
out.failure_reason = run.failure_reason;
|
|
314
|
-
if (typeof run
|
|
728
|
+
if (typeof run?.failure_detail === "string" && run.failure_detail)
|
|
315
729
|
out.failure_detail = run.failure_detail;
|
|
316
730
|
return out;
|
|
317
731
|
}
|
|
318
732
|
/**
|
|
319
|
-
* The error for a run that ended `failed`, or null for any other state. A run
|
|
320
|
-
*
|
|
321
|
-
*
|
|
733
|
+
* The error for a run that ended `failed`, or null for any other state. A run
|
|
734
|
+
* the account's credits, its subscription or the key's credit cap stopped gets
|
|
735
|
+
* the code and advice the matching HTTP refusal gets (POLICY_MAX_CREDITS_EXCEEDED,
|
|
736
|
+
* AUTH_INVALID, KEY_CREDIT_CAP_REACHED); any other reason is BATCH_FAILED. All
|
|
737
|
+
* carry `server_code` (the failure_reason) and `job_id`, and say how to run the
|
|
738
|
+
* tasks the failure left unfinished.
|
|
322
739
|
*/
|
|
323
740
|
export function runFailure(job) {
|
|
324
741
|
if (job.latest_run?.status !== "failed")
|
|
325
742
|
return null;
|
|
326
743
|
const { failure_reason, failure_detail } = failureFields(job);
|
|
327
|
-
const
|
|
744
|
+
const id = job.job_id;
|
|
745
|
+
const where = `batch job ${id}`;
|
|
746
|
+
const rerun = `zenrows batch rerun ${id} --status ${UNFINISHED}`;
|
|
747
|
+
const ids = { ...(failure_reason ? { server_code: failure_reason } : {}), job_id: id };
|
|
748
|
+
const detail = failure_detail ? `${failure_detail.replace(/[.\s]+$/, "")}. ` : "";
|
|
328
749
|
if (isKeyCapReached(failure_reason)) {
|
|
329
|
-
|
|
750
|
+
const cap = keyCapReached(where, { status: null, detail: failure_detail }).toJSON();
|
|
751
|
+
return new ToolkitError({
|
|
752
|
+
...cap,
|
|
753
|
+
...ids,
|
|
754
|
+
next_action: thenRerun(cap.next_action, rerun),
|
|
755
|
+
suggested_commands: [...(cap.suggested_commands ?? []), rerun],
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
if (failure_reason === "insufficient_credits") {
|
|
759
|
+
const acct = readAccount();
|
|
760
|
+
const quota = quotaExhausted(where, storedClaimHint(acct), { detail: failure_detail }).toJSON();
|
|
761
|
+
return new ToolkitError({
|
|
762
|
+
...quota,
|
|
763
|
+
...ids,
|
|
764
|
+
likely_cause: `${detail}Stopped ${where}: the account ran out of credits.`,
|
|
765
|
+
next_action: thenRerun(quota.next_action, rerun),
|
|
766
|
+
suggested_commands: [...(quota.suggested_commands ?? []), rerun],
|
|
767
|
+
});
|
|
768
|
+
}
|
|
769
|
+
if (failure_reason === "subscription_inactive") {
|
|
770
|
+
const sub = subscriptionInactive(`${detail}Stopped ${where}`).toJSON();
|
|
771
|
+
return new ToolkitError({
|
|
772
|
+
...sub,
|
|
773
|
+
...ids,
|
|
774
|
+
next_action: thenRerun(sub.next_action, rerun),
|
|
775
|
+
suggested_commands: [rerun],
|
|
776
|
+
});
|
|
330
777
|
}
|
|
331
778
|
const why = [failure_reason, failure_detail].filter(Boolean).join(": ");
|
|
332
779
|
return new ToolkitError({
|
|
333
780
|
code: "BATCH_FAILED",
|
|
334
|
-
message: `Batch job ${
|
|
781
|
+
message: `Batch job ${id} stopped before it finished.`,
|
|
335
782
|
likely_cause: why || "The run ended in status failed without a reason.",
|
|
336
|
-
next_action:
|
|
337
|
-
suggested_commands: [`zenrows batch
|
|
783
|
+
next_action: `Check the run with \`zenrows batch status ${id}\`. Once the cause is fixed, run the unfinished tasks: \`${rerun}\`.`,
|
|
784
|
+
suggested_commands: [`zenrows batch status ${id}`, rerun],
|
|
785
|
+
...ids,
|
|
338
786
|
});
|
|
339
787
|
}
|
|
788
|
+
/** Where batch advice continues after a quota error's own `next_action`. */
|
|
789
|
+
const ADVICE_SUFFIX = / (An earlier attempt of this request|Then run the unfinished tasks)/;
|
|
340
790
|
/**
|
|
341
|
-
*
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
*
|
|
791
|
+
* `refreshClaimAdvice` for a batch error: it rewrites a quota error's claim
|
|
792
|
+
* advice from the current claim state, which would drop what batch adds after
|
|
793
|
+
* that advice (the safe-repeat steps of an unknown submit, or the rerun of a
|
|
794
|
+
* failed run's unfinished tasks). Refresh the quota part and keep the rest.
|
|
345
795
|
*/
|
|
346
|
-
function
|
|
347
|
-
|
|
348
|
-
|
|
796
|
+
export async function refreshQuotaAdvice(err, opts = {}) {
|
|
797
|
+
if (!(err instanceof ToolkitError))
|
|
798
|
+
return refreshClaimAdvice(err, opts);
|
|
799
|
+
const m = ADVICE_SUFFIX.exec(err.next_action);
|
|
800
|
+
if (!m)
|
|
801
|
+
return refreshClaimAdvice(err, opts);
|
|
802
|
+
const head = err.next_action.slice(0, m.index);
|
|
803
|
+
const fresh = await refreshClaimAdvice(new ToolkitError({ ...err.toJSON(), next_action: head }), opts);
|
|
804
|
+
if (!(fresh instanceof ToolkitError) || fresh.next_action === head)
|
|
805
|
+
return err;
|
|
806
|
+
const next = `${fresh.next_action.replace(/[.\s]*$/, ".")}${err.next_action.slice(m.index)}`;
|
|
807
|
+
return new ToolkitError({ ...err.toJSON(), next_action: next });
|
|
808
|
+
}
|
|
809
|
+
/** `next` followed by the command that runs the tasks a failure left unfinished. */
|
|
810
|
+
function thenRerun(next, rerun) {
|
|
811
|
+
return `${next}${/[.!?]$/.test(next) ? "" : "."} Then run the unfinished tasks: \`${rerun}\`.`;
|
|
812
|
+
}
|
|
813
|
+
/**
|
|
814
|
+
* Print a job's latest run, structured under --json, and return the exit code:
|
|
815
|
+
* 1 when the run ended `failed`, else 0. `stopped` and `deleted` are deliberate
|
|
816
|
+
* outcomes (someone cancelled or removed the run), so they exit 0 but print as
|
|
817
|
+
* a warning. `extra.note` says why a wait ended without a terminal run.
|
|
818
|
+
*/
|
|
819
|
+
function printJob(job, json, headline, extra = {}) {
|
|
820
|
+
const run = job.latest_run;
|
|
821
|
+
const stats = run?.stats;
|
|
349
822
|
const failure = failureFields(job);
|
|
350
823
|
const err = runFailure(job);
|
|
824
|
+
const paused = isPaused(run);
|
|
825
|
+
const answer = pickAnswerFields(job);
|
|
826
|
+
const ingest = run?.ingest_status;
|
|
351
827
|
if (json) {
|
|
352
|
-
log.out(JSON.stringify({
|
|
828
|
+
log.out(JSON.stringify({
|
|
829
|
+
ok: !err,
|
|
830
|
+
job_id: job.job_id,
|
|
831
|
+
jobId: job.job_id,
|
|
832
|
+
...(run?.run_id ? { run_id: run.run_id } : {}),
|
|
833
|
+
status: run?.status ?? null,
|
|
834
|
+
...(paused ? { paused: true } : {}),
|
|
835
|
+
stats,
|
|
836
|
+
...failure,
|
|
837
|
+
...(ingest ? { ingest_status: ingest } : {}),
|
|
838
|
+
...answer,
|
|
839
|
+
job,
|
|
840
|
+
...(extra.note ? { note: extra.note } : {}),
|
|
841
|
+
...(err ? { error: err.toJSON() } : {}),
|
|
842
|
+
}, null, 2));
|
|
353
843
|
return err ? 1 : 0;
|
|
354
844
|
}
|
|
355
|
-
const line = `${headline} · status: ${run
|
|
845
|
+
const line = `${headline} · status: ${run?.status ?? "no run yet"}${paused ? " (paused)" : ""}`;
|
|
356
846
|
if (err)
|
|
357
847
|
log.error(line);
|
|
358
|
-
else if (run
|
|
848
|
+
else if (run?.status === "stopped" || run?.status === "deleted" || paused || extra.note)
|
|
359
849
|
log.warn(line);
|
|
360
850
|
else
|
|
361
851
|
log.success(line);
|
|
852
|
+
if (run?.run_id)
|
|
853
|
+
log.info(` run: ${run.run_id}`);
|
|
362
854
|
if (stats) {
|
|
363
855
|
log.info(` ${stats.completed}/${stats.total} completed · ${stats.successful} successful · ${stats.failed} failed`);
|
|
364
856
|
}
|
|
857
|
+
if (answer.accepted_tasks !== undefined)
|
|
858
|
+
log.info(` accepted_tasks: ${answer.accepted_tasks}`);
|
|
859
|
+
if (answer.rerun_of) {
|
|
860
|
+
log.info(` rerun of ${answer.rerun_of}: ${answer.retried_tasks ?? 0} task(s) retried, ${answer.inherited_tasks ?? 0} carried over`);
|
|
861
|
+
}
|
|
862
|
+
if (ingest === "pending")
|
|
863
|
+
log.dim(" ingest_status: pending (the API is still receiving the tasks; results stay partial until done)");
|
|
365
864
|
if (failure.failure_reason)
|
|
366
865
|
log.info(` failure_reason: ${failure.failure_reason}`);
|
|
367
866
|
if (failure.failure_detail)
|
|
368
867
|
log.info(` failure_detail: ${failure.failure_detail}`);
|
|
868
|
+
if (!err && run?.status === "stopped" && stats && stats.completed < stats.total) {
|
|
869
|
+
log.dim(` ${stats.total - stats.completed} task(s) did not finish: zenrows batch rerun ${job.job_id} --status ${UNFINISHED}`);
|
|
870
|
+
}
|
|
871
|
+
if (extra.note) {
|
|
872
|
+
log.dim(` ${extra.note}`);
|
|
873
|
+
log.dim(` next: zenrows batch ${paused ? "resume" : "wait"} ${job.job_id}`);
|
|
874
|
+
}
|
|
369
875
|
if (err)
|
|
370
876
|
printError(err, false);
|
|
371
877
|
return err ? 1 : 0;
|
|
@@ -374,13 +880,13 @@ function normalizeResultStatus(v) {
|
|
|
374
880
|
if (!v)
|
|
375
881
|
return undefined;
|
|
376
882
|
const s = v.toLowerCase();
|
|
377
|
-
if (s === "successful" || s === "failed" || s === "all")
|
|
883
|
+
if (s === "successful" || s === "failed" || s === "processing" || s === "all")
|
|
378
884
|
return s;
|
|
379
885
|
throw new ToolkitError({
|
|
380
886
|
code: "INVALID_USAGE",
|
|
381
887
|
message: `Invalid --status: ${v}`,
|
|
382
|
-
likely_cause: "Only successful | failed | all are supported.",
|
|
383
|
-
next_action: "Use --status successful | failed | all (default: all).",
|
|
888
|
+
likely_cause: "Only successful | failed | processing | all are supported.",
|
|
889
|
+
next_action: "Use --status successful | failed | processing | all (default: all). processing lists the tasks not finished yet; expired rows come back with successful.",
|
|
384
890
|
});
|
|
385
891
|
}
|
|
386
892
|
function normalizeOutput(v) {
|
|
@@ -409,15 +915,51 @@ function normalizeOutput(v) {
|
|
|
409
915
|
const mapped = map[key];
|
|
410
916
|
return mapped ? mapped : undefined; // html → undefined (no response_type)
|
|
411
917
|
}
|
|
412
|
-
|
|
918
|
+
/**
|
|
919
|
+
* `--timeout` as a duration, returned in milliseconds; undefined when unset.
|
|
920
|
+
* Takes `500ms`, `30s`, `5m` or `1h`; a bare number keeps its earlier meaning,
|
|
921
|
+
* milliseconds.
|
|
922
|
+
*/
|
|
923
|
+
export function timeoutFlag(v) {
|
|
924
|
+
if (v === undefined)
|
|
925
|
+
return undefined;
|
|
926
|
+
const m = typeof v === "string" ? /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h)?\s*$/i.exec(v) : null;
|
|
927
|
+
const unit = { ms: 1, s: 1000, m: 60_000, h: 3_600_000 }[(m?.[2] ?? "ms").toLowerCase()];
|
|
928
|
+
const ms = m ? Number(m[1]) * unit : NaN;
|
|
929
|
+
if (!Number.isFinite(ms) || ms <= 0) {
|
|
930
|
+
throw usageError(`Invalid --timeout value '${String(v)}'.`, "Pass a duration such as --timeout 30s, 5m or 1h (a bare number is milliseconds).", "--timeout takes a positive duration.");
|
|
931
|
+
}
|
|
932
|
+
return Math.round(ms);
|
|
933
|
+
}
|
|
934
|
+
/** `--limit` for a paged read: a whole number from 1 to `max`. */
|
|
935
|
+
function pageLimit(v, max) {
|
|
936
|
+
if (v === undefined)
|
|
937
|
+
return undefined;
|
|
938
|
+
const n = asNumber(v);
|
|
939
|
+
if (n === undefined || !Number.isInteger(n) || n < 1 || n > max) {
|
|
940
|
+
throw usageError(`Invalid --limit value '${String(v)}'.`, `Pass e.g. --limit ${Math.min(max, 50)}.`, `--limit takes a whole number from 1 to ${max}.`);
|
|
941
|
+
}
|
|
942
|
+
return n;
|
|
943
|
+
}
|
|
944
|
+
function oneOf(flag, v, allowed) {
|
|
945
|
+
if (v === undefined)
|
|
946
|
+
return undefined;
|
|
947
|
+
if (allowed.includes(v))
|
|
948
|
+
return v;
|
|
949
|
+
throw usageError(`Invalid --${flag} '${v}'.`, `Pass --${flag} ${allowed.join(" or ")}, or leave it out.`);
|
|
950
|
+
}
|
|
951
|
+
function usageError(message, next_action, likely_cause = "A flag value is outside what the Batch API accepts.") {
|
|
952
|
+
return new ToolkitError({ code: "INVALID_USAGE", message, likely_cause, next_action });
|
|
953
|
+
}
|
|
954
|
+
function requireId(id, sub) {
|
|
413
955
|
if (id)
|
|
414
956
|
return id;
|
|
415
957
|
throw new ToolkitError({
|
|
416
958
|
code: "INVALID_USAGE",
|
|
417
959
|
message: "Missing job id.",
|
|
418
960
|
likely_cause: "No <id> positional was provided.",
|
|
419
|
-
next_action:
|
|
420
|
-
suggested_commands: ["zenrows batch
|
|
961
|
+
next_action: `Usage: zenrows batch ${sub} <id>. List your jobs with \`zenrows batch list\`.`,
|
|
962
|
+
suggested_commands: ["zenrows batch list"],
|
|
421
963
|
});
|
|
422
964
|
}
|
|
423
965
|
function needFile() {
|