@zenrows/cli 1.7.0 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +83 -12
  2. package/agent-plugin/README.md +1 -1
  3. package/agent-plugin/skills/batch-jobs/SKILL.md +16 -3
  4. package/agent-plugin/skills/cost-control/SKILL.md +2 -2
  5. package/agent-plugin/skills/extract/SKILL.md +49 -9
  6. package/agent-plugin/skills/protected-fetch/SKILL.md +1 -1
  7. package/agent-plugin/skills/zenrows/SKILL.md +3 -2
  8. package/dist/src/adapters/batch.js +237 -59
  9. package/dist/src/adapters/extract.js +18 -19
  10. package/dist/src/cli/commands/batch.js +665 -126
  11. package/dist/src/cli/commands/crawl.js +1 -1
  12. package/dist/src/cli/commands/extract-prepare.js +168 -0
  13. package/dist/src/cli/commands/extract.js +21 -13
  14. package/dist/src/core/batch-api.js +398 -145
  15. package/dist/src/core/config.js +1 -1
  16. package/dist/src/core/errors.js +6 -0
  17. package/dist/src/core/http.js +78 -20
  18. package/dist/src/core/policy.js +13 -6
  19. package/dist/src/core/prepared-domains-api.js +289 -0
  20. package/evals/extract-smoke/README.md +1 -1
  21. package/package.json +1 -1
  22. package/recipes/batch-estimate/RECIPE.md +6 -3
  23. package/registry/capabilities.json +3 -3
  24. package/registry/skills.json +2 -2
  25. package/registry/templates.json +1 -1
  26. package/skills/batch-jobs/SKILL.md +160 -43
  27. package/skills/crawl/SKILL.md +1 -1
  28. package/skills/extract/SKILL.md +50 -6
  29. package/skills/trace-debug/SKILL.md +17 -2
  30. package/skills/zenrows/SKILL.md +9 -5
  31. package/templates/batch-jsonl-pipeline/README.md +10 -5
  32. package/templates/extract-pricing-pages/README.md +3 -5
  33. package/workflows/competitor-intelligence/WORKFLOW.md +2 -2
@@ -1,52 +1,89 @@
1
1
  /**
2
2
  * `zenrows batch`: Batch API (status: available).
3
3
  *
4
- * Local (no key, always works): `estimate`/`create --dry-run`-style spec
5
- * validation + credit estimate. Cloud (needs a key):
6
- * `create`, `status`, `results`, `cancel`, `wait`, `retry-failed`. A 403 maps to
7
- * BATCH_ACCESS_DENIED.
4
+ * Local (no key, always works): `estimate` validates a JSONL spec and prices
5
+ * it. Cloud (needs a key): `create`, `list`, `status`, `wait`, `results`,
6
+ * `content`, `cancel`, `pause`, `resume`, `rerun`, `retry-failed`, `delete`.
7
+ * A 403 maps to BATCH_ACCESS_DENIED. Schedules, webhooks, exports, CSV inputs,
8
+ * open jobs, runs and task history are API/SDK-only.
8
9
  */
9
- import { statSync } from "node:fs";
10
10
  import { join } from "node:path";
11
11
  import { log, ANSI, c } from "../../core/logger.js";
12
- import { estimateCredits, toJobBody, validateJsonl, withAdaptiveStealth } from "../../adapters/batch.js";
12
+ import { estimateCredits, externalIdError, LIMITS, metadataError, toJobBody, validateJsonl, withAdaptiveStealth, } from "../../adapters/batch.js";
13
13
  import { loadConfig } from "../../core/config.js";
14
14
  import { requireApiKey } from "../../core/auth.js";
15
15
  import { ensureApiKey } from "../../core/ensure-key.js";
16
16
  import { assertUsable } from "../../core/capabilities.js";
17
17
  import { assertDomainAllowed, assertWithinLimits, loadPolicy } from "../../core/policy.js";
18
18
  import { newRunId, writeRun } from "../../core/artifacts.js";
19
- import { createJob, downloadResults, getJob, listResults, rerunJob, stopJob, waitForJob } from "../../core/batch-api.js";
19
+ import { RERUN_STATUSES, assertIdempotencyKey, createJob, deleteJob, downloadResults, getJob, getRun, getTaskContent, isPaused, isTerminal, listJobs, listResults, newIdempotencyKey, pauseJob, rerunJob, resumeJob, stopJob, subscriptionInactive, waitForJob, } from "../../core/batch-api.js";
20
+ import { readAccount } from "../../core/agent-account.js";
20
21
  import { asNumber, asString, parse } from "../command.js";
21
- import { ToolkitError, isKeyCapReached, keyCapReached } from "../../core/errors.js";
22
+ import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.js";
22
23
  import { printError, writeOut } from "../output.js";
23
- import { refreshClaimAdvice } from "../../core/claim-link.js";
24
+ import { refreshClaimAdvice, storedClaimHint } from "../../core/claim-link.js";
25
+ /** What `rerun` selects with no --status: the unfinished tasks, never the successful ones. */
26
+ const DEFAULT_RERUN = "failed,pending";
27
+ /** The task statuses that recover every task a run did not finish successfully. */
28
+ const UNFINISHED = "failed,pending,processing";
24
29
  export const batch = {
25
30
  name: "batch",
26
31
  summary: "Run JSONL batch jobs on Zenrows Batch.",
27
- usage: "zenrows batch <estimate|create <file.jsonl>|status <id>|results <id>|cancel <id>|wait <id>|retry-failed <id>>",
32
+ usage: "zenrows batch <estimate <file.jsonl>|create <file.jsonl>|list|status <id>|wait <id>|results <id>|content <id> <task_id>|cancel <id>|pause <id>|resume <id>|rerun <id>|retry-failed <id>|delete <id>>",
28
33
  help: [
29
34
  "Local (no key):",
30
- " estimate <file.jsonl> [--manual] validate the spec + estimate credits (upper bound)",
35
+ " estimate <file.jsonl> [flags] validate the spec and price it (credits: min and upper bound);",
36
+ " takes the same --manual / --js-render / --premium-proxy /",
37
+ " --proxy-country flags as create, so both price the same request",
31
38
  "Cloud (needs a key):",
32
39
  " create <file.jsonl> [flags] validate, then submit the job",
33
- " (default) Adaptive Stealth Mode (mode=auto) on every task that",
34
- " doesn't force js_render / premium_proxy itself",
40
+ " (default) Adaptive Stealth Mode (mode=auto) on every GET task that",
41
+ " doesn't force js_render / premium_proxy or set mode itself",
35
42
  " --manual no Adaptive Stealth Mode; plain requests unless flags below",
36
43
  " --js-render job-level: force JavaScript rendering (turns auto off)",
37
44
  " --premium-proxy job-level: force residential IPs (turns auto off)",
38
45
  " --proxy-country <cc> job-level: geo-target (auto mode, or with --premium-proxy)",
39
46
  " --output <fmt> job-level response_type (markdown|plaintext|pdf|html)",
40
- " --follow poll until the run finishes (alias: --wait)",
47
+ " --name <name> job name (up to 100 characters)",
48
+ " --external-id <id> job-level external id (letters, digits, . _ -; up to 128)",
49
+ " --metadata <key=value> job-level metadata entry (repeatable, up to 20)",
50
+ " --idempotency-key <key> so a repeated create starts no second job (one is generated",
51
+ " otherwise; the API keeps it 24 hours)",
52
+ " --follow wait until the run ends (alias: --wait; Ctrl-C stops the",
53
+ " wait, not the job)",
54
+ " --timeout <duration> with --follow: stop waiting after this long (10s, 5m, 1h;",
55
+ " a bare number is milliseconds; default 10m)",
41
56
  " --no-signup do not auto-create a Free plan account if no key exists",
42
- " status <id> show run status + stats",
43
- " results <id> [--status s] list results (successful|failed|all); paginated",
57
+ " list [--status open|closed] [--type regular|scheduled] [--limit n] [--cursor c]",
58
+ " your jobs, newest first (limit up to 100)",
59
+ " status <id> show the job's latest run: status, stats, pause state",
60
+ " wait <id> [--timeout <duration>] poll until the run ends, is paused, or the time runs out",
61
+ " (default 10m); a run still going then is printed, not stopped",
62
+ " results <id> [--status s] list results (successful|failed|processing|all); partial",
63
+ " while the run is in progress",
64
+ " --run <run_id> read an earlier run instead of the latest",
44
65
  " --out <file> write results as JSONL instead of printing",
45
66
  " --download <dir> fetch each result body into <dir> (+ _manifest.jsonl)",
46
- " cancel <id> stop an in-flight run",
47
- " wait <id> [--timeout <ms>] poll until the run finishes",
48
- " retry-failed <id> rerun only the failed tasks (new run)",
49
- " --json print structured output",
67
+ " content <id> <task_id> [--run <run_id>] [--out <file>]",
68
+ " print or save one task's result body",
69
+ " cancel <id> stop the in-flight run (an ended run answers as it ended)",
70
+ " pause <id> / resume <id> pause the in-flight run, or resume it",
71
+ " rerun <id> start a new run of the failed and pending tasks",
72
+ " (successful tasks carry over and are not billed again)",
73
+ " --status <list> rerun only tasks with these statuses",
74
+ " (pending,processing,failed,successful)",
75
+ " --all rerun every task (a full rerun; not with --status)",
76
+ " rerun also takes --idempotency-key, --follow and --timeout",
77
+ " retry-failed <id> rerun only the failed tasks (rerun --status failed); takes",
78
+ " --idempotency-key, --follow and --timeout like rerun",
79
+ " --include-pending also the tasks a stop or a credit failure left unfinished",
80
+ " (rerun --status failed,pending,processing)",
81
+ " delete <id> --yes delete the job, its runs and stored results; frees its slot",
82
+ " --json print structured output: create/status/wait/cancel/pause/",
83
+ " resume/rerun print {ok, job_id, run_id, status, stats, job};",
84
+ " a wait that runs out or meets a paused run adds note",
85
+ "Schedules, webhooks, exports, CSV inputs and open jobs are available through the Batch API",
86
+ "and the SDKs: https://docs.zenrows.com",
50
87
  ].join("\n"),
51
88
  async run(argv, ctx) {
52
89
  const [sub, ...rest] = argv;
@@ -56,51 +93,100 @@ export const batch = {
56
93
  return estimateCmd(rest, ctx);
57
94
  case "create":
58
95
  return await createCmd(rest, ctx);
96
+ case "list":
97
+ return await listCmd(rest, ctx);
59
98
  case "status":
60
99
  return await statusCmd(rest, ctx);
100
+ case "wait":
101
+ return await waitCmd(rest, ctx);
61
102
  case "results":
62
103
  return await resultsCmd(rest, ctx);
104
+ case "content":
105
+ return await contentCmd(rest, ctx);
63
106
  case "cancel":
107
+ case "stop":
64
108
  return await cancelCmd(rest, ctx);
65
- case "wait":
66
- return await waitCmd(rest, ctx);
109
+ case "pause":
110
+ return await pauseCmd(rest, ctx, "pause");
111
+ case "resume":
112
+ return await pauseCmd(rest, ctx, "resume");
113
+ case "rerun":
114
+ return await rerunCmd(rest, ctx, "rerun");
67
115
  case "retry-failed":
68
- return await retryCmd(rest, ctx);
116
+ return await rerunCmd(rest, ctx, "retry-failed");
117
+ case "delete":
118
+ return await deleteCmd(rest, ctx);
69
119
  default:
70
120
  throw new ToolkitError({
71
121
  code: "INVALID_USAGE",
72
122
  message: `Unknown batch subcommand: ${sub ?? "(none)"}`,
73
123
  likely_cause: "Subcommand not recognized.",
74
- next_action: "Use estimate | create <file.jsonl> | status <id> | results <id> | cancel <id> | wait <id> | retry-failed <id>.",
124
+ next_action: "Use estimate | create <file.jsonl> | list | status <id> | wait <id> | results <id> | content <id> <task_id> | cancel <id> | pause <id> | resume <id> | rerun <id> | retry-failed <id> | delete <id>.",
125
+ suggested_commands: ["zenrows batch --help"],
75
126
  });
76
127
  }
77
128
  }
78
129
  catch (caught) {
79
130
  // A quota error's claim advice is built from local state; check it is still current.
80
- const err = await refreshClaimAdvice(caught);
131
+ const err = await refreshQuotaAdvice(caught);
81
132
  printError(err, ctx.json);
82
133
  return 1;
83
134
  }
84
135
  },
85
136
  };
137
+ /** Flags that shape the request each task sends; shared by `estimate` and `create`. */
138
+ const PARAM_FLAGS = {
139
+ "js-render": { type: "boolean" },
140
+ "premium-proxy": { type: "boolean" },
141
+ "proxy-country": { type: "string" },
142
+ manual: { type: "boolean" },
143
+ output: { type: "string" },
144
+ };
145
+ /** Job-level `zenrows_params` from the CLI flags. */
146
+ function jobParamsOf(values) {
147
+ const jobParams = {};
148
+ if (values["js-render"] === true)
149
+ jobParams.js_render = true;
150
+ if (values["premium-proxy"] === true)
151
+ jobParams.premium_proxy = true;
152
+ const proxyCountry = asString(values["proxy-country"]);
153
+ if (proxyCountry)
154
+ jobParams.proxy_country = proxyCountry;
155
+ const responseType = normalizeOutput(asString(values.output));
156
+ if (responseType)
157
+ jobParams.response_type = responseType;
158
+ return jobParams;
159
+ }
86
160
  /** Jobs as they will be sent: Adaptive Stealth Mode applied unless manual or disabled in config. */
87
161
  function effectiveJobs(jobs, jobParams, manual) {
88
162
  return manual || loadConfig().defaultMode !== "auto" ? jobs : withAdaptiveStealth(jobs, jobParams);
89
163
  }
90
164
  function estimateCmd(rest, ctx) {
91
- const { values, positionals } = parse(rest, { manual: { type: "boolean" } });
165
+ const { values, positionals } = parse(rest, { ...PARAM_FLAGS, json: { type: "boolean" } });
166
+ const json = ctx.json || values.json === true;
92
167
  const file = positionals[0];
93
168
  if (!file)
94
169
  throw needFile();
95
170
  const v = validateJsonl(file);
96
- const est = estimateCredits(effectiveJobs(v.jobs, {}, values.manual === true));
97
- if (ctx.json) {
98
- log.out(JSON.stringify({ ok: v.errors.length === 0, ...v, estimatedCredits: est.credits }, null, 2));
171
+ const jobParams = jobParamsOf(values);
172
+ const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true);
173
+ // Price exactly what create would submit, and catch what it would refuse.
174
+ if (!v.errors.length)
175
+ toJobBody(jobs, jobParams);
176
+ const est = estimateCredits(jobs, jobParams);
177
+ if (json) {
178
+ log.out(JSON.stringify({
179
+ ok: v.errors.length === 0,
180
+ ...v,
181
+ estimatedCredits: est.credits,
182
+ estimated_credits: est.credits,
183
+ estimated_credits_min: est.min,
184
+ }, null, 2));
99
185
  }
100
186
  else {
101
187
  log.info(c(ANSI.bold, `Job spec: ${file}`));
102
188
  log.info(`valid jobs: ${v.validJobs}/${v.totalLines}`);
103
- log.info(`estimated credits, upper bound (1x basic / 5x js / 10x proxy / 25x both or auto): ${est.credits}`);
189
+ log.info(`estimated credits: ${est.min} to ${est.credits} (1 basic / 5 js / 10 proxy / 25 both; auto costs 1 to 25)`);
104
190
  if (v.errors.length) {
105
191
  log.warn(`${v.errors.length} invalid line(s):`);
106
192
  v.errors.slice(0, 10).forEach((e) => log.dim(` line ${e.line}: ${e.reason}`));
@@ -111,15 +197,51 @@ function estimateCmd(rest, ctx) {
111
197
  }
112
198
  return v.errors.length ? 1 : 0;
113
199
  }
200
+ /** `--name`, `--external-id` and `--metadata key=value`, checked against the API's limits. */
201
+ function jobFieldsOf(values) {
202
+ const fields = {};
203
+ const name = asString(values.name);
204
+ if (name !== undefined) {
205
+ // The API counts the name in characters (code points), not UTF-16 units.
206
+ if (!name.trim() || [...name].length > LIMITS.nameLength) {
207
+ throw usageError(`Invalid --name: it must be 1 to ${LIMITS.nameLength} characters.`, "Pass a shorter name.");
208
+ }
209
+ fields.name = name;
210
+ }
211
+ const externalId = asString(values["external-id"]);
212
+ if (externalId !== undefined) {
213
+ const e = externalIdError(externalId);
214
+ if (e)
215
+ throw usageError(`Invalid --external-id: ${e}.`, "Pass e.g. --external-id nightly-2026-10-16.");
216
+ fields.external_id = externalId;
217
+ }
218
+ const pairs = Array.isArray(values.metadata) ? values.metadata : [];
219
+ if (pairs.length) {
220
+ const metadata = {};
221
+ for (const p of pairs) {
222
+ const s = String(p);
223
+ const eq = s.indexOf("=");
224
+ if (eq <= 0)
225
+ throw usageError(`Invalid --metadata '${s}': use key=value.`, "Pass e.g. --metadata source=catalog.");
226
+ metadata[s.slice(0, eq)] = s.slice(eq + 1);
227
+ }
228
+ const e = metadataError(metadata);
229
+ if (e)
230
+ throw usageError(`Invalid --metadata: ${e}.`, "Fix the entry, then retry.");
231
+ fields.metadata = metadata;
232
+ }
233
+ return fields;
234
+ }
114
235
  async function createCmd(rest, ctx) {
115
236
  const { values, positionals } = parse(rest, {
116
- "js-render": { type: "boolean" },
117
- "premium-proxy": { type: "boolean" },
118
- "proxy-country": { type: "string" },
119
- manual: { type: "boolean" },
120
- output: { type: "string" },
237
+ ...PARAM_FLAGS,
238
+ name: { type: "string" },
239
+ "external-id": { type: "string" },
240
+ metadata: { type: "string", multiple: true },
241
+ "idempotency-key": { type: "string" },
121
242
  follow: { type: "boolean" },
122
243
  wait: { type: "boolean" }, // back-compat alias for --follow
244
+ timeout: { type: "string" },
123
245
  "no-signup": { type: "boolean" },
124
246
  json: { type: "boolean" },
125
247
  });
@@ -130,55 +252,57 @@ async function createCmd(rest, ctx) {
130
252
  const file = positionals[0];
131
253
  if (!file)
132
254
  throw needFile();
133
- // Capability gate first — a beta-disabled backend never gets a spec read.
255
+ const waitMs = timeoutFlag(values.timeout);
256
+ const fields = jobFieldsOf(values);
257
+ const givenKey = asString(values["idempotency-key"]);
258
+ if (givenKey !== undefined)
259
+ assertIdempotencyKey(givenKey);
260
+ // Capability gate first: a disabled backend never gets a spec read.
134
261
  assertUsable("batch");
135
262
  const v = validateJsonl(file);
136
263
  if (v.errors.length) {
137
- log.warn(`${v.errors.length} invalid line(s) — fix them before submitting:`);
264
+ log.warn(`${v.errors.length} invalid line(s), fix them before submitting:`);
138
265
  v.errors.slice(0, 10).forEach((e) => log.dim(` line ${e.line}: ${e.reason}`));
139
266
  throw new ToolkitError({
140
267
  code: "INVALID_USAGE",
141
268
  message: `Job spec has ${v.errors.length} invalid line(s).`,
142
- likely_cause: "One or more JSONL lines are not valid JSON or lack a valid `url`.",
269
+ likely_cause: `One or more JSONL lines are not valid JSON or break the API's task limits (first: line ${v.errors[0].line}: ${v.errors[0].reason}).`,
143
270
  next_action: "Fix the flagged lines (see `zenrows batch estimate`), then retry.",
144
271
  suggested_commands: [`zenrows batch estimate ${file}`],
145
272
  });
146
273
  }
147
- // Pre-flight governance — all local, before any network call:
274
+ // Pre-flight governance, all local, before any network call:
148
275
  // 1. every task URL must pass the allow/deny domain policy,
149
- // 2. the run must fit the batch-only page/credit caps (a batch is the one
150
- // primitive that fans out into many requests, so the caps bind here).
151
- const jobParams = {};
152
- if (values["js-render"] === true)
153
- jobParams.js_render = true;
154
- if (values["premium-proxy"] === true)
155
- jobParams.premium_proxy = true;
156
- const proxyCountry = asString(values["proxy-country"]);
157
- if (proxyCountry)
158
- jobParams.proxy_country = proxyCountry;
159
- const responseType = normalizeOutput(asString(values.output));
160
- if (responseType)
161
- jobParams.response_type = responseType;
276
+ // 2. the run must fit the page/credit caps, priced on the params as submitted.
277
+ const jobParams = jobParamsOf(values);
162
278
  const jobs = effectiveJobs(v.jobs, jobParams, values.manual === true);
163
279
  const policy = loadPolicy();
164
280
  for (const job of jobs)
165
281
  assertDomainAllowed(job.url, policy);
166
- const est = estimateCredits(jobs);
282
+ const est = estimateCredits(jobs, jobParams);
167
283
  assertWithinLimits({ pages: v.validJobs, credits: est.credits }, policy, "batch");
168
- // toJobBody validates proxy_country/premium_proxy BEFORE any HTTP call.
169
- const body = toJobBody(jobs, jobParams);
284
+ // toJobBody validates params (proxy_country/premium_proxy, booleans, POST) BEFORE any HTTP call.
285
+ const body = toJobBody(jobs, jobParams, fields);
170
286
  const apiKey = await ensureApiKey(values["no-signup"] ? { ...policy, auto_signup: false } : policy, {
171
287
  onProvision: (a) => {
172
- log.info("No API key found — created a Zenrows Free plan account for you.");
288
+ log.info("No API key found: created a Zenrows Free plan account for you.");
173
289
  log.dim(`Claim it anytime (keeps your usage): ${a.claimUrl}`);
174
290
  },
175
291
  });
292
+ const idempotencyKey = givenKey ?? newIdempotencyKey();
176
293
  const runId = newRunId();
177
294
  const startedAt = new Date().toISOString();
178
- log.step(`Submitting batch job (${body.tasks.length} tasks, ~${est.credits} credits)…`);
295
+ const request = { file, tasks: body.tasks.length, estimatedCredits: est.credits, jobParams, ...fields };
296
+ log.step(`Submitting batch job (${body.tasks.length} tasks, ${est.min} to ${est.credits} credits)…`);
179
297
  try {
180
- const job = await createJob(body, { apiKey });
181
- const finished = follow ? await waitForJob(job.job_id, { apiKey }) : job;
298
+ let job;
299
+ try {
300
+ job = await createJob(body, { apiKey, idempotencyKey });
301
+ }
302
+ catch (err) {
303
+ throw withRepeatCommand(err, "create", rest);
304
+ }
305
+ const finished = follow ? await followJob(job.job_id, apiKey, waitMs, job) : job;
182
306
  const runError = runFailure(finished);
183
307
  const runDir = writeRun({
184
308
  runId,
@@ -187,11 +311,19 @@ async function createCmd(rest, ctx) {
187
311
  startedAt,
188
312
  finishedAt: new Date().toISOString(),
189
313
  status: runError ? "error" : "ok",
190
- request: { file, tasks: body.tasks.length, estimatedCredits: est.credits, jobParams },
191
- result: { jobId: job.job_id, status: finished.latest_run?.status ?? "unknown", ...failureFields(finished) },
314
+ request,
315
+ result: {
316
+ jobId: job.job_id,
317
+ job_id: job.job_id,
318
+ run_id: finished.latest_run?.run_id,
319
+ status: finished.latest_run?.status ?? "unknown",
320
+ ...failureFields(finished),
321
+ },
192
322
  ...(runError ? { error: runError.toJSON() } : {}),
193
323
  });
194
- const code = printJob(finished, json, `Submitted job ${job.job_id}`);
324
+ const code = follow
325
+ ? printJob(finished, json, waitHeadline(finished), { note: waitNote(finished) })
326
+ : printJob(finished, json, `Submitted job ${job.job_id}`);
195
327
  if (runDir && !json)
196
328
  log.dim(` artifact: ${runDir}`);
197
329
  return code;
@@ -204,171 +336,542 @@ async function createCmd(rest, ctx) {
204
336
  startedAt,
205
337
  finishedAt: new Date().toISOString(),
206
338
  status: "error",
207
- request: { file, tasks: body.tasks.length, estimatedCredits: est.credits, jobParams },
339
+ request,
208
340
  error: err instanceof ToolkitError ? err.toJSON() : { message: String(err) },
209
341
  });
210
342
  throw err;
211
343
  }
212
344
  }
345
+ /**
346
+ * A write whose outcome is unknown carries its Idempotency-Key: suggest the
347
+ * exact command that repeats it under that key, which cannot create a duplicate.
348
+ */
349
+ function withRepeatCommand(err, sub, rest) {
350
+ if (!(err instanceof ToolkitError) || !err.idempotency_key)
351
+ return err;
352
+ const args = [];
353
+ for (let i = 0; i < rest.length; i++) {
354
+ const a = rest[i];
355
+ if (a === "--idempotency-key") {
356
+ i++;
357
+ continue;
358
+ }
359
+ if (a.startsWith("--idempotency-key="))
360
+ continue;
361
+ args.push(shellQuote(a));
362
+ }
363
+ const repeat = ["zenrows", "batch", sub, ...args, "--idempotency-key", shellQuote(err.idempotency_key)].join(" ");
364
+ return new ToolkitError({ ...err.toJSON(), suggested_commands: [repeat, "zenrows batch list"] });
365
+ }
366
+ /** `s` as one POSIX shell word: bare when it is plainly safe, else single-quoted. */
367
+ export function shellQuote(s) {
368
+ return /^[A-Za-z0-9._\-/:=,@%+]+$/.test(s) ? s : `'${s.replace(/'/g, "'\\''")}'`;
369
+ }
370
+ async function listCmd(rest, ctx) {
371
+ const { values } = parse(rest, {
372
+ status: { type: "string" },
373
+ type: { type: "string" },
374
+ limit: { type: "string" },
375
+ cursor: { type: "string" },
376
+ json: { type: "boolean" },
377
+ });
378
+ const json = ctx.json || values.json === true;
379
+ const status = oneOf("status", asString(values.status), ["open", "closed"]);
380
+ const type = oneOf("type", asString(values.type), ["regular", "scheduled"]);
381
+ const limit = pageLimit(values.limit, 100);
382
+ const cursor = asString(values.cursor);
383
+ assertUsable("batch");
384
+ const apiKey = requireApiKey();
385
+ const page = await listJobs({ apiKey, limit, cursor, status, type });
386
+ if (json) {
387
+ log.out(JSON.stringify({ ok: true, count: page.jobs.length, jobs: page.jobs, next_cursor: page.next_cursor }, null, 2));
388
+ return 0;
389
+ }
390
+ if (!page.jobs.length)
391
+ log.info("No jobs.");
392
+ for (const j of page.jobs) {
393
+ const run = j.latest_run;
394
+ const runState = run ? `${run.status}${isPaused(run) ? " (paused)" : ""}` : "no run yet";
395
+ const progress = run?.stats ? `${run.stats.completed}/${run.stats.total}` : "";
396
+ const created = typeof j.created_at === "string" ? j.created_at : "";
397
+ const name = typeof j.name === "string" ? j.name : "";
398
+ log.out(`${j.job_id} ${String(j.type ?? "")} ${String(j.status ?? "")} ${runState} ${progress} ${created} ${name}`.trimEnd());
399
+ }
400
+ if (page.next_cursor)
401
+ log.dim(` more: zenrows batch list --cursor ${page.next_cursor}`);
402
+ return 0;
403
+ }
213
404
  async function statusCmd(rest, ctx) {
214
- const { positionals } = parse(rest, {});
215
- const id = requireId(positionals[0]);
405
+ const { values, positionals } = parse(rest, { json: { type: "boolean" } });
406
+ const id = requireId(positionals[0], "status");
216
407
  assertUsable("batch");
217
408
  const apiKey = requireApiKey();
218
409
  const job = await getJob(id, { apiKey });
219
- return printJob(job, ctx.json, `Job ${id}`);
410
+ return printJob(job, ctx.json || values.json === true, `Job ${id}`, { note: isPaused(job.latest_run) ? pausedNote(id) : undefined });
411
+ }
412
+ async function waitCmd(rest, ctx) {
413
+ const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } });
414
+ const json = ctx.json || values.json === true;
415
+ const id = requireId(positionals[0], "wait");
416
+ const waitMs = timeoutFlag(values.timeout);
417
+ assertUsable("batch");
418
+ const apiKey = requireApiKey();
419
+ const job = await followJob(id, apiKey, waitMs);
420
+ return printJob(job, json, waitHeadline(job), { note: waitNote(job) });
421
+ }
422
+ /**
423
+ * Wait for a job's run to end. Ctrl-C ends the wait, not the job: say so, give
424
+ * the command that resumes the wait, and exit 130 as shells do on SIGINT. A
425
+ * submit's own answer is printed instead of a read when the wait ends before
426
+ * the first poll, so its `accepted_tasks` survive.
427
+ */
428
+ async function followJob(id, apiKey, waitMs, created) {
429
+ const onSigint = () => {
430
+ log.warn(`Stopped waiting. Job ${id} is still running; it was not stopped.`);
431
+ log.dim(` resume: zenrows batch wait ${id}`);
432
+ process.exit(130);
433
+ };
434
+ process.once("SIGINT", onSigint);
435
+ try {
436
+ const job = await waitForJob(id, { apiKey, waitMs });
437
+ // Keep the submit/rerun answer's own fields (accepted_tasks, rerun_of, ...).
438
+ return created ? { ...pickAnswerFields(created), ...job } : job;
439
+ }
440
+ finally {
441
+ process.off("SIGINT", onSigint);
442
+ }
443
+ }
444
+ /** The fields only a submit or rerun answer carries. */
445
+ function pickAnswerFields(job) {
446
+ const out = {};
447
+ for (const k of ["accepted_tasks", "rerun_of", "retried_tasks", "inherited_tasks"]) {
448
+ if (job[k] !== undefined)
449
+ out[k] = job[k];
450
+ }
451
+ return out;
452
+ }
453
+ function waitHeadline(job) {
454
+ if (isTerminal(job.latest_run))
455
+ return `Job ${job.job_id} finished`;
456
+ if (isPaused(job.latest_run))
457
+ return `Stopped waiting: job ${job.job_id} is paused`;
458
+ return `Stopped waiting: job ${job.job_id} is still running`;
459
+ }
460
+ /** Set when a wait ended without a terminal run: it ran out, or the run is paused. */
461
+ function waitNote(job) {
462
+ if (isTerminal(job.latest_run))
463
+ return undefined;
464
+ if (isPaused(job.latest_run))
465
+ return pausedNote(job.job_id);
466
+ return `The wait ran out and the run is still in progress. Run zenrows batch wait ${job.job_id}.`;
467
+ }
468
+ function pausedNote(id) {
469
+ return `The run is paused: it picks up no new task until it is resumed. Run zenrows batch resume ${id}.`;
220
470
  }
221
471
  async function resultsCmd(rest, ctx) {
222
472
  const { values, positionals } = parse(rest, {
223
473
  status: { type: "string" },
474
+ run: { type: "string" },
224
475
  out: { type: "string" },
225
476
  download: { type: "string" },
226
477
  json: { type: "boolean" },
227
478
  });
228
479
  const json = ctx.json || values.json === true;
229
- const id = requireId(positionals[0]);
480
+ const id = requireId(positionals[0], "results");
230
481
  const status = normalizeResultStatus(asString(values.status));
482
+ const runId = asString(values.run);
231
483
  assertUsable("batch");
232
484
  const apiKey = requireApiKey();
233
- const rows = await listResults(id, { apiKey, status });
485
+ const rows = await listResults(id, { apiKey, status, runId });
486
+ const state = await runState(id, runId, apiKey);
487
+ const note = state.unknown
488
+ ? `The run's state could not be read, so these results may be partial. Check it with zenrows batch status ${id}, then read them again.`
489
+ : state.partial
490
+ ? partialNote(state.status, id)
491
+ : undefined;
492
+ const meta = { jobId: id, job_id: id, ...(runId ? { run_id: runId } : {}), ...(state.status ? { status: state.status } : {}) };
234
493
  const downloadDir = asString(values.download);
235
494
  if (downloadDir) {
236
495
  const out = await downloadResults(rows, downloadDir);
496
+ const expired = out.skipped.filter((s) => s.reason.startsWith("result expired")).length;
237
497
  if (json) {
238
498
  log.out(JSON.stringify({
239
499
  ok: out.failed.length === 0,
240
- jobId: id,
500
+ ...meta,
241
501
  dir: out.dir,
242
502
  downloaded: out.downloaded.length,
243
503
  failed: out.failed.length,
244
504
  skipped: out.skipped.length,
505
+ expired,
506
+ partial: state.partial,
507
+ ...(note ? { note } : {}),
245
508
  results: out,
246
509
  }, null, 2));
247
510
  }
248
511
  else {
249
512
  log.success(`Downloaded ${out.downloaded.length}/${rows.length} result(s) → ${out.dir}`);
250
513
  log.dim(` index: ${join(out.dir, "_manifest.jsonl")}`);
251
- if (out.skipped.length)
252
- log.dim(` skipped ${out.skipped.length} (no body — e.g. failed tasks)`);
514
+ if (out.skipped.length) {
515
+ log.dim(` skipped ${out.skipped.length} with no body (failed, expired or unfinished tasks${expired ? `; ${expired} expired` : ""})`);
516
+ }
517
+ if (expired)
518
+ log.dim(` expired results are gone after 14 days by default; rerun the job in full: zenrows batch rerun ${id} --all`);
253
519
  if (out.failed.length) {
254
520
  log.warn(` ${out.failed.length} download(s) failed:`);
255
521
  out.failed.slice(0, 10).forEach((f) => log.dim(` ${f.external_id ?? f.task_id}: ${f.reason}`));
256
522
  }
523
+ if (note)
524
+ log.warn(` ${note}`);
257
525
  }
258
526
  return out.failed.length ? 1 : 0;
259
527
  }
260
528
  const jsonl = rows.map((r) => JSON.stringify(r)).join("\n");
261
529
  const outFile = asString(values.out);
262
530
  if (outFile) {
263
- if (statSync(outFile, { throwIfNoEntry: false })?.isDirectory()) {
264
- throw new ToolkitError({
265
- code: "INVALID_USAGE",
266
- message: `--out expects a file, but '${outFile}' is a directory.`,
267
- likely_cause: "--out writes the results listing to a single JSONL file; you passed a directory.",
268
- next_action: "Use a file path (e.g. --out results.jsonl), or download each result body into a directory with --download <dir>.",
269
- suggested_commands: [`zenrows batch results ${id} --download ${outFile}`],
270
- });
271
- }
272
531
  writeOut(outFile, jsonl + (jsonl ? "\n" : ""));
273
- log.success(`Wrote ${rows.length} result(s) → ${outFile}`);
532
+ if (json) {
533
+ log.out(JSON.stringify({ ok: true, ...meta, count: rows.length, file: outFile, partial: state.partial, ...(note ? { note } : {}) }, null, 2));
534
+ }
535
+ else {
536
+ log.success(`Wrote ${rows.length} result(s) → ${outFile}`);
537
+ }
274
538
  }
275
539
  else if (json) {
276
- log.out(JSON.stringify({ ok: true, jobId: id, count: rows.length, results: rows }, null, 2));
540
+ log.out(JSON.stringify({ ok: true, ...meta, count: rows.length, results: rows, partial: state.partial, ...(note ? { note } : {}) }, null, 2));
277
541
  }
278
542
  else {
279
- log.info(`${rows.length} result(s) for job ${id}:`);
543
+ log.info(`${rows.length} result(s) for job ${id}${state.status ? ` (status: ${state.status})` : ""}:`);
280
544
  log.out(jsonl);
281
545
  }
546
+ if (note && !json)
547
+ log.warn(` ${note}`);
282
548
  return 0;
283
549
  }
550
+ /**
551
+ * Whether the run whose results were read can still change: in progress, or
552
+ * still receiving its tasks. An error here never fails the listing.
553
+ */
554
+ async function runState(id, runId, apiKey) {
555
+ try {
556
+ const run = runId ? await getRun(id, runId, { apiKey }) : (await getJob(id, { apiKey })).latest_run;
557
+ if (!run)
558
+ return { partial: false };
559
+ return { status: run.status, partial: !isTerminal(run) || run.ingest_status === "pending" };
560
+ }
561
+ catch {
562
+ // Unknown is not complete: report the rows as possibly partial.
563
+ return { partial: true, unknown: true };
564
+ }
565
+ }
566
+ function partialNote(status, id) {
567
+ return status && isTerminal({ status })
568
+ ? `The run is still receiving its tasks: these results are partial. Read them again once zenrows batch status ${id} shows ingest_status done.`
569
+ : `The run is still in progress: these results are partial. Wait with zenrows batch wait ${id}, then read them again.`;
570
+ }
571
+ async function contentCmd(rest, ctx) {
572
+ const { values, positionals } = parse(rest, { run: { type: "string" }, out: { type: "string" }, json: { type: "boolean" } });
573
+ const json = ctx.json || values.json === true;
574
+ const id = requireId(positionals[0], "content");
575
+ const taskId = positionals[1];
576
+ if (!taskId) {
577
+ throw new ToolkitError({
578
+ code: "INVALID_USAGE",
579
+ message: "Missing task id.",
580
+ likely_cause: "No <task_id> positional was provided.",
581
+ next_action: `Usage: zenrows batch content ${id} <task_id>. Each result row carries its task_id.`,
582
+ suggested_commands: [`zenrows batch results ${id}`],
583
+ });
584
+ }
585
+ const runId = asString(values.run);
586
+ assertUsable("batch");
587
+ const apiKey = requireApiKey();
588
+ const { contentType, body } = await getTaskContent(id, taskId, { apiKey, runId });
589
+ const ids = { job_id: id, task_id: taskId, ...(runId ? { run_id: runId } : {}) };
590
+ const outFile = asString(values.out);
591
+ if (outFile) {
592
+ writeOut(outFile, body);
593
+ if (json) {
594
+ log.out(JSON.stringify({ ok: true, ...ids, content_type: contentType, bytes: body.length, file: outFile }, null, 2));
595
+ }
596
+ else {
597
+ log.success(`Wrote ${body.length} bytes (${contentType || "unknown type"}) → ${outFile}`);
598
+ }
599
+ }
600
+ else if (json) {
601
+ log.out(JSON.stringify({ ok: true, ...ids, content_type: contentType, bytes: body.length, content: body.toString("utf8") }, null, 2));
602
+ }
603
+ else {
604
+ process.stdout.write(body);
605
+ if (body.length && body[body.length - 1] !== 0x0a && isText(contentType))
606
+ process.stdout.write("\n");
607
+ }
608
+ return 0;
609
+ }
610
+ function isText(contentType) {
611
+ return /^(text\/|application\/(json|xml))/i.test(contentType);
612
+ }
284
613
  async function cancelCmd(rest, ctx) {
285
- const { positionals } = parse(rest, {});
286
- const id = requireId(positionals[0]);
614
+ const { values, positionals } = parse(rest, { json: { type: "boolean" } });
615
+ const json = ctx.json || values.json === true;
616
+ const id = requireId(positionals[0], "cancel");
287
617
  assertUsable("batch");
288
618
  const apiKey = requireApiKey();
289
- const job = await stopJob(id, { apiKey });
290
- return printJob(job, ctx.json, `Stopped job ${id}`);
619
+ let run;
620
+ try {
621
+ run = await stopJob(id, { apiKey });
622
+ }
623
+ catch (err) {
624
+ // An ended run cannot be stopped: answer with the job as it ended, like crawl's stop.
625
+ if (err instanceof ToolkitError && err.server_code === "run_not_stoppable") {
626
+ const job = await getJob(id, { apiKey });
627
+ return printJob(job, json, `Nothing to stop: job ${id} had already ended`);
628
+ }
629
+ throw err;
630
+ }
631
+ return printJob(asJob(id, run), json, `Stopped job ${id}`);
291
632
  }
292
- async function waitCmd(rest, ctx) {
293
- const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } });
633
+ async function pauseCmd(rest, ctx, op) {
634
+ const { values, positionals } = parse(rest, { json: { type: "boolean" } });
635
+ const json = ctx.json || values.json === true;
636
+ const id = requireId(positionals[0], op);
637
+ assertUsable("batch");
638
+ const apiKey = requireApiKey();
639
+ const run = op === "pause" ? await pauseJob(id, { apiKey }) : await resumeJob(id, { apiKey });
640
+ const job = asJob(id, run);
641
+ return op === "pause"
642
+ ? printJob(job, json, `Paused job ${id}`, { note: isPaused(run) ? pausedNote(id) : undefined })
643
+ : printJob(job, json, `Resumed job ${id}`);
644
+ }
645
+ /** stop, pause and resume answer with the run, not the job. */
646
+ function asJob(id, run) {
647
+ return { job_id: run.job_id ?? id, latest_run: run };
648
+ }
649
+ async function rerunCmd(rest, ctx, sub) {
650
+ const { values, positionals } = parse(rest, {
651
+ ...(sub === "rerun" ? { status: { type: "string" }, all: { type: "boolean" } } : { "include-pending": { type: "boolean" } }),
652
+ "idempotency-key": { type: "string" },
653
+ follow: { type: "boolean" },
654
+ timeout: { type: "string" },
655
+ json: { type: "boolean" },
656
+ });
294
657
  const json = ctx.json || values.json === true;
295
- const id = requireId(positionals[0]);
658
+ const id = requireId(positionals[0], sub);
659
+ if (values.all === true && values.status !== undefined) {
660
+ throw usageError("--all and --status cannot be combined.", "Pass --all to rerun every task, or --status <list> to pick statuses, not both.", "--all reruns every task; --status reruns only some.");
661
+ }
662
+ const status = sub === "rerun"
663
+ ? values.all === true
664
+ ? undefined
665
+ : (rerunStatus(asString(values.status)) ?? DEFAULT_RERUN)
666
+ : values["include-pending"] === true
667
+ ? UNFINISHED
668
+ : "failed";
669
+ const waitMs = timeoutFlag(values.timeout);
670
+ const givenKey = asString(values["idempotency-key"]);
671
+ if (givenKey !== undefined)
672
+ assertIdempotencyKey(givenKey);
296
673
  assertUsable("batch");
297
674
  const apiKey = requireApiKey();
298
- const job = await waitForJob(id, { apiKey, timeoutMs: asNumber(values.timeout) });
299
- return printJob(job, json, `Job ${id} finished`);
675
+ let job;
676
+ try {
677
+ job = await rerunJob(id, { apiKey, status, idempotencyKey: givenKey ?? newIdempotencyKey() });
678
+ }
679
+ catch (err) {
680
+ throw withRepeatCommand(err, sub, rest);
681
+ }
682
+ const what = status ? `tasks with status ${status}` : "every task";
683
+ if (values.follow === true) {
684
+ const finished = await followJob(job.job_id ?? id, apiKey, waitMs, job);
685
+ return printJob(finished, json, waitHeadline(finished), { note: waitNote(finished) });
686
+ }
687
+ return printJob(job, json, `Started a new run of job ${id} (${what})`);
300
688
  }
301
- async function retryCmd(rest, ctx) {
302
- const { positionals } = parse(rest, {});
303
- const id = requireId(positionals[0]);
689
+ /** `--status` for a rerun: a comma-separated list of task statuses; undefined when unset. */
690
+ function rerunStatus(v) {
691
+ if (v === undefined)
692
+ return undefined;
693
+ const parts = [...new Set(v.split(",").map((s) => s.trim().toLowerCase()).filter(Boolean))];
694
+ const bad = parts.find((p) => !RERUN_STATUSES.includes(p));
695
+ if (!parts.length || bad !== undefined) {
696
+ throw usageError(`Invalid --status '${v}'.`, `Pass a comma-separated list of ${RERUN_STATUSES.join(", ")} (e.g. --status ${UNFINISHED}), leave it out to rerun failed and pending tasks, or pass --all to rerun every task.`, `A rerun selects tasks of the previous run by status: ${RERUN_STATUSES.join(", ")}.`);
697
+ }
698
+ return parts.join(",");
699
+ }
700
+ async function deleteCmd(rest, ctx) {
701
+ const { values, positionals } = parse(rest, { json: { type: "boolean" } });
702
+ const json = ctx.json || values.json === true;
703
+ const id = requireId(positionals[0], "delete");
704
+ if (!ctx.yes) {
705
+ throw new ToolkitError({
706
+ code: "INVALID_USAGE",
707
+ message: `Deleting job ${id} needs --yes.`,
708
+ likely_cause: "Delete stops the job's run and removes the job, its runs and its stored results for good.",
709
+ next_action: "Re-run with --yes to confirm.",
710
+ suggested_commands: [`zenrows batch delete ${id} --yes`],
711
+ });
712
+ }
304
713
  assertUsable("batch");
305
714
  const apiKey = requireApiKey();
306
- // "Reruns and retrying failures": POST /jobs/{id}/rerun?status=failed replays
307
- // only the failures; already-successful tasks carry over.
308
- const job = await rerunJob(id, { apiKey, status: "failed" });
309
- return printJob(job, ctx.json, `Reran failed tasks for job ${id}`);
715
+ await deleteJob(id, { apiKey });
716
+ if (json)
717
+ log.out(JSON.stringify({ ok: true, job_id: id, deleted: true }, null, 2));
718
+ else
719
+ log.success(`Deleted job ${id}. Its stored results are removed in the background.`);
720
+ return 0;
310
721
  }
311
722
  /** The run's failure_reason / failure_detail, when the API reports them. */
312
723
  function failureFields(job) {
313
- const run = job.latest_run ?? {};
724
+ const run = job.latest_run;
314
725
  const out = {};
315
- if (typeof run.failure_reason === "string" && run.failure_reason)
726
+ if (typeof run?.failure_reason === "string" && run.failure_reason)
316
727
  out.failure_reason = run.failure_reason;
317
- if (typeof run.failure_detail === "string" && run.failure_detail)
728
+ if (typeof run?.failure_detail === "string" && run.failure_detail)
318
729
  out.failure_detail = run.failure_detail;
319
730
  return out;
320
731
  }
321
732
  /**
322
- * The error for a run that ended `failed`, or null for any other state. A run the
323
- * key's credit cap stopped (`api_key_cap_reached`) gets the cap guidance: the
324
- * account still has credits and its other keys keep working.
733
+ * The error for a run that ended `failed`, or null for any other state. A run
734
+ * the account's credits, its subscription or the key's credit cap stopped gets
735
+ * the code and advice the matching HTTP refusal gets (POLICY_MAX_CREDITS_EXCEEDED,
736
+ * AUTH_INVALID, KEY_CREDIT_CAP_REACHED); any other reason is BATCH_FAILED. All
737
+ * carry `server_code` (the failure_reason) and `job_id`, and say how to run the
738
+ * tasks the failure left unfinished.
325
739
  */
326
740
  export function runFailure(job) {
327
741
  if (job.latest_run?.status !== "failed")
328
742
  return null;
329
743
  const { failure_reason, failure_detail } = failureFields(job);
330
- const where = `batch job ${job.job_id}`;
744
+ const id = job.job_id;
745
+ const where = `batch job ${id}`;
746
+ const rerun = `zenrows batch rerun ${id} --status ${UNFINISHED}`;
747
+ const ids = { ...(failure_reason ? { server_code: failure_reason } : {}), job_id: id };
748
+ const detail = failure_detail ? `${failure_detail.replace(/[.\s]+$/, "")}. ` : "";
331
749
  if (isKeyCapReached(failure_reason)) {
332
- return keyCapReached(where, { status: null, detail: failure_detail });
750
+ const cap = keyCapReached(where, { status: null, detail: failure_detail }).toJSON();
751
+ return new ToolkitError({
752
+ ...cap,
753
+ ...ids,
754
+ next_action: thenRerun(cap.next_action, rerun),
755
+ suggested_commands: [...(cap.suggested_commands ?? []), rerun],
756
+ });
757
+ }
758
+ if (failure_reason === "insufficient_credits") {
759
+ const acct = readAccount();
760
+ const quota = quotaExhausted(where, storedClaimHint(acct), { detail: failure_detail }).toJSON();
761
+ return new ToolkitError({
762
+ ...quota,
763
+ ...ids,
764
+ likely_cause: `${detail}Stopped ${where}: the account ran out of credits.`,
765
+ next_action: thenRerun(quota.next_action, rerun),
766
+ suggested_commands: [...(quota.suggested_commands ?? []), rerun],
767
+ });
768
+ }
769
+ if (failure_reason === "subscription_inactive") {
770
+ const sub = subscriptionInactive(`${detail}Stopped ${where}`).toJSON();
771
+ return new ToolkitError({
772
+ ...sub,
773
+ ...ids,
774
+ next_action: thenRerun(sub.next_action, rerun),
775
+ suggested_commands: [rerun],
776
+ });
333
777
  }
334
778
  const why = [failure_reason, failure_detail].filter(Boolean).join(": ");
335
779
  return new ToolkitError({
336
780
  code: "BATCH_FAILED",
337
- message: `Batch job ${job.job_id} failed.`,
781
+ message: `Batch job ${id} stopped before it finished.`,
338
782
  likely_cause: why || "The run ended in status failed without a reason.",
339
- next_action: "Inspect the per-task results, fix the cause, then rerun the failed tasks.",
340
- suggested_commands: [`zenrows batch results ${job.job_id} --status failed`, `zenrows batch retry-failed ${job.job_id}`],
783
+ next_action: `Check the run with \`zenrows batch status ${id}\`. Once the cause is fixed, run the unfinished tasks: \`${rerun}\`.`,
784
+ suggested_commands: [`zenrows batch status ${id}`, rerun],
785
+ ...ids,
341
786
  });
342
787
  }
788
+ /** Where batch advice continues after a quota error's own `next_action`. */
789
+ const ADVICE_SUFFIX = / (An earlier attempt of this request|Then run the unfinished tasks)/;
343
790
  /**
344
- * Print a job's status + stats, structured under --json, and return the exit
345
- * code: 1 when the run ended `failed`, else 0. `stopped` and `deleted` are
346
- * deliberate outcomes (someone cancelled or removed the run), so they exit 0
347
- * but print as a warning, not a success.
791
+ * `refreshClaimAdvice` for a batch error: it rewrites a quota error's claim
792
+ * advice from the current claim state, which would drop what batch adds after
793
+ * that advice (the safe-repeat steps of an unknown submit, or the rerun of a
794
+ * failed run's unfinished tasks). Refresh the quota part and keep the rest.
348
795
  */
349
- function printJob(job, json, headline) {
350
- const run = job.latest_run ?? {};
351
- const stats = run.stats;
796
+ export async function refreshQuotaAdvice(err, opts = {}) {
797
+ if (!(err instanceof ToolkitError))
798
+ return refreshClaimAdvice(err, opts);
799
+ const m = ADVICE_SUFFIX.exec(err.next_action);
800
+ if (!m)
801
+ return refreshClaimAdvice(err, opts);
802
+ const head = err.next_action.slice(0, m.index);
803
+ const fresh = await refreshClaimAdvice(new ToolkitError({ ...err.toJSON(), next_action: head }), opts);
804
+ if (!(fresh instanceof ToolkitError) || fresh.next_action === head)
805
+ return err;
806
+ const next = `${fresh.next_action.replace(/[.\s]*$/, ".")}${err.next_action.slice(m.index)}`;
807
+ return new ToolkitError({ ...err.toJSON(), next_action: next });
808
+ }
809
+ /** `next` followed by the command that runs the tasks a failure left unfinished. */
810
+ function thenRerun(next, rerun) {
811
+ return `${next}${/[.!?]$/.test(next) ? "" : "."} Then run the unfinished tasks: \`${rerun}\`.`;
812
+ }
813
+ /**
814
+ * Print a job's latest run, structured under --json, and return the exit code:
815
+ * 1 when the run ended `failed`, else 0. `stopped` and `deleted` are deliberate
816
+ * outcomes (someone cancelled or removed the run), so they exit 0 but print as
817
+ * a warning. `extra.note` says why a wait ended without a terminal run.
818
+ */
819
+ function printJob(job, json, headline, extra = {}) {
820
+ const run = job.latest_run;
821
+ const stats = run?.stats;
352
822
  const failure = failureFields(job);
353
823
  const err = runFailure(job);
824
+ const paused = isPaused(run);
825
+ const answer = pickAnswerFields(job);
826
+ const ingest = run?.ingest_status;
354
827
  if (json) {
355
- log.out(JSON.stringify({ ok: !err, jobId: job.job_id, status: run.status, stats, ...failure, ...(err ? { error: err.toJSON() } : {}) }, null, 2));
828
+ log.out(JSON.stringify({
829
+ ok: !err,
830
+ job_id: job.job_id,
831
+ jobId: job.job_id,
832
+ ...(run?.run_id ? { run_id: run.run_id } : {}),
833
+ status: run?.status ?? null,
834
+ ...(paused ? { paused: true } : {}),
835
+ stats,
836
+ ...failure,
837
+ ...(ingest ? { ingest_status: ingest } : {}),
838
+ ...answer,
839
+ job,
840
+ ...(extra.note ? { note: extra.note } : {}),
841
+ ...(err ? { error: err.toJSON() } : {}),
842
+ }, null, 2));
356
843
  return err ? 1 : 0;
357
844
  }
358
- const line = `${headline} · status: ${run.status ?? "unknown"}`;
845
+ const line = `${headline} · status: ${run?.status ?? "no run yet"}${paused ? " (paused)" : ""}`;
359
846
  if (err)
360
847
  log.error(line);
361
- else if (run.status === "stopped" || run.status === "deleted")
848
+ else if (run?.status === "stopped" || run?.status === "deleted" || paused || extra.note)
362
849
  log.warn(line);
363
850
  else
364
851
  log.success(line);
852
+ if (run?.run_id)
853
+ log.info(` run: ${run.run_id}`);
365
854
  if (stats) {
366
855
  log.info(` ${stats.completed}/${stats.total} completed · ${stats.successful} successful · ${stats.failed} failed`);
367
856
  }
857
+ if (answer.accepted_tasks !== undefined)
858
+ log.info(` accepted_tasks: ${answer.accepted_tasks}`);
859
+ if (answer.rerun_of) {
860
+ log.info(` rerun of ${answer.rerun_of}: ${answer.retried_tasks ?? 0} task(s) retried, ${answer.inherited_tasks ?? 0} carried over`);
861
+ }
862
+ if (ingest === "pending")
863
+ log.dim(" ingest_status: pending (the API is still receiving the tasks; results stay partial until done)");
368
864
  if (failure.failure_reason)
369
865
  log.info(` failure_reason: ${failure.failure_reason}`);
370
866
  if (failure.failure_detail)
371
867
  log.info(` failure_detail: ${failure.failure_detail}`);
868
+ if (!err && run?.status === "stopped" && stats && stats.completed < stats.total) {
869
+ log.dim(` ${stats.total - stats.completed} task(s) did not finish: zenrows batch rerun ${job.job_id} --status ${UNFINISHED}`);
870
+ }
871
+ if (extra.note) {
872
+ log.dim(` ${extra.note}`);
873
+ log.dim(` next: zenrows batch ${paused ? "resume" : "wait"} ${job.job_id}`);
874
+ }
372
875
  if (err)
373
876
  printError(err, false);
374
877
  return err ? 1 : 0;
@@ -377,13 +880,13 @@ function normalizeResultStatus(v) {
377
880
  if (!v)
378
881
  return undefined;
379
882
  const s = v.toLowerCase();
380
- if (s === "successful" || s === "failed" || s === "all")
883
+ if (s === "successful" || s === "failed" || s === "processing" || s === "all")
381
884
  return s;
382
885
  throw new ToolkitError({
383
886
  code: "INVALID_USAGE",
384
887
  message: `Invalid --status: ${v}`,
385
- likely_cause: "Only successful | failed | all are supported.",
386
- next_action: "Use --status successful | failed | all (default: all).",
888
+ likely_cause: "Only successful | failed | processing | all are supported.",
889
+ next_action: "Use --status successful | failed | processing | all (default: all). processing lists the tasks not finished yet; expired rows come back with successful.",
387
890
  });
388
891
  }
389
892
  function normalizeOutput(v) {
@@ -412,15 +915,51 @@ function normalizeOutput(v) {
412
915
  const mapped = map[key];
413
916
  return mapped ? mapped : undefined; // html → undefined (no response_type)
414
917
  }
415
- function requireId(id) {
918
+ /**
919
+ * `--timeout` as a duration, returned in milliseconds; undefined when unset.
920
+ * Takes `500ms`, `30s`, `5m` or `1h`; a bare number keeps its earlier meaning,
921
+ * milliseconds.
922
+ */
923
+ export function timeoutFlag(v) {
924
+ if (v === undefined)
925
+ return undefined;
926
+ const m = typeof v === "string" ? /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h)?\s*$/i.exec(v) : null;
927
+ const unit = { ms: 1, s: 1000, m: 60_000, h: 3_600_000 }[(m?.[2] ?? "ms").toLowerCase()];
928
+ const ms = m ? Number(m[1]) * unit : NaN;
929
+ if (!Number.isFinite(ms) || ms <= 0) {
930
+ throw usageError(`Invalid --timeout value '${String(v)}'.`, "Pass a duration such as --timeout 30s, 5m or 1h (a bare number is milliseconds).", "--timeout takes a positive duration.");
931
+ }
932
+ return Math.round(ms);
933
+ }
934
+ /** `--limit` for a paged read: a whole number from 1 to `max`. */
935
+ function pageLimit(v, max) {
936
+ if (v === undefined)
937
+ return undefined;
938
+ const n = asNumber(v);
939
+ if (n === undefined || !Number.isInteger(n) || n < 1 || n > max) {
940
+ throw usageError(`Invalid --limit value '${String(v)}'.`, `Pass e.g. --limit ${Math.min(max, 50)}.`, `--limit takes a whole number from 1 to ${max}.`);
941
+ }
942
+ return n;
943
+ }
944
+ function oneOf(flag, v, allowed) {
945
+ if (v === undefined)
946
+ return undefined;
947
+ if (allowed.includes(v))
948
+ return v;
949
+ throw usageError(`Invalid --${flag} '${v}'.`, `Pass --${flag} ${allowed.join(" or ")}, or leave it out.`);
950
+ }
951
+ function usageError(message, next_action, likely_cause = "A flag value is outside what the Batch API accepts.") {
952
+ return new ToolkitError({ code: "INVALID_USAGE", message, likely_cause, next_action });
953
+ }
954
+ function requireId(id, sub) {
416
955
  if (id)
417
956
  return id;
418
957
  throw new ToolkitError({
419
958
  code: "INVALID_USAGE",
420
959
  message: "Missing job id.",
421
960
  likely_cause: "No <id> positional was provided.",
422
- next_action: "Usage: zenrows batch status <id>",
423
- suggested_commands: ["zenrows batch status <id>"],
961
+ next_action: `Usage: zenrows batch ${sub} <id>. List your jobs with \`zenrows batch list\`.`,
962
+ suggested_commands: ["zenrows batch list"],
424
963
  });
425
964
  }
426
965
  function needFile() {