@zenrows/cli 1.6.2 → 1.8.0

package/dist/src/core/batch-api.js

@@ -4,19 +4,29 @@
  * A SEPARATE host from Fetch and Extract (`api.zenrows.com/v1`): batch
  * jobs live at `https://async.api.zenrows.com/v1`. Auth is the `X-API-Key`
  * header (same key as Fetch and Extract), never the `apikey` query param. Bodies
- * are JSON; errors come back as `application/problem+json` (RFC 7807) with a
- * stable `code` we branch on. The key is registered as a secret so it is
- * redacted from any logged output. `fetchImpl` is injectable for tests, mirroring
- * `usage.ts` / `agent-account.ts`.
+ * are JSON; errors come back as `application/problem+json` (RFC 9457) with a
+ * stable `code` we branch on and carry as `server_code`. The key is registered
+ * as a secret so it is redacted from any logged output. `fetchImpl` is
+ * injectable for tests, mirroring `usage.ts` / `agent-account.ts`.
  *
- * Deferred (documented, not wired here): CSV upload (`/job_inputs` + presigned
- * PUT), open/queue jobs (`/tasks`, `last_batch`), scheduled jobs
- * (`schedule` / `PUT /schedule` / `schedule/state`), webhooks + HMAC, ZIP
- * archive export, and the `Idempotency-Key` header.
+ * Retries: reads (GET, DELETE) retry 429/502/503/504 and connection errors;
+ * stop, pause and resume retry 502/503/504 and connection errors, since
+ * repeating them is harmless; a submit or rerun retries the same way only
+ * under its `Idempotency-Key`, which `createJob` and `rerunJob` always send and
+ * reuse across the retries of one call. The API keeps a key for 24 hours and
+ * replays the original answer to a repeat with the same body, so a retry never
+ * starts a second job or run. A keyed write that still has no answer after its
+ * retries fails with `idempotency_key` set: repeating it with that key is safe.
+ *
+ * Not in the CLI (the API and the SDKs have them): CSV inputs, open jobs and
+ * add-tasks, scheduled jobs, webhooks and HMAC keys, ZIP exports, the runs
+ * list, and task history.
  */
 import { mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
+import { randomUUID } from "node:crypto";
 import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.js";
+import { storedClaimHint } from "./claim-link.js";
 import { readAccount } from "./agent-account.js";
 import { registerSecret } from "./logger.js";
 import { CLI_USER_AGENT } from "./config.js";
@@ -32,193 +42,413 @@ export function batchBase() {
     const base = env && env.trim() ? env.trim() : DEFAULT_BATCH_API_BASE;
     return base.replace(/\/+$/, "");
 }
-/** Terminal run states — a run stops progressing once it reaches one of these. */
+/** Terminal run states: a run stops progressing once it reaches one of these. */
 export const TERMINAL_STATUSES = new Set(["completed", "stopped", "failed", "deleted"]);
+/** Task statuses a rerun can select (`?status=` on `/rerun`). */
+export const RERUN_STATUSES = ["pending", "processing", "failed", "successful"];
+/** True when the run reached a terminal state. */
+export function isTerminal(run) {
+    return !!run && TERMINAL_STATUSES.has(run.status);
+}
+/** True when a live run is paused: it picks up no new task until resumed. */
+export function isPaused(run) {
+    return run?.pause_state === "paused" && !isTerminal(run);
+}
+const MAX_RETRIES = 3;
+/** The longest a Retry-After may make one retry wait, so a large value cannot hang a command. */
+const MAX_RETRY_AFTER_SECONDS = 30;
+const READ_RETRY_STATUSES = new Set([429, 502, 503, 504]);
+// 429 on a write is an account limit (active jobs): a retry only waits for a slot that may never free.
+const WRITE_RETRY_STATUSES = new Set([502, 503, 504]);
+/** 250 ms x 2^attempt, capped at 10 s, +/-20% jitter. */
+function backoffMs(attempt) {
+    return Math.min(250 * 2 ** attempt, 10_000) * (0.8 + Math.random() * 0.4);
+}
+/** `Retry-After` in whole seconds, as milliseconds, at most `MAX_RETRY_AFTER_SECONDS`. */
+function retryAfterMs(value) {
+    return value && /^\d+$/.test(value.trim()) ? Math.min(Number(value), MAX_RETRY_AFTER_SECONDS) * 1000 : undefined;
+}
+const defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** A fresh Idempotency-Key for one logical submit or rerun. */
+export function newIdempotencyKey() {
+    return randomUUID();
+}
+/** The longest Idempotency-Key the Batch API accepts. */
+export const MAX_IDEMPOTENCY_KEY_LENGTH = 128;
 /**
- * Perform a Batch API request. Sets `X-API-Key`, adds a JSON content-type when a
- * body is present, parses the JSON response, and on a non-2xx parses the
- * problem+json body into a normalized ToolkitError. Returns `null` for empty
- * (204 / no-body) responses.
+ * Throw before any request if `key` cannot work as an Idempotency-Key: HTTP
+ * strips a header's surrounding whitespace, so a blank or padded key would
+ * reach the API as a different key, and control characters cannot be sent.
  */
-export async function batchRequest(method, path, opts) {
+export function assertIdempotencyKey(key) {
+    let reason;
+    if (key.trim() === "")
+        reason = "it is empty";
+    else if (key !== key.trim())
+        reason = "it starts or ends with whitespace";
+    else if (/[\u0000-\u001f\u007f]/.test(key))
+        reason = "it contains a control character";
+    else if (key.length > MAX_IDEMPOTENCY_KEY_LENGTH)
+        reason = `it is longer than ${MAX_IDEMPOTENCY_KEY_LENGTH} characters`;
+    if (!reason)
+        return;
+    throw new ToolkitError({
+        code: "INVALID_USAGE",
+        message: `Invalid --idempotency-key: ${reason}.`,
+        likely_cause: `An Idempotency-Key is 1 to ${MAX_IDEMPOTENCY_KEY_LENGTH} characters, with no surrounding whitespace and no control characters.`,
+        next_action: "Pass a key such as a UUID, or leave --idempotency-key out to have one generated.",
+    });
+}
+/** Errors a poll may ride out: the API was unreachable or answered 429/5xx. */
+const transientErrors = new WeakSet();
+/** True for an error worth polling past (a connection failure, a 429 or a 5xx). */
+export function isTransient(err) {
+    return err instanceof ToolkitError && transientErrors.has(err);
+}
+function transient(err) {
+    transientErrors.add(err);
+    return err;
+}
+/** One HTTP attempt with its own timeout; the body is read inside it. */
+async function attempt(url, init, opts) {
+    const controller = new AbortController();
+    const perAttempt = opts.timeoutMs ?? 60_000;
+    const ms = opts.deadline !== undefined ? Math.max(1, Math.min(perAttempt, opts.deadline - Date.now())) : perAttempt;
+    const timeout = setTimeout(() => controller.abort(), ms);
+    try {
+        const res = await (opts.fetchImpl ?? fetch)(url, { ...init, signal: controller.signal });
+        return { status: res.status, headers: res.headers, body: Buffer.from(await res.arrayBuffer()) };
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+/**
+ * Perform a Batch API request and return the raw response. Sets `X-API-Key`,
+ * adds a JSON content-type when a body is present, retries transient failures
+ * as `opts.retry` allows, and on a non-2xx parses the problem+json body into a
+ * normalized ToolkitError.
+ */
+async function batchFetch(method, path, opts) {
     registerSecret(opts.apiKey);
     const url = new URL(batchBase() + path);
     for (const [k, v] of Object.entries(opts.query ?? {})) {
         if (v !== undefined && v !== null)
-            url.searchParams.set(k, v);
+            url.searchParams.set(k, String(v));
     }
-    const doFetch = opts.fetchImpl ?? fetch;
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000);
     const headers = {
         "X-API-Key": opts.apiKey,
         Accept: "application/json",
         "User-Agent": CLI_USER_AGENT,
         ...agentClientHeader(),
     };
+    if (opts.idempotencyKey)
+        headers["Idempotency-Key"] = opts.idempotencyKey;
     if (opts.body !== undefined)
         headers["Content-Type"] = "application/json";
-    let res;
-    try {
-        res = await doFetch(url.toString(), {
-            method,
-            headers,
-            body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined,
-            signal: controller.signal,
-        });
-    }
-    catch (err) {
-        clearTimeout(timeout);
-        throw new ToolkitError({
-            code: "BACKEND_UNAVAILABLE",
-            message: "Could not reach the Zenrows Batch API.",
-            likely_cause: err instanceof Error ? err.message : String(err),
-            next_action: "Check connectivity and retry. Override the host with ZENROWS_BATCH_API_BASE if you are testing against staging.",
-            suggested_commands: ["zenrows status"],
-        });
-    }
-    clearTimeout(timeout);
-    const text = await res.text();
-    if (res.status < 200 || res.status >= 300) {
-        throw problemToError(res.status, text, method, path);
+    const init = { method, headers, body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined };
+    const retryOn = opts.retry === "read" ? READ_RETRY_STATUSES : opts.retry === "write" ? WRITE_RETRY_STATUSES : undefined;
+    const sleep = opts.sleepImpl ?? defaultSleep;
+    const key = opts.idempotencyKey;
+    // Set once an attempt of a keyed write got no answer: the API may have done the work.
+    let mayHaveLanded = false;
+    // A retry that would sleep past the call's deadline is not started.
+    const fits = (waitMs) => opts.deadline === undefined || Date.now() + waitMs < opts.deadline;
+    for (let n = 0;; n++) {
+        const canRetry = retryOn !== undefined && n < MAX_RETRIES;
+        let res;
+        try {
+            res = await attempt(url.toString(), init, opts);
+        }
+        catch (err) {
+            const wait = backoffMs(n);
+            if (canRetry && fits(wait)) {
+                mayHaveLanded = true;
+                await sleep(wait);
+                continue;
+            }
+            if (key)
+                mayHaveLanded = true;
+            const failure = transient(new ToolkitError({
+                code: "BACKEND_UNAVAILABLE",
+                message: "Could not reach the Zenrows Batch API.",
+                likely_cause: err instanceof Error ? err.message : String(err),
+                next_action: "Check connectivity and retry. Override the host with ZENROWS_BATCH_API_BASE if you are testing against staging.",
+                suggested_commands: ["zenrows status"],
+                ...(opts.jobId ? { job_id: opts.jobId } : {}),
+            }));
+            throw key ? outcomeUnknown(failure, key) : failure;
+        }
+        if (canRetry && retryOn.has(res.status)) {
+            const wait = retryAfterMs(res.headers.get("retry-after")) ?? backoffMs(n);
+            if (fits(wait)) {
+                if (res.status >= 500)
+                    mayHaveLanded = true;
+                await sleep(wait);
+                continue;
+            }
+        }
+        if (res.status < 200 || res.status >= 300) {
+            let failure = problemToError(res.status, res.body.toString("utf8"), method, path, opts.jobId);
+            if (res.status === 429 || res.status >= 500)
+                failure = transient(failure);
+            if (key && res.status >= 500)
+                throw outcomeUnknown(failure, key);
+            // An earlier attempt may have been accepted, and the API checks credit
+            // (402) and other gates before it looks at the key: whatever this attempt
+            // answers, the safe next step is still to repeat with the same key.
+            if (key && mayHaveLanded)
+                throw outcomeUnknown(failure, key, { keepAdvice: true });
+            throw failure;
+        }
+        return res;
     }
+}
+/** `batchFetch` + JSON parse. Returns `null` for an empty body. */
+export async function batchRequest(method, path, opts) {
+    return batchJson(method, path, opts);
+}
+async function batchJson(method, path, opts) {
+    const { status, body } = await batchFetch(method, path, opts);
+    const text = body.toString("utf8");
     if (!text)
         return null;
     try {
         return JSON.parse(text);
     }
     catch {
-        throw new ToolkitError({
+        const failure = new ToolkitError({
             code: "BATCH_FAILED",
-            message: "The Batch API response was not valid JSON.",
+            message: `The Batch API answered ${method} ${path} (HTTP ${status}) with a body that is not JSON.`,
             likely_cause: text.slice(0, 240),
-            next_action: "Retry, or inspect the job in the Zenrows dashboard.",
+            next_action: "Retry, or check the job with `zenrows batch list`.",
+            ...(opts.jobId ? { job_id: opts.jobId } : {}),
         });
+        throw opts.idempotencyKey ? outcomeUnknown(failure, opts.idempotencyKey) : failure;
     }
 }
-/** Map an RFC 7807 problem body + HTTP status to a normalized ToolkitError. */
-function problemToError(status, body, method, path) {
+/**
+ * A keyed write (submit or rerun) whose outcome is unknown: the API may have
+ * done the work although no usable answer arrived. Carry the key and say what
+ * is safe; the command layer adds the exact command to repeat.
+ */
+function outcomeUnknown(err, key, opts = {}) {
+    const lead = opts.keepAdvice
+        ? `${err.next_action.replace(/[.\s]*$/, ".")} An earlier attempt of this request got no answer and may have been accepted. Once that is resolved, repeat it with the same Idempotency-Key (${key}) `
+        : `The request may have been accepted even though no usable answer arrived. Repeat it with the same Idempotency-Key (${key}) `;
+    const out = new ToolkitError({
+        ...err.toJSON(),
+        idempotency_key: key,
+        next_action: lead +
+            "within 24 hours: the API then answers with the original job or run instead of creating a second one. " +
+            "Do not repeat it with a new key, which can create a duplicate. Check `zenrows batch list` first.",
+    });
+    return isTransient(err) ? transient(out) : out;
+}
+/** `s` without a trailing period, so sentences can be joined. */
+function trimDot(s) {
+    return s.replace(/[.\s]+$/, "");
+}
+/** The error for an account whose subscription is not active (401 subscription_inactive, 402 expired). */
+export function subscriptionInactive(cause) {
+    // Same AUTH_INVALID code as any 401 so callers that branch on it keep
+    // working; only the guidance changes, since re-entering the key won't help.
+    return new ToolkitError({
+        code: "AUTH_INVALID",
+        message: "Zenrows refused the Batch API request because the subscription is not active.",
+        likely_cause: `${trimDot(cause)}. The API key is valid, but the subscription is expired, past due, unpaid, incomplete or paused.`,
+        next_action: `Fix the subscription in Billing (${BILLING_URL}), or in Azure Marketplace if you subscribed there, then retry.`,
+    });
+}
+/** Map a problem+json body + HTTP status to a normalized ToolkitError carrying `server_code` and `job_id`. */
+function problemToError(status, text, method, path, jobId) {
     let problem = {};
     try {
-        problem = JSON.parse(body);
+        problem = JSON.parse(text);
     }
     catch {
-        // non-JSON error body — fall through with an empty problem
+        // non-JSON error body: fall through with an empty problem
     }
-    const serverCode = problem.code ?? "";
-    const detail = problem.detail || problem.title || body.slice(0, 240) || `HTTP ${status}`;
-    const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${method} ${path}: ${detail}`;
+    const serverCode = problem.code || undefined;
+    const detail = trimDot(problem.detail || problem.title || text.slice(0, 240) || `HTTP ${status}`);
+    const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${method} ${path}: ${detail}.`;
+    const err = classifyProblem(status, problem, detail, cause, serverCode, method, path, jobId ?? "<id>");
+    return new ToolkitError({
+        ...err.toJSON(),
+        ...(serverCode ? { server_code: serverCode } : {}),
+        ...(jobId ? { job_id: jobId } : {}),
+    });
+}
+function classifyProblem(status, problem, detail, cause, serverCode, method, path, id) {
+    const err = (code, message, next_action, suggested_commands, likely = cause) => new ToolkitError({ code, message, likely_cause: likely, next_action, suggested_commands });
     if (status === 403) {
-        return new ToolkitError({
-            code: "BATCH_ACCESS_DENIED",
-            message: "The Batch API rejected this request (access denied).",
-            likely_cause: `${cause}. Batch is not enabled for this account or API key.`,
-            next_action: "Check that your Zenrows API key is active, and contact Zenrows support if the 403 persists. Meanwhile validate/estimate specs locally and fan out with `zenrows fetch` per URL.",
-            suggested_commands: ["zenrows batch estimate jobs.jsonl"],
-        });
-    }
-    if (status === 401 && serverCode === "subscription_inactive") {
-        // The key is fine; the subscription needs a billing action. Same
-        // AUTH_INVALID code as any 401 so callers that branch on it keep
-        // working; only the guidance changes, since re-entering the key won't help.
-        return new ToolkitError({
-            code: "AUTH_INVALID",
-            message: "Zenrows refused the Batch API request because the subscription is not active.",
-            likely_cause: `${cause.replace(/\.?$/, ".")} The API key is valid, but the subscription is past due, unpaid, incomplete or paused.`,
-            next_action: `Fix the subscription in Billing (${BILLING_URL}), or in Azure Marketplace if you subscribed there, then retry.`,
-        });
+        return err("BATCH_ACCESS_DENIED", "The Batch API rejected this request (access denied).", "Check that your Zenrows API key is active, and contact Zenrows support if the 403 persists. Meanwhile validate/estimate specs locally and fan out with `zenrows fetch` per URL.", ["zenrows batch estimate jobs.jsonl"], `${cause} Batch is not enabled for this account or API key.`);
     }
+    if (status === 401 && serverCode === "subscription_inactive")
+        return subscriptionInactive(cause);
     if (status === 401) {
-        return new ToolkitError({
-            code: "AUTH_INVALID",
-            message: "Zenrows rejected the API key for the Batch API.",
-            likely_cause: cause,
-            next_action: "Re-check your key and log in again.",
-            suggested_commands: ["zenrows login --api-key <your-key>"],
-        });
+        return err("AUTH_INVALID", "Zenrows rejected the API key for the Batch API.", "Re-check your key and log in again.", [
+            "zenrows login --api-key <your-key>",
+        ]);
     }
     if (status === 404) {
-        return new ToolkitError({
-            code: "BATCH_NOT_FOUND",
-            message: "Batch job, run, or task not found.",
-            likely_cause: `${cause}. The id may be wrong or the job is not owned by this account.`,
-            next_action: "Check the job id, or list your recent jobs in the Zenrows dashboard.",
-        });
+        const task = /\/tasks\//.test(path);
+        return err("BATCH_NOT_FOUND", task ? "Batch job, run or task not found." : "Batch job or run not found.", task
+            ? `Check the ids. List the job's task ids with \`zenrows batch results ${id}\`.`
+            : "Check the job id, or list your jobs with `zenrows batch list`.", [task ? `zenrows batch results ${id}` : "zenrows batch list"], `${cause} The id may be wrong, the job deleted, or not owned by this account.`);
     }
     if (status === 429) {
-        return new ToolkitError({
-            code: "BATCH_QUOTA_EXCEEDED",
-            message: "Batch quota exceeded.",
-            likely_cause: `${cause}. An account limit was reached (e.g. the maximum of 3 concurrent active jobs).`,
-            next_action: "Wait for an in-flight job to finish (or stop one with `zenrows batch cancel <id>`), then retry.",
-            suggested_commands: ["zenrows batch status <id>"],
-        });
+        if (/scheduled job/i.test(detail)) {
+            return err("BATCH_QUOTA_EXCEEDED", "The account has reached its limit of scheduled jobs.", "Nothing was created. A scheduled job holds its slot until it is deleted, whatever its runs do: delete one you no longer need, then retry.", ["zenrows batch list --type scheduled", "zenrows batch delete <id> --yes"], `${cause} The account holds 5 scheduled jobs at most by default.`);
+        }
+        if (/task cap/i.test(detail)) {
+            return err("BATCH_QUOTA_EXCEEDED", "The open job has reached its task cap.", "Nothing was added. An open job holds 100,000 tasks at most: close it and start a new job for the rest.");
+        }
+        return err("BATCH_QUOTA_EXCEEDED", "The account has reached its limit of active jobs.", "Nothing was created. Active jobs are shared with Crawl. A closed job frees its slot when its run ends: wait for one (`zenrows batch wait <id>`) or stop it (`zenrows batch cancel <id>`). An open or scheduled job frees it only when deleted (`zenrows batch delete <id> --yes`). Then retry.", ["zenrows batch list", "zenrows crawl list"], `${cause} The account holds 3 active jobs at most by default, Batch and Crawl together.`);
     }
     if (status === 402 && isKeyCapReached(serverCode)) {
         return keyCapReached(`${method} ${path}`, { status: 402, detail: problem.detail || problem.title || undefined });
     }
+    if (status === 402 && /subscription has expired/i.test(detail))
+        return subscriptionInactive(cause);
     if (status === 402) {
-        // Out of credits ("Subscription has no credit available") — the same
-        // exhausted state as the scraper/usage 402. Surface the credits error with
-        // a claim link (unclaimed Free plan) or dashboard/upgrade pointer, not a
-        // generic "fix the problem and retry".
+        // Out of credits ("Subscription has no credit available"): the same
+        // exhausted state as the scraper/usage 402, with a claim link (unclaimed
+        // Free plan) or dashboard/upgrade pointer.
         const acct = readAccount();
-        return quotaExhausted(`${method} ${path}`, acct?.unclaimed ? acct.claimUrl : undefined, {
+        return quotaExhausted(`${method} ${path}`, storedClaimHint(acct), {
             status: 402,
             detail: problem.detail || problem.title || undefined,
         });
     }
-    // 400 invalid_argument, 409 conflict, 5xx, etc.
+    if (status === 409)
+        return conflictError(serverCode, detail, cause, id);
+    if (status === 410) {
+        return err("BATCH_RESULT_EXPIRED", "This result has expired.", `Results are kept 14 days by default; after that a task reads as expired and its body is gone. Scrape it again with a full rerun: \`zenrows batch rerun ${id} --all\`.`, [`zenrows batch rerun ${id} --all`]);
+    }
+    if (status === 422 && /\/content$/.test(path)) {
+        return err("BATCH_TASK_FAILED", "This task failed, so it has no content.", `Read why in its result row (\`zenrows batch results ${id} --status failed\`), then rerun the failures with \`zenrows batch retry-failed ${id}\`.`, [`zenrows batch results ${id} --status failed`]);
+    }
+    // 400 invalid_argument, 501, 5xx, etc.
     const invalid = problem.invalid_tasks?.length
         ? ` invalid_tasks: ${problem.invalid_tasks
             .slice(0, 10)
-            .map((t) => `#${t.index}: ${t.reason}`)
+            .map((t) => `#${t.index}: ${t.reason}${t.value ? ` (${t.value})` : ""}`)
             .join("; ")}`
         : "";
-    return new ToolkitError({
-        code: "BATCH_FAILED",
-        message: `Batch request failed (HTTP ${status}).`,
-        likely_cause: `${cause}.${invalid}`,
-        next_action: status >= 500
-            ? "This is transient — retry with a short backoff."
-            : "Fix the reported problem and retry. For validation errors, correct the flagged tasks.",
-    });
+    return err("BATCH_FAILED", `Batch request failed (HTTP ${status}).`, status === 501
+        ? "This operation is not available for this job."
+        : status >= 500
+            ? "This is transient: retry with a short backoff."
+            : "Fix the reported problem and retry. For validation errors, correct the flagged tasks (#n is the task's 0-based position in the submit).", undefined, `${cause}${invalid}`);
 }
-/** Create a job. Accepts both 201 (<10k tasks) and 202 (≥10k) as success. */
+/** One code and next action per 409 the Batch API answers. */
+function conflictError(serverCode, detail, cause, id) {
+    const e = (code, message, next_action, suggested_commands = [`zenrows batch status ${id}`]) => new ToolkitError({ code, message, likely_cause: cause, next_action, suggested_commands });
+    switch (serverCode) {
+        case "run_not_terminal":
+            if (/ingestion did not complete/i.test(detail)) {
+                return e("BATCH_RUN_IN_PROGRESS", "The job's last run never finished receiving its tasks.", `It cannot be rerun. Delete the job (\`zenrows batch delete ${id} --yes\`) and submit the URLs again as a new job.`);
+            }
+            return e("BATCH_RUN_IN_PROGRESS", "The job's current run is still in progress.", `Wait for it with \`zenrows batch wait ${id}\`, or stop it with \`zenrows batch cancel ${id}\`, then retry. A large submit or a rerun also reads as in progress while it is still receiving its tasks.`, [`zenrows batch wait ${id}`]);
+        case "run_not_stoppable":
+            return e("BATCH_RUN_ENDED", "The job's run already ended.", `Only a pending or running run can be stopped, paused or resumed. Start a new run with \`zenrows batch rerun ${id}\`.`);
+        case "no_matching_tasks":
+            return e("BATCH_NO_MATCHING_TASKS", "No task of the last run has the statuses you asked to rerun.", `Tasks a stop or a credit failure left unfinished are pending or processing: rerun them with \`zenrows batch rerun ${id} --status failed,pending,processing\`, or rerun every task with \`zenrows batch rerun ${id} --all\`.`, [`zenrows batch rerun ${id} --status failed,pending,processing`]);
+        case "partial_rerun_window_expired":
+            return e("BATCH_RERUN_WINDOW_EXPIRED", "Rerunning only some tasks is no longer allowed for this run.", `A partial rerun (the default, or --status) works for 10 days after the run started. Rerun every task with \`zenrows batch rerun ${id} --all\` instead.`, [`zenrows batch rerun ${id} --all`]);
+        case "idempotency_key_conflict":
+            return e("BATCH_REQUEST_IN_FLIGHT", "This Idempotency-Key is already in use.", "Either the first request with this key is still being processed, or the key was used for a different request body. Wait a moment and repeat the same command with the same key; use a new key only for a deliberately different request.", ["zenrows batch list"]);
+        case "task_not_ready":
+            return e("BATCH_TASK_NOT_READY", "This task has not finished yet.", `Wait for the run with \`zenrows batch wait ${id}\`, then read the task again.`, [`zenrows batch wait ${id}`]);
+        case "conflict":
+            if (/no longer valid/i.test(detail)) {
+                return e("BATCH_CONFLICT", "The API key that created this job no longer works.", "The job cannot run again. Submit its URLs as a new job.", []);
+            }
+            return e("BATCH_CONFLICT", "The job changed while the request ran.", "Retry the same command.");
+        default:
+            return e("BATCH_CONFLICT", "The job's state does not allow this request.", `Check the job with \`zenrows batch status ${id}\`; the cause above says what blocked it.`);
+    }
+}
+const jobPath = (id, rest = "") => `/jobs/${encodeURIComponent(id)}${rest}`;
+/**
+ * Create a job. Both 201 (tasks stored) and 202 (a large submit, still
+ * receiving its tasks: `latest_run.ingest_status` is `pending`) are success.
+ */
 export function createJob(body, opts) {
-    return batchRequest("POST", "/jobs", { apiKey: opts.apiKey, body, fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs });
+    const idempotencyKey = opts.idempotencyKey ?? newIdempotencyKey();
+    assertIdempotencyKey(idempotencyKey);
+    return batchJson("POST", "/jobs", { ...opts, body, idempotencyKey, retry: "write" });
 }
 /** Read a job (`latest_run.status` + `latest_run.stats`). */
 export function getJob(id, opts) {
-    return batchRequest("GET", `/jobs/${encodeURIComponent(id)}`, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs });
+    return batchJson("GET", jobPath(id), { ...opts, retry: "read", jobId: id });
+}
+/** Read one run of a job. */
+export function getRun(id, runId, opts) {
+    return batchJson("GET", jobPath(id, `/runs/${encodeURIComponent(runId)}`), { ...opts, retry: "read", jobId: id });
 }
-/** Abort an in-flight run. */
+/** One page of the account's jobs, newest first. */
+export async function listJobs(opts) {
+    const page = await batchJson("GET", "/jobs", {
+        ...opts,
+        retry: "read",
+        query: { limit: opts.limit, cursor: opts.cursor, status: opts.status, type: opts.type },
+    });
+    return { jobs: page?.jobs ?? [], next_cursor: page?.next_cursor || null };
+}
+/** Stop the job's in-flight run. The API answers with the refreshed run, not the job. */
 export function stopJob(id, opts) {
-    return batchRequest("POST", `/jobs/${encodeURIComponent(id)}/stop`, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl, timeoutMs: opts.timeoutMs });
+    return batchJson("POST", jobPath(id, "/stop"), { ...opts, retry: "write", jobId: id });
+}
+/** Pause the job's in-flight run: no new task is picked up until it is resumed. Answers with the run. */
+export function pauseJob(id, opts) {
+    return batchJson("POST", jobPath(id, "/pause"), { ...opts, retry: "write", jobId: id });
+}
+/** Resume a paused run. Needs credit on the job's API key. Answers with the run. */
+export function resumeJob(id, opts) {
+    return batchJson("POST", jobPath(id, "/resume"), { ...opts, retry: "write", jobId: id });
 }
 /**
- * Replay a job as a new run. `status="failed"` re-runs only the failures
- * (append `,pending` to also include tasks that never started). Already-
- * successful tasks carry over, so you only re-scrape what failed.
+ * Delete a job, its runs and its stored results (202, asynchronous). Stops an
+ * in-flight run and frees the job's slot. Repeating it is harmless.
+ */
+export async function deleteJob(id, opts) {
+    await batchFetch("DELETE", jobPath(id), { ...opts, retry: "read", jobId: id });
+}
+/**
+ * Start a new run of a job. With no `status`, every task runs again (a full
+ * rerun). `status` is a comma-separated list of task statuses to rerun
+ * (`pending,processing,failed,successful`); the rest carry over from the
+ * previous run. After a stop or a credit failure, unstarted tasks stay
+ * `pending` and tasks cut off mid-flight can stay `processing`, so
+ * `failed,pending,processing` recovers everything that did not succeed.
  */
 export function rerunJob(id, opts) {
-    return batchRequest("POST", `/jobs/${encodeURIComponent(id)}/rerun`, {
-        apiKey: opts.apiKey,
+    const idempotencyKey = opts.idempotencyKey ?? newIdempotencyKey();
+    assertIdempotencyKey(idempotencyKey);
+    return batchJson("POST", jobPath(id, "/rerun"), {
+        ...opts,
         query: opts.status ? { status: opts.status } : undefined,
-        fetchImpl: opts.fetchImpl,
-        timeoutMs: opts.timeoutMs,
+        idempotencyKey,
+        retry: "write",
+        jobId: id,
     });
 }
-/** List all result rows for a run, paginating over the `cursor` until `next_cursor` is null. */
+/**
+ * List all result rows of the latest run (or `runId`), following `next_cursor`
+ * until it is null.
+ */
 export async function listResults(id, opts) {
+    const path = opts.runId ? jobPath(id, `/runs/${encodeURIComponent(opts.runId)}/results`) : jobPath(id, "/results");
     const all = [];
     let cursor;
     do {
-        const page = await batchRequest("GET", `/jobs/${encodeURIComponent(id)}/results`, {
-            apiKey: opts.apiKey,
+        const page = await batchJson("GET", path, {
+            ...opts,
+            retry: "read",
+            jobId: id,
             query: { status: opts.status, cursor },
-            fetchImpl: opts.fetchImpl,
-            timeoutMs: opts.timeoutMs,
         });
         if (page?.results)
             all.push(...page.results);
@@ -226,6 +456,13 @@ export async function listResults(id, opts) {
     } while (cursor);
     return all;
 }
+/** One task's result body (latest run, or `runId`), as returned by the content endpoint. */
+export async function getTaskContent(id, taskId, opts) {
+    const task = `/tasks/${encodeURIComponent(taskId)}/content`;
+    const path = opts.runId ? jobPath(id, `/runs/${encodeURIComponent(opts.runId)}${task}`) : jobPath(id, task);
+    const res = await batchFetch("GET", path, { ...opts, retry: "read", jobId: id });
+    return { contentType: res.headers.get("content-type") ?? "", body: res.body };
+}
 /** Map a result `type` to a file extension for downloaded bodies. */
 const EXT_BY_TYPE = {
     json: "json",
@@ -252,15 +489,25 @@ function uniqueName(used, base, ext) {
     used.add(name);
     return name;
 }
+/** Why a row has no body to download. */
+function skipReason(row) {
+    if (row.status === "failed")
+        return "task failed: no result body";
+    if (row.status === "expired")
+        return "result expired (kept 14 days by default): a full rerun (--all) scrapes it again";
+    if (row.status === "pending" || row.status === "processing")
+        return "task not finished yet";
+    return "no result_url";
+}
 /**
  * Stream each result body into its own file under `dir` (created if needed),
- * plus a self-describing `_manifest.jsonl` index (result_url dropped — it is a
+ * plus a self-describing `_manifest.jsonl` index (result_url dropped: it is a
  * short-lived presigned link). Fetches run `concurrency` at a time and a single
  * failure never aborts the rest: per-task outcomes come back in the result.
  *
- * Only tasks with a `result_url` have a body to fetch; failed tasks carry an
- * `error` instead and are reported as `skipped`. Because the presigned links
- * expire, download soon after `listResults`. Deliberately opt-in (not the
+ * Only tasks with a `result_url` have a body to fetch; failed, expired and
+ * unfinished tasks are reported as `skipped`. The presigned links are valid for
+ * 2 hours, so download soon after `listResults`. Deliberately opt-in (not the
  * `results` default): a large job's bodies can be many GB, and the URL manifest
  * is the right primitive to pull selectively.
  */
@@ -281,10 +528,7 @@ export async function downloadResults(rows, dir, opts = {}) {
             const { row, file } = planned[next++];
             const ref = { task_id: row.task_id, external_id: row.external_id };
             if (!file) {
-                outcome.skipped.push({
-                    ...ref,
-                    reason: row.status === "failed" ? "task failed — no result body" : "no result_url",
-                });
+                outcome.skipped.push({ ...ref, reason: skipReason(row) });
                 continue;
             }
             try {
@@ -305,33 +549,43 @@ export async function downloadResults(rows, dir, opts = {}) {
     const fileByTask = new Map(outcome.downloaded.map((d) => [d.task_id, d.file]));
     const manifest = rows.map((r) => {
         const rest = { ...r };
-        delete rest.result_url; // expires — the local `file` is the durable pointer
+        delete rest.result_url; // expires: the local `file` is the durable pointer
         return { ...rest, file: fileByTask.get(r.task_id) ?? null };
     });
     writeFileSync(join(dir, "_manifest.jsonl"), manifest.map((m) => JSON.stringify(m)).join("\n") + (manifest.length ? "\n" : ""));
     return outcome;
 }
 /**
- * Poll `GET /jobs/{id}` until `latest_run.status` is terminal, backing off
- * 2s → ×1.5 → capped at 15s. Throws BATCH_FAILED on timeout. `sleepImpl` is
- * injectable for tests.
+ * Poll `GET /jobs/{id}` until the latest run is terminal, the run is paused (a
+ * paused run never progresses, so the wait returns at once), or `waitMs`
+ * (default 10 minutes) runs out, backing off 2s, x1.5, capped at 15s. Returns
+ * the last job read in every case; the caller tells them apart with
+ * `isTerminal` / `isPaused`. A transient error (connection, 429, 5xx) is
+ * ridden out until the deadline. `sleepImpl` is injectable for tests.
  */
 export async function waitForJob(id, opts) {
-    const sleep = opts.sleepImpl ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
-    const deadline = Date.now() + (opts.timeoutMs ?? 600_000);
+    const sleep = opts.sleepImpl ?? defaultSleep;
+    const deadline = Date.now() + (opts.waitMs ?? 600_000);
     let delay = 2000;
+    const read = { ...opts, deadline };
+    let last;
+    let lastErr;
     for (;;) {
-        const job = await getJob(id, { apiKey: opts.apiKey, fetchImpl: opts.fetchImpl });
-        if (job.latest_run && TERMINAL_STATUSES.has(job.latest_run.status))
-            return job;
-        if (Date.now() > deadline) {
-            throw new ToolkitError({
-                code: "BATCH_FAILED",
-                message: `Timed out waiting for batch job ${id} to finish.`,
-                likely_cause: `The run did not reach a terminal state within ${Math.round((opts.timeoutMs ?? 600_000) / 1000)}s.`,
-                next_action: "Re-check progress with `zenrows batch status <id>`, or raise --timeout.",
-                suggested_commands: [`zenrows batch status ${id}`],
-            });
+        try {
+            last = await getJob(id, read);
+            if (isTerminal(last.latest_run) || isPaused(last.latest_run))
+                return last;
+        }
+        catch (err) {
+            if (!isTransient(err))
+                throw err;
+            lastErr = err;
+        }
+        if (Date.now() + delay > deadline) {
+            // Out of time: report the last state read, or the error if none was.
+            if (!last)
+                throw lastErr;
+            return last;
         }
         await sleep(delay);
         delay = Math.min(delay * 1.5, 15_000);