@zenrows/cli 1.2.3 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/adapters/protected-fetch.js +4 -2
- package/dist/src/cli/commands/batch.js +67 -17
- package/dist/src/cli/commands/init.js +1 -1
- package/dist/src/cli/commands/status.js +1 -0
- package/dist/src/cli/commands/usage.js +18 -0
- package/dist/src/core/agent-account.js +6 -4
- package/dist/src/core/agent-client.js +98 -0
- package/dist/src/core/batch-api.js +8 -2
- package/dist/src/core/browser-api.js +11 -3
- package/dist/src/core/config.js +9 -5
- package/dist/src/core/errors.js +28 -0
- package/dist/src/core/http.js +21 -3
- package/dist/src/core/usage.js +3 -1
- package/package.json +1 -1
- package/skills/batch-jobs/SKILL.md +8 -0
- package/skills/extract/SKILL.md +5 -2
- package/skills/protected-fetch/SKILL.md +22 -7
- package/skills/trace-debug/SKILL.md +18 -2
|
@@ -93,11 +93,13 @@ export function buildParams(opts, config) {
|
|
|
93
93
|
params.response_type = RESPONSE_TYPE[opts.output];
|
|
94
94
|
return params;
|
|
95
95
|
}
|
|
96
|
-
export async function runFetch(opts, config, policy, apiKey
|
|
96
|
+
export async function runFetch(opts, config, policy, apiKey,
|
|
97
|
+
/** The workspace `config` came from, when it is not the one above the cwd. */
|
|
98
|
+
projectRoot) {
|
|
97
99
|
assertDomainAllowed(opts.url, policy);
|
|
98
100
|
validateAutoManual(opts, config);
|
|
99
101
|
const params = buildParams(opts, config);
|
|
100
102
|
const mode = params.mode === "auto" ? "auto" : "manual";
|
|
101
|
-
const result = await scrape(config.apiBase, apiKey, params, { timeoutMs: opts.timeoutMs });
|
|
103
|
+
const result = await scrape(config.apiBase, apiKey, params, { timeoutMs: opts.timeoutMs, projectRoot });
|
|
102
104
|
return { result, params, mode };
|
|
103
105
|
}
|
|
@@ -17,7 +17,7 @@ import { assertDomainAllowed, assertWithinLimits, loadPolicy } from "../../core/
|
|
|
17
17
|
import { newRunId, writeRun } from "../../core/artifacts.js";
|
|
18
18
|
import { createJob, downloadResults, getJob, listResults, rerunJob, stopJob, waitForJob } from "../../core/batch-api.js";
|
|
19
19
|
import { asNumber, asString, parse } from "../command.js";
|
|
20
|
-
import { ToolkitError } from "../../core/errors.js";
|
|
20
|
+
import { ToolkitError, isKeyCapReached, keyCapReached } from "../../core/errors.js";
|
|
21
21
|
import { printError, writeOut } from "../output.js";
|
|
22
22
|
export const batch = {
|
|
23
23
|
name: "batch",
|
|
@@ -166,20 +166,22 @@ async function createCmd(rest, ctx) {
|
|
|
166
166
|
try {
|
|
167
167
|
const job = await createJob(body, { apiKey });
|
|
168
168
|
const finished = follow ? await waitForJob(job.job_id, { apiKey }) : job;
|
|
169
|
+
const runError = runFailure(finished);
|
|
169
170
|
const runDir = writeRun({
|
|
170
171
|
runId,
|
|
171
172
|
command: "zenrows batch create",
|
|
172
173
|
capability: "batch",
|
|
173
174
|
startedAt,
|
|
174
175
|
finishedAt: new Date().toISOString(),
|
|
175
|
-
status: "ok",
|
|
176
|
+
status: runError ? "error" : "ok",
|
|
176
177
|
request: { file, tasks: body.tasks.length, estimatedCredits: est.credits, jobParams },
|
|
177
|
-
result: { jobId: job.job_id, status: finished.latest_run?.status ?? "unknown" },
|
|
178
|
+
result: { jobId: job.job_id, status: finished.latest_run?.status ?? "unknown", ...failureFields(finished) },
|
|
179
|
+
...(runError ? { error: runError.toJSON() } : {}),
|
|
178
180
|
});
|
|
179
|
-
printJob(finished, json, `Submitted job ${job.job_id}`);
|
|
181
|
+
const code = printJob(finished, json, `Submitted job ${job.job_id}`);
|
|
180
182
|
if (runDir && !json)
|
|
181
183
|
log.dim(` artifact: ${runDir}`);
|
|
182
|
-
return
|
|
184
|
+
return code;
|
|
183
185
|
}
|
|
184
186
|
catch (err) {
|
|
185
187
|
writeRun({
|
|
@@ -201,8 +203,7 @@ async function statusCmd(rest, ctx) {
|
|
|
201
203
|
assertUsable("batch");
|
|
202
204
|
const apiKey = requireApiKey();
|
|
203
205
|
const job = await getJob(id, { apiKey });
|
|
204
|
-
printJob(job, ctx.json, `Job ${id}`);
|
|
205
|
-
return 0;
|
|
206
|
+
return printJob(job, ctx.json, `Job ${id}`);
|
|
206
207
|
}
|
|
207
208
|
async function resultsCmd(rest, ctx) {
|
|
208
209
|
const { values, positionals } = parse(rest, {
|
|
@@ -273,8 +274,7 @@ async function cancelCmd(rest, ctx) {
|
|
|
273
274
|
assertUsable("batch");
|
|
274
275
|
const apiKey = requireApiKey();
|
|
275
276
|
const job = await stopJob(id, { apiKey });
|
|
276
|
-
printJob(job, ctx.json, `Stopped job ${id}`);
|
|
277
|
-
return 0;
|
|
277
|
+
return printJob(job, ctx.json, `Stopped job ${id}`);
|
|
278
278
|
}
|
|
279
279
|
async function waitCmd(rest, ctx) {
|
|
280
280
|
const { values, positionals } = parse(rest, { timeout: { type: "string" }, json: { type: "boolean" } });
|
|
@@ -283,8 +283,7 @@ async function waitCmd(rest, ctx) {
|
|
|
283
283
|
assertUsable("batch");
|
|
284
284
|
const apiKey = requireApiKey();
|
|
285
285
|
const job = await waitForJob(id, { apiKey, timeoutMs: asNumber(values.timeout) });
|
|
286
|
-
printJob(job, json, `Job ${id} finished`);
|
|
287
|
-
return 0;
|
|
286
|
+
return printJob(job, json, `Job ${id} finished`);
|
|
288
287
|
}
|
|
289
288
|
async function retryCmd(rest, ctx) {
|
|
290
289
|
const { positionals } = parse(rest, {});
|
|
@@ -294,21 +293,72 @@ async function retryCmd(rest, ctx) {
|
|
|
294
293
|
// "Reruns and retrying failures": POST /jobs/{id}/rerun?status=failed replays
|
|
295
294
|
// only the failures; already-successful tasks carry over.
|
|
296
295
|
const job = await rerunJob(id, { apiKey, status: "failed" });
|
|
297
|
-
printJob(job, ctx.json, `Reran failed tasks for job ${id}`);
|
|
298
|
-
|
|
296
|
+
return printJob(job, ctx.json, `Reran failed tasks for job ${id}`);
|
|
297
|
+
}
|
|
298
|
+
/** The run's failure_reason / failure_detail, when the API reports them. */
|
|
299
|
+
function failureFields(job) {
|
|
300
|
+
const run = job.latest_run ?? {};
|
|
301
|
+
const out = {};
|
|
302
|
+
if (typeof run.failure_reason === "string" && run.failure_reason)
|
|
303
|
+
out.failure_reason = run.failure_reason;
|
|
304
|
+
if (typeof run.failure_detail === "string" && run.failure_detail)
|
|
305
|
+
out.failure_detail = run.failure_detail;
|
|
306
|
+
return out;
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* The error for a run that ended `failed`, or null for any other state. A run the
|
|
310
|
+
* key's credit cap stopped (`api_key_cap_reached`) gets the cap guidance: the
|
|
311
|
+
* account still has credits and its other keys keep working.
|
|
312
|
+
*/
|
|
313
|
+
export function runFailure(job) {
|
|
314
|
+
if (job.latest_run?.status !== "failed")
|
|
315
|
+
return null;
|
|
316
|
+
const { failure_reason, failure_detail } = failureFields(job);
|
|
317
|
+
const where = `batch job ${job.job_id}`;
|
|
318
|
+
if (isKeyCapReached(failure_reason)) {
|
|
319
|
+
return keyCapReached(where, { status: null, detail: failure_detail });
|
|
320
|
+
}
|
|
321
|
+
const why = [failure_reason, failure_detail].filter(Boolean).join(": ");
|
|
322
|
+
return new ToolkitError({
|
|
323
|
+
code: "BATCH_FAILED",
|
|
324
|
+
message: `Batch job ${job.job_id} failed.`,
|
|
325
|
+
likely_cause: why || "The run ended in status failed without a reason.",
|
|
326
|
+
next_action: "Inspect the per-task results, fix the cause, then rerun the failed tasks.",
|
|
327
|
+
suggested_commands: [`zenrows batch results ${job.job_id} --status failed`, `zenrows batch retry-failed ${job.job_id}`],
|
|
328
|
+
});
|
|
299
329
|
}
|
|
300
|
-
/**
|
|
330
|
+
/**
|
|
331
|
+
* Print a job's status + stats, structured under --json, and return the exit
|
|
332
|
+
* code: 1 when the run ended `failed`, else 0. `stopped` and `deleted` are
|
|
333
|
+
* deliberate outcomes (someone cancelled or removed the run), so they exit 0
|
|
334
|
+
* but print as a warning, not a success.
|
|
335
|
+
*/
|
|
301
336
|
function printJob(job, json, headline) {
|
|
302
337
|
const run = job.latest_run ?? {};
|
|
303
338
|
const stats = run.stats;
|
|
339
|
+
const failure = failureFields(job);
|
|
340
|
+
const err = runFailure(job);
|
|
304
341
|
if (json) {
|
|
305
|
-
log.out(JSON.stringify({ ok:
|
|
306
|
-
return;
|
|
342
|
+
log.out(JSON.stringify({ ok: !err, jobId: job.job_id, status: run.status, stats, ...failure, ...(err ? { error: err.toJSON() } : {}) }, null, 2));
|
|
343
|
+
return err ? 1 : 0;
|
|
307
344
|
}
|
|
308
|
-
|
|
345
|
+
const line = `${headline} · status: ${run.status ?? "unknown"}`;
|
|
346
|
+
if (err)
|
|
347
|
+
log.error(line);
|
|
348
|
+
else if (run.status === "stopped" || run.status === "deleted")
|
|
349
|
+
log.warn(line);
|
|
350
|
+
else
|
|
351
|
+
log.success(line);
|
|
309
352
|
if (stats) {
|
|
310
353
|
log.info(` ${stats.completed}/${stats.total} completed · ${stats.successful} successful · ${stats.failed} failed`);
|
|
311
354
|
}
|
|
355
|
+
if (failure.failure_reason)
|
|
356
|
+
log.info(` failure_reason: ${failure.failure_reason}`);
|
|
357
|
+
if (failure.failure_detail)
|
|
358
|
+
log.info(` failure_detail: ${failure.failure_detail}`);
|
|
359
|
+
if (err)
|
|
360
|
+
printError(err, false);
|
|
361
|
+
return err ? 1 : 0;
|
|
312
362
|
}
|
|
313
363
|
function normalizeResultStatus(v) {
|
|
314
364
|
if (!v)
|
|
@@ -141,7 +141,7 @@ export const init = {
|
|
|
141
141
|
section("Test Protected Fetch");
|
|
142
142
|
try {
|
|
143
143
|
const apiKey = requireApiKey(root);
|
|
144
|
-
const { result } = await runFetch({ url: SMOKE_URL }, loadConfig(root), loadPolicy(root), apiKey);
|
|
144
|
+
const { result } = await runFetch({ url: SMOKE_URL }, loadConfig(root), loadPolicy(root), apiKey, root);
|
|
145
145
|
log.success(`Protected Fetch OK — HTTP ${result.status}, ${result.body.length} bytes, ${formatRequestCost(result.costUsd, result.costCredits)}.`);
|
|
146
146
|
}
|
|
147
147
|
catch (err) {
|
|
@@ -80,6 +80,7 @@ export const status = {
|
|
|
80
80
|
},
|
|
81
81
|
};
|
|
82
82
|
/** Reachability probe that does not consume credits. */
|
|
83
|
+
// Deliberately sends no X-ZenRows-Client: unauthenticated, non-billable reachability check.
|
|
83
84
|
async function probe(apiBase) {
|
|
84
85
|
try {
|
|
85
86
|
const controller = new AbortController();
|
|
@@ -27,6 +27,22 @@ export function formatPlanStatus(status) {
|
|
|
27
27
|
return "active";
|
|
28
28
|
return status;
|
|
29
29
|
}
|
|
30
|
+
const CAP_WINDOW_LABEL = {
|
|
31
|
+
day: "daily",
|
|
32
|
+
week: "weekly",
|
|
33
|
+
month: "monthly",
|
|
34
|
+
billing_period: "billing-period",
|
|
35
|
+
};
|
|
36
|
+
/** One line per credit cap on the calling key; none when the key is uncapped. */
|
|
37
|
+
export function formatKeyCaps(caps) {
|
|
38
|
+
return (caps ?? []).map((c) => {
|
|
39
|
+
const label = `This key: ${CAP_WINDOW_LABEL[c.window] ?? c.window} cap ${fmt(c.credits)}`;
|
|
40
|
+
const reset = c.resets_at ? `, resets ${c.resets_at}` : "";
|
|
41
|
+
if (c.unavailable || c.used_credits === undefined)
|
|
42
|
+
return `${label}, usage unavailable right now${reset}`;
|
|
43
|
+
return `${label}, ${fmt(c.used_credits)} used, ${fmt(c.remaining_credits ?? Math.max(0, c.credits - c.used_credits))} left${reset}`;
|
|
44
|
+
});
|
|
45
|
+
}
|
|
30
46
|
export const usage = {
|
|
31
47
|
name: "usage",
|
|
32
48
|
summary: "Show plan usage, credits, and concurrency for the current API key.",
|
|
@@ -63,6 +79,8 @@ export const usage = {
|
|
|
63
79
|
}
|
|
64
80
|
if (u.period_ends_at)
|
|
65
81
|
log.info(`Billing period ends: ${u.period_ends_at}`);
|
|
82
|
+
for (const line of formatKeyCaps(u.api_key?.caps))
|
|
83
|
+
log.info(line);
|
|
66
84
|
if (Array.isArray(u.top_ups) && u.top_ups.length)
|
|
67
85
|
log.info(`Top-ups: ${u.top_ups.length}`);
|
|
68
86
|
return 0;
|
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
* accountId, the human-facing claim URL, and claim metadata.
|
|
7
7
|
*/
|
|
8
8
|
import { chmodSync, existsSync, unlinkSync } from "node:fs";
|
|
9
|
-
import { attributionEnabled, getOrCreateTelemetryId, loadConfig, CLI_VERSION } from "./config.js";
|
|
9
|
+
import { attributionEnabled, getOrCreateTelemetryId, loadConfig, CLI_USER_AGENT, CLI_VERSION } from "./config.js";
|
|
10
|
+
import { agentClientHeader } from "./agent-client.js";
|
|
10
11
|
import { ToolkitError } from "./errors.js";
|
|
11
12
|
import { AGENT_SIGNUP_API_URL, WELL_KNOWN_PROTECTED_RESOURCE } from "./open-url.js";
|
|
12
13
|
import { detectClient } from "./provenance.js";
|
|
@@ -42,7 +43,7 @@ export async function discoverSignupUrl(projectRoot, opts = {}) {
|
|
|
42
43
|
const doFetch = opts.fetchImpl ?? fetch;
|
|
43
44
|
const res = await doFetch(url, {
|
|
44
45
|
method: "GET",
|
|
45
|
-
headers: { Accept: "application/json", "User-Agent":
|
|
46
|
+
headers: { Accept: "application/json", "User-Agent": CLI_USER_AGENT, ...agentClientHeader({ projectRoot }) },
|
|
46
47
|
});
|
|
47
48
|
if (!res.ok)
|
|
48
49
|
return null;
|
|
@@ -128,7 +129,8 @@ export async function signupAgent(opts = {}) {
|
|
|
128
129
|
// `telemetry:"off"` / ZENROWS_TELEMETRY=off suppresses every X-ZR-* header.
|
|
129
130
|
const headers = {
|
|
130
131
|
"content-type": "application/json",
|
|
131
|
-
"User-Agent":
|
|
132
|
+
"User-Agent": CLI_USER_AGENT,
|
|
133
|
+
...agentClientHeader(),
|
|
132
134
|
};
|
|
133
135
|
if (attributionEnabled()) {
|
|
134
136
|
const p = detectClient();
|
|
@@ -211,7 +213,7 @@ export async function fetchAccountStatus(apiKey, opts = {}) {
|
|
|
211
213
|
const doFetch = opts.fetchImpl ?? fetch;
|
|
212
214
|
const res = await doFetch(url, {
|
|
213
215
|
method: "GET",
|
|
214
|
-
headers: { "X-API-Key": apiKey, Accept: "application/json", "User-Agent":
|
|
216
|
+
headers: { "X-API-Key": apiKey, Accept: "application/json", "User-Agent": CLI_USER_AGENT, ...agentClientHeader() },
|
|
215
217
|
});
|
|
216
218
|
if (res.status !== 200) {
|
|
217
219
|
const body = await res.text();
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which AI coding agent is running the CLI, sent to Zenrows as the
|
|
3
|
+
* `X-ZenRows-Client` header so the dashboard Activity Log can say "CLI, driven
|
|
4
|
+
* by Claude Code" instead of just "CLI".
|
|
5
|
+
*
|
|
6
|
+
* Only the derived name is sent, never an environment value: the header carries
|
|
7
|
+
* one of the fixed names below, or the user's own `ZENROWS_CLIENT` override.
|
|
8
|
+
* When no agent is detected, no header is sent at all. `ZENROWS_TELEMETRY=off`
|
|
9
|
+
* (or config `telemetry: "off"`) suppresses it, override included.
|
|
10
|
+
*
|
|
11
|
+
* The gateway resolves the value against its own list of client names and
|
|
12
|
+
* stores anything it does not know as "other", so a name that is new here is
|
|
13
|
+
* harmless there.
|
|
14
|
+
*/
|
|
15
|
+
import { attributionEnabled } from "./config.js";
|
|
16
|
+
/** The request header the Zenrows gateway reads the client name from. */
|
|
17
|
+
export const CLIENT_HEADER = "X-ZenRows-Client";
|
|
18
|
+
/** Env var a user sets to name the client themselves. It wins over detection. */
|
|
19
|
+
export const CLIENT_OVERRIDE_ENV = "ZENROWS_CLIENT";
|
|
20
|
+
/**
|
|
21
|
+
* The gateway considers at most 32 bytes of the header.
|
|
22
|
+
* Anything we send must also be a legal header value, or `fetch` throws on
|
|
23
|
+
* every request, so the override is held to a plain-token charset.
|
|
24
|
+
*/
|
|
25
|
+
const CLIENT_NAME = /^[a-z0-9][a-z0-9._-]{0,31}$/;
|
|
26
|
+
/**
|
|
27
|
+
* Each agent's signal is a variable the agent itself sets in the shells it runs
|
|
28
|
+
* commands in, verified against the agent's docs or source. Agents we found no
|
|
29
|
+
* such variable for (Windsurf, Aider, Copilot CLI, the Copilot cloud agent) are
|
|
30
|
+
* left out rather than guessed.
|
|
31
|
+
*
|
|
32
|
+
* The first match wins. Every variable here is set by the agent itself when it
|
|
33
|
+
* launches a command, never by an IDE or extension in a terminal a person types
|
|
34
|
+
* in, so a human at the keyboard is never labelled as an agent.
|
|
35
|
+
*/
|
|
36
|
+
const AGENT_SIGNALS = [
|
|
37
|
+
// Cursor agent terminals: "Use the CURSOR_AGENT environment variable in your
|
|
38
|
+
// shell config to detect when Cursor is running".
|
|
39
|
+
// https://cursor.com/docs/agent/tools/terminal
|
|
40
|
+
{ client: "cursor", vars: ["CURSOR_AGENT"] },
|
|
41
|
+
// GitHub Copilot agent mode in VS Code sets COPILOT_AGENT=1 in agent terminals.
|
|
42
|
+
// Sent as "vscode", the name the gateway already has for VS Code's agent.
|
|
43
|
+
// https://github.com/microsoft/vscode/pull/316267
|
|
44
|
+
// https://github.com/microsoft/vscode/blob/45373f06ff77cc97a7754a376548d8937fb3af54/src/vs/workbench/contrib/terminalContrib/chatAgentTools/browser/toolTerminalCreator.ts#L156-L159
|
|
45
|
+
{ client: "vscode", vars: ["COPILOT_AGENT"] },
|
|
46
|
+
// OpenAI Codex CLI injects CODEX_THREAD_ID into every shell-tool environment,
|
|
47
|
+
// and CODEX_SANDBOX when the command runs sandboxed.
|
|
48
|
+
// https://github.com/openai/codex/blob/b741e480e203f037ca726bc2a76d99a8e8668e66/codex-rs/protocol/src/shell_environment.rs#L151-L154
|
|
49
|
+
// https://github.com/openai/codex/blob/b741e480e203f037ca726bc2a76d99a8e8668e66/codex-rs/core/src/spawn.rs#L23-L26
|
|
50
|
+
{ client: "codex", vars: ["CODEX_THREAD_ID", "CODEX_SANDBOX"] },
|
|
51
|
+
// Gemini CLI sets GEMINI_CLI=1 for every shell command it executes.
|
|
52
|
+
// https://google-gemini.github.io/gemini-cli/docs/cli/commands.html
|
|
53
|
+
// https://github.com/google-gemini/gemini-cli/blob/fb972b2f87fe7d5b06d37eac711490162d98de2c/packages/core/src/services/shellExecutionService.ts#L582-L585
|
|
54
|
+
{ client: "gemini-cli", vars: ["GEMINI_CLI"] },
|
|
55
|
+
// Claude Code sets CLAUDE_CODE_CHILD_SESSION=1 in what its Bash, PowerShell and
|
|
56
|
+
// Monitor tools, hooks and status line spawn, "only set by Claude Code itself
|
|
57
|
+
// ... and not by IDE extensions" (v2.1.172+). Not CLAUDECODE: the IDE extensions
|
|
58
|
+
// export that into every integrated terminal, so a person typing `zenrows` there
|
|
59
|
+
// would read as Claude Code. Older versions send no name rather than a wrong one,
|
|
60
|
+
// and so does a stdio MCP server, which gets neither variable's guarantee.
|
|
61
|
+
// https://code.claude.com/docs/en/env-vars
|
|
62
|
+
{ client: "claude-code", vars: ["CLAUDE_CODE_CHILD_SESSION"] },
|
|
63
|
+
];
|
|
64
|
+
/** Every variable detection reads, so tests can keep the ambient shell out. */
|
|
65
|
+
export const AGENT_ENV_VARS = AGENT_SIGNALS.flatMap((s) => s.vars);
|
|
66
|
+
/**
|
|
67
|
+
* The client name to send, or `undefined` to send nothing. Pure: reads only
|
|
68
|
+
* `env`. A valid `ZENROWS_CLIENT` override is sent even when no agent is
|
|
69
|
+
* detected: setting it is an explicit opt-in. It replaces detection entirely, so
|
|
70
|
+
* an override that is not a valid name sends nothing rather than a detected name
|
|
71
|
+
* the user asked not to send.
|
|
72
|
+
*/
|
|
73
|
+
export function detectAgentClient(env) {
|
|
74
|
+
const override = env[CLIENT_OVERRIDE_ENV]?.trim();
|
|
75
|
+
if (override) {
|
|
76
|
+
// Same normalisation the gateway applies: lower-case, spaces to dashes.
|
|
77
|
+
const name = override.toLowerCase().split(/\s+/).join("-");
|
|
78
|
+
return CLIENT_NAME.test(name) ? name : undefined;
|
|
79
|
+
}
|
|
80
|
+
return AGENT_SIGNALS.find((s) => s.vars.some((v) => isSet(env[v])))?.client;
|
|
81
|
+
}
|
|
82
|
+
/** Empty, `0` and `false` (any case) count as unset, the way `CLAUDE_CODE_CHILD_SESSION=0` reads. */
|
|
83
|
+
function isSet(value) {
|
|
84
|
+
const v = value?.trim().toLowerCase();
|
|
85
|
+
return Boolean(v) && v !== "0" && v !== "false";
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* The header to spread into a request to a Zenrows API: `{ "X-ZenRows-Client":
|
|
89
|
+
* name }` when there is a name and attribution is on, `{}` otherwise. Pass
|
|
90
|
+
* `projectRoot` when the caller works on a workspace other than the one above
|
|
91
|
+
* the cwd (`init --workspace`), so that workspace's `telemetry: "off"` applies.
|
|
92
|
+
*/
|
|
93
|
+
export function agentClientHeader(opts = {}) {
|
|
94
|
+
if (!attributionEnabled(opts.projectRoot))
|
|
95
|
+
return {};
|
|
96
|
+
const client = detectAgentClient(opts.env ?? process.env);
|
|
97
|
+
return client ? { [CLIENT_HEADER]: client } : {};
|
|
98
|
+
}
|
|
@@ -16,9 +16,11 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
18
18
|
import { join } from "node:path";
|
|
19
|
-
import { ToolkitError, quotaExhausted } from "./errors.js";
|
|
19
|
+
import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.js";
|
|
20
20
|
import { readAccount } from "./agent-account.js";
|
|
21
21
|
import { registerSecret } from "./logger.js";
|
|
22
|
+
import { CLI_USER_AGENT } from "./config.js";
|
|
23
|
+
import { agentClientHeader } from "./agent-client.js";
|
|
22
24
|
/** Confirmed Batch API base (no trailing slash). */
|
|
23
25
|
export const DEFAULT_BATCH_API_BASE = "https://async.api.zenrows.com/v1";
|
|
24
26
|
/** Env var to override the Batch API base (local/staging testing). */
|
|
@@ -50,7 +52,8 @@ export async function batchRequest(method, path, opts) {
|
|
|
50
52
|
const headers = {
|
|
51
53
|
"X-API-Key": opts.apiKey,
|
|
52
54
|
Accept: "application/json",
|
|
53
|
-
"User-Agent":
|
|
55
|
+
"User-Agent": CLI_USER_AGENT,
|
|
56
|
+
...agentClientHeader(),
|
|
54
57
|
};
|
|
55
58
|
if (opts.body !== undefined)
|
|
56
59
|
headers["Content-Type"] = "application/json";
|
|
@@ -139,6 +142,9 @@ function problemToError(status, body, method, path) {
|
|
|
139
142
|
suggested_commands: ["zenrows batch status <id>"],
|
|
140
143
|
});
|
|
141
144
|
}
|
|
145
|
+
if (status === 402 && isKeyCapReached(serverCode)) {
|
|
146
|
+
return keyCapReached(`${method} ${path}`, { status: 402, detail: problem.detail || problem.title || undefined });
|
|
147
|
+
}
|
|
142
148
|
if (status === 402) {
|
|
143
149
|
// Out of credits ("Subscription has no credit available") — the same
|
|
144
150
|
// exhausted state as the scraper/usage 402. Surface the credits error with
|
|
@@ -11,10 +11,11 @@
|
|
|
11
11
|
* Sessions are billed by bandwidth + session time, so callers must always close
|
|
12
12
|
* them (`closeSession`) — the `browser run` command does so in a `finally`.
|
|
13
13
|
*/
|
|
14
|
-
import { ToolkitError, quotaExhausted } from "./errors.js";
|
|
14
|
+
import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.js";
|
|
15
15
|
import { readAccount } from "./agent-account.js";
|
|
16
16
|
import { registerSecret } from "./logger.js";
|
|
17
|
-
import {
|
|
17
|
+
import { CLI_USER_AGENT } from "./config.js";
|
|
18
|
+
import { agentClientHeader } from "./agent-client.js";
|
|
18
19
|
/** Managed Browser session API base (no trailing slash). Tied to the MCP host. */
|
|
19
20
|
export const DEFAULT_BROWSER_BASE = "https://mcp.zenrows.com";
|
|
20
21
|
/** Env var to override the Browser API base (local/staging testing). */
|
|
@@ -42,7 +43,8 @@ export async function browserRequest(method, path, opts) {
|
|
|
42
43
|
const headers = {
|
|
43
44
|
Authorization: `Bearer ${opts.apiKey}`,
|
|
44
45
|
Accept: "application/json",
|
|
45
|
-
"User-Agent":
|
|
46
|
+
"User-Agent": CLI_USER_AGENT,
|
|
47
|
+
...agentClientHeader(),
|
|
46
48
|
};
|
|
47
49
|
if (opts.body !== undefined)
|
|
48
50
|
headers["Content-Type"] = "application/json";
|
|
@@ -105,6 +107,12 @@ export function browserProblemToError(status, body, method, path) {
|
|
|
105
107
|
suggested_commands: ["zenrows login --api-key <your-key>"],
|
|
106
108
|
});
|
|
107
109
|
}
|
|
110
|
+
if (status === 402 && isKeyCapReached(serverCode, body)) {
|
|
111
|
+
return keyCapReached(`${method} ${path}`, {
|
|
112
|
+
status: 402,
|
|
113
|
+
detail: parsed.error || parsed.detail || parsed.title || undefined,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
108
116
|
if (status === 402) {
|
|
109
117
|
// Out of credits — same exhausted state as the scraper/batch 402.
|
|
110
118
|
const acct = readAccount();
|
package/dist/src/core/config.js
CHANGED
|
@@ -11,19 +11,23 @@ export const CONFIG_VERSION = "0.1.0";
|
|
|
11
11
|
* entrypoint) so core modules — telemetry, signup provenance — can read it
|
|
12
12
|
* without an import cycle. `VERSION` in `cli/index.ts` re-exports this.
|
|
13
13
|
*/
|
|
14
|
-
export const CLI_VERSION = "1.
|
|
14
|
+
export const CLI_VERSION = "1.3.1";
|
|
15
|
+
/** The User-Agent on every request to a Zenrows API. The gateway reads the version from it. */
|
|
16
|
+
export const CLI_USER_AGENT = `zenrows-cli/${CLI_VERSION}`;
|
|
15
17
|
/** Env var to override the Fetch and Extract API base (local/staging testing). */
|
|
16
18
|
export const API_BASE_ENV = "ZENROWS_API_BASE";
|
|
17
19
|
/**
|
|
18
20
|
* Env var to opt out of anonymous attribution. The toolkit never POSTs to a
|
|
19
21
|
* telemetry endpoint; attribution is only anonymous provenance headers on the
|
|
20
|
-
* signup request
|
|
21
|
-
*
|
|
22
|
+
* signup request, the `X-ZenRows-Client` agent name on API requests
|
|
23
|
+
* (`agent-client.ts`), and `utm_*` params on the browser URLs a human opens.
|
|
24
|
+
* Setting this to `off` (or config `telemetry: "off"`) suppresses all of it.
|
|
22
25
|
*/
|
|
23
26
|
export const TELEMETRY_ENV = "ZENROWS_TELEMETRY";
|
|
24
27
|
/**
|
|
25
|
-
* Whether to attach anonymous attribution (signup provenance headers
|
|
26
|
-
* on browser URLs). Off when
|
|
28
|
+
* Whether to attach anonymous attribution (signup provenance headers, the
|
|
29
|
+
* `X-ZenRows-Client` agent name, `utm_*` on browser URLs). Off when
|
|
30
|
+
* `ZENROWS_TELEMETRY=off` or config `telemetry:"off"`.
|
|
27
31
|
* There is no telemetry beacon — this only gates what rides on requests/URLs
|
|
28
32
|
* the toolkit already makes.
|
|
29
33
|
*/
|
package/dist/src/core/errors.js
CHANGED
|
@@ -57,3 +57,31 @@ export function quotaExhausted(url, claimUrl, opts = {}) {
|
|
|
57
57
|
suggested_commands: ["zenrows usage"],
|
|
58
58
|
});
|
|
59
59
|
}
|
|
60
|
+
/** Where an account manages its API keys and their credit caps. */
|
|
61
|
+
export const API_KEYS_SETTINGS_URL = "https://app.zenrows.com/settings/api-keys";
|
|
62
|
+
/**
|
|
63
|
+
* True when a 402 is a per-key credit cap (gateway AUTH014, Batch
|
|
64
|
+
* `api_key_cap_reached`), not an account out of credits. The account still has
|
|
65
|
+
* credits, so the out-of-credits advice (top up, upgrade, claim) is wrong here.
|
|
66
|
+
*/
|
|
67
|
+
export function isKeyCapReached(code, body) {
|
|
68
|
+
if (code === "AUTH014" || code === "api_key_cap_reached")
|
|
69
|
+
return true;
|
|
70
|
+
return !!body && /\b(AUTH014|api_key_cap_reached)\b/.test(body);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* The error for a request refused because this API key reached one of its credit
|
|
74
|
+
* caps. `opts.status: null` is for a Batch run the cap stopped: the run's status
|
|
75
|
+
* call succeeded, so there is no HTTP 402 to cite, only `url` (the job).
|
|
76
|
+
*/
|
|
77
|
+
export function keyCapReached(url, opts = {}) {
|
|
78
|
+
const detail = opts.detail ? `${opts.detail.replace(/\.\s*$/, "")}. ` : "";
|
|
79
|
+
const where = opts.status === null ? `Stopped ${url}` : `HTTP ${opts.status ?? 402} for ${url}`;
|
|
80
|
+
return new ToolkitError({
|
|
81
|
+
code: "KEY_CREDIT_CAP_REACHED",
|
|
82
|
+
message: "This API key reached one of its credit caps.",
|
|
83
|
+
likely_cause: `${detail}${where}`,
|
|
84
|
+
next_action: `The account still has credits and its other API keys keep working. Wait for the cap to reset (\`zenrows usage\` shows when), or raise or remove this key's cap at ${API_KEYS_SETTINGS_URL}.`,
|
|
85
|
+
suggested_commands: ["zenrows usage"],
|
|
86
|
+
});
|
|
87
|
+
}
|
package/dist/src/core/http.js
CHANGED
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
* query parameter (per docs) and is registered as a secret so it is redacted
|
|
6
6
|
* from any logged URL.
|
|
7
7
|
*/
|
|
8
|
-
import { ToolkitError, quotaExhausted } from "./errors.js";
|
|
8
|
+
import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.js";
|
|
9
9
|
import { readAccount } from "./agent-account.js";
|
|
10
10
|
import { ENV_KEY, resolveApiKey } from "./auth.js";
|
|
11
11
|
import { registerSecret } from "./logger.js";
|
|
12
|
-
import {
|
|
12
|
+
import { CLI_USER_AGENT } from "./config.js";
|
|
13
|
+
import { agentClientHeader } from "./agent-client.js";
|
|
13
14
|
/**
|
|
14
15
|
* Client-side timeout for a Fetch and Extract request, deliberately ABOVE the
|
|
15
16
|
* gateway's own 90s request budget.
|
|
@@ -56,7 +57,7 @@ export async function scrape(apiBase, apiKey, params, opts = {}) {
|
|
|
56
57
|
try {
|
|
57
58
|
res = await fetch(full, {
|
|
58
59
|
method: "GET",
|
|
59
|
-
headers: { "User-Agent":
|
|
60
|
+
headers: { "User-Agent": CLI_USER_AGENT, "Accept-Encoding": "gzip, deflate", ...agentClientHeader({ projectRoot: opts.projectRoot }) },
|
|
60
61
|
signal: controller.signal,
|
|
61
62
|
});
|
|
62
63
|
}
|
|
@@ -142,6 +143,11 @@ export async function scrape(apiBase, apiKey, params, opts = {}) {
|
|
|
142
143
|
suggested_commands: [`zenrows extract ${params.url} --autoparse`],
|
|
143
144
|
});
|
|
144
145
|
}
|
|
146
|
+
// AUTH014: this key hit one of its credit caps; the account still has credits.
|
|
147
|
+
if (isKeyCapReached(zrErrorCode(body) ?? undefined)) {
|
|
148
|
+
// `detail` names the cap and its reset date; the title alone says neither.
|
|
149
|
+
throw keyCapReached(redacted, { status: 402, detail: zrErrorProblemDetail(body) ?? zrErrorDetail(body) ?? undefined });
|
|
150
|
+
}
|
|
145
151
|
// Zenrows returns 402 with a JSON error envelope (e.g. AUTH004 "reached its
|
|
146
152
|
// usage limit" / "Subscription has no credit available") when the account is
|
|
147
153
|
// out of credits. This is NOT scraped content — surface it as a credits
|
|
@@ -358,6 +364,18 @@ export function zrErrorCode(body) {
|
|
|
358
364
|
return null;
|
|
359
365
|
}
|
|
360
366
|
}
|
|
367
|
+
/** The problem body's `detail` field alone, prefixed with its code. */
|
|
368
|
+
function zrErrorProblemDetail(body) {
|
|
369
|
+
try {
|
|
370
|
+
const j = JSON.parse(body);
|
|
371
|
+
if (!j.detail)
|
|
372
|
+
return null;
|
|
373
|
+
return j.code ? `(${j.code}) ${j.detail}` : j.detail;
|
|
374
|
+
}
|
|
375
|
+
catch {
|
|
376
|
+
return null;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
361
379
|
/**
|
|
362
380
|
* Clean one-line detail from a Zenrows JSON error body
|
|
363
381
|
* (e.g. `(AUTH003) Invalid apikey provided`).
|
package/dist/src/core/usage.js
CHANGED
|
@@ -11,6 +11,8 @@ import { isQuotaError, zrErrorDetail } from "./http.js";
|
|
|
11
11
|
import { readAccount } from "./agent-account.js";
|
|
12
12
|
import { ENV_KEY, resolveApiKey } from "./auth.js";
|
|
13
13
|
import { registerSecret } from "./logger.js";
|
|
14
|
+
import { CLI_USER_AGENT } from "./config.js";
|
|
15
|
+
import { agentClientHeader } from "./agent-client.js";
|
|
14
16
|
/** Build the plan-usage URL from the configured API base. */
|
|
15
17
|
export function usageUrl(apiBase) {
|
|
16
18
|
const base = apiBase.endsWith("/") ? apiBase : apiBase + "/";
|
|
@@ -27,7 +29,7 @@ export async function fetchUsage(apiBase, apiKey, opts = {}) {
|
|
|
27
29
|
try {
|
|
28
30
|
res = await doFetch(url, {
|
|
29
31
|
method: "GET",
|
|
30
|
-
headers: { "X-API-Key": apiKey, Accept: "application/json", "User-Agent":
|
|
32
|
+
headers: { "X-API-Key": apiKey, Accept: "application/json", "User-Agent": CLI_USER_AGENT, ...agentClientHeader() },
|
|
31
33
|
signal: controller.signal,
|
|
32
34
|
});
|
|
33
35
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zenrows/cli",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.1",
|
|
4
4
|
"description": "Open-source CLI, MCP, skills, templates, workflows, recipes, and evals layer for giving AI agents reliable access to protected web data through Zenrows cloud infrastructure.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -54,6 +54,14 @@ zenrows batch retry-failed <id> # rerun only the failed tasks (new run)
|
|
|
54
54
|
`--premium-proxy` (or `mode=auto`) and is rejected before any request. Job-level
|
|
55
55
|
flags apply to every task; per-task keys in the JSONL override them.
|
|
56
56
|
|
|
57
|
+
A run ends in one of `completed`, `failed`, `stopped`, or `deleted`. `status`,
|
|
58
|
+
`wait`, and `create --wait` exit non-zero (`ok: false` under `--json`) only for
|
|
59
|
+
`failed`, and print the run's `failure_reason` and `failure_detail`. When
|
|
60
|
+
`failure_reason` is `api_key_cap_reached`, the key hit its own credit cap
|
|
61
|
+
(`KEY_CREDIT_CAP_REACHED`): the account still has credits, other keys keep
|
|
62
|
+
working, and the tasks not yet run stay pending. Wait for the cap to reset
|
|
63
|
+
(`zenrows usage`) or raise the cap; do not resubmit the job with the same key.
|
|
64
|
+
|
|
57
65
|
Deferred (documented, not yet in the CLI): CSV upload, open/queue jobs,
|
|
58
66
|
scheduled jobs, webhooks/HMAC, and ZIP export.
|
|
59
67
|
|
package/skills/extract/SKILL.md
CHANGED
|
@@ -28,7 +28,10 @@ zenrows extract <url> --validate # fail if not valid JSON
|
|
|
28
28
|
|
|
29
29
|
## Rules
|
|
30
30
|
- Validate on a single page before scaling across many URLs.
|
|
31
|
-
- If Autoparse misses fields, switch to `--css` with explicit selectors
|
|
32
|
-
|
|
31
|
+
- If Autoparse misses fields, switch to `--css` with explicit selectors. Auto
|
|
32
|
+
mode already renders JS when the target needs it, so reach for
|
|
33
|
+
`--manual --js-render` only after auto mode has returned an incomplete page:
|
|
34
|
+
it costs several times a basic call ([[cost-control]]) and it makes the
|
|
35
|
+
escalation yours.
|
|
33
36
|
|
|
34
37
|
See [[protected-fetch]] for retrieval semantics.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: protected-fetch
|
|
3
|
-
description: Use Zenrows Protected Fetch for anti-bot-protected page retrieval.
|
|
3
|
+
description: Use Zenrows Protected Fetch in auto mode for anti-bot-protected page retrieval. Auto mode handles the anti-bot escalation, so never enable JS rendering or premium proxies yourself on a first attempt.
|
|
4
4
|
version: 0.1.0
|
|
5
5
|
requires_backend_capabilities: [protected_fetch]
|
|
6
6
|
---
|
|
@@ -16,18 +16,33 @@ cannot. This is the **core primitive** — backed by Zenrows **Fetch**
|
|
|
16
16
|
- The target has anti-bot protection, needs JS rendering, or geo-specific access.
|
|
17
17
|
|
|
18
18
|
## How to call
|
|
19
|
+
|
|
20
|
+
Auto mode is the answer for anti-bot targets, Cloudflare included. It escalates
|
|
21
|
+
for you and bills only for the configuration that succeeds.
|
|
22
|
+
|
|
19
23
|
```
|
|
20
|
-
zenrows fetch <url> # Adaptive Stealth Mode
|
|
24
|
+
zenrows fetch <url> # Adaptive Stealth Mode. Start here, always
|
|
21
25
|
zenrows fetch <url> --output markdown # convert to Markdown
|
|
22
|
-
zenrows fetch <url> --
|
|
23
|
-
zenrows fetch <url> --proxy-country us # geo-target (auto mode; in manual mode needs --premium-proxy)
|
|
26
|
+
zenrows fetch <url> --proxy-country us # geo-target, works in auto mode
|
|
24
27
|
zenrows fetch <url> --wait-for ".price" # wait for a selector
|
|
25
28
|
```
|
|
26
29
|
|
|
30
|
+
Manual mode exists for the rare case where auto mode has already failed and a
|
|
31
|
+
trace shows why. It costs more and it makes the escalation your problem:
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
zenrows fetch <url> --manual --js-render --premium-proxy # the most expensive path, see [[cost-control]]
|
|
35
|
+
```
|
|
36
|
+
|
|
27
37
|
## Rules
|
|
28
|
-
- Start with **auto mode**.
|
|
29
|
-
|
|
30
|
-
|
|
38
|
+
- Start with **auto mode**. Enabling `--js-render` and `--premium-proxy` yourself
|
|
39
|
+
is the most expensive configuration this API offers, by a wide margin, and auto
|
|
40
|
+
mode reaches the same place only when the target actually needs it. The
|
|
41
|
+
multipliers are in [[cost-control]].
|
|
42
|
+
- A hard target is not a reason to skip auto mode. It is the reason auto mode
|
|
43
|
+
exists.
|
|
44
|
+
- In auto mode `js_render` and `premium_proxy` are managed for you. Passing them
|
|
45
|
+
manually requires `--manual`, otherwise you get `PARAM_CONFLICT_AUTO_MANUAL`.
|
|
31
46
|
- `--proxy-country` works in auto mode on its own; in `--manual` mode it also
|
|
32
47
|
needs `--premium-proxy` (else `PARAM_PROXY_COUNTRY_REQUIRES_PREMIUM`).
|
|
33
48
|
- You are billed only for the configuration that succeeds.
|
|
@@ -19,8 +19,12 @@ zenrows trace export <run-id> # JSON for sharing
|
|
|
19
19
|
```
|
|
20
20
|
|
|
21
21
|
## Failure → action map
|
|
22
|
-
- `FETCH_FAILED` / empty content → retry
|
|
23
|
-
|
|
22
|
+
- `FETCH_FAILED` / empty content → first retry in auto mode (`mode=auto`), which
|
|
23
|
+
escalates for you and bills only for what succeeds. For slow pages add
|
|
24
|
+
`--wait-for <selector>`. Only when auto mode has failed on its own, take
|
|
25
|
+
manual control with `--manual --js-render`, then `--premium-proxy`. Each step
|
|
26
|
+
multiplies the cost of the request, and both together are the most expensive
|
|
27
|
+
configuration available ([[cost-control]]).
|
|
24
28
|
- `REQUEST_TIMEOUT` → the CLI stopped waiting; the API was reached. Raise
|
|
25
29
|
`--timeout` (default 120000ms, above the API's own 90s budget), or drop
|
|
26
30
|
`--wait-for` so the request finishes inside that budget and the API returns
|
|
@@ -28,6 +32,18 @@ zenrows trace export <run-id> # JSON for sharing
|
|
|
28
32
|
- `BACKEND_UNAVAILABLE` → a genuine transport failure (DNS/TCP/TLS). Check
|
|
29
33
|
connectivity and `zenrows config show`.
|
|
30
34
|
- `AUTH_INVALID` → re-check the key, `zenrows login --api-key …`.
|
|
35
|
+
- `KEY_CREDIT_CAP_REACHED` → this API key hit one of its own credit caps (HTTP
|
|
36
|
+
402, gateway code `AUTH014`, Batch `api_key_cap_reached`). The account still
|
|
37
|
+
has credits and its other keys keep working. Do not retry and do not escalate:
|
|
38
|
+
`zenrows usage` shows the cap and when it resets. Wait for the reset, or have
|
|
39
|
+
the account owner raise or remove the cap at
|
|
40
|
+
https://app.zenrows.com/settings/api-keys.
|
|
41
|
+
- `POLICY_MAX_CREDITS_EXCEEDED` → the account itself is out of credits (or a
|
|
42
|
+
local policy credit limit was hit). Do not retry-loop; `zenrows usage` shows
|
|
43
|
+
when credits renew. Top up or upgrade, or claim the account if it is an
|
|
44
|
+
auto-created Free plan.
|
|
45
|
+
- `BATCH_FAILED` → the batch run ended `failed`; `failure_reason` and
|
|
46
|
+
`failure_detail` in `zenrows batch status <id> --json` say why.
|
|
31
47
|
- `PARAM_CONFLICT_AUTO_MANUAL` → drop the managed flags or add `--manual`.
|
|
32
48
|
- `CAPABILITY_UNAVAILABLE` → the primitive is not available on this account (e.g. beta/invite-only); use the local-spec path where offered.
|
|
33
49
|
|