@zenrows/mcp 2.1.2 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -7
- package/dist/auth/claim-hint.d.ts +16 -0
- package/dist/auth/claim-hint.js +71 -0
- package/dist/auth/ensure-key.d.ts +59 -0
- package/dist/auth/ensure-key.js +184 -0
- package/dist/batch-api.d.ts +89 -0
- package/dist/batch-api.js +182 -0
- package/dist/http.js +4 -4
- package/dist/index.js +30 -4
- package/dist/server.js +51 -15
- package/dist/tools/account.d.ts +18 -0
- package/dist/tools/account.js +98 -0
- package/dist/tools/batch.d.ts +2 -0
- package/dist/tools/batch.js +245 -0
- package/dist/tools/browser.js +194 -47
- package/dist/tools/extract.d.ts +40 -0
- package/dist/tools/extract.js +232 -0
- package/package.json +6 -3
package/README.md
CHANGED
|
@@ -41,7 +41,7 @@ https://mcp.zenrows.com/mcp
|
|
|
41
41
|
|
|
42
42
|
**Transport:** Streamable HTTP
|
|
43
43
|
|
|
44
|
-
**Authentication:** OAuth
|
|
44
|
+
**Authentication:** OAuth or API key as Bearer token. Pass your Zenrows API key in the `Authorization` header on every request (or complete OAuth in clients that support it).
|
|
45
45
|
|
|
46
46
|
```
|
|
47
47
|
Authorization: Bearer YOUR_ZENROWS_API_KEY
|
|
@@ -49,6 +49,8 @@ Authorization: Bearer YOUR_ZENROWS_API_KEY
|
|
|
49
49
|
|
|
50
50
|
Most MCP clients accept this through an `authorization` shorthand field on the tool config and forward it as the Bearer token automatically. Some clients use a free-form `headers` field instead. Either approach works.
|
|
51
51
|
|
|
52
|
+
> Remote MCP does **not** auto-create accounts. Use OAuth “Create Free account” in the client, or pass an existing API key.
|
|
53
|
+
|
|
52
54
|
#### Example: OpenAI Responses API
|
|
53
55
|
|
|
54
56
|
```python
|
|
@@ -84,11 +86,15 @@ Use the local stdio configuration when your MCP client runs the server as a loca
|
|
|
84
86
|
|
|
85
87
|
**Package:** [`@zenrows/mcp`](https://www.npmjs.com/package/@zenrows/mcp) on npm
|
|
86
88
|
|
|
87
|
-
**Authentication:**
|
|
89
|
+
**Authentication:**
|
|
90
|
+
|
|
91
|
+
1. `ZENROWS_API_KEY` environment variable, or
|
|
92
|
+
2. Key previously stored in `~/.zenrows/secrets.json`, or
|
|
93
|
+
3. **Auto-signup** (default): if neither is set, stdio provisions a Free plan account via `POST /api/agent/signup`, persists the key + claim metadata under `~/.zenrows/` (`secrets.json` + `account.json`, mode `0600`), and prints a claim URL on stderr. Opt out with `ZENROWS_AUTO_SIGNUP=false`.
|
|
88
94
|
|
|
89
95
|
**Requirements:** [Node.js](https://nodejs.org/) installed (for `npx` to work).
|
|
90
96
|
|
|
91
|
-
**Configuration:**
|
|
97
|
+
**Configuration (with your own key):**
|
|
92
98
|
|
|
93
99
|
```json
|
|
94
100
|
{
|
|
@@ -104,16 +110,33 @@ Use the local stdio configuration when your MCP client runs the server as a loca
|
|
|
104
110
|
}
|
|
105
111
|
```
|
|
106
112
|
|
|
113
|
+
**Zero-config (auto-signup):**
|
|
114
|
+
|
|
115
|
+
```json
|
|
116
|
+
{
|
|
117
|
+
"mcpServers": {
|
|
118
|
+
"zenrows": {
|
|
119
|
+
"command": "npx",
|
|
120
|
+
"args": ["-y", "@zenrows/mcp"]
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
```
|
|
125
|
+
|
|
107
126
|
The exact location of this config varies by client. See the [per-client setup guides](https://docs.zenrows.com/mcp/overview#per-client-setup-guides) for the file path for your client.
|
|
108
127
|
|
|
109
128
|
---
|
|
110
129
|
|
|
111
130
|
## Tools
|
|
112
131
|
|
|
113
|
-
The Zenrows MCP exposes
|
|
132
|
+
The Zenrows MCP exposes these tool families:
|
|
114
133
|
|
|
115
|
-
|
|
116
|
-
|
|
134
|
+
| Tool | Purpose |
|
|
135
|
+
|------|---------|
|
|
136
|
+
| **`scrape`** | Full-page content → Markdown, plain text, HTML, PDF, or screenshot (plus helper outputs). |
|
|
137
|
+
| **`extract`** | Structured JSON (`extract=auto`, autoparse, or `css_extractor`) + optional stealth flags. `extract=auto` is open beta (currently free; billing may apply later). |
|
|
138
|
+
| **`batch_create` / `batch_status` / `batch_results` / `batch_cancel` / `batch_wait`** | Cloud Batch API fan-out (`async.api.zenrows.com`). Beta; may return `BATCH_ACCESS_DENIED`. Not `browser_batch`. |
|
|
139
|
+
| **`browser_*`** | 30+ tools for full browser automation (navigation, clicks, forms, JS, cookies, tabs, sessions). |
|
|
117
140
|
|
|
118
141
|
The AI selects the right tool from your prompt. You don't call tools directly in code.
|
|
119
142
|
|
|
@@ -127,7 +150,7 @@ See the [full tool reference](https://docs.zenrows.com/mcp/overview#tools) for e
|
|
|
127
150
|
git clone https://github.com/ZenRows/zenrows-mcp
|
|
128
151
|
cd zenrows-mcp
|
|
129
152
|
npm install
|
|
130
|
-
cp .env.example .env #
|
|
153
|
+
cp .env.example .env # Optional: add your API key (stdio can auto-signup)
|
|
131
154
|
npm run dev # Run with .env loaded (requires Node.js 20.6+)
|
|
132
155
|
npm run build # Compile to dist/
|
|
133
156
|
npm run inspect # Open the MCP inspector UI
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export declare function isQuotaOrPlanError(opts?: {
|
|
2
|
+
status?: number;
|
|
3
|
+
body?: string;
|
|
4
|
+
code?: string;
|
|
5
|
+
message?: string;
|
|
6
|
+
}): boolean;
|
|
7
|
+
/**
|
|
8
|
+
* If the local agent account is still unclaimed and this looks like a quota/plan
|
|
9
|
+
* failure, append the claim URL so agents can nudge the user.
|
|
10
|
+
*/
|
|
11
|
+
export declare function appendClaimHint(text: string, opts?: {
|
|
12
|
+
status?: number;
|
|
13
|
+
body?: string;
|
|
14
|
+
code?: string;
|
|
15
|
+
message?: string;
|
|
16
|
+
}): string;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Append a claim-account nudge when an unclaimed Free agent hits quota/plan limits.
|
|
3
|
+
*/
|
|
4
|
+
import { readAccount } from "./ensure-key.js";
|
|
5
|
+
const CLAIM_NUDGE = "Claim your Free account to keep usage and upgrade: ";
|
|
6
|
+
/**
|
|
7
|
+
* Codes that are not an allowance problem, so a claim nudge would be noise.
|
|
8
|
+
*
|
|
9
|
+
* AUTH006 is the concurrency limit, and is belt-and-braces: it comes back as 429, so it
|
|
10
|
+
* would not reach the 402 branch below in the first place. It is listed to keep the
|
|
11
|
+
* intent legible rather than because anything depends on it.
|
|
12
|
+
*
|
|
13
|
+
* AUTH004 used to be listed here on the belief that it
|
|
14
|
+
* was concurrency too — it is not. AUTH004 is "Usage Exceeded": the allowance itself is
|
|
15
|
+
* spent (docs `api-error-codes#AUTH004`; gateway logs carry `err: "usage exceeded"`,
|
|
16
|
+
* `msg: "user allowance failure"`). Skipping it suppressed the nudge at the one moment it
|
|
17
|
+
* is worth the most — an unclaimed Free agent that has just run through its allowance and
|
|
18
|
+
* would otherwise lose the account along with its usage history.
|
|
19
|
+
*/
|
|
20
|
+
const SKIP_CODES = new Set(["AUTH006"]);
|
|
21
|
+
function extractCode(body, code) {
|
|
22
|
+
if (code && typeof code === "string")
|
|
23
|
+
return code;
|
|
24
|
+
if (!body)
|
|
25
|
+
return undefined;
|
|
26
|
+
try {
|
|
27
|
+
const j = JSON.parse(body);
|
|
28
|
+
if (typeof j.code === "string")
|
|
29
|
+
return j.code;
|
|
30
|
+
const m = typeof j.error === "string" ? j.error.match(/\((AUTH\d+)\)/) : null;
|
|
31
|
+
return m?.[1];
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
const m = body.match(/\b(AUTH\d+|BATCH_QUOTA_EXCEEDED)\b/);
|
|
35
|
+
return m?.[1];
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export function isQuotaOrPlanError(opts = {}) {
|
|
39
|
+
const code = extractCode(opts.body, opts.code);
|
|
40
|
+
if (code && SKIP_CODES.has(code))
|
|
41
|
+
return false;
|
|
42
|
+
if (opts.status === 402)
|
|
43
|
+
return true;
|
|
44
|
+
if (code === "BATCH_QUOTA_EXCEEDED")
|
|
45
|
+
return true;
|
|
46
|
+
const hay = `${opts.message ?? ""} ${opts.body ?? ""} ${code ?? ""}`;
|
|
47
|
+
if (/\bHTTP\s*402\b/i.test(hay) || /\berror\s+402\b/i.test(hay))
|
|
48
|
+
return true;
|
|
49
|
+
return /\b(no credit|credits?\s+(exhausted|exceeded|available)|quota exceeded|subscription has no credit)\b/i.test(hay);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* If the local agent account is still unclaimed and this looks like a quota/plan
|
|
53
|
+
* failure, append the claim URL so agents can nudge the user.
|
|
54
|
+
*/
|
|
55
|
+
export function appendClaimHint(text, opts = {}) {
|
|
56
|
+
const probe = {
|
|
57
|
+
status: opts.status,
|
|
58
|
+
body: opts.body ?? text,
|
|
59
|
+
code: opts.code,
|
|
60
|
+
message: opts.message ?? text,
|
|
61
|
+
};
|
|
62
|
+
if (!isQuotaOrPlanError(probe))
|
|
63
|
+
return text;
|
|
64
|
+
const acct = readAccount();
|
|
65
|
+
if (!acct?.unclaimed || !acct.claimUrl)
|
|
66
|
+
return text;
|
|
67
|
+
const hint = `${CLAIM_NUDGE}${acct.claimUrl}`;
|
|
68
|
+
if (text.includes(acct.claimUrl) || text.includes(CLAIM_NUDGE.trim()))
|
|
69
|
+
return text;
|
|
70
|
+
return `${text}\n\n${hint}`;
|
|
71
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
export declare const ENV_KEY = "ZENROWS_API_KEY";
|
|
2
|
+
export declare const AUTO_SIGNUP_ENV = "ZENROWS_AUTO_SIGNUP";
|
|
3
|
+
export declare const SIGNUP_URL_ENV = "ZENROWS_AGENT_SIGNUP_URL";
|
|
4
|
+
export declare const DISCOVERY_URL_ENV = "ZENROWS_DISCOVERY_URL";
|
|
5
|
+
export declare const AGENT_SIGNUP_API_URL = "https://app.zenrows.com/api/agent/signup";
|
|
6
|
+
export declare const WELL_KNOWN_PROTECTED_RESOURCE = "/.well-known/oauth-protected-resource";
|
|
7
|
+
export interface AgentAccount {
|
|
8
|
+
accountId: string;
|
|
9
|
+
unclaimed: boolean;
|
|
10
|
+
claimUrl: string;
|
|
11
|
+
createdAt: string;
|
|
12
|
+
}
|
|
13
|
+
export interface SignupResponse {
|
|
14
|
+
apiKey: string;
|
|
15
|
+
accountId: string;
|
|
16
|
+
claimUrl: string;
|
|
17
|
+
}
|
|
18
|
+
export declare class AuthError extends Error {
|
|
19
|
+
code: string;
|
|
20
|
+
constructor(code: string, message: string);
|
|
21
|
+
}
|
|
22
|
+
/** Override home for tests / custom installs (absolute path to the `.zenrows` dir parent, or the dir itself if it ends with `.zenrows`). */
|
|
23
|
+
export declare const ZENROWS_HOME_ENV = "ZENROWS_HOME";
|
|
24
|
+
/** Test-only: clear discovery cache between cases. */
|
|
25
|
+
export declare function _resetDiscoveryCache(): void;
|
|
26
|
+
export declare function getZenrowsDir(): string;
|
|
27
|
+
export declare function readStoredApiKey(): string | undefined;
|
|
28
|
+
export declare function readAccount(): AgentAccount | null;
|
|
29
|
+
export declare function saveApiKey(apiKey: string): void;
|
|
30
|
+
export declare function writeAccount(acct: AgentAccount): void;
|
|
31
|
+
/** Resolve key: env → ~/.zenrows/secrets.json. Does not signup. */
|
|
32
|
+
export declare function resolveApiKey(): {
|
|
33
|
+
key?: string;
|
|
34
|
+
source: "env" | "secrets-file" | "none";
|
|
35
|
+
};
|
|
36
|
+
export declare function autoSignupEnabled(): boolean;
|
|
37
|
+
export declare function discoverSignupUrl(opts?: {
|
|
38
|
+
fetchImpl?: typeof fetch;
|
|
39
|
+
}): Promise<string | null>;
|
|
40
|
+
export declare function signupCandidates(opts?: {
|
|
41
|
+
fetchImpl?: typeof fetch;
|
|
42
|
+
}): Promise<string[]>;
|
|
43
|
+
export declare function signupAgent(opts?: {
|
|
44
|
+
url?: string;
|
|
45
|
+
fetchImpl?: typeof fetch;
|
|
46
|
+
userAgent?: string;
|
|
47
|
+
}): Promise<SignupResponse>;
|
|
48
|
+
/**
|
|
49
|
+
* Ensure an API key is available for stdio.
|
|
50
|
+
* Returns the key and optional claim metadata when a new account was provisioned.
|
|
51
|
+
*/
|
|
52
|
+
export declare function ensureApiKey(opts?: {
|
|
53
|
+
fetchImpl?: typeof fetch;
|
|
54
|
+
userAgent?: string;
|
|
55
|
+
onProvision?: (a: AgentAccount) => void;
|
|
56
|
+
}): Promise<{
|
|
57
|
+
apiKey: string;
|
|
58
|
+
provisioned?: AgentAccount;
|
|
59
|
+
}>;
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve-or-provision the Zenrows API key for stdio MCP.
|
|
3
|
+
*
|
|
4
|
+
* Persistence lives under ~/.zenrows/ (secrets.json + account.json, mode 0600).
|
|
5
|
+
* Remote HTTP transport must NOT call this — Bearer/OAuth only.
|
|
6
|
+
*/
|
|
7
|
+
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
8
|
+
import { homedir } from "node:os";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
export const ENV_KEY = "ZENROWS_API_KEY";
|
|
11
|
+
export const AUTO_SIGNUP_ENV = "ZENROWS_AUTO_SIGNUP";
|
|
12
|
+
export const SIGNUP_URL_ENV = "ZENROWS_AGENT_SIGNUP_URL";
|
|
13
|
+
export const DISCOVERY_URL_ENV = "ZENROWS_DISCOVERY_URL";
|
|
14
|
+
export const AGENT_SIGNUP_API_URL = "https://app.zenrows.com/api/agent/signup";
|
|
15
|
+
export const WELL_KNOWN_PROTECTED_RESOURCE = "/.well-known/oauth-protected-resource";
|
|
16
|
+
export class AuthError extends Error {
|
|
17
|
+
code;
|
|
18
|
+
constructor(code, message) {
|
|
19
|
+
super(message);
|
|
20
|
+
this.name = "AuthError";
|
|
21
|
+
this.code = code;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
/** Override home for tests / custom installs (absolute path to the `.zenrows` dir parent, or the dir itself if it ends with `.zenrows`). */
|
|
25
|
+
export const ZENROWS_HOME_ENV = "ZENROWS_HOME";
|
|
26
|
+
function zenrowsDir() {
|
|
27
|
+
const override = process.env[ZENROWS_HOME_ENV]?.trim();
|
|
28
|
+
if (override) {
|
|
29
|
+
return override.endsWith(".zenrows") ? override : join(override, ".zenrows");
|
|
30
|
+
}
|
|
31
|
+
return join(homedir(), ".zenrows");
|
|
32
|
+
}
|
|
33
|
+
/** Test-only: clear discovery cache between cases. */
|
|
34
|
+
export function _resetDiscoveryCache() {
|
|
35
|
+
discoveredSignupUrl = undefined;
|
|
36
|
+
}
|
|
37
|
+
export function getZenrowsDir() {
|
|
38
|
+
return zenrowsDir();
|
|
39
|
+
}
|
|
40
|
+
function secretsPath() {
|
|
41
|
+
return join(zenrowsDir(), "secrets.json");
|
|
42
|
+
}
|
|
43
|
+
function accountPath() {
|
|
44
|
+
return join(zenrowsDir(), "account.json");
|
|
45
|
+
}
|
|
46
|
+
function ensureDir() {
|
|
47
|
+
const dir = zenrowsDir();
|
|
48
|
+
if (!existsSync(dir))
|
|
49
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
50
|
+
}
|
|
51
|
+
function readJsonFile(file) {
|
|
52
|
+
if (!existsSync(file))
|
|
53
|
+
return null;
|
|
54
|
+
try {
|
|
55
|
+
return JSON.parse(readFileSync(file, "utf8"));
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
function writeJsonSecure(file, data) {
|
|
62
|
+
ensureDir();
|
|
63
|
+
writeFileSync(file, JSON.stringify(data, null, 2) + "\n", { mode: 0o600 });
|
|
64
|
+
try {
|
|
65
|
+
chmodSync(file, 0o600);
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
// best-effort on platforms without POSIX permissions
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export function readStoredApiKey() {
|
|
72
|
+
const stored = readJsonFile(secretsPath());
|
|
73
|
+
const key = stored?.apiKey?.trim();
|
|
74
|
+
return key || undefined;
|
|
75
|
+
}
|
|
76
|
+
export function readAccount() {
|
|
77
|
+
return readJsonFile(accountPath());
|
|
78
|
+
}
|
|
79
|
+
export function saveApiKey(apiKey) {
|
|
80
|
+
writeJsonSecure(secretsPath(), { apiKey: apiKey.trim() });
|
|
81
|
+
}
|
|
82
|
+
export function writeAccount(acct) {
|
|
83
|
+
writeJsonSecure(accountPath(), acct);
|
|
84
|
+
}
|
|
85
|
+
/** Resolve key: env → ~/.zenrows/secrets.json. Does not signup. */
|
|
86
|
+
export function resolveApiKey() {
|
|
87
|
+
const env = process.env[ENV_KEY]?.trim();
|
|
88
|
+
if (env)
|
|
89
|
+
return { key: env, source: "env" };
|
|
90
|
+
const stored = readStoredApiKey();
|
|
91
|
+
if (stored)
|
|
92
|
+
return { key: stored, source: "secrets-file" };
|
|
93
|
+
return { source: "none" };
|
|
94
|
+
}
|
|
95
|
+
export function autoSignupEnabled() {
|
|
96
|
+
return process.env[AUTO_SIGNUP_ENV] !== "false";
|
|
97
|
+
}
|
|
98
|
+
let discoveredSignupUrl;
|
|
99
|
+
export async function discoverSignupUrl(opts = {}) {
|
|
100
|
+
try {
|
|
101
|
+
const base = process.env[DISCOVERY_URL_ENV]?.trim() || new URL(AGENT_SIGNUP_API_URL).origin;
|
|
102
|
+
const url = base.replace(/\/$/, "") + WELL_KNOWN_PROTECTED_RESOURCE;
|
|
103
|
+
const doFetch = opts.fetchImpl ?? fetch;
|
|
104
|
+
const res = await doFetch(url, {
|
|
105
|
+
method: "GET",
|
|
106
|
+
headers: { Accept: "application/json", "User-Agent": "zenrows/mcp" },
|
|
107
|
+
});
|
|
108
|
+
if (!res.ok)
|
|
109
|
+
return null;
|
|
110
|
+
const json = (await res.json());
|
|
111
|
+
const endpoint = json?.agent_auth?.signup_endpoint;
|
|
112
|
+
if (typeof endpoint === "string" && endpoint.trim())
|
|
113
|
+
return endpoint.trim();
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
export async function signupCandidates(opts = {}) {
|
|
121
|
+
const fromEnv = process.env[SIGNUP_URL_ENV];
|
|
122
|
+
if (fromEnv && fromEnv.trim())
|
|
123
|
+
return [fromEnv.trim()];
|
|
124
|
+
if (discoveredSignupUrl === undefined) {
|
|
125
|
+
discoveredSignupUrl = await discoverSignupUrl(opts);
|
|
126
|
+
}
|
|
127
|
+
const urls = [];
|
|
128
|
+
if (discoveredSignupUrl && discoveredSignupUrl !== AGENT_SIGNUP_API_URL) {
|
|
129
|
+
urls.push(discoveredSignupUrl);
|
|
130
|
+
}
|
|
131
|
+
urls.push(AGENT_SIGNUP_API_URL);
|
|
132
|
+
return urls;
|
|
133
|
+
}
|
|
134
|
+
export async function signupAgent(opts = {}) {
|
|
135
|
+
const urls = opts.url ? [opts.url] : await signupCandidates({ fetchImpl: opts.fetchImpl });
|
|
136
|
+
const doFetch = opts.fetchImpl ?? fetch;
|
|
137
|
+
const headers = {
|
|
138
|
+
"content-type": "application/json",
|
|
139
|
+
"User-Agent": opts.userAgent ?? "zenrows/mcp",
|
|
140
|
+
"X-ZR-Source": "mcp",
|
|
141
|
+
};
|
|
142
|
+
let lastMessage = "No signup endpoint was reachable.";
|
|
143
|
+
for (const url of urls) {
|
|
144
|
+
let res;
|
|
145
|
+
try {
|
|
146
|
+
res = await doFetch(url, { method: "POST", headers });
|
|
147
|
+
}
|
|
148
|
+
catch (err) {
|
|
149
|
+
lastMessage = err instanceof Error ? err.message : String(err);
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
if (res.status === 201)
|
|
153
|
+
return (await res.json());
|
|
154
|
+
const body = await res.text();
|
|
155
|
+
if (res.status === 429) {
|
|
156
|
+
throw new AuthError("SIGNUP_RATE_LIMITED", "Zenrows blocked auto-signup: too many new accounts from this network. Wait and retry, or set ZENROWS_API_KEY.");
|
|
157
|
+
}
|
|
158
|
+
lastMessage = `HTTP ${res.status}: ${body.slice(0, 240)}`;
|
|
159
|
+
}
|
|
160
|
+
throw new AuthError("SIGNUP_FAILED", `Automatic account provisioning failed. ${lastMessage}`);
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Ensure an API key is available for stdio.
|
|
164
|
+
* Returns the key and optional claim metadata when a new account was provisioned.
|
|
165
|
+
*/
|
|
166
|
+
export async function ensureApiKey(opts = {}) {
|
|
167
|
+
const existing = resolveApiKey();
|
|
168
|
+
if (existing.key)
|
|
169
|
+
return { apiKey: existing.key };
|
|
170
|
+
if (!autoSignupEnabled()) {
|
|
171
|
+
throw new AuthError("AUTH_MISSING", "ZENROWS_API_KEY is required (auto-signup disabled via ZENROWS_AUTO_SIGNUP=false).");
|
|
172
|
+
}
|
|
173
|
+
const res = await signupAgent({ fetchImpl: opts.fetchImpl, userAgent: opts.userAgent });
|
|
174
|
+
saveApiKey(res.apiKey);
|
|
175
|
+
const account = {
|
|
176
|
+
accountId: res.accountId,
|
|
177
|
+
unclaimed: true,
|
|
178
|
+
claimUrl: res.claimUrl,
|
|
179
|
+
createdAt: new Date().toISOString(),
|
|
180
|
+
};
|
|
181
|
+
writeAccount(account);
|
|
182
|
+
opts.onProvision?.(account);
|
|
183
|
+
return { apiKey: res.apiKey, provisioned: account };
|
|
184
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client for the Zenrows Batch API (https://async.api.zenrows.com/v1).
|
|
3
|
+
* Auth via X-API-Key header. Errors are application/problem+json (RFC 7807).
|
|
4
|
+
*/
|
|
5
|
+
export declare const DEFAULT_BATCH_API_BASE = "https://async.api.zenrows.com/v1";
|
|
6
|
+
export declare const BATCH_API_BASE_ENV = "ZENROWS_BATCH_API_BASE";
|
|
7
|
+
export declare function batchBase(): string;
|
|
8
|
+
export interface JobStats {
|
|
9
|
+
total: number;
|
|
10
|
+
completed: number;
|
|
11
|
+
successful: number;
|
|
12
|
+
failed: number;
|
|
13
|
+
}
|
|
14
|
+
export interface JobRun {
|
|
15
|
+
status: string;
|
|
16
|
+
stats: JobStats;
|
|
17
|
+
run_id?: string;
|
|
18
|
+
[k: string]: unknown;
|
|
19
|
+
}
|
|
20
|
+
export interface Job {
|
|
21
|
+
job_id: string;
|
|
22
|
+
latest_run: JobRun;
|
|
23
|
+
[k: string]: unknown;
|
|
24
|
+
}
|
|
25
|
+
export interface ResultRow {
|
|
26
|
+
external_id?: string;
|
|
27
|
+
task_id: string;
|
|
28
|
+
status?: string;
|
|
29
|
+
result_url?: string;
|
|
30
|
+
[k: string]: unknown;
|
|
31
|
+
}
|
|
32
|
+
export interface ResultsPage {
|
|
33
|
+
results: ResultRow[];
|
|
34
|
+
next_cursor: string | null;
|
|
35
|
+
}
|
|
36
|
+
export interface ProblemJson {
|
|
37
|
+
type?: string;
|
|
38
|
+
title?: string;
|
|
39
|
+
status?: number;
|
|
40
|
+
detail?: string;
|
|
41
|
+
code?: string;
|
|
42
|
+
invalid_tasks?: Array<{
|
|
43
|
+
index: number;
|
|
44
|
+
reason: string;
|
|
45
|
+
}>;
|
|
46
|
+
}
|
|
47
|
+
export declare const TERMINAL_STATUSES: ReadonlySet<string>;
|
|
48
|
+
export declare class BatchError extends Error {
|
|
49
|
+
code: string;
|
|
50
|
+
status?: number;
|
|
51
|
+
detail?: string;
|
|
52
|
+
constructor(opts: {
|
|
53
|
+
code: string;
|
|
54
|
+
message: string;
|
|
55
|
+
status?: number;
|
|
56
|
+
detail?: string;
|
|
57
|
+
});
|
|
58
|
+
toJSON(): {
|
|
59
|
+
code: string;
|
|
60
|
+
message: string;
|
|
61
|
+
status?: number;
|
|
62
|
+
detail?: string;
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
interface RequestOpts {
|
|
66
|
+
apiKey: string;
|
|
67
|
+
body?: unknown;
|
|
68
|
+
query?: Record<string, string | undefined>;
|
|
69
|
+
timeoutMs?: number;
|
|
70
|
+
userAgent?: string;
|
|
71
|
+
fetchImpl?: typeof fetch;
|
|
72
|
+
}
|
|
73
|
+
export declare function batchRequest<T>(method: string, path: string, opts: RequestOpts): Promise<T>;
|
|
74
|
+
interface CallOpts {
|
|
75
|
+
apiKey: string;
|
|
76
|
+
timeoutMs?: number;
|
|
77
|
+
userAgent?: string;
|
|
78
|
+
fetchImpl?: typeof fetch;
|
|
79
|
+
}
|
|
80
|
+
export declare function createJob(body: unknown, opts: CallOpts): Promise<Job>;
|
|
81
|
+
export declare function getJob(id: string, opts: CallOpts): Promise<Job>;
|
|
82
|
+
export declare function stopJob(id: string, opts: CallOpts): Promise<Job>;
|
|
83
|
+
export declare function listResults(id: string, opts: CallOpts & {
|
|
84
|
+
status?: "successful" | "failed" | "all";
|
|
85
|
+
}): Promise<ResultRow[]>;
|
|
86
|
+
export declare function waitForJob(id: string, opts: CallOpts & {
|
|
87
|
+
pollTimeoutMs?: number;
|
|
88
|
+
}): Promise<Job>;
|
|
89
|
+
export {};
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client for the Zenrows Batch API (https://async.api.zenrows.com/v1).
|
|
3
|
+
* Auth via X-API-Key header. Errors are application/problem+json (RFC 7807).
|
|
4
|
+
*/
|
|
5
|
+
export const DEFAULT_BATCH_API_BASE = "https://async.api.zenrows.com/v1";
|
|
6
|
+
export const BATCH_API_BASE_ENV = "ZENROWS_BATCH_API_BASE";
|
|
7
|
+
export function batchBase() {
|
|
8
|
+
const env = process.env[BATCH_API_BASE_ENV];
|
|
9
|
+
const base = env && env.trim() ? env.trim() : DEFAULT_BATCH_API_BASE;
|
|
10
|
+
return base.replace(/\/+$/, "");
|
|
11
|
+
}
|
|
12
|
+
export const TERMINAL_STATUSES = new Set(["completed", "stopped", "deleted"]);
|
|
13
|
+
export class BatchError extends Error {
|
|
14
|
+
code;
|
|
15
|
+
status;
|
|
16
|
+
detail;
|
|
17
|
+
constructor(opts) {
|
|
18
|
+
super(opts.message);
|
|
19
|
+
this.name = "BatchError";
|
|
20
|
+
this.code = opts.code;
|
|
21
|
+
this.status = opts.status;
|
|
22
|
+
this.detail = opts.detail;
|
|
23
|
+
}
|
|
24
|
+
toJSON() {
|
|
25
|
+
return { code: this.code, message: this.message, status: this.status, detail: this.detail };
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
export async function batchRequest(method, path, opts) {
|
|
29
|
+
const url = new URL(batchBase() + path);
|
|
30
|
+
for (const [k, v] of Object.entries(opts.query ?? {})) {
|
|
31
|
+
if (v !== undefined && v !== null)
|
|
32
|
+
url.searchParams.set(k, v);
|
|
33
|
+
}
|
|
34
|
+
const controller = new AbortController();
|
|
35
|
+
const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000);
|
|
36
|
+
const headers = {
|
|
37
|
+
"X-API-Key": opts.apiKey,
|
|
38
|
+
Accept: "application/json",
|
|
39
|
+
"User-Agent": opts.userAgent ?? "zenrows/mcp",
|
|
40
|
+
};
|
|
41
|
+
if (opts.body !== undefined)
|
|
42
|
+
headers["Content-Type"] = "application/json";
|
|
43
|
+
const doFetch = opts.fetchImpl ?? fetch;
|
|
44
|
+
let res;
|
|
45
|
+
try {
|
|
46
|
+
res = await doFetch(url.toString(), {
|
|
47
|
+
method,
|
|
48
|
+
headers,
|
|
49
|
+
body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined,
|
|
50
|
+
signal: controller.signal,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
clearTimeout(timeout);
|
|
55
|
+
throw new BatchError({
|
|
56
|
+
code: "BACKEND_UNAVAILABLE",
|
|
57
|
+
message: `Could not reach the Zenrows Batch API: ${err instanceof Error ? err.message : String(err)}`,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
clearTimeout(timeout);
|
|
61
|
+
const text = await res.text();
|
|
62
|
+
if (res.status < 200 || res.status >= 300) {
|
|
63
|
+
throw problemToError(res.status, text, method, path);
|
|
64
|
+
}
|
|
65
|
+
if (!text)
|
|
66
|
+
return null;
|
|
67
|
+
try {
|
|
68
|
+
return JSON.parse(text);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
throw new BatchError({
|
|
72
|
+
code: "BATCH_FAILED",
|
|
73
|
+
message: "The Batch API response was not valid JSON.",
|
|
74
|
+
detail: text.slice(0, 240),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
function problemToError(status, body, method, path) {
|
|
79
|
+
let problem = {};
|
|
80
|
+
try {
|
|
81
|
+
problem = JSON.parse(body);
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
// non-JSON — fall through
|
|
85
|
+
}
|
|
86
|
+
const serverCode = problem.code ?? "";
|
|
87
|
+
const detail = problem.detail || problem.title || body.slice(0, 240) || `HTTP ${status}`;
|
|
88
|
+
const cause = `HTTP ${status}${serverCode ? ` (${serverCode})` : ""} for ${method} ${path}: ${detail}`;
|
|
89
|
+
if (status === 403) {
|
|
90
|
+
return new BatchError({
|
|
91
|
+
code: "BATCH_ACCESS_DENIED",
|
|
92
|
+
message: "The Batch API rejected this request (access denied). The Batch API is in beta and this account does not have beta access. Request access from Zenrows, or fan out with scrape/extract per URL.",
|
|
93
|
+
status,
|
|
94
|
+
detail: cause,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
if (status === 401) {
|
|
98
|
+
return new BatchError({
|
|
99
|
+
code: "AUTH_INVALID",
|
|
100
|
+
message: "Zenrows rejected the API key for the Batch API.",
|
|
101
|
+
status,
|
|
102
|
+
detail: cause,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
if (status === 404) {
|
|
106
|
+
return new BatchError({
|
|
107
|
+
code: "BATCH_NOT_FOUND",
|
|
108
|
+
message: "Batch job, run, or task not found.",
|
|
109
|
+
status,
|
|
110
|
+
detail: cause,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
if (status === 429) {
|
|
114
|
+
return new BatchError({
|
|
115
|
+
code: "BATCH_QUOTA_EXCEEDED",
|
|
116
|
+
message: "Batch quota exceeded (e.g. max concurrent active jobs). Wait for an in-flight job to finish or cancel one, then retry.",
|
|
117
|
+
status,
|
|
118
|
+
detail: cause,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
if (status === 402) {
|
|
122
|
+
return new BatchError({
|
|
123
|
+
code: "BATCH_QUOTA_EXCEEDED",
|
|
124
|
+
message: "Subscription has no credit available for the Batch API.",
|
|
125
|
+
status,
|
|
126
|
+
detail: cause,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
const invalid = problem.invalid_tasks?.length
|
|
130
|
+
? ` invalid_tasks: ${problem.invalid_tasks
|
|
131
|
+
.slice(0, 10)
|
|
132
|
+
.map((t) => `#${t.index}: ${t.reason}`)
|
|
133
|
+
.join("; ")}`
|
|
134
|
+
: "";
|
|
135
|
+
return new BatchError({
|
|
136
|
+
code: "BATCH_FAILED",
|
|
137
|
+
message: `Batch request failed (HTTP ${status}).${invalid}`,
|
|
138
|
+
status,
|
|
139
|
+
detail: cause,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
export function createJob(body, opts) {
|
|
143
|
+
return batchRequest("POST", "/jobs", { ...opts, body });
|
|
144
|
+
}
|
|
145
|
+
export function getJob(id, opts) {
|
|
146
|
+
return batchRequest("GET", `/jobs/${encodeURIComponent(id)}`, opts);
|
|
147
|
+
}
|
|
148
|
+
export function stopJob(id, opts) {
|
|
149
|
+
return batchRequest("POST", `/jobs/${encodeURIComponent(id)}/stop`, opts);
|
|
150
|
+
}
|
|
151
|
+
export async function listResults(id, opts) {
|
|
152
|
+
const all = [];
|
|
153
|
+
let cursor;
|
|
154
|
+
do {
|
|
155
|
+
const page = await batchRequest("GET", `/jobs/${encodeURIComponent(id)}/results`, {
|
|
156
|
+
...opts,
|
|
157
|
+
query: { status: opts.status, cursor },
|
|
158
|
+
});
|
|
159
|
+
if (page?.results)
|
|
160
|
+
all.push(...page.results);
|
|
161
|
+
cursor = page?.next_cursor ?? undefined;
|
|
162
|
+
} while (cursor);
|
|
163
|
+
return all;
|
|
164
|
+
}
|
|
165
|
+
export async function waitForJob(id, opts) {
|
|
166
|
+
const deadline = Date.now() + (opts.pollTimeoutMs ?? opts.timeoutMs ?? 600_000);
|
|
167
|
+
let delay = 2000;
|
|
168
|
+
for (;;) {
|
|
169
|
+
const job = await getJob(id, opts);
|
|
170
|
+
if (job.latest_run && TERMINAL_STATUSES.has(job.latest_run.status))
|
|
171
|
+
return job;
|
|
172
|
+
if (Date.now() > deadline) {
|
|
173
|
+
throw new BatchError({
|
|
174
|
+
code: "BATCH_FAILED",
|
|
175
|
+
message: `Timed out waiting for batch job ${id} to finish.`,
|
|
176
|
+
detail: `The run did not reach a terminal state within ${Math.round((opts.pollTimeoutMs ?? opts.timeoutMs ?? 600_000) / 1000)}s.`,
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
180
|
+
delay = Math.min(delay * 1.5, 15_000);
|
|
181
|
+
}
|
|
182
|
+
}
|