@zenrows/cli 1.6.2 → 1.8.0

package/dist/src/cli/commands/browser.js

@@ -23,6 +23,7 @@ import { newRunId, writeRun } from "../../core/artifacts.js";
 import { ToolkitError } from "../../core/errors.js";
 import { parse, asString, asNumber } from "../command.js";
 import { printError, writeOut } from "../output.js";
+import { refreshClaimAdvice } from "../../core/claim-link.js";
 import { createSession, sessionCall, closeSession, decodeBinary, connectUrl, normalizeSelectValue, } from "../../core/browser-api.js";
 const VERBS = {
     navigate: { verb: "navigate", opts: { url: { type: "string" } }, required: ["url"], body: (v) => ({ url: asString(v.url) }) },
@@ -155,7 +156,9 @@ export const browser = {
                     });
             }
         }
-        catch (err) {
+        catch (caught) {
+            // A quota error's claim advice is built from local state; check it is still current.
+            const err = await refreshClaimAdvice(caught);
             printError(err, ctx.json);
             return err instanceof ToolkitError && err.code === "POLICY_BROWSER_DISABLED" ? 2 : 1;
         }

package/dist/src/cli/commands/crawl.js

@@ -18,6 +18,7 @@ import { asNumber, asString, parse } from "../command.js";
 import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.js";
 import { readAccount } from "../../core/agent-account.js";
 import { printError, writeOut, writeOutStream } from "../output.js";
+import { refreshClaimAdvice, storedClaimHint } from "../../core/claim-link.js";
 /** The API's default for `max_pages` (and `max_items`) when the flag is unset. */
 const DEFAULT_MAX_PAGES = 100;
 /** Largest `depth` / `max_items` / `max_pages` the API takes. */
@@ -33,7 +34,7 @@ export const crawl = {
         "                                    leave it out for no depth limit (the caps below stop the crawl)",
         "    --max-items <n>                 stop after keeping n URLs (API default 100)",
         "    --max-pages <n>                 stop after fetching n pages (API default 100; bounds cost);",
-        "                                    the local policy max_pages_per_run caps it (default 1000)",
+        "                                    the local policy max_pages_per_run caps it (default 10000)",
         "    --include-patterns <p>          keep only URLs containing this substring (repeatable)",
         "    --exclude-patterns <p>          drop URLs containing this substring (repeatable)",
         "    --output-format <html|markdown> also fetch each kept URL's page, as HTML or Markdown",
@@ -95,7 +96,9 @@ export const crawl = {
                     });
             }
         }
-        catch (err) {
+        catch (caught) {
+            // A quota error's claim advice is built from local state; check it is still current.
+            const err = await refreshClaimAdvice(caught);
             printError(err, ctx.json);
             return 1;
         }
@@ -462,7 +465,7 @@ export function crawlFailure(c) {
     }
     if (code === "insufficient_credits") {
         const acct = readAccount();
-        const quota = quotaExhausted(where, acct?.unclaimed ? acct.claimUrl : undefined, { status: null, detail }).toJSON();
+        const quota = quotaExhausted(where, storedClaimHint(acct), { status: null, detail }).toJSON();
         return new ToolkitError({ ...quota, ...ids, code: "CRAWL_INSUFFICIENT_CREDITS", suggested_commands: [...(quota.suggested_commands ?? []), getCmd] });
     }
     const why = [code, c.error?.detail].filter(Boolean).join(": ");

package/dist/src/cli/commands/extract.js

@@ -17,6 +17,7 @@ import { runExtract } from "../../adapters/extract.js";
 import { formatRequestCost } from "../../core/http.js";
 import { parse, asString, asNumber } from "../command.js";
 import { printError, writeOut } from "../output.js";
+import { refreshClaimAdvice } from "../../core/claim-link.js";
 export const extract = {
     name: "extract",
     summary: "Turn a protected page into structured data (Extract / Autoparse / CSS / Markdown).",
@@ -175,7 +176,9 @@ export const extract = {
             await maybeNudgeClaim(config, {});
             return 0;
         }
-        catch (err) {
+        catch (caught) {
+            // A quota error's claim advice is built from local state; check it is still current.
+            const err = await refreshClaimAdvice(caught);
             writeRun({
                 runId,
                 command: "zenrows extract",

package/dist/src/cli/commands/fetch.js

@@ -16,6 +16,7 @@ import { runFetch } from "../../adapters/protected-fetch.js";
 import { formatRequestCost } from "../../core/http.js";
 import { parse, asString, asNumber } from "../command.js";
 import { printError, writeOut } from "../output.js";
+import { refreshClaimAdvice } from "../../core/claim-link.js";
 export const fetch_ = {
     name: "fetch",
     summary: "Retrieve a protected page (Fetch).",
@@ -152,7 +153,9 @@ export const fetch_ = {
             await maybeNudgeClaim(config, {});
             return 0;
         }
-        catch (err) {
+        catch (caught) {
+            // A quota error's claim advice is built from local state; check it is still current.
+            const err = await refreshClaimAdvice(caught);
             writeRun({
                 runId,
                 command: "zenrows fetch",

package/dist/src/cli/commands/signup.js

@@ -21,12 +21,13 @@ export const signup = {
         if (values.agent) {
             const account = readAccount();
             if (account) {
-                log.info("An agent account already exists. Claim it: " + account.claimUrl);
+                log.info("An agent account already exists. Claim it with: zenrows account claim");
                 return 0;
             }
             const res = await signupAgent();
             saveApiKey(res.apiKey);
-            writeAccount({ accountId: res.accountId, unclaimed: true, claimUrl: res.claimUrl, createdAt: new Date().toISOString() });
+            const now = new Date().toISOString();
+            writeAccount({ accountId: res.accountId, unclaimed: true, claimUrl: res.claimUrl, createdAt: now, claimUrlIssuedAt: now });
             log.success("Created a Zenrows Free plan account.");
             log.dim(`Claim it anytime: ${res.claimUrl}`);
             if (ctx.json)

package/dist/src/cli/index.js

@@ -8,6 +8,7 @@
  */
 import { CLI_VERSION } from "../core/config.js";
 import { ToolkitError } from "../core/errors.js";
+import { refreshClaimAdvice } from "../core/claim-link.js";
 import { log, ANSI, c } from "../core/logger.js";
 import { makeAssetCommand } from "./asset-command.js";
 import { printError } from "./output.js";
@@ -82,7 +83,7 @@ export async function main(rawArgv) {
         return await cmd.run(argv, ctx);
     }
     catch (err) {
-        printError(err, json);
+        printError(await refreshClaimAdvice(err), json);
         // Any error exits 1 — the exit code doesn't encode error kind. Machine
         // consumers read the precise `error.code` from the --json output instead.
         return 1;

package/dist/src/core/agent-account.js

@@ -5,13 +5,13 @@
  * It never holds the API key (that lives in secrets.json, 0600) — only the
  * accountId, the human-facing claim URL, and claim metadata.
  */
-import { chmodSync, existsSync, unlinkSync } from "node:fs";
+import { existsSync, unlinkSync } from "node:fs";
 import { attributionEnabled, getOrCreateTelemetryId, loadConfig, CLI_USER_AGENT, CLI_VERSION } from "./config.js";
 import { agentClientHeader } from "./agent-client.js";
 import { ToolkitError } from "./errors.js";
 import { AGENT_SIGNUP_API_URL, WELL_KNOWN_PROTECTED_RESOURCE } from "./open-url.js";
 import { detectClient } from "./provenance.js";
-import { findWorkspace, readJson, workspacePaths, writeJson } from "./workspace.js";
+import { findWorkspace, readJson, workspacePaths, writePrivateJson } from "./workspace.js";
 /** Env var to override the signup endpoint (highest priority; for local testing). */
 export const SIGNUP_URL_ENV = "ZENROWS_AGENT_SIGNUP_URL";
 /** Env var to override the discovery base origin (for local testing). */
@@ -49,7 +49,9 @@ export async function discoverSignupUrl(projectRoot, opts = {}) {
             return null;
         const json = (await res.json());
         const endpoint = json?.agent_auth?.signup_endpoint;
-        if (typeof endpoint === "string" && endpoint.trim())
+        // The key is later sent to URLs derived from this one, so only a Zenrows
+        // https endpoint is accepted from a document fetched over the network.
+        if (typeof endpoint === "string" && isZenrowsHttpsUrl(endpoint.trim()))
             return endpoint.trim();
         return null;
     }
@@ -89,7 +91,7 @@ export async function signupCandidates(projectRoot, opts = {}) {
 export async function resolveSignupUrl(projectRoot, opts = {}) {
     return (await signupCandidates(projectRoot, opts))[0];
 }
-function accountPath(projectRoot) {
+export function accountPath(projectRoot) {
     const ws = projectRoot ? workspacePaths(projectRoot) : (findWorkspace() ?? workspacePaths());
     return ws.account;
 }
@@ -98,14 +100,8 @@ export function readAccount(projectRoot) {
 }
 export function writeAccount(acct, projectRoot) {
     const file = accountPath(projectRoot);
-    writeJson(file, acct);
-    try {
-        // account.json holds the single-use claim token — restrict to owner-only.
-        chmodSync(file, 0o600);
-    }
-    catch {
-        // best-effort on platforms without POSIX permissions
-    }
+    // account.json holds the single-use claim token: owner-only from creation.
+    writePrivateJson(file, acct);
 }
 /** Remove local agent account metadata (used by `zenrows logout`). */
 export function clearAccount(projectRoot) {
@@ -201,7 +197,108 @@ function signupError(status, body) {
  * `signupUrl` override applies here too (local/staging testing).
  */
 export async function resolveAccountUrl(projectRoot, opts = {}) {
-    return (await resolveSignupUrl(projectRoot, opts)).replace(/\/signup\/?$/, "/account");
+    return siblingOfSignup(await resolveSignupUrl(projectRoot, opts), "account");
+}
+/**
+ * `…/agent/signup` → `…/agent/<name>`. A signup URL that does not end in
+ * `/signup` gives no safe sibling, so the built-in endpoint's sibling is used
+ * rather than sending the key somewhere unexpected.
+ */
+export function siblingOfSignup(signupUrl, name) {
+    const re = /\/signup\/?$/;
+    // These two endpoints receive the API key. A signup URL from the environment
+    // or a project's config (which a cloned repository could ship) is only
+    // trusted when it is Zenrows or this machine; anything else gets the
+    // built-in endpoint.
+    const base = re.test(signupUrl) && isAgentKeyDestination(signupUrl) ? signupUrl : AGENT_SIGNUP_API_URL;
+    return base.replace(re, `/${name}`);
+}
+/**
+ * Where the CLI may send the API key for the agent account endpoints: a
+ * Zenrows https URL, or http(s) on loopback (local testing). No userinfo.
+ */
+export function isAgentKeyDestination(value) {
+    if (isZenrowsHttpsUrl(value))
+        return true;
+    try {
+        const u = new URL(value);
+        const host = u.hostname.toLowerCase();
+        return ((u.protocol === "https:" || u.protocol === "http:") &&
+            u.username === "" &&
+            u.password === "" &&
+            (host === "127.0.0.1" || host === "localhost" || host === "[::1]"));
+    }
+    catch {
+        return false;
+    }
+}
+/** True for an https URL on zenrows.com or a subdomain, with no userinfo. */
+export function isZenrowsHttpsUrl(value) {
+    if (typeof value !== "string")
+        return false;
+    try {
+        const u = new URL(value);
+        const host = u.hostname.toLowerCase();
+        return (u.protocol === "https:" &&
+            u.username === "" &&
+            u.password === "" &&
+            (host === "zenrows.com" || host.endsWith(".zenrows.com")));
+    }
+    catch {
+        return false;
+    }
+}
+/** The fresh-claim-link endpoint, derived from the signup URL like the status one. */
+export async function resolveClaimUrl(projectRoot, opts = {}) {
+    return siblingOfSignup(await resolveSignupUrl(projectRoot, opts), "claim");
+}
+/**
+ * Mint a fresh claim link for the account behind `apiKey`. Each call
+ * invalidates the previous link and the endpoint is rate-limited per key, so
+ * callers only ask when the link they hold is missing or near expiry.
+ * A 409 `already_claimed` means a person owns the account. Throws ToolkitError
+ * on any other failure.
+ */
+export async function requestClaimLink(apiKey, opts = {}) {
+    const url = opts.url ?? (await resolveClaimUrl(opts.projectRoot, { fetchImpl: opts.fetchImpl }));
+    const doFetch = opts.fetchImpl ?? fetch;
+    const res = await doFetch(url, {
+        method: "POST",
+        headers: { "X-API-Key": apiKey, Accept: "application/json", "User-Agent": CLI_USER_AGENT, ...agentClientHeader() },
+    });
+    const body = await res.text();
+    if (res.status === 200) {
+        let json = null;
+        try {
+            json = JSON.parse(body);
+        }
+        catch {
+            // not JSON: reported below
+        }
+        if (!json || typeof json.claimUrl !== "string") {
+            throw new ToolkitError({
+                code: "FETCH_FAILED",
+                message: "The claim-link endpoint answered HTTP 200 without a claim link (unexpected response body).",
+                likely_cause: body.slice(0, 240),
+                next_action: "Retry later.",
+            });
+        }
+        return {
+            status: "unclaimed",
+            claimUrl: json.claimUrl,
+            ...(typeof json.expiresAt === "string" ? { expiresAt: json.expiresAt } : {}),
+        };
+    }
+    if (res.status === 409 && /already_claimed/.test(body))
+        return { status: "claimed" };
+    if (res.status === 409 && /not_agent_account/.test(body))
+        return { status: "not_agent" };
+    throw new ToolkitError({
+        code: res.status === 401 || res.status === 403 ? "AUTH_INVALID" : "FETCH_FAILED",
+        message: `Claim link request failed (HTTP ${res.status}).`,
+        likely_cause: body.slice(0, 240),
+        next_action: res.status === 429 ? "Too many claim links requested for this key; retry in an hour." : "Retry later.",
+    });
 }
 /**
  * Ask the backend whether the account behind `apiKey` has been claimed
@@ -209,7 +306,7 @@ export async function resolveAccountUrl(projectRoot, opts = {}) {
  * `X-API-Key` header. Throws ToolkitError on failure.
  */
 export async function fetchAccountStatus(apiKey, opts = {}) {
-    const url = opts.url ?? (await resolveAccountUrl());
+    const url = opts.url ?? (await resolveAccountUrl(opts.projectRoot, { fetchImpl: opts.fetchImpl }));
     const doFetch = opts.fetchImpl ?? fetch;
     const res = await doFetch(url, {
         method: "GET",

package/dist/src/core/auth.js

@@ -4,11 +4,11 @@
  * Resolution order: explicit secrets file → ZENROWS_API_KEY env var.
  * The key is never printed in full and never written into run artifacts.
  */
-import { chmodSync, existsSync } from "node:fs";
+import { existsSync } from "node:fs";
 import { ToolkitError } from "./errors.js";
 import { registerSecret } from "./logger.js";
 import { mask } from "./redact.js";
-import { findWorkspace, readJson, workspacePaths, writeJson } from "./workspace.js";
+import { findWorkspace, readJson, workspacePaths, writePrivateJson } from "./workspace.js";
 export const ENV_KEY = "ZENROWS_API_KEY";
 function secretsPath(projectRoot) {
     const ws = projectRoot ? workspacePaths(projectRoot) : (findWorkspace() ?? workspacePaths());
@@ -49,20 +49,14 @@ export function saveApiKey(apiKey, projectRoot) {
         });
     }
     const file = secretsPath(projectRoot);
-    writeJson(file, { apiKey: trimmed });
-    try {
-        chmodSync(file, 0o600);
-    }
-    catch {
-        // best-effort on platforms without POSIX permissions
-    }
+    writePrivateJson(file, { apiKey: trimmed });
     registerSecret(trimmed);
 }
 export function clearApiKey(projectRoot) {
     const file = secretsPath(projectRoot);
     if (!existsSync(file))
         return false;
-    writeJson(file, {});
+    writePrivateJson(file, {});
     return true;
 }
 /** Throw AUTH_MISSING if no key is configured. */