@zenrows/cli 1.11.0 → 1.12.0

package/dist/src/cli/commands/crawl.js

@@ -13,7 +13,7 @@ import { ensureApiKey } from "../../core/ensure-key.js";
 import { assertUsable } from "../../core/capabilities.js";
 import { assertDomainAllowed, assertWithinLimits, loadPolicy } from "../../core/policy.js";
 import { newRunId, writeRun } from "../../core/artifacts.js";
-import { contentIdOf, createCrawl, downloadCrawl, getCrawl, getCrawlContent, isTerminal, listAllResults, listCrawls, stopCrawl, waitForCrawl, } from "../../core/crawl-api.js";
+import { contentIdOf, createCrawl, downloadCrawl, getCrawl, getCrawlContent, isTerminal, listAllResults, listCrawls, stopCrawl, waitForCrawl, CRAWL_WEBHOOK_EVENTS, WEBHOOK_METADATA_LIMITS, } from "../../core/crawl-api.js";
 import { asNumber, asString, parse } from "../command.js";
 import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "../../core/errors.js";
 import { readAccount } from "../../core/agent-account.js";
@@ -42,6 +42,13 @@ export const crawl = {
         "                                    fetch counted by --max-pages",
         "    --discovery links               how the crawl finds the URLs it follows; links (the",
         "                                    default) follows the links in each page's HTML",
+        "    --webhook <url>                 post the crawl's events to this https URL, signed with the",
+        "                                    account's webhook signing key (Settings → API Keys)",
+        "    --webhook-event <event>         with --webhook: send only this event (repeatable); one of",
+        "                                    crawl.started, crawl.completed, crawl.stopped, crawl.failed;",
+        "                                    leave it out for all four",
+        "    --webhook-metadata <key=value>  with --webhook: sent back with every event (repeatable,",
+        "                                    up to 20)",
         "    --follow                        wait until the crawl ends (Ctrl-C stops the wait, not the crawl)",
         "    --timeout <s>                   with --follow: stop waiting after s seconds (default 600)",
         "    --idempotency-key <key>         so a retried create starts no second crawl",
@@ -113,6 +120,9 @@ async function createCmd(rest, ctx) {
         "exclude-patterns": { type: "string", multiple: true },
         "output-format": { type: "string" },
         discovery: { type: "string", multiple: true },
+        webhook: { type: "string" },
+        "webhook-event": { type: "string", multiple: true },
+        "webhook-metadata": { type: "string", multiple: true },
         follow: { type: "boolean" },
         timeout: { type: "string" },
         "idempotency-key": { type: "string" },
@@ -171,6 +181,9 @@ async function createCmd(rest, ctx) {
         }
         params.discovery = ["links"];
     }
+    const webhook = webhookOf(values);
+    if (webhook)
+        params.webhook = webhook;
     const timeoutMs = timeoutFlag(values.timeout);
     const idempotencyKey = asString(values["idempotency-key"]);
     assertUsable("crawl");
@@ -531,6 +544,11 @@ function printCrawl(c, json, headline, extra = {}) {
     if (cov) {
         log.info(`  ${cov.items_found} item(s) found · ${cov.pages_fetched} page(s) fetched · ${cov.pages_failed} failed`);
     }
+    const hook = c.webhook;
+    if (hook?.url) {
+        const meta = hook.metadata && Object.keys(hook.metadata).length ? ` · metadata: ${JSON.stringify(hook.metadata)}` : "";
+        log.info(`  webhook: ${hook.url} · events: ${(hook.events ?? []).join(", ") || "all"}${meta}`);
+    }
     if (extra.note)
         log.dim(`  next: zenrows crawl wait ${c.crawl_id}`);
     else if (note)
@@ -587,6 +605,77 @@ function pageLimit(v, max) {
     }
     return n;
 }
+/**
+ * `--webhook`, `--webhook-event` and `--webhook-metadata key=value` as the create's
+ * `webhook`, or undefined without `--webhook`. Checks the shapes and the API's limits
+ * before any request; whether the URL's host resolves is the API's to judge.
+ */
+function webhookOf(values) {
+    const url = asString(values.webhook);
+    const events = patterns(values["webhook-event"]);
+    const pairs = patterns(values["webhook-metadata"]);
+    if (url === undefined) {
+        if (events.length || pairs.length) {
+            throw webhookUsage(`${events.length ? "--webhook-event" : "--webhook-metadata"} needs --webhook.`, "Pass --webhook <https-url> with it, or leave it out.");
+        }
+        return undefined;
+    }
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        parsed = undefined;
+    }
+    if (parsed?.protocol !== "https:") {
+        throw webhookUsage(`Invalid --webhook value '${url}': it must be an absolute https URL.`, "Pass e.g. --webhook https://example.com/hooks/zenrows.");
+    }
+    const webhook = { url };
+    if (events.length) {
+        const known = new Set(CRAWL_WEBHOOK_EVENTS);
+        const unknown = events.find((e) => !known.has(e));
+        if (unknown !== undefined) {
+            throw webhookUsage(`Invalid --webhook-event value '${unknown}'.`, `Pass one of ${CRAWL_WEBHOOK_EVENTS.join(", ")}, or leave it out for all four.`);
+        }
+        const repeated = events.find((e, i) => events.indexOf(e) !== i);
+        if (repeated !== undefined) {
+            throw webhookUsage(`--webhook-event '${repeated}' is given twice.`, "Name each event once.");
+        }
+        webhook.events = events;
+    }
+    if (pairs.length) {
+        const { keys, keyLength, valueBytes } = WEBHOOK_METADATA_LIMITS;
+        const metadata = {};
+        for (const p of pairs) {
+            const eq = p.indexOf("=");
+            if (eq <= 0)
+                throw webhookUsage(`Invalid --webhook-metadata '${p}': use key=value.`, "Pass e.g. --webhook-metadata site=books.");
+            const key = p.slice(0, eq);
+            const value = p.slice(eq + 1);
+            if ([...key].length > keyLength)
+                throw webhookUsage(`--webhook-metadata key '${key}' is longer than ${keyLength} characters.`, "Use a shorter key.");
+            if (key in metadata)
+                throw webhookUsage(`--webhook-metadata key '${key}' is given twice.`, "Name each key once.");
+            if (Buffer.byteLength(JSON.stringify(value)) > valueBytes) {
+                throw webhookUsage(`--webhook-metadata '${key}' is longer than ${valueBytes} bytes as JSON.`, "Use a shorter value.");
+            }
+            metadata[key] = value;
+        }
+        if (Object.keys(metadata).length > keys)
+            throw webhookUsage(`--webhook-metadata has more than ${keys} keys.`, `Pass at most ${keys}.`);
+        webhook.metadata = metadata;
+    }
+    return webhook;
+}
+function webhookUsage(message, next_action) {
+    return new ToolkitError({
+        code: "INVALID_USAGE",
+        message,
+        likely_cause: "The webhook flags do not match what Crawl accepts.",
+        next_action,
+        suggested_commands: ["zenrows crawl --help"],
+    });
+}
 function patterns(v) {
     const list = Array.isArray(v) ? v : typeof v === "string" ? [v] : [];
     return list.filter((p) => typeof p === "string" && p.length > 0);

package/dist/src/core/batch-api.js

@@ -581,7 +581,7 @@ export async function waitForJob(id, opts) {
                 throw err;
             lastErr = err;
         }
-        if (Date.now() + delay > deadline) {
+        if (Date.now() + delay >= deadline) {
             // Out of time: report the last state read, or the error if none was.
             if (!last)
                 throw lastErr;

package/dist/src/core/config.js

@@ -14,7 +14,7 @@ export const CONFIG_VERSION = "0.1.0";
  * entrypoint) so core modules — telemetry, signup provenance — can read it
  * without an import cycle. `VERSION` in `cli/index.ts` re-exports this.
  */
-export const CLI_VERSION = "1.11.0"; // x-release-please-version
+export const CLI_VERSION = "1.12.0"; // x-release-please-version
 /** The User-Agent on every request to a Zenrows API. The gateway reads the version from it. */
 export const CLI_USER_AGENT = `zenrows-cli/${CLI_VERSION}`;
 /** Env var to override the Fetch and Extract API base (local/staging testing). */

package/dist/src/core/crawl-api.js

@@ -10,8 +10,9 @@
  * Crawl SDKs do (see `retryStatuses`). `fetchImpl` / `sleepImpl` are injectable
  * for tests.
  *
- * `output_format` takes `html` or `markdown` (absent = URLs only). Response fields we do
- * not model pass through untouched.
+ * `output_format` takes `html` or `markdown` (absent = URLs only). A `webhook` gets the
+ * crawl's events (`crawl.started` and how it ended) posted to a URL, signed with the
+ * account's webhook signing key. Response fields we do not model pass through untouched.
  */
 import { Readable } from "node:stream";
 import { ToolkitError, isKeyCapReached, keyCapReached, quotaExhausted } from "./errors.js";
@@ -36,6 +37,10 @@ export function crawlBase() {
 }
 /** The API's "Crawl is not enabled for this account" code (403). */
 export const CRAWL_NOT_ENABLED_CODE = "REQS008";
+/** The events a crawl's webhook can send. */
+export const CRAWL_WEBHOOK_EVENTS = ["crawl.started", "crawl.completed", "crawl.stopped", "crawl.failed"];
+/** The API's limits on a webhook's `metadata`. */
+export const WEBHOOK_METADATA_LIMITS = { keys: 20, keyLength: 64, valueBytes: 1024 };
 /** A crawl stops progressing once its status is anything but `running`. */
 export function isTerminal(status) {
     return !!status && status !== "running";
@@ -273,7 +278,9 @@ function classifyProblem(status, problem, body, request, serverCode, retryAfter)
             likely_cause: cause,
             next_action: serverCode === "idempotency_key_reused"
                 ? "This Idempotency-Key was already used for a different request. Send a new key, or no key. Do not retry as is."
-                : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.",
+                : serverCode === "webhook_signing_key_required"
+                    ? "Nothing was created. Create a webhook signing key in the dashboard (Settings → API Keys), then retry."
+                    : "Fix the reported parameter and retry. See `zenrows crawl --help` for the accepted flags and ranges.",
         });
     }
     return new ToolkitError({
@@ -300,6 +307,14 @@ export function createCrawl(params, opts) {
         body.output_format = params.output_format;
     if (params.discovery?.length)
         body.discovery = params.discovery;
+    if (params.webhook !== undefined) {
+        const webhook = { url: params.webhook.url };
+        if (params.webhook.events?.length)
+            webhook.events = params.webhook.events;
+        if (params.webhook.metadata !== undefined)
+            webhook.metadata = params.webhook.metadata;
+        body.webhook = webhook;
+    }
     return crawlJson("POST", "/crawls", {
         ...opts,
         body,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenrows/cli",
-  "version": "1.11.0",
+  "version": "1.12.0",
   "description": "Open-source CLI, MCP, skills, templates, workflows, recipes, and evals layer for giving AI agents reliable access to protected web data through Zenrows cloud infrastructure.",
   "type": "module",
   "license": "MIT",

package/skills/crawl/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: crawl
-description: "Collect the URLs behind one start page with Crawl (new): create a crawl, wait for it, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs."
+description: "Collect the URLs behind one start page with Crawl (new): create a crawl, wait for it or get notified by webhook, read the kept URLs and optionally each page's HTML, stop it. Use when the user has a listing or section URL but not the item URLs."
 version: 0.1.0
 requires_backend_capabilities: [crawl]
 ---
@@ -71,6 +71,36 @@ of URLs. A result's `content_status` is `pending`, `fetched` or `failed`; only
 `fetched` results have content (else `CRAWL_CONTENT_NOT_FOUND`). `download`
 reports the crawl's `status`, and `partial: true` while it runs.
 
+## Webhooks
+
+Instead of polling, have Crawl post its events to a URL you run:
+
+```
+zenrows crawl create <url> --depth 1 --webhook https://example.com/hooks/zenrows \
+  --webhook-event crawl.completed --webhook-event crawl.failed \
+  --webhook-metadata site=books --json
+```
+
+- `--webhook <url>` takes an absolute `https` URL. It is fixed at create.
+- `--webhook-event` repeats; each of `crawl.started`, `crawl.completed`,
+  `crawl.stopped`, `crawl.failed` at most once. Leave it out for all four.
+  Exactly one of the last three is sent per crawl.
+- `--webhook-metadata key=value` repeats (up to 20 keys of 1 to 64
+  characters, each value at most 1,024 bytes as JSON); every event carries it
+  back, to match the crawl to your own records.
+- The crawl shows `webhook` as `{url, events, metadata}`, `events` being the
+  effective list; a crawl without one has none.
+- Events are signed JSON (`schema: zenrows.webhook/v1`, `event_id`,
+  `event_type`, `occurred_at`, `resource {type: crawl, id, url}`, `metadata`).
+  Verify the `X-Signature: t=…,v1=…,kid=…` header: `v1` is the hex
+  HMAC-SHA256 of `"<t>." + body` with the account's webhook signing key
+  (Settings → API Keys). Events can repeat (drop one whose `event_id` you
+  handled) and arrive out of order (sort by `occurred_at`).
+- The account needs a webhook signing key: without one the create gets 422
+  `webhook_signing_key_required` (create the key, then retry). A bad URL,
+  event or metadata gets 422 `invalid_webhook`. Both are
+  `CRAWL_INVALID_REQUEST`, and nothing was created.
+
 ## While it runs, and after
 
 ```