@zenrows/cli 1.6.1 → 1.6.2

package/README.md

@@ -187,7 +187,10 @@ to open. `--max-items` and `--max-pages` cap the crawl (API default 100 each),
 and `--max-pages` bounds the cost, since each page is one fetch. The local policy `max_pages_per_run` caps `--max-pages` (default
 1000). `--output-format html` uses up the page budget: each kept page is one
 more fetch, and `--max-pages` counts it. `--include-patterns` /
-`--exclude-patterns` are repeatable substring filters on the URL. `get` and
+`--exclude-patterns` are repeatable substring filters on the URL. A crawl whose
+pages held nothing new to follow ends `completed` with no results and exits 0.
+When patterns were set, it prints a note (under `--json`, in `note`) to check
+them against the URLs the start page links to. `get` and
 `list` read one page and take `--cursor` and `--limit`. `results` follows every
 page (`--limit` is the page size); on a running crawl it returns the URLs kept
 so far, with `partial: true`. `download` reports the crawl's `status` (`null`

package/dist/src/cli/commands/crawl.js

@@ -62,7 +62,8 @@ export const crawl = {
         "                                    a crawl still running then is printed, not stopped",
         "  --json                            print structured output: create/get/wait/stop print",
         "                                    {ok, crawl_id, status, crawl}; get adds results and next_cursor;",
-        "                                    a wait that runs out adds note",
+        "                                    a wait that runs out, or a completed crawl with patterns",
+        "                                    and 0 results, adds note",
     ].join("\n"),
     async run(argv, ctx) {
         const [sub, ...rest] = argv;
@@ -466,8 +467,8 @@ export function crawlFailure(c) {
     }
     const why = [code, c.error?.detail].filter(Boolean).join(": ");
     const next = {
-        seed_unreachable: "The start URL could not be fetched (the site may be blocking it). Check it with `zenrows fetch <url>`.",
-        no_items_found: "Nothing matched. Loosen --include-patterns / --exclude-patterns, raise or drop --depth, or check the start page with `zenrows fetch <url>`.",
+        seed_unreachable: "The start URL could not be fetched or returned an empty page (the site may be blocking it). Check it with `zenrows fetch <url>`.",
+        no_items_found: "The crawl read the start page but found no data it could use, so it had nothing to follow. Start from a listing or product page.",
     };
     return new ToolkitError({
         code: "CRAWL_FAILED",
@@ -478,6 +479,18 @@ export function crawlFailure(c) {
         ...ids,
     });
 }
+/**
+ * Set only when a crawl with patterns completed and kept nothing. The API ends a
+ * crawl whose pages held nothing new to follow `completed`, and does not count
+ * the links the patterns rejected, so the patterns are the first thing to check.
+ */
+function noMatchNote(c) {
+    const crawl = c;
+    const patterns = [...(crawl.include_patterns ?? []), ...(crawl.exclude_patterns ?? [])];
+    if (crawl.status !== "completed" || crawl.coverage?.items_found !== 0 || !patterns.length)
+        return undefined;
+    return "The crawl completed with no results. Check --include-patterns / --exclude-patterns against the URLs the start page links to.";
+}
 /**
  * Print a crawl (or a stop answer), structured under --json, and return the exit
  * code: 1 when it ended `failed`, or `stopped` because credits ran out or the
@@ -487,6 +500,7 @@ export function crawlFailure(c) {
  */
 function printCrawl(c, json, headline, extra = {}) {
     const err = crawlFailure(c);
+    const note = extra.note ?? noMatchNote(c);
     const crawlFields = { ...c };
     // A wait polls with one result to stay cheap; that page is not "the results".
     delete crawlFields.results;
@@ -498,6 +512,7 @@ function printCrawl(c, json, headline, extra = {}) {
             status: c.status,
             crawl: crawlFields,
             ...extra,
+            note,
             ...(err ? { error: err.toJSON() } : {}),
         }, null, 2));
         return err ? 1 : 0;
@@ -515,6 +530,8 @@ function printCrawl(c, json, headline, extra = {}) {
     }
     if (extra.note)
         log.dim(`  next: zenrows crawl wait ${c.crawl_id}`);
+    else if (note)
+        log.warn(`  ${note}`);
     if (err)
         printError(err, false);
     return err ? 1 : 0;

package/dist/src/core/config.js

@@ -14,7 +14,7 @@ export const CONFIG_VERSION = "0.1.0";
  * entrypoint) so core modules — telemetry, signup provenance — can read it
  * without an import cycle. `VERSION` in `cli/index.ts` re-exports this.
  */
-export const CLI_VERSION = "1.6.1"; // x-release-please-version
+export const CLI_VERSION = "1.6.2"; // x-release-please-version
 /** The User-Agent on every request to a Zenrows API. The gateway reads the version from it. */
 export const CLI_USER_AGENT = `zenrows-cli/${CLI_VERSION}`;
 /** Env var to override the Fetch and Extract API base (local/staging testing). */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenrows/cli",
-  "version": "1.6.1",
+  "version": "1.6.2",
   "description": "Open-source CLI, MCP, skills, templates, workflows, recipes, and evals layer for giving AI agents reliable access to protected web data through Zenrows cloud infrastructure.",
   "type": "module",
   "license": "MIT",

package/skills/crawl/SKILL.md

@@ -39,6 +39,10 @@ zenrows crawl results <id> --out urls.jsonl  # same, as JSONL
 - `--include-patterns` / `--exclude-patterns` are substring matches on the
   normalized URL and repeat:
   `--include-patterns /product/ --exclude-patterns ?add-to-cart`.
+- A crawl whose pages held nothing new to follow (no link, only links to other
+  sites, or only links the patterns reject) ends `completed` with no results
+  and exits 0. When patterns were set, it prints a note (under `--json`, in
+  `note`) to check them against the URLs the start page links to.
 - `create` returns at once. `--follow` (or `zenrows crawl wait <id>`) polls
   until the crawl ends or `--timeout` seconds run out (default 600). When the
   time runs out, the command exits 0 and prints the crawl with status
@@ -85,7 +89,10 @@ credit cap; the pages fetched before that stay readable. `failed`, and a
 `stopped` crawl with an `error`, exit non-zero: `CRAWL_INSUFFICIENT_CREDITS` or
 `CRAWL_KEY_CAP_REACHED` when credits ended it (the same codes a refused
 `create` gets, also when an older deployment reports it `failed`), else
-`CRAWL_FAILED`. `server_code` carries the API's `error.code`.
+`CRAWL_FAILED`. `server_code` carries the API's `error.code`:
+`seed_unreachable` when the start URL could not be fetched or returned an empty
+page, `no_items_found` when the crawl read the start page but found no data it
+could use.
 `stop` on a crawl that already ended prints the status it ended with.
 
 With `--json`, `create`, `get`, `wait` and `stop` print