@nitpicker/crawler 0.15.0 → 0.17.0

package/lib/permanent-error-kinds.js

@@ -6,11 +6,12 @@
  * Used by `resetFailedPages` to exclude pages whose latest recorded error
  * falls in this set, so `--retry-failed` actually converges: without the
  * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
- * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every
- * iteration, the crawler would re-attempt them, they would fail again the
- * same way, and the retry-target count would stay constant forever.
+ * `ECONNREFUSED` / HTTP parse-error / redirect-loop pages would be reset to
+ * pending on every iteration, the crawler would re-attempt them, they would
+ * fail again the same way, and the retry-target count would stay constant
+ * forever.
  *
- * Why these five and not others:
+ * Why these six and not others:
  * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
  *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
  *   `dns-transient` precisely so it is NOT in this set.
@@ -27,6 +28,10 @@
  *   the listener; either no process is listening on the port or its accept
  *   queue rejected the connection. Either way the answer is final until the
  *   server operator intervenes.
+ * - **redirect-loop** — `Maximum number of redirects exceeded` /
+ *   `ERR_TOO_MANY_REDIRECTS` means the site's own redirect chain never
+ *   terminates; the exact same chain is served on every future fetch until
+ *   the site operator fixes it.
  *
  * Notably absent (intentionally retryable):
  * - `connection-reset` / `connection-timeout` — could be middlebox or
@@ -45,4 +50,5 @@ export const PERMANENT_ERROR_KINDS = new Set([
     'client-blocked',
     'parse-error',
     'connection-refused',
+    'redirect-loop',
 ]);

package/lib/types.d.ts

@@ -18,6 +18,8 @@ export interface InventoryRunAggregates {
     nonHtmlCount: number;
     /** URLs dropped because they fell outside the archived scope. Stored as `scope_skipped`. */
     outOfScope: number;
+    /** Novel in-scope URLs recorded as terminal skipped pages (`is_skipped=1`, `skip_reason='excluded'`) instead of being imported, because they matched the effective `excludes` / `excludeUrls` config — the same inputs the scrape phase's fetch-time `shouldSkipUrl` gate uses. Stored as `exclude_skipped`. */
+    excludeSkipped: number;
     /**
      * SHA-256 hex digest of the source `.txt`, **pre-computed by the caller**
      * (typically the CLI's `inventoryCrawl`). Stored verbatim as
@@ -66,6 +68,7 @@ export interface InventoryRunAggregates {
  * | `local-network` | **yes** | no | local machine's network is unreachable / changed (WiFi, sleep, ICMP-unreachable, …) |
  * | `parse-error` | mostly persistent | no | HTTP response could not be parsed (proxy, garbage, MITM) |
  * | `client-blocked` | persistent (per browser) | no | Chromium-side `ERR_BLOCKED_BY_*` family — the browser actively refused the request (ad/tracker heuristics, CSP, CORP, administrator block list, …) |
+ * | `redirect-loop` | no | no | the redirect chain exceeded `follow-redirects`' `maxRedirects` limit — the site's own redirect configuration never converges |
  * | `protocol` | yes | no | puppeteer protocol layer (frame detached, target closed, …) |
  * | `timeout` | yes | no | puppeteer navigation timeout or HEAD pre-flight race timeout (`Timeout: <url>`) |
  * | `unknown` | unknown | no | catch-all for messages no matcher recognised |
@@ -90,7 +93,7 @@ export interface InventoryRunAggregates {
  * entry), or a recoverable URL never reaching the puppeteer fallback (no
  * PUPPETEER_FALLBACK entry).
  */
-export type ErrorKind = 'dns' | 'dns-transient' | 'connection-refused' | 'connection-reset' | 'connection-timeout' | 'tls' | 'local-network' | 'parse-error' | 'client-blocked' | 'timeout' | 'protocol' | 'unknown';
+export type ErrorKind = 'dns' | 'dns-transient' | 'connection-refused' | 'connection-reset' | 'connection-timeout' | 'tls' | 'local-network' | 'parse-error' | 'client-blocked' | 'redirect-loop' | 'timeout' | 'protocol' | 'unknown';
 /**
  * Event map for the `CrawlerOrchestrator` class.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.15.0",
+	"version": "0.17.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -39,7 +39,7 @@
 		"libsql": "0.5.29",
 		"puppeteer": "25.3.0",
 		"robots-parser": "3.0.1",
-		"tar": "7.5.20"
+		"tar": "7.5.21"
 	},
 	"devDependencies": {
 		"@types/debug": "4.1.13",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "d1485df6e43375c6a44edfe07a0293fd39d7810d"
+	"gitHead": "2cdef7cbb4da489270e0f1c9ec2e8166b3d84e91"
 }