@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/network-related-error-kinds.js

@@ -0,0 +1,28 @@
+/**
+ * `ErrorKind`s that plausibly indicate trouble with the operator's own
+ * network path rather than a genuinely dead or misconfigured target site.
+ *
+ * Single source of truth shared by two consumers that must agree on
+ * exactly the same set:
+ *
+ * - `NetworkOutageDetector` — only these kinds contribute to the sliding
+ *   window that detects a suspect outage.
+ * - `evict-network-classified-destination-cache-entries.ts` — on recovery,
+ *   only `destinationCache` entries whose cached `Error` classifies into
+ *   one of these kinds are evicted (a cached `tls` or `client-blocked`
+ *   failure is a site-specific fact and must survive an outage recovery
+ *   unrelated to it).
+ *
+ * `dns` is included despite `permanent-error-kinds.ts` treating it as a
+ * permanent, site-specific verdict in isolation — that classification is
+ * exactly what outage detection exists to override when the surrounding
+ * evidence (many hosts, tight time window) points to the local network
+ * instead.
+ */
+export const NETWORK_RELATED_ERROR_KINDS = new Set([
+    'dns',
+    'dns-transient',
+    'local-network',
+    'connection-timeout',
+    'connection-reset',
+]);

package/lib/permanent-error-kinds.d.ts

@@ -7,11 +7,12 @@ import type { ErrorKind } from './types.js';
  * Used by `resetFailedPages` to exclude pages whose latest recorded error
  * falls in this set, so `--retry-failed` actually converges: without the
  * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
- * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every
- * iteration, the crawler would re-attempt them, they would fail again the
- * same way, and the retry-target count would stay constant forever.
+ * `ECONNREFUSED` / HTTP parse-error / redirect-loop pages would be reset to
+ * pending on every iteration, the crawler would re-attempt them, they would
+ * fail again the same way, and the retry-target count would stay constant
+ * forever.
  *
- * Why these five and not others:
+ * Why these six and not others:
  * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
  *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
  *   `dns-transient` precisely so it is NOT in this set.
@@ -28,6 +29,10 @@ import type { ErrorKind } from './types.js';
  *   the listener; either no process is listening on the port or its accept
  *   queue rejected the connection. Either way the answer is final until the
  *   server operator intervenes.
+ * - **redirect-loop** — `Maximum number of redirects exceeded` /
+ *   `ERR_TOO_MANY_REDIRECTS` means the site's own redirect chain never
+ *   terminates; the exact same chain is served on every future fetch until
+ *   the site operator fixes it.
  *
  * Notably absent (intentionally retryable):
  * - `connection-reset` / `connection-timeout` — could be middlebox or

package/lib/permanent-error-kinds.js

@@ -6,11 +6,12 @@
  * Used by `resetFailedPages` to exclude pages whose latest recorded error
  * falls in this set, so `--retry-failed` actually converges: without the
  * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
- * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every
- * iteration, the crawler would re-attempt them, they would fail again the
- * same way, and the retry-target count would stay constant forever.
+ * `ECONNREFUSED` / HTTP parse-error / redirect-loop pages would be reset to
+ * pending on every iteration, the crawler would re-attempt them, they would
+ * fail again the same way, and the retry-target count would stay constant
+ * forever.
  *
- * Why these five and not others:
+ * Why these six and not others:
  * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
  *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
  *   `dns-transient` precisely so it is NOT in this set.
@@ -27,6 +28,10 @@
  *   the listener; either no process is listening on the port or its accept
  *   queue rejected the connection. Either way the answer is final until the
  *   server operator intervenes.
+ * - **redirect-loop** — `Maximum number of redirects exceeded` /
+ *   `ERR_TOO_MANY_REDIRECTS` means the site's own redirect chain never
+ *   terminates; the exact same chain is served on every future fetch until
+ *   the site operator fixes it.
  *
  * Notably absent (intentionally retryable):
  * - `connection-reset` / `connection-timeout` — could be middlebox or
@@ -45,4 +50,5 @@ export const PERMANENT_ERROR_KINDS = new Set([
     'client-blocked',
     'parse-error',
     'connection-refused',
+    'redirect-loop',
 ]);

package/lib/types.d.ts

@@ -10,7 +10,7 @@ import type { CrawlerError, PageData } from './utils/types/types.js';
  * documented per-property rather than scattered across the two emit sites.
  */
 export interface InventoryRunAggregates {
-    /** Total non-empty lines in the input list (= `inventoryUrls.length` before any filtering). Stored verbatim as `inventory_runs.total_lines`. */
+    /** `inventoryUrls.length` as received by `CrawlerOrchestrator.inventory` — the CLI (`inventoryCrawl`) has already warned-and-dropped unparseable-URL lines before this point, so this counts valid URLs, not raw source-file lines. Stored verbatim as `inventory_runs.total_lines`. */
     inventoryUrlsCount: number;
     /** Number of novel URLs classified as HTML and queued for render. Stored as `new_pages` (excludes anchor-discovered descendants — those add later via the crawler graph and are NOT counted here). */
     htmlSeedsCount: number;
@@ -32,6 +32,14 @@ export interface InventoryRunAggregates {
      * `source_file_sha256` will be `NULL`.
      */
     sourceFileSha256: string | null;
+    /**
+     * Number of source-file lines the CLI warned-and-dropped for failing
+     * URL validation, before `inventoryUrlsCount` was ever counted. Stored
+     * verbatim as `inventory_runs.invalid_skipped`. `null` for programmatic
+     * callers that built `inventoryUrls` in-memory — there is no source
+     * file, so no line was ever dropped as invalid.
+     */
+    invalidSkipped: number | null;
 }
 /**
  * Coarse cause of a crawl/scrape failure.
@@ -58,6 +66,7 @@ export interface InventoryRunAggregates {
  * | `local-network` | **yes** | no | local machine's network is unreachable / changed (WiFi, sleep, ICMP-unreachable, …) |
  * | `parse-error` | mostly persistent | no | HTTP response could not be parsed (proxy, garbage, MITM) |
  * | `client-blocked` | persistent (per browser) | no | Chromium-side `ERR_BLOCKED_BY_*` family — the browser actively refused the request (ad/tracker heuristics, CSP, CORP, administrator block list, …) |
+ * | `redirect-loop` | no | no | the redirect chain exceeded `follow-redirects`' `maxRedirects` limit — the site's own redirect configuration never converges |
  * | `protocol` | yes | no | puppeteer protocol layer (frame detached, target closed, …) |
  * | `timeout` | yes | no | puppeteer navigation timeout or HEAD pre-flight race timeout (`Timeout: <url>`) |
  * | `unknown` | unknown | no | catch-all for messages no matcher recognised |
@@ -82,7 +91,7 @@ export interface InventoryRunAggregates {
  * entry), or a recoverable URL never reaching the puppeteer fallback (no
  * PUPPETEER_FALLBACK entry).
  */
-export type ErrorKind = 'dns' | 'dns-transient' | 'connection-refused' | 'connection-reset' | 'connection-timeout' | 'tls' | 'local-network' | 'parse-error' | 'client-blocked' | 'timeout' | 'protocol' | 'unknown';
+export type ErrorKind = 'dns' | 'dns-transient' | 'connection-refused' | 'connection-reset' | 'connection-timeout' | 'tls' | 'local-network' | 'parse-error' | 'client-blocked' | 'redirect-loop' | 'timeout' | 'protocol' | 'unknown';
 /**
  * Event map for the `CrawlerOrchestrator` class.
  *

package/lib/utils/array/each-splitted.d.ts

@@ -7,4 +7,4 @@
  * @param callback - A function to invoke on each chunk. May be synchronous or asynchronous.
  * @returns A promise that resolves when all chunk callbacks have completed.
  */
-export declare function eachSplitted<T>(a: T[], count: number, callback: (items: T[]) => void | Promise<void>): Promise<void>;
+export declare function eachSplitted<T>(a: readonly T[], count: number, callback: (items: T[]) => void | Promise<void>): Promise<void>;

package/lib/utils/compute-file-sha256.d.ts

@@ -1,24 +1,24 @@
 /**
- * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
- * O(1) regardless of file size.
+ * Compute the SHA-256 digest of an in-memory byte buffer.
  *
- * Used by `CrawlerOrchestrator.inventory` to fingerprint the source
- * URL list `.txt` and store the digest on the `inventory_runs` row. The
- * digest is the content-identity key for "was this exact list applied
- * before" — recorded for operator audit; nothing dedupes on it at write
- * time.
+ * Used by `CrawlerOrchestrator.inventory` to fingerprint the source URL
+ * list — the digest is both recorded on the `inventory_runs` audit row
+ * (the content-identity key for "was this exact list applied before") and
+ * used as the file name under which the raw list is archived
+ * (`Archive.saveInventorySourceList`).
  *
- * Returns `null` instead of throwing when the file cannot be read (e.g.
- * vanished mid-run, permissions issue) so the inventory run's audit row
- * can still be written with a NULL digest. Hashing failure is an audit
- * loss, not a correctness failure — the actual ingestion has already
- * succeeded by the time this is called.
- * @param filePath - Absolute or relative path to the file to hash.
- * @returns Lower-case hex digest (64 chars), or `null` if reading failed.
+ * Takes an already-read `Buffer` rather than a file path: the CLI reads the
+ * source file exactly once (`inventoryCrawl`) and derives the digest, the
+ * parsed URL list, and the archived copy all from that single buffer. A
+ * separate read-then-hash pass would let the file change between the two
+ * reads and desync the archived bytes from the hash naming them.
+ * @param bytes - The exact bytes to hash.
+ * @returns Lower-case hex digest (64 chars).
  * @example
  * ```ts
- * const sha = await computeFileSha256('/tmp/list.txt');
- * if (sha) console.log(sha); // 64-char hex string
+ * const bytes = await fs.readFile('/tmp/list.txt');
+ * const sha = computeFileSha256(bytes);
+ * console.log(sha); // 64-char hex string
  * ```
  */
-export declare function computeFileSha256(filePath: string): Promise<string | null>;
+export declare function computeFileSha256(bytes: Buffer): string;

package/lib/utils/compute-file-sha256.js

@@ -1,56 +1,27 @@
 import { createHash } from 'node:crypto';
-import { createReadStream } from 'node:fs';
-import { crawlerLog } from '../debug.js';
 /**
- * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
- * O(1) regardless of file size.
+ * Compute the SHA-256 digest of an in-memory byte buffer.
  *
- * Used by `CrawlerOrchestrator.inventory` to fingerprint the source
- * URL list `.txt` and store the digest on the `inventory_runs` row. The
- * digest is the content-identity key for "was this exact list applied
- * before" — recorded for operator audit; nothing dedupes on it at write
- * time.
+ * Used by `CrawlerOrchestrator.inventory` to fingerprint the source URL
+ * list — the digest is both recorded on the `inventory_runs` audit row
+ * (the content-identity key for "was this exact list applied before") and
+ * used as the file name under which the raw list is archived
+ * (`Archive.saveInventorySourceList`).
  *
- * Returns `null` instead of throwing when the file cannot be read (e.g.
- * vanished mid-run, permissions issue) so the inventory run's audit row
- * can still be written with a NULL digest. Hashing failure is an audit
- * loss, not a correctness failure — the actual ingestion has already
- * succeeded by the time this is called.
- * @param filePath - Absolute or relative path to the file to hash.
- * @returns Lower-case hex digest (64 chars), or `null` if reading failed.
+ * Takes an already-read `Buffer` rather than a file path: the CLI reads the
+ * source file exactly once (`inventoryCrawl`) and derives the digest, the
+ * parsed URL list, and the archived copy all from that single buffer. A
+ * separate read-then-hash pass would let the file change between the two
+ * reads and desync the archived bytes from the hash naming them.
+ * @param bytes - The exact bytes to hash.
+ * @returns Lower-case hex digest (64 chars).
  * @example
  * ```ts
- * const sha = await computeFileSha256('/tmp/list.txt');
- * if (sha) console.log(sha); // 64-char hex string
+ * const bytes = await fs.readFile('/tmp/list.txt');
+ * const sha = computeFileSha256(bytes);
+ * console.log(sha); // 64-char hex string
  * ```
  */
-export async function computeFileSha256(filePath) {
-    return await new Promise((resolve) => {
-        const hash = createHash('sha256');
-        const stream = createReadStream(filePath);
-        let settled = false;
-        const settle = (value) => {
-            if (settled) {
-                return;
-            }
-            settled = true;
-            resolve(value);
-        };
-        stream.on('data', (chunk) => hash.update(chunk));
-        stream.on('end', () => settle(hash.digest('hex')));
-        stream.on('error', (error) => {
-            // Trace the failure so operators inspecting `--inventory`
-            // audit rows with `source_file_sha256 = NULL` can recover
-            // the underlying cause via `DEBUG=Nitpicker:Crawler:*`.
-            crawlerLog('compute-file-sha256 failed for %s: %s', filePath, error.message);
-            settle(null);
-        });
-        // `'close'` is the last-resort settler. If the stream is
-        // destroyed externally between `'data'` and `'end'` (e.g. a
-        // test or signal handler), neither `'end'` nor `'error'` may
-        // fire — leaving the promise pending forever. Falling back to
-        // `null` matches the documented contract ("hashing failure
-        // yields null").
-        stream.on('close', () => settle(null));
-    });
+export function computeFileSha256(bytes) {
+    return createHash('sha256').update(bytes).digest('hex');
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.14.0",
+	"version": "0.16.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -27,7 +27,7 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "4.0.0",
+		"@d-zero/beholder": "4.1.0",
 		"@d-zero/dealer": "1.9.4",
 		"@d-zero/fs": "0.2.6",
 		"@d-zero/shared": "0.22.2",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "98222c2bbce55ab6d253787a251f040b51c7a9a0"
+	"gitHead": "bef8b6d48e3ca5167fee643d6aba8644a065df4a"
 }