@nitpicker/crawler 0.14.0 → 0.15.0

package/lib/network-related-error-kinds.js

@@ -0,0 +1,28 @@
+/**
+ * `ErrorKind`s that plausibly indicate trouble with the operator's own
+ * network path rather than a genuinely dead or misconfigured target site.
+ *
+ * Single source of truth shared by two consumers that must agree on
+ * exactly the same set:
+ *
+ * - `NetworkOutageDetector` — only these kinds contribute to the sliding
+ *   window that detects a suspect outage.
+ * - `evict-network-classified-destination-cache-entries.ts` — on recovery,
+ *   only `destinationCache` entries whose cached `Error` classifies into
+ *   one of these kinds are evicted (a cached `tls` or `client-blocked`
+ *   failure is a site-specific fact and must survive an outage recovery
+ *   unrelated to it).
+ *
+ * `dns` is included despite `permanent-error-kinds.ts` treating it as a
+ * permanent, site-specific verdict in isolation — that classification is
+ * exactly what outage detection exists to override when the surrounding
+ * evidence (many hosts, tight time window) points to the local network
+ * instead.
+ */
+export const NETWORK_RELATED_ERROR_KINDS = new Set([
+    'dns',
+    'dns-transient',
+    'local-network',
+    'connection-timeout',
+    'connection-reset',
+]);

package/lib/types.d.ts

@@ -10,7 +10,7 @@ import type { CrawlerError, PageData } from './utils/types/types.js';
  * documented per-property rather than scattered across the two emit sites.
  */
 export interface InventoryRunAggregates {
-    /** Total non-empty lines in the input list (= `inventoryUrls.length` before any filtering). Stored verbatim as `inventory_runs.total_lines`. */
+    /** `inventoryUrls.length` as received by `CrawlerOrchestrator.inventory` — the CLI (`inventoryCrawl`) has already warned-and-dropped unparseable-URL lines before this point, so this counts valid URLs, not raw source-file lines. Stored verbatim as `inventory_runs.total_lines`. */
     inventoryUrlsCount: number;
     /** Number of novel URLs classified as HTML and queued for render. Stored as `new_pages` (excludes anchor-discovered descendants — those add later via the crawler graph and are NOT counted here). */
     htmlSeedsCount: number;
@@ -32,6 +32,14 @@ export interface InventoryRunAggregates {
      * `source_file_sha256` will be `NULL`.
      */
     sourceFileSha256: string | null;
+    /**
+     * Number of source-file lines the CLI warned-and-dropped for failing
+     * URL validation, before `inventoryUrlsCount` was ever counted. Stored
+     * verbatim as `inventory_runs.invalid_skipped`. `null` for programmatic
+     * callers that built `inventoryUrls` in-memory — there is no source
+     * file, so no line was ever dropped as invalid.
+     */
+    invalidSkipped: number | null;
 }
 /**
  * Coarse cause of a crawl/scrape failure.

package/lib/utils/array/each-splitted.d.ts

@@ -7,4 +7,4 @@
  * @param callback - A function to invoke on each chunk. May be synchronous or asynchronous.
  * @returns A promise that resolves when all chunk callbacks have completed.
  */
-export declare function eachSplitted<T>(a: T[], count: number, callback: (items: T[]) => void | Promise<void>): Promise<void>;
+export declare function eachSplitted<T>(a: readonly T[], count: number, callback: (items: T[]) => void | Promise<void>): Promise<void>;

package/lib/utils/compute-file-sha256.d.ts

@@ -1,24 +1,24 @@
 /**
- * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
- * O(1) regardless of file size.
+ * Compute the SHA-256 digest of an in-memory byte buffer.
  *
- * Used by `CrawlerOrchestrator.inventory` to fingerprint the source
- * URL list `.txt` and store the digest on the `inventory_runs` row. The
- * digest is the content-identity key for "was this exact list applied
- * before" — recorded for operator audit; nothing dedupes on it at write
- * time.
+ * Used by `CrawlerOrchestrator.inventory` to fingerprint the source URL
+ * list — the digest is both recorded on the `inventory_runs` audit row
+ * (the content-identity key for "was this exact list applied before") and
+ * used as the file name under which the raw list is archived
+ * (`Archive.saveInventorySourceList`).
  *
- * Returns `null` instead of throwing when the file cannot be read (e.g.
- * vanished mid-run, permissions issue) so the inventory run's audit row
- * can still be written with a NULL digest. Hashing failure is an audit
- * loss, not a correctness failure — the actual ingestion has already
- * succeeded by the time this is called.
- * @param filePath - Absolute or relative path to the file to hash.
- * @returns Lower-case hex digest (64 chars), or `null` if reading failed.
+ * Takes an already-read `Buffer` rather than a file path: the CLI reads the
+ * source file exactly once (`inventoryCrawl`) and derives the digest, the
+ * parsed URL list, and the archived copy all from that single buffer. A
+ * separate read-then-hash pass would let the file change between the two
+ * reads and desync the archived bytes from the hash naming them.
+ * @param bytes - The exact bytes to hash.
+ * @returns Lower-case hex digest (64 chars).
  * @example
  * ```ts
- * const sha = await computeFileSha256('/tmp/list.txt');
- * if (sha) console.log(sha); // 64-char hex string
+ * const bytes = await fs.readFile('/tmp/list.txt');
+ * const sha = computeFileSha256(bytes);
+ * console.log(sha); // 64-char hex string
  * ```
  */
-export declare function computeFileSha256(filePath: string): Promise<string | null>;
+export declare function computeFileSha256(bytes: Buffer): string;

package/lib/utils/compute-file-sha256.js

@@ -1,56 +1,27 @@
 import { createHash } from 'node:crypto';
-import { createReadStream } from 'node:fs';
-import { crawlerLog } from '../debug.js';
 /**
- * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
- * O(1) regardless of file size.
+ * Compute the SHA-256 digest of an in-memory byte buffer.
  *
- * Used by `CrawlerOrchestrator.inventory` to fingerprint the source
- * URL list `.txt` and store the digest on the `inventory_runs` row. The
- * digest is the content-identity key for "was this exact list applied
- * before" — recorded for operator audit; nothing dedupes on it at write
- * time.
+ * Used by `CrawlerOrchestrator.inventory` to fingerprint the source URL
+ * list — the digest is both recorded on the `inventory_runs` audit row
+ * (the content-identity key for "was this exact list applied before") and
+ * used as the file name under which the raw list is archived
+ * (`Archive.saveInventorySourceList`).
  *
- * Returns `null` instead of throwing when the file cannot be read (e.g.
- * vanished mid-run, permissions issue) so the inventory run's audit row
- * can still be written with a NULL digest. Hashing failure is an audit
- * loss, not a correctness failure — the actual ingestion has already
- * succeeded by the time this is called.
- * @param filePath - Absolute or relative path to the file to hash.
- * @returns Lower-case hex digest (64 chars), or `null` if reading failed.
+ * Takes an already-read `Buffer` rather than a file path: the CLI reads the
+ * source file exactly once (`inventoryCrawl`) and derives the digest, the
+ * parsed URL list, and the archived copy all from that single buffer. A
+ * separate read-then-hash pass would let the file change between the two
+ * reads and desync the archived bytes from the hash naming them.
+ * @param bytes - The exact bytes to hash.
+ * @returns Lower-case hex digest (64 chars).
  * @example
  * ```ts
- * const sha = await computeFileSha256('/tmp/list.txt');
- * if (sha) console.log(sha); // 64-char hex string
+ * const bytes = await fs.readFile('/tmp/list.txt');
+ * const sha = computeFileSha256(bytes);
+ * console.log(sha); // 64-char hex string
  * ```
  */
-export async function computeFileSha256(filePath) {
-    return await new Promise((resolve) => {
-        const hash = createHash('sha256');
-        const stream = createReadStream(filePath);
-        let settled = false;
-        const settle = (value) => {
-            if (settled) {
-                return;
-            }
-            settled = true;
-            resolve(value);
-        };
-        stream.on('data', (chunk) => hash.update(chunk));
-        stream.on('end', () => settle(hash.digest('hex')));
-        stream.on('error', (error) => {
-            // Trace the failure so operators inspecting `--inventory`
-            // audit rows with `source_file_sha256 = NULL` can recover
-            // the underlying cause via `DEBUG=Nitpicker:Crawler:*`.
-            crawlerLog('compute-file-sha256 failed for %s: %s', filePath, error.message);
-            settle(null);
-        });
-        // `'close'` is the last-resort settler. If the stream is
-        // destroyed externally between `'data'` and `'end'` (e.g. a
-        // test or signal handler), neither `'end'` nor `'error'` may
-        // fire — leaving the promise pending forever. Falling back to
-        // `null` matches the documented contract ("hashing failure
-        // yields null").
-        stream.on('close', () => settle(null));
-    });
+export function computeFileSha256(bytes) {
+    return createHash('sha256').update(bytes).digest('hex');
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.14.0",
+	"version": "0.15.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -27,7 +27,7 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "4.0.0",
+		"@d-zero/beholder": "4.1.0",
 		"@d-zero/dealer": "1.9.4",
 		"@d-zero/fs": "0.2.6",
 		"@d-zero/shared": "0.22.2",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "98222c2bbce55ab6d253787a251f040b51c7a9a0"
+	"gitHead": "d1485df6e43375c6a44edfe07a0293fd39d7810d"
 }