@nitpicker/crawler 0.13.0 → 0.15.0

package/lib/utils/compute-file-sha256.js

@@ -1,56 +1,27 @@
 import { createHash } from 'node:crypto';
-import { createReadStream } from 'node:fs';
-import { crawlerLog } from '../debug.js';
 /**
- * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
- * O(1) regardless of file size.
+ * Compute the SHA-256 digest of an in-memory byte buffer.
  *
- * Used by `CrawlerOrchestrator.inventory` to fingerprint the source
- * URL list `.txt` and store the digest on the `inventory_runs` row. The
- * digest is the content-identity key for "was this exact list applied
- * before" — recorded for operator audit; nothing dedupes on it at write
- * time.
+ * Used by `CrawlerOrchestrator.inventory` to fingerprint the source URL
+ * list — the digest is both recorded on the `inventory_runs` audit row
+ * (the content-identity key for "was this exact list applied before") and
+ * used as the file name under which the raw list is archived
+ * (`Archive.saveInventorySourceList`).
  *
- * Returns `null` instead of throwing when the file cannot be read (e.g.
- * vanished mid-run, permissions issue) so the inventory run's audit row
- * can still be written with a NULL digest. Hashing failure is an audit
- * loss, not a correctness failure — the actual ingestion has already
- * succeeded by the time this is called.
- * @param filePath - Absolute or relative path to the file to hash.
- * @returns Lower-case hex digest (64 chars), or `null` if reading failed.
+ * Takes an already-read `Buffer` rather than a file path: the CLI reads the
+ * source file exactly once (`inventoryCrawl`) and derives the digest, the
+ * parsed URL list, and the archived copy all from that single buffer. A
+ * separate read-then-hash pass would let the file change between the two
+ * reads and desync the archived bytes from the hash naming them.
+ * @param bytes - The exact bytes to hash.
+ * @returns Lower-case hex digest (64 chars).
  * @example
  * ```ts
- * const sha = await computeFileSha256('/tmp/list.txt');
- * if (sha) console.log(sha); // 64-char hex string
+ * const bytes = await fs.readFile('/tmp/list.txt');
+ * const sha = computeFileSha256(bytes);
+ * console.log(sha); // 64-char hex string
  * ```
  */
-export async function computeFileSha256(filePath) {
-    return await new Promise((resolve) => {
-        const hash = createHash('sha256');
-        const stream = createReadStream(filePath);
-        let settled = false;
-        const settle = (value) => {
-            if (settled) {
-                return;
-            }
-            settled = true;
-            resolve(value);
-        };
-        stream.on('data', (chunk) => hash.update(chunk));
-        stream.on('end', () => settle(hash.digest('hex')));
-        stream.on('error', (error) => {
-            // Trace the failure so operators inspecting `--inventory`
-            // audit rows with `source_file_sha256 = NULL` can recover
-            // the underlying cause via `DEBUG=Nitpicker:Crawler:*`.
-            crawlerLog('compute-file-sha256 failed for %s: %s', filePath, error.message);
-            settle(null);
-        });
-        // `'close'` is the last-resort settler. If the stream is
-        // destroyed externally between `'data'` and `'end'` (e.g. a
-        // test or signal handler), neither `'end'` nor `'error'` may
-        // fire — leaving the promise pending forever. Falling back to
-        // `null` matches the documented contract ("hashing failure
-        // yields null").
-        stream.on('close', () => settle(null));
-    });
+export function computeFileSha256(bytes) {
+    return createHash('sha256').update(bytes).digest('hex');
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.13.0",
+	"version": "0.15.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -27,7 +27,7 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "3.1.5",
+		"@d-zero/beholder": "4.1.0",
 		"@d-zero/dealer": "1.9.4",
 		"@d-zero/fs": "0.2.6",
 		"@d-zero/shared": "0.22.2",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "2cd462ea6b183f4462b7caba27b4db04aaba7c70"
+	"gitHead": "d1485df6e43375c6a44edfe07a0293fd39d7810d"
 }