@slicedearth/whoisleuth-cli 1.43.0 → 1.47.4

package/cli/bulk-checkpoint.mjs

@@ -1,10 +1,12 @@
 import { createHash } from 'node:crypto';
-import { open } from 'node:fs/promises';
+import { lstat } from 'node:fs/promises';
 import { recordOrNull } from "../lib/bounded-contract-normalizers.mjs";
-import { CliUsageError } from "./errors.mjs";
+import { readBoundedRegularTextFile } from "../lib/bounded-file.mjs";
+import { boundedCliInputError, CliUsageError } from "./errors.mjs";
 import { writePrivateFile } from "./output-file.mjs";
 export const CLI_BULK_CHECKPOINT_SCHEMA = 'whoisleuth.cli.bulk-checkpoint';
-export const CLI_BULK_CHECKPOINT_VERSION = 1;
+export const CLI_BULK_CHECKPOINT_VERSION = 2;
+const SUPPORTED_BULK_CHECKPOINT_VERSIONS = new Set([1, CLI_BULK_CHECKPOINT_VERSION]);
 export const MAX_BULK_CHECKPOINT_BYTES = 16 * 1024 * 1024;
 const MAX_CHECKPOINT_JSON_DEPTH = 12;
 const MAX_CHECKPOINT_OBJECT_KEYS = 256;
@@ -51,7 +53,7 @@ function normalizeCompactResult(value) {
         return null;
     return result;
 }
-function normalizeCheckpointResult(value, queries, classifyQuery) {
+function normalizeCheckpointResult(value, queries, classifyQuery, version = CLI_BULK_CHECKPOINT_VERSION) {
     const item = recordOrNull(value);
     const index = item?.index;
     if (!Number.isSafeInteger(index) || Number(index) < 0 || Number(index) >= queries.length)
@@ -59,10 +61,17 @@ function normalizeCheckpointResult(value, queries, classifyQuery) {
     const query = queries[Number(index)];
     if (typeof item?.query !== 'string' || item.query !== query)
         return null;
+    const observedAt = version >= 2
+        ? item.observedAt === null
+            ? null
+            : validTimestamp(item.observedAt) ? new Date(item.observedAt).toISOString() : undefined
+        : null;
+    if (observedAt === undefined)
+        return null;
     if (item.ok === false) {
         if (typeof item.error !== 'string' || !item.error || item.error.length > 300)
             return null;
-        return { index: Number(index), query, ok: false, error: item.error };
+        return { index: Number(index), query, ok: false, error: item.error, observedAt };
     }
     const result = normalizeCompactResult(item.result);
     if (item.ok !== true || !result)
@@ -71,7 +80,7 @@ function normalizeCheckpointResult(value, queries, classifyQuery) {
     const storedClassified = recordOrNull(item.classified);
     if (storedClassified?.type !== classified.type || storedClassified.value !== classified.value)
         return null;
-    return { index: Number(index), query, ok: true, classified, result };
+    return { index: Number(index), query, ok: true, classified, result, observedAt };
 }
 function parseBulkCheckpoint(text, options) {
     if (typeof text !== 'string' || Buffer.byteLength(text, 'utf8') > MAX_BULK_CHECKPOINT_BYTES) {
@@ -86,8 +95,11 @@ function parseBulkCheckpoint(text, options) {
     }
     const document = recordOrNull(parsed);
     const expectedDigest = checkpointDigest(options.queries, options.deep);
-    if (document?.schema !== CLI_BULK_CHECKPOINT_SCHEMA || document.version !== CLI_BULK_CHECKPOINT_VERSION) {
-        throw new CliUsageError(`Bulk checkpoint must use ${CLI_BULK_CHECKPOINT_SCHEMA} version ${CLI_BULK_CHECKPOINT_VERSION}.`);
+    const version = typeof document?.version === 'number' && Number.isSafeInteger(document.version)
+        ? document.version
+        : 0;
+    if (document?.schema !== CLI_BULK_CHECKPOINT_SCHEMA || !SUPPORTED_BULK_CHECKPOINT_VERSIONS.has(version)) {
+        throw new CliUsageError(`Bulk checkpoint must use ${CLI_BULK_CHECKPOINT_SCHEMA} version 1 or ${CLI_BULK_CHECKPOINT_VERSION}.`);
     }
     if (document.mode !== (options.deep ? 'deep' : 'fast')
         || document.inputDigestSha256 !== expectedDigest
@@ -100,11 +112,11 @@ function parseBulkCheckpoint(text, options) {
     const results = [];
     const seen = new Set();
     for (const candidate of document.results.slice(0, options.queries.length + 1)) {
-        const result = normalizeCheckpointResult(candidate, options.queries, options.classifyQuery);
+        const result = normalizeCheckpointResult(candidate, options.queries, options.classifyQuery, version);
         if (!result || seen.has(result.index))
             throw new CliUsageError('Bulk checkpoint contains an invalid or duplicate result.');
         seen.add(result.index);
-        results.push(result);
+        results.push({ ...result, collectionOrigin: 'resumed_checkpoint' });
     }
     return {
         schema: CLI_BULK_CHECKPOINT_SCHEMA,
@@ -118,33 +130,32 @@ function parseBulkCheckpoint(text, options) {
     };
 }
 async function readCheckpointFile(path) {
-    const handle = await open(path, 'r');
     try {
-        const stats = await handle.stat();
-        if (!stats.isFile() || stats.size > MAX_BULK_CHECKPOINT_BYTES) {
-            throw new CliUsageError(`Bulk checkpoint input is limited to ${MAX_BULK_CHECKPOINT_BYTES} bytes.`);
-        }
-        return await handle.readFile({ encoding: 'utf8' });
+        return await readBoundedRegularTextFile(path, {
+            maximumBytes: MAX_BULK_CHECKPOINT_BYTES,
+            label: 'Bulk checkpoint input',
+            allowSymbolicLink: true,
+        });
     }
-    finally {
-        await handle.close();
+    catch (error) {
+        throw boundedCliInputError(error, 'Bulk checkpoint input');
     }
 }
 async function checkpointFileExists(path) {
     try {
-        const handle = await open(path, 'r');
-        await handle.close();
+        await lstat(path);
         return true;
     }
     catch (error) {
         const code = error && typeof error === 'object' && 'code' in error ? error.code : null;
         if (code === 'ENOENT')
             return false;
-        throw error;
+        throw boundedCliInputError(error, 'Bulk checkpoint input');
     }
 }
 async function createBulkCheckpointWriter(options) {
     const now = options.now || (() => new Date().toISOString());
+    const writeCheckpoint = options.writeFile ?? writePrivateFile;
     const exists = await checkpointFileExists(options.path);
     if (options.resume && !exists)
         throw new CliUsageError('Bulk checkpoint does not exist; remove --resume to start a new checkpoint.');
@@ -155,7 +166,8 @@ async function createBulkCheckpointWriter(options) {
         : null;
     const startedAt = resumed?.startedAt || now();
     const results = new Map((resumed?.results || []).map((item) => [item.index, item]));
-    let chain = Promise.resolve();
+    let activeWrite = null;
+    let dirty = false;
     let writeFailure = null;
     let created = resumed !== null;
     function document() {
@@ -168,19 +180,20 @@ async function createBulkCheckpointWriter(options) {
             queryCount: options.queries.length,
             startedAt,
             updatedAt,
-            results: [...results.values()].sort((left, right) => left.index - right.index),
+            results: [...results.values()]
+                .sort((left, right) => left.index - right.index)
+                .map(({ collectionOrigin: _collectionOrigin, ...result }) => result),
         };
     }
-    function scheduleWrite() {
-        chain = chain.then(async () => {
-            if (writeFailure)
-                return;
+    async function drainWrites() {
+        while (dirty && !writeFailure) {
+            dirty = false;
             try {
                 const content = `${JSON.stringify(document(), null, 2)}\n`;
                 if (Buffer.byteLength(content, 'utf8') > MAX_BULK_CHECKPOINT_BYTES) {
                     throw new CliUsageError(`Bulk checkpoints are limited to ${MAX_BULK_CHECKPOINT_BYTES} bytes.`);
                 }
-                await writePrivateFile(options.path, content, {
+                await writeCheckpoint(options.path, content, {
                     force: created,
                     existingFileMessage: 'Bulk checkpoint already exists; use --resume or choose another path.',
                 });
@@ -189,6 +202,23 @@ async function createBulkCheckpointWriter(options) {
             catch (error) {
                 writeFailure = error;
             }
+        }
+    }
+    function scheduleWrite() {
+        dirty = true;
+        if (activeWrite || writeFailure)
+            return;
+        const pending = drainWrites();
+        activeWrite = pending;
+        void pending.then(() => {
+            if (activeWrite === pending)
+                activeWrite = null;
+            if (dirty && !writeFailure)
+                scheduleWrite();
+        }, (error) => {
+            writeFailure ||= error;
+            if (activeWrite === pending)
+                activeWrite = null;
         });
     }
     if (!resumed)
@@ -196,16 +226,22 @@ async function createBulkCheckpointWriter(options) {
     return Object.freeze({
         initialResults: Object.freeze([...results.values()].sort((left, right) => left.index - right.index)),
         record(result) {
-            const normalized = normalizeCheckpointResult(result, options.queries, options.classifyQuery);
+            const observedAt = validTimestamp(result.observedAt) ? new Date(result.observedAt).toISOString() : now();
+            const normalized = normalizeCheckpointResult({ ...result, observedAt }, options.queries, options.classifyQuery);
             if (!normalized) {
                 writeFailure ||= new CliUsageError('Bulk checkpoint result is not a bounded compact lookup result.');
                 return;
             }
-            results.set(normalized.index, normalized);
+            results.set(normalized.index, { ...normalized, collectionOrigin: 'current_run' });
             scheduleWrite();
         },
         async flush() {
-            await chain;
+            while (activeWrite || dirty) {
+                if (!activeWrite)
+                    scheduleWrite();
+                if (activeWrite)
+                    await activeWrite;
+            }
             if (writeFailure)
                 throw writeFailure;
         },

package/cli/bulk-command-runner.mjs

@@ -1,8 +1,7 @@
-import { createReadStream } from 'node:fs';
 import { classifyQuery } from "../lib/classify.mjs";
 import { runUnifiedLookup } from "../lib/lookup.mjs";
 import { createBulkCheckpointWriter } from "./bulk-checkpoint.mjs";
-import { MAX_BULK_INPUT_BYTES, parseBulkQueries, readTextStreamBounded, runBulkLookups, } from "./bulk.mjs";
+import { MAX_BULK_INPUT_BYTES, parseBulkQueries, runBulkLookups, } from "./bulk.mjs";
 import { formatBulkCsv, formatBulkDomainList, formatBulkQueryList, selectBulkItems } from "./bulk-output.mjs";
 import { boundedCliErrorMessage, CliUsageError } from "./errors.mjs";
 import EXIT_CODES from "./exit-codes.mjs";
@@ -24,9 +23,7 @@ async function runBulkCommand(args, dependencies, context) {
     try {
         input = dependencies.readBulkInput
             ? await dependencies.readBulkInput(args.source)
-            : await readTextStreamBounded(args.source
-                ? createReadStream(args.source, { highWaterMark: 64 * 1024 })
-                : dependencies.stdin || process.stdin, MAX_BULK_INPUT_BYTES);
+            : await context.readInput(args.source, MAX_BULK_INPUT_BYTES, 'Bulk input');
     }
     catch (error) {
         if (error instanceof CliUsageError)
@@ -56,6 +53,8 @@ async function runBulkCommand(args, dependencies, context) {
         : null;
     const indicator = context.beginProgress(`Collecting 0 of ${parsed.queries.length} targets`);
     let completed = checkpoint?.initialResults.length || 0;
+    const resumedItems = new Map((checkpoint?.initialResults ?? []).map((item) => [item.index, item]));
+    const settledItems = new Map();
     if (completed)
         indicator.update(`Resumed ${completed} of ${parsed.queries.length} targets`);
     let items;
@@ -69,12 +68,23 @@ async function runBulkCommand(args, dependencies, context) {
             ...(checkpoint ? { initialResults: checkpoint.initialResults } : {}),
             ...(dependencies.signal ? { signal: dependencies.signal } : {}),
             onItemSettled: (item) => {
+                const observed = {
+                    ...item,
+                    observedAt: context.now(),
+                    collectionOrigin: 'current_run',
+                };
+                settledItems.set(item.index, observed);
                 completed += 1;
                 indicator.update(`Collected ${completed} of ${parsed.queries.length} targets`);
                 eventProgress.emit({ event: 'item_settled', index: item.index, ok: item.ok });
-                checkpoint?.record(item);
+                checkpoint?.record(observed);
             },
         });
+        items = items.map((item) => resumedItems.get(item.index) ?? settledItems.get(item.index) ?? {
+            ...item,
+            observedAt: null,
+            collectionOrigin: 'current_run',
+        });
     }
     finally {
         context.endProgress();

package/cli/bulk.mjs

@@ -3,17 +3,28 @@ import { abortable } from "../lib/abort.mjs";
 import { CliUsageError } from "./arguments.mjs";
 import { MAX_DEEP_BULK_QUERIES, MAX_FAST_BULK_QUERIES, } from "../lib/bulk-limits.mjs";
 const MAX_BULK_INPUT_BYTES = 1024 * 1024;
-async function readTextStreamBounded(stream, limit = MAX_BULK_INPUT_BYTES) {
+async function readTextStreamBounded(stream, limit = MAX_BULK_INPUT_BYTES, signal) {
     if (!stream || stream.isTTY)
         return '';
+    const abort = () => stream.destroy?.(new DOMException('Aborted', 'AbortError'));
+    if (signal?.aborted)
+        throw new DOMException('Aborted', 'AbortError');
+    signal?.addEventListener('abort', abort, { once: true });
     const chunks = [];
     let total = 0;
-    for await (const chunk of stream) {
-        const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-        total += buffer.length;
-        if (total > limit)
-            throw new CliUsageError(`Bulk input is limited to ${limit} bytes.`);
-        chunks.push(buffer);
+    try {
+        for await (const chunk of stream) {
+            if (signal?.aborted)
+                throw new DOMException('Aborted', 'AbortError');
+            const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            total += buffer.length;
+            if (total > limit)
+                throw new CliUsageError(`Bulk input is limited to ${limit} bytes.`);
+            chunks.push(buffer);
+        }
+    }
+    finally {
+        signal?.removeEventListener('abort', abort);
     }
     return Buffer.concat(chunks).toString('utf8');
 }