@slicedearth/whoisleuth-cli 1.45.0 → 2.0.0

package/cli/bulk-checkpoint.mjs

@@ -1,12 +1,13 @@
 import { createHash } from 'node:crypto';
-import { open } from 'node:fs/promises';
+import { lstat } from 'node:fs/promises';
 import { recordOrNull } from "../lib/bounded-contract-normalizers.mjs";
 import { readBoundedRegularTextFile } from "../lib/bounded-file.mjs";
+import { scanBoundedJson } from "../lib/bounded-json.mjs";
+import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
 import { boundedCliInputError, CliUsageError } from "./errors.mjs";
 import { writePrivateFile } from "./output-file.mjs";
 export const CLI_BULK_CHECKPOINT_SCHEMA = 'whoisleuth.cli.bulk-checkpoint';
 export const CLI_BULK_CHECKPOINT_VERSION = 2;
-const SUPPORTED_BULK_CHECKPOINT_VERSIONS = new Set([1, CLI_BULK_CHECKPOINT_VERSION]);
 export const MAX_BULK_CHECKPOINT_BYTES = 16 * 1024 * 1024;
 const MAX_CHECKPOINT_JSON_DEPTH = 12;
 const MAX_CHECKPOINT_OBJECT_KEYS = 256;
@@ -19,8 +20,8 @@ function checkpointDigest(queries, deep) {
         .update(queries.join('\n'))
         .digest('hex');
 }
-function validTimestamp(value) {
-    return typeof value === 'string' && value.length <= 64 && Number.isFinite(Date.parse(value));
+function normalizedTimestamp(value) {
+    return normalizeExplicitIsoTimestamp(value);
 }
 function isBoundedCheckpointJson(value, depth = 0) {
     if (depth > MAX_CHECKPOINT_JSON_DEPTH)
@@ -53,7 +54,7 @@ function normalizeCompactResult(value) {
         return null;
     return result;
 }
-function normalizeCheckpointResult(value, queries, classifyQuery, version = CLI_BULK_CHECKPOINT_VERSION) {
+function normalizeCheckpointResult(value, queries, classifyQuery) {
     const item = recordOrNull(value);
     const index = item?.index;
     if (!Number.isSafeInteger(index) || Number(index) < 0 || Number(index) >= queries.length)
@@ -61,11 +62,9 @@ function normalizeCheckpointResult(value, queries, classifyQuery, version = CLI_
     const query = queries[Number(index)];
     if (typeof item?.query !== 'string' || item.query !== query)
         return null;
-    const observedAt = version >= 2
-        ? item.observedAt === null
-            ? null
-            : validTimestamp(item.observedAt) ? new Date(item.observedAt).toISOString() : undefined
-        : null;
+    const observedAt = item.observedAt === null
+        ? null
+        : normalizedTimestamp(item.observedAt) ?? undefined;
     if (observedAt === undefined)
         return null;
     if (item.ok === false) {
@@ -86,33 +85,37 @@ function parseBulkCheckpoint(text, options) {
     if (typeof text !== 'string' || Buffer.byteLength(text, 'utf8') > MAX_BULK_CHECKPOINT_BYTES) {
         throw new CliUsageError(`Bulk checkpoint input is limited to ${MAX_BULK_CHECKPOINT_BYTES} bytes.`);
     }
+    const normalized = text.replace(/^\uFEFF/u, '');
     let parsed;
     try {
-        parsed = JSON.parse(text.replace(/^\uFEFF/u, ''));
+        scanBoundedJson(normalized);
+        parsed = JSON.parse(normalized);
     }
     catch {
-        throw new CliUsageError('Bulk checkpoint must be valid JSON.');
+        throw new CliUsageError('Bulk checkpoint must be valid bounded JSON without duplicate keys.');
     }
     const document = recordOrNull(parsed);
     const expectedDigest = checkpointDigest(options.queries, options.deep);
     const version = typeof document?.version === 'number' && Number.isSafeInteger(document.version)
         ? document.version
         : 0;
-    if (document?.schema !== CLI_BULK_CHECKPOINT_SCHEMA || !SUPPORTED_BULK_CHECKPOINT_VERSIONS.has(version)) {
-        throw new CliUsageError(`Bulk checkpoint must use ${CLI_BULK_CHECKPOINT_SCHEMA} version 1 or ${CLI_BULK_CHECKPOINT_VERSION}.`);
+    if (document?.schema !== CLI_BULK_CHECKPOINT_SCHEMA || version !== CLI_BULK_CHECKPOINT_VERSION) {
+        throw new CliUsageError(`Bulk checkpoint must use ${CLI_BULK_CHECKPOINT_SCHEMA} version ${CLI_BULK_CHECKPOINT_VERSION}.`);
     }
     if (document.mode !== (options.deep ? 'deep' : 'fast')
         || document.inputDigestSha256 !== expectedDigest
         || document.queryCount !== options.queries.length) {
         throw new CliUsageError('Bulk checkpoint does not match the current input or scan mode.');
     }
-    if (!validTimestamp(document.startedAt) || !validTimestamp(document.updatedAt) || !Array.isArray(document.results)) {
+    const startedAt = normalizedTimestamp(document.startedAt);
+    const updatedAt = normalizedTimestamp(document.updatedAt);
+    if (!startedAt || !updatedAt || !Array.isArray(document.results)) {
         throw new CliUsageError('Bulk checkpoint metadata is invalid.');
     }
     const results = [];
     const seen = new Set();
     for (const candidate of document.results.slice(0, options.queries.length + 1)) {
-        const result = normalizeCheckpointResult(candidate, options.queries, options.classifyQuery, version);
+        const result = normalizeCheckpointResult(candidate, options.queries, options.classifyQuery);
         if (!result || seen.has(result.index))
             throw new CliUsageError('Bulk checkpoint contains an invalid or duplicate result.');
         seen.add(result.index);
@@ -124,8 +127,8 @@ function parseBulkCheckpoint(text, options) {
         mode: options.deep ? 'deep' : 'fast',
         inputDigestSha256: expectedDigest,
         queryCount: options.queries.length,
-        startedAt: new Date(document.startedAt).toISOString(),
-        updatedAt: new Date(document.updatedAt).toISOString(),
+        startedAt,
+        updatedAt,
         results: results.sort((left, right) => left.index - right.index),
     };
 }
@@ -143,8 +146,7 @@ async function readCheckpointFile(path) {
 }
 async function checkpointFileExists(path) {
     try {
-        const handle = await open(path, 'r');
-        await handle.close();
+        await lstat(path);
         return true;
     }
     catch (error) {
@@ -165,14 +167,20 @@ async function createBulkCheckpointWriter(options) {
     const resumed = options.resume
         ? parseBulkCheckpoint(await readCheckpointFile(options.path), options)
         : null;
-    const startedAt = resumed?.startedAt || now();
+    const currentTimestamp = () => {
+        const timestamp = normalizedTimestamp(now());
+        if (!timestamp)
+            throw new CliUsageError('Bulk checkpoint collection time is invalid.');
+        return timestamp;
+    };
+    const startedAt = resumed?.startedAt || currentTimestamp();
     const results = new Map((resumed?.results || []).map((item) => [item.index, item]));
     let activeWrite = null;
     let dirty = false;
     let writeFailure = null;
     let created = resumed !== null;
     function document() {
-        const updatedAt = now();
+        const updatedAt = currentTimestamp();
         return {
             schema: CLI_BULK_CHECKPOINT_SCHEMA,
             version: CLI_BULK_CHECKPOINT_VERSION,
@@ -227,7 +235,11 @@ async function createBulkCheckpointWriter(options) {
     return Object.freeze({
         initialResults: Object.freeze([...results.values()].sort((left, right) => left.index - right.index)),
         record(result) {
-            const observedAt = validTimestamp(result.observedAt) ? new Date(result.observedAt).toISOString() : now();
+            const observedAt = normalizedTimestamp(result.observedAt) ?? normalizedTimestamp(now());
+            if (!observedAt) {
+                writeFailure ||= new CliUsageError('Bulk checkpoint collection time is invalid.');
+                return;
+            }
             const normalized = normalizeCheckpointResult({ ...result, observedAt }, options.queries, options.classifyQuery);
             if (!normalized) {
                 writeFailure ||= new CliUsageError('Bulk checkpoint result is not a bounded compact lookup result.');

package/cli/bulk-command-runner.mjs

@@ -1,8 +1,7 @@
-import { createReadStream } from 'node:fs';
 import { classifyQuery } from "../lib/classify.mjs";
 import { runUnifiedLookup } from "../lib/lookup.mjs";
 import { createBulkCheckpointWriter } from "./bulk-checkpoint.mjs";
-import { MAX_BULK_INPUT_BYTES, parseBulkQueries, readTextStreamBounded, runBulkLookups, } from "./bulk.mjs";
+import { MAX_BULK_INPUT_BYTES, parseBulkQueries, runBulkLookups, } from "./bulk.mjs";
 import { formatBulkCsv, formatBulkDomainList, formatBulkQueryList, selectBulkItems } from "./bulk-output.mjs";
 import { boundedCliErrorMessage, CliUsageError } from "./errors.mjs";
 import EXIT_CODES from "./exit-codes.mjs";
@@ -24,9 +23,7 @@ async function runBulkCommand(args, dependencies, context) {
     try {
         input = dependencies.readBulkInput
             ? await dependencies.readBulkInput(args.source)
-            : await readTextStreamBounded(args.source
-                ? createReadStream(args.source, { highWaterMark: 64 * 1024 })
-                : dependencies.stdin || process.stdin, MAX_BULK_INPUT_BYTES, dependencies.signal);
+            : await context.readInput(args.source, MAX_BULK_INPUT_BYTES, 'Bulk input');
     }
     catch (error) {
         if (error instanceof CliUsageError)

package/cli/bulk.mjs

@@ -1,6 +1,8 @@
 import { Buffer } from 'node:buffer';
 import { abortable } from "../lib/abort.mjs";
+import { decodeBoundedUtf8 } from "../lib/bounded-file.mjs";
 import { CliUsageError } from "./arguments.mjs";
+import { hasUnsafeCliText } from "./errors.mjs";
 import { MAX_DEEP_BULK_QUERIES, MAX_FAST_BULK_QUERIES, } from "../lib/bulk-limits.mjs";
 const MAX_BULK_INPUT_BYTES = 1024 * 1024;
 async function readTextStreamBounded(stream, limit = MAX_BULK_INPUT_BYTES, signal) {
@@ -26,7 +28,12 @@ async function readTextStreamBounded(stream, limit = MAX_BULK_INPUT_BYTES, signa
     finally {
         signal?.removeEventListener('abort', abort);
     }
-    return Buffer.concat(chunks).toString('utf8');
+    try {
+        return decodeBoundedUtf8(Buffer.concat(chunks), 'Bulk input');
+    }
+    catch (cause) {
+        throw new CliUsageError(cause instanceof Error ? cause.message : 'Bulk input must contain valid UTF-8 text.');
+    }
 }
 function parseBulkQueries(text, { deep = false } = {}) {
     if (typeof text !== 'string')
@@ -43,7 +50,7 @@ function parseBulkQueries(text, { deep = false } = {}) {
         const query = line.trim();
         if (!query)
             continue;
-        if (query.length > 1024 || /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/.test(query)) {
+        if (query.length > 1024 || hasUnsafeCliText(query)) {
             throw new CliUsageError('Bulk input contains an overlong query or unsupported control character.');
         }
         const key = query.toLowerCase();