@slicedearth/whoisleuth-cli 2.2.0 → 2.4.0

package/cli/source-reliability.mjs

@@ -1,35 +1,23 @@
 import { Buffer } from 'node:buffer';
 import { canonicalArtifactJson } from "../packages/evidence/artifact-integrity.mjs";
 import { scanBoundedJson } from "../lib/bounded-json.mjs";
+import { LOOKUP_SOURCE_STATES, lookupDiagnosticStates } from "../lib/lookup-diagnostics.mjs";
 import { normalizeExplicitIsoTimestamp, normalizeLegacyIsoTimestamp, } from "../packages/evidence/observation.mjs";
 import { CLI_LOOKUP_SCHEMA, MAX_CLI_LOOKUP_BYTES, } from "../packages/contracts/cli-lookup.mjs";
 import { normalizeCliLookupDocument } from "./saved-lookup.mjs";
+import { CLI_BULK_SCHEMA, CLI_BULK_ITEM_SCHEMA, CLI_BULK_SCHEMA_VERSION } from "./formatters/json.mjs";
+import { LOOKUP_THREAT_INTELLIGENCE_PROVIDERS } from "../lib/lookup-threat-provider-inventory.mjs";
+import { MAX_THREAT_INTELLIGENCE_PROVIDERS } from "../lib/lookup-response-contract.mjs";
+import { THREAT_INTELLIGENCE_SCHEMA, THREAT_INTELLIGENCE_CONTRACT_VERSION } from "../lib/threat-intelligence-types.mjs";
 export const SOURCE_RELIABILITY_REPORT_SCHEMA = 'whoisleuth.source-reliability-report';
 export const SOURCE_RELIABILITY_REPORT_VERSION = 1;
 export const MAX_SOURCE_RELIABILITY_INPUT_BYTES = 12 * 1024 * 1024;
 export const MAX_SOURCE_RELIABILITY_DOCUMENTS = 100;
 export const MAX_SOURCE_RELIABILITY_SOURCES = 64;
-export const MAX_SOURCE_RELIABILITY_TRAVERSAL_NODES = 25_000;
 const SOURCE_RE = /^[a-z][a-z0-9_-]{0,39}$/u;
-const STATES = new Set([
-    'success',
-    'partial',
-    'not_found',
-    'skipped',
-    'error',
-    'unsupported',
-    'not_applicable',
-    'unavailable',
-    'rate_limited',
-    'complete',
-    'disabled',
-    'stale',
-]);
-const LOOKUP_SCHEMAS = new Set([
-    CLI_LOOKUP_SCHEMA,
-    'whoisleuth.cli.bulk',
-    'whoisleuth.cli.bulk.item',
-]);
+const STATES = new Set(LOOKUP_SOURCE_STATES);
+const BULK_HEALTH_VERSIONS = new Set([1, 2, CLI_BULK_SCHEMA_VERSION]);
+const PROVIDER_IDS = new Set(LOOKUP_THREAT_INTELLIGENCE_PROVIDERS.map((provider) => provider.id));
 const REPORT_KEYS = new Set([
     'schema',
     'version',
@@ -175,9 +163,9 @@ function parseLookupDocuments(values) {
                 throw new TypeError('Source reliability input requires a supported bounded CLI Lookup document.');
             }
         }
-        const supported = document.version === 1;
-        if (!LOOKUP_SCHEMAS.has(schema) || !supported) {
-            throw new TypeError('Source reliability input requires CLI Lookup version 1 or 2, version 1 Bulk or Bulk-item documents, or only source reliability reports.');
+        if ((schema !== CLI_BULK_SCHEMA && schema !== CLI_BULK_ITEM_SCHEMA)
+            || typeof document.version !== 'number' || !BULK_HEALTH_VERSIONS.has(document.version)) {
+            throw new TypeError('Source reliability input requires supported CLI Lookup, Bulk or Bulk-item documents, or only source reliability reports.');
         }
         return document;
     });
@@ -232,35 +220,21 @@ function collectTiming(document, store) {
     }
 }
 function collectDiagnostics(document, store) {
-    const diagnostics = record(document.diagnostics);
-    if (!diagnostics)
-        return;
-    const fixed = [
-        ['rdap', record(diagnostics.rdap)?.status],
-        ['registrar_rdap', record(record(diagnostics.rdap)?.registrar)?.status],
-        ['whois', record(diagnostics.whois)?.status],
-        ['availability', record(diagnostics.availability)?.status],
-        ['reverse_dns', record(diagnostics.reverseDns)?.status],
-        ['network_context', record(diagnostics.network)?.status],
-        ['security_txt', record(diagnostics.securityTxt)?.status],
-        ['sslbl', record(diagnostics.sslbl)?.status],
-    ];
-    for (const [source, value] of fixed) {
-        const state = safeState(value);
+    for (const [source, state] of Object.entries(lookupDiagnosticStates(document.diagnostics))) {
         if (state)
             addState(store, source, state);
     }
 }
-function collectObservation(value, store) {
-    if (value.version !== 1
+function collectObservation(raw, source, store) {
+    const value = record(raw);
+    if (!value || value.version !== 1 || value.source !== source
         || typeof value.observedAt !== 'string'
         || typeof value.complete !== 'boolean'
         || typeof value.truncated !== 'boolean'
         || !Array.isArray(value.limitations))
         return;
-    const source = safeSource(value.source);
     const state = safeState(value.status);
-    if (!source || !state)
+    if (!state)
         return;
     const item = accumulator(store, source);
     item.observationSamples += 1;
@@ -271,48 +245,65 @@ function collectObservation(value, store) {
     if (value.truncated === true)
         item.truncated += 1;
 }
-function traverse(document, store) {
-    let visited = 0;
-    const stack = [{ value: document, depth: 0 }];
-    while (stack.length) {
-        const current = stack.pop();
-        if (!current)
+function collectLookupHealth(document, store) {
+    collectDiagnostics(document, store);
+    collectTiming(document, store);
+    const availability = record(document.availability);
+    const dns = record(availability?.dns);
+    const technology = record(availability?.technologyProfile);
+    // Only collector-owned envelopes have health semantics. Raw publications,
+    // parsed registration extensions and nested provider findings remain opaque.
+    const observations = [
+        ['dns', dns], ['dns', dns?.caaPolicy], ['dns_delegation', dns?.delegation],
+        ['http', availability?.http], ['tls', availability?.tls],
+        ['html', availability?.pageIdentity],
+    ];
+    const fullLookup = document.schema === CLI_LOOKUP_SCHEMA;
+    if (fullLookup)
+        observations.push(['html', availability?.credentialSurfaceProfile], ['html', availability?.structuredDataIdentity], ['derived', technology], ['derived', technology?.browserLibraryProfile], ['derived', availability?.pageRoleProfile], ['derived', availability?.clientBehaviorProfile], ['derived', availability?.securityPosture], ['reverse_dns', document.reverseDns], ['ip_rdap', document.networkContext], ['security_txt', document.securityTxt], ['sslbl', document.sslbl]);
+    for (const [source, observation] of observations)
+        collectObservation(observation, source, store);
+    if (!fullLookup)
+        return;
+    const providers = record(document.threatIntelligence)?.providers;
+    if (providers === undefined)
+        return;
+    if (!Array.isArray(providers) || providers.length > MAX_THREAT_INTELLIGENCE_PROVIDERS) {
+        throw new TypeError(`Source reliability input exceeds the ${MAX_THREAT_INTELLIGENCE_PROVIDERS}-provider health boundary.`);
+    }
+    const seen = new Set();
+    for (const rawProvider of providers) {
+        const provider = record(rawProvider);
+        const source = record(provider?.provider)?.id;
+        if (provider?.schema !== THREAT_INTELLIGENCE_SCHEMA || provider.version !== THREAT_INTELLIGENCE_CONTRACT_VERSION
+            || typeof source !== 'string' || !PROVIDER_IDS.has(source)
+            || record(provider.observation)?.source !== source)
             continue;
-        if (current.depth > 12) {
-            throw new TypeError('Source reliability input exceeds the 12-level traversal limit.');
-        }
-        if (visited >= MAX_SOURCE_RELIABILITY_TRAVERSAL_NODES) {
-            throw new TypeError(`Source reliability input exceeds the ${MAX_SOURCE_RELIABILITY_TRAVERSAL_NODES}-node traversal limit.`);
-        }
-        visited += 1;
-        const object = record(current.value);
-        if (object) {
-            collectDiagnostics(object, store);
-            collectTiming(object, store);
-            collectObservation(object, store);
-            if (object.state === 'rate_limited') {
-                const source = safeSource(record(object.observation)?.source);
-                if (source)
-                    addState(store, source, 'rate_limited');
-            }
-            const children = Object.values(object);
-            if (children.length > 200) {
-                throw new TypeError('Source reliability input contains an object with more than 200 fields.');
-            }
-            for (const child of children) {
-                if (child && typeof child === 'object')
-                    stack.push({ value: child, depth: current.depth + 1 });
-            }
-        }
-        else if (Array.isArray(current.value)) {
-            if (current.value.length > 1_000) {
-                throw new TypeError('Source reliability input contains an array with more than 1000 items.');
-            }
-            for (const child of current.value) {
-                if (child && typeof child === 'object')
-                    stack.push({ value: child, depth: current.depth + 1 });
-            }
+        if (seen.has(source))
+            throw new TypeError('Source reliability input contains duplicate provider health identities.');
+        seen.add(source);
+        collectObservation(provider?.observation, source, store);
+        if (provider?.state === 'rate_limited')
+            addState(store, source, 'rate_limited');
+    }
+}
+function collectDocumentHealth(document, store) {
+    if (document.schema === CLI_LOOKUP_SCHEMA) {
+        collectLookupHealth(document, store);
+        return;
+    }
+    const items = document.schema === CLI_BULK_SCHEMA ? document.results : [document];
+    if (!Array.isArray(items) || items.length > 1_000) {
+        throw new TypeError('Source reliability Bulk input requires at most 1000 result items.');
+    }
+    for (const rawItem of items) {
+        const item = record(rawItem);
+        if (!item || item.schema !== CLI_BULK_ITEM_SCHEMA || item.version !== document.version
+            || typeof item.ok !== 'boolean') {
+            throw new TypeError('Source reliability Bulk input contains an invalid result envelope.');
         }
+        if (item.ok)
+            collectLookupHealth(item, store);
     }
 }
 function percentile(sorted, ratio) {
@@ -663,7 +654,7 @@ export function buildSourceReliabilityReport(raw, generatedAt = new Date().toISO
             const sampleDate = lookupDocumentTimestamp(document);
             if (sampleDate)
                 sampleDates.push(sampleDate);
-            traverse(document, store);
+            collectDocumentHealth(document, store);
         }
     }
     sampleDates.sort();

package/cli/terminal-input.mjs

@@ -0,0 +1,168 @@
+import { StringDecoder } from 'node:string_decoder';
+import { CliUsageError, hasUnsafeCliText } from "./errors.mjs";
+export function canReadInteractiveLine(input, output, environment = process.env) {
+    return input?.isTTY === true && output?.isTTY === true
+        && typeof input.setRawMode === 'function' && typeof input.resume === 'function'
+        && typeof input.pause === 'function' && typeof input.on === 'function'
+        && typeof input.off === 'function' && typeof output.write === 'function'
+        && !environment.CI;
+}
+const MAX_INTERACTIVE_ANSWER_SCALARS = 1_024;
+const MAX_INTERACTIVE_ANSWER_BYTES = 4_096;
+function boundedInteractiveAnswer(value) {
+    const supplied = typeof value === 'string' ? value : '';
+    if (Buffer.byteLength(supplied, 'utf8') > MAX_INTERACTIVE_ANSWER_BYTES
+        || Array.from(supplied).length > MAX_INTERACTIVE_ANSWER_SCALARS
+        || hasUnsafeCliText(supplied)) {
+        throw new CliUsageError('Interactive input must be bounded text without control characters.');
+    }
+    return supplied.trim();
+}
+function abortError() {
+    return new DOMException('Aborted', 'AbortError');
+}
+function readBoundedInteractiveLine(prompt, options) {
+    const { input, output, signal } = options;
+    if (signal?.aborted)
+        return Promise.reject(signal.reason || abortError());
+    const previousRaw = input.isRaw === true;
+    const wasPaused = input.isPaused?.() === true;
+    let receivedBytes = 0;
+    let receivedScalars = 0;
+    let settled = false;
+    let characters = [];
+    const decoder = new StringDecoder('utf8');
+    return new Promise((resolve, reject) => {
+        const cleanup = () => {
+            let failure = null;
+            const attempt = (operation) => {
+                try {
+                    operation();
+                }
+                catch (error) {
+                    failure ||= error instanceof Error ? error : new Error('Interactive terminal cleanup failed.');
+                }
+            };
+            attempt(() => input.off?.('data', onData));
+            attempt(() => input.off?.('end', onEnd));
+            attempt(() => signal?.removeEventListener('abort', onAbort));
+            attempt(() => input.setRawMode?.(previousRaw));
+            if (wasPaused)
+                attempt(() => input.pause?.());
+            return failure;
+        };
+        const finish = (value, error) => {
+            if (settled)
+                return;
+            settled = true;
+            const cleanupFailure = cleanup();
+            characters = [];
+            if (error && cleanupFailure) {
+                reject(new AggregateError([error, cleanupFailure], `${error.message}; terminal cleanup also failed: ${cleanupFailure.message}`));
+            }
+            else if (error || cleanupFailure)
+                reject(error || cleanupFailure);
+            else
+                resolve(value || '');
+        };
+        const rejectUsage = () => finish(undefined, new CliUsageError('Interactive input must be bounded text without control characters.'));
+        const onData = (chunk) => {
+            if (settled)
+                return;
+            let rawChunk;
+            if (Buffer.isBuffer(chunk)) {
+                if (chunk.length > MAX_INTERACTIVE_ANSWER_BYTES - receivedBytes) {
+                    rejectUsage();
+                    return;
+                }
+                rawChunk = chunk;
+            }
+            else {
+                const supplied = String(chunk ?? '');
+                if (supplied.length > MAX_INTERACTIVE_ANSWER_BYTES - receivedBytes
+                    || Buffer.byteLength(supplied, 'utf8') > MAX_INTERACTIVE_ANSWER_BYTES - receivedBytes) {
+                    rejectUsage();
+                    return;
+                }
+                rawChunk = Buffer.from(supplied, 'utf8');
+            }
+            receivedBytes += rawChunk.length;
+            const value = decoder.write(rawChunk);
+            receivedScalars += Array.from(value).length;
+            if (receivedBytes > MAX_INTERACTIVE_ANSWER_BYTES
+                || receivedScalars > MAX_INTERACTIVE_ANSWER_SCALARS
+                || value.includes('\ufffd')) {
+                rejectUsage();
+                return;
+            }
+            for (const character of value) {
+                if (character === '\r' || character === '\n') {
+                    try {
+                        output.write('\n');
+                    }
+                    catch (error) {
+                        finish(undefined, error instanceof Error ? error : new Error('Interactive terminal output failed.'));
+                        return;
+                    }
+                    finish(characters.join(''));
+                    return;
+                }
+                if (character === '\u0003' || character === '\u0004' || character === '\u001b') {
+                    finish(undefined, abortError());
+                    return;
+                }
+                if (character === '\u0008' || character === '\u007f') {
+                    if (characters.length) {
+                        characters.pop();
+                        try {
+                            output.write('\b \b');
+                        }
+                        catch (error) {
+                            finish(undefined, error instanceof Error ? error : new Error('Interactive terminal output failed.'));
+                            return;
+                        }
+                    }
+                    continue;
+                }
+                if (hasUnsafeCliText(character)) {
+                    rejectUsage();
+                    return;
+                }
+                characters.push(character);
+                try {
+                    output.write(character);
+                }
+                catch (error) {
+                    finish(undefined, error instanceof Error ? error : new Error('Interactive terminal output failed.'));
+                    return;
+                }
+            }
+        };
+        const onEnd = () => {
+            const remainder = decoder.end();
+            if (remainder)
+                rejectUsage();
+            else
+                finish(undefined, abortError());
+        };
+        const onAbort = () => finish(undefined, signal?.reason instanceof Error ? signal.reason : abortError());
+        try {
+            output.write(prompt);
+            input.on?.('data', onData);
+            input.on?.('end', onEnd);
+            signal?.addEventListener('abort', onAbort, { once: true });
+            if (signal?.aborted) {
+                onAbort();
+                return;
+            }
+            input.setRawMode?.(true);
+            if (settled)
+                return;
+            input.resume?.();
+        }
+        catch (error) {
+            finish(undefined, error instanceof Error ? error : new Error('Interactive terminal input failed.'));
+        }
+    });
+}
+export { MAX_INTERACTIVE_ANSWER_BYTES, MAX_INTERACTIVE_ANSWER_SCALARS, boundedInteractiveAnswer, readBoundedInteractiveLine };

package/cli/workflow-command-runner.mjs

@@ -1,15 +1,39 @@
 import { formatCliJunit } from "./ci-report.mjs";
 import { formatDomainControlMonitor, runDomainControlMonitor } from "./domain-control-monitor.mjs";
-import { boundedCliErrorMessage, CliUsageError } from "./errors.mjs";
+import { boundedCliErrorMessage, createCliDiagnosticOutput, CliUsageError } from "./errors.mjs";
 import EXIT_CODES from "./exit-codes.mjs";
 import { evaluateCliFailPolicies, formatFailPolicyNotice } from "./fail-policy.mjs";
 import { formatJsonDocument } from "./formatters/json.mjs";
 import { buildInvestigationPlan, buildWorkflowRecipeCatalogue, formatInvestigationPlan, formatWorkflowRecipeCatalogue, } from "./investigation-plan.mjs";
-import { MAX_INVESTIGATION_RUN_BYTES, formatInvestigationRun, runInvestigationRecipe, } from "./investigation-run.mjs";
+import { MAX_INVESTIGATION_RUN_BYTES, formatInvestigationRun, investigationRunExitCode, runInvestigationRecipe, } from "./investigation-run.mjs";
 import { MAX_OFFLINE_EVIDENCE_INPUT_BYTES } from "./offline-evidence-review.mjs";
+import { MAX_OFFLINE_ARTIFACT_BYTES } from "./artifact-verify.mjs";
+import { MAX_RETAINED_ARTIFACT_DIFF_BYTES } from "./retained-artifact-diff.mjs";
+import { MAX_SAVED_LOOKUP_INPUT_BYTES } from "./saved-lookup.mjs";
+import { MAX_COMPARE_INPUT_BYTES } from "./compare.mjs";
+import { MAX_SOURCE_RELIABILITY_INPUT_BYTES } from "./source-reliability.mjs";
+import { MAX_SHARING_REVIEW_BYTES } from "./sharing-review.mjs";
 import { createBufferedOutput } from "./output-file.mjs";
+import { boundedInteractiveAnswer, canReadInteractiveLine, readBoundedInteractiveLine } from "./terminal-input.mjs";
 import { WORKFLOW_INLINE_COMMANDS } from "./inline-command-families.mjs";
 import { runDiscriminatedCommandHandler } from "./discriminated-command-handlers.mjs";
+function boundWorkflowInputs(command, inputs, dependencies, context) {
+    if (!inputs.size)
+        return {};
+    const read = (source, fallback, maximumBytes, label) => typeof source === 'string' && inputs.has(source)
+        ? inputs.get(source)
+        : fallback ? fallback(source) : context.readInput(source, maximumBytes, label);
+    switch (command) {
+        case 'export': return { readExportInput: (source) => read(source, dependencies.readExportInput, MAX_SAVED_LOOKUP_INPUT_BYTES, 'Evidence export input') };
+        case 'verify-artifact': return { readArtifactInput: (source) => read(source, dependencies.readArtifactInput, MAX_OFFLINE_ARTIFACT_BYTES, 'Artefact input') };
+        case 'sharing-review': return { readArtifactInput: (source) => read(source, dependencies.readArtifactInput, MAX_SHARING_REVIEW_BYTES, 'Sharing review input') };
+        case 'compare': return { readCompareInput: (source) => read(source, dependencies.readCompareInput, MAX_COMPARE_INPUT_BYTES, 'Comparison input') };
+        case 'source-report': return { readSourceReliabilityInput: (source) => read(source, dependencies.readSourceReliabilityInput, MAX_SOURCE_RELIABILITY_INPUT_BYTES, 'Source reliability input') };
+        case 'diff':
+        case 'timeline': return { readDiffInput: (source) => read(source, dependencies.readDiffInput, command === 'diff' ? MAX_RETAINED_ARTIFACT_DIFF_BYTES : MAX_SAVED_LOOKUP_INPUT_BYTES, command === 'diff' ? 'Retained diff input' : 'Lookup timeline input') };
+        default: throw new CliUsageError('This fixed-workflow command does not accept retained artefacts.');
+    }
+}
 async function runMonitorOnceCommand(args, dependencies, context) {
     context.setFailureLabel('One-shot domain control review');
     let manifestInput;
@@ -82,12 +106,18 @@ async function runWorkflowPlanCommand(args, dependencies, context) {
 }
 async function runWorkflowRecipeCommand(args, dependencies, context) {
     context.setFailureLabel('Investigation workflow');
+    const input = (dependencies.stdin ?? process.stdin);
+    if (args.interactive && !canReadInteractiveLine(input, context.stderr, dependencies.environment)) {
+        throw new CliUsageError('--interactive requires terminal input and terminal stderr. Use --select for unattended workflows.');
+    }
     let resumeInput = null;
     if (args.resumeSource) {
         try {
-            resumeInput = dependencies.readDiffInput
-                ? await dependencies.readDiffInput(args.resumeSource)
-                : await context.readInput(args.resumeSource, MAX_INVESTIGATION_RUN_BYTES, 'Investigation resume state');
+            resumeInput = dependencies.workflowResumeInput !== undefined
+                ? dependencies.workflowResumeInput
+                : dependencies.readDiffInput
+                    ? await dependencies.readDiffInput(args.resumeSource)
+                    : await context.readInput(args.resumeSource, MAX_INVESTIGATION_RUN_BYTES, 'Investigation resume state');
         }
         catch (error) {
             if (error instanceof CliUsageError)
@@ -98,17 +128,42 @@ async function runWorkflowRecipeCommand(args, dependencies, context) {
     const document = await runInvestigationRecipe(args.recipe, args.subject, {
         approveNetwork: args.approveNetwork,
         selections: args.selections,
+        artifactBindings: args.artifactBindings,
+        confirmedReviews: args.confirmedReviews,
+        ...(args.interactive ? { selectInputs: async (request) => {
+                context.writeStderr(`${request.label}\nEnter literal input paths or values. Leave blank to pause; no approval is implied.\n`);
+                const answers = [];
+                for (const placeholder of request.inputs) {
+                    const prompt = `${request.stepId} ${placeholder}: `;
+                    const answer = boundedInteractiveAnswer(await (dependencies.workflowQuestion
+                        ? dependencies.workflowQuestion(prompt)
+                        : readBoundedInteractiveLine(prompt, { input: input, output: context.stderr,
+                            ...(dependencies.signal ? { signal: dependencies.signal } : {}) })));
+                    if (!answer)
+                        break;
+                    answers.push(answer);
+                }
+                return answers;
+            } } : {}),
         resumeInput,
         generatedAt: context.now(),
         ...(dependencies.signal ? { signal: dependencies.signal } : {}),
-        execute: async (command, stepArguments) => {
+        execute: async (command, stepArguments, inputs) => {
             const stepStdout = createBufferedOutput();
-            const stepStderr = createBufferedOutput();
-            const exitCode = await context.executeCli([command, ...stepArguments], {
-                stdout: stepStdout.stream,
-                stderr: stepStderr.stream,
-            });
-            return { exitCode, stdout: stepStdout.value() };
+            const stepStderr = createCliDiagnosticOutput();
+            try {
+                const exitCode = await context.executeCli([command, ...stepArguments], {
+                    stdout: stepStdout.stream,
+                    stderr: stepStderr.stream,
+                    ...boundWorkflowInputs(command, inputs, dependencies, context),
+                });
+                return { exitCode, stdout: stepStdout.value() };
+            }
+            finally {
+                const diagnostic = stepStderr.value();
+                if (diagnostic)
+                    context.writeStderr(`${command}: ${diagnostic}\n`);
+            }
         },
     });
     if (!args.quiet) {
@@ -116,7 +171,7 @@ async function runWorkflowRecipeCommand(args, dependencies, context) {
             ? formatJsonDocument(document)
             : context.terminal(formatInvestigationRun(document), args.color));
     }
-    return document.state === 'step_failed' ? EXIT_CODES.PARTIAL_FAILURE : EXIT_CODES.SUCCESS;
+    return investigationRunExitCode(document);
 }
 const WORKFLOW_COMMAND_HANDLERS = Object.freeze({
     'monitor-once': runMonitorOnceCommand,