@slicedearth/whoisleuth-cli 1.32.0

package/lib/lookup-progress.mjs

@@ -0,0 +1,291 @@
+// Portable bounded NDJSON Lookup progress contract. Partial events are
+// presentation-only; only a validated final ordinary Lookup result may cross
+// the persistence boundary.
+export const LOOKUP_PROGRESS_SCHEMA = 'whoisleuth.lookup-progress';
+export const LOOKUP_PROGRESS_VERSION = 1;
+export const MAX_LOOKUP_PROGRESS_SOURCES = 16;
+export const MAX_LOOKUP_PROGRESS_EVENTS = MAX_LOOKUP_PROGRESS_SOURCES + 2;
+export const MAX_LOOKUP_PROGRESS_FRAGMENT_BYTES = 256 * 1024;
+export const MAX_LOOKUP_PROGRESS_FINAL_BYTES = 8 * 1024 * 1024;
+export const MAX_LOOKUP_PROGRESS_STREAM_BYTES = 12 * 1024 * 1024;
+const SOURCE_IDS = Object.freeze([
+    'rdap',
+    'whois',
+    'domain_evidence',
+    'reverse_dns',
+    'registrar_rdap',
+    'network_context',
+    'security_txt',
+    'external_intelligence',
+    'malware_host_intelligence',
+    'malware_ioc_intelligence',
+]);
+const SOURCE_ID_SET = new Set(SOURCE_IDS);
+const SOURCE_STATES = new Set([
+    'success',
+    'partial',
+    'not_found',
+    'skipped',
+    'error',
+    'unsupported',
+    'unavailable',
+    'rate_limited',
+]);
+const SOURCE_ID_RE = /^[a-z][a-z0-9_]{0,39}$/u;
+const TEXT_ENCODER = new TextEncoder();
+function byteLength(value) {
+    return TEXT_ENCODER.encode(value).byteLength;
+}
+function record(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : null;
+}
+function exactKeys(value, expected, label) {
+    const keys = Object.keys(value);
+    if (keys.length !== expected.length || keys.some((key) => !expected.includes(key))) {
+        throw new TypeError(`${label} has an invalid field set.`);
+    }
+}
+function sequence(value) {
+    if (!Number.isSafeInteger(value) || Number(value) < 0 || Number(value) >= MAX_LOOKUP_PROGRESS_EVENTS) {
+        throw new TypeError('Lookup progress sequence is invalid.');
+    }
+    return Number(value);
+}
+function source(value) {
+    if (typeof value !== 'string' || !SOURCE_ID_RE.test(value) || !SOURCE_ID_SET.has(value)) {
+        throw new TypeError('Lookup progress source is invalid.');
+    }
+    return value;
+}
+function sources(value) {
+    if (!Array.isArray(value) || value.length === 0 || value.length > MAX_LOOKUP_PROGRESS_SOURCES) {
+        throw new TypeError('Lookup progress sources are missing or exceed their bound.');
+    }
+    const normalized = value.map(source);
+    if (new Set(normalized).size !== normalized.length) {
+        throw new TypeError('Lookup progress sources contain duplicates.');
+    }
+    return normalized;
+}
+function cloneBounded(value, maximumBytes, label) {
+    let serialized;
+    try {
+        serialized = JSON.stringify(value);
+    }
+    catch {
+        throw new TypeError(`${label} is not JSON serializable.`);
+    }
+    if (serialized === undefined || byteLength(serialized) > maximumBytes) {
+        throw new TypeError(`${label} exceeds its ${maximumBytes}-byte bound.`);
+    }
+    return JSON.parse(serialized);
+}
+export function createLookupProgressStart(mode, rawSources) {
+    const planned = sources(rawSources);
+    return Object.freeze({
+        schema: LOOKUP_PROGRESS_SCHEMA,
+        version: LOOKUP_PROGRESS_VERSION,
+        event: 'start',
+        sequence: 0,
+        mode,
+        sources: Object.freeze(planned),
+        persistence: 'prohibited',
+    });
+}
+export function createLookupProgressSource(sequenceValue, sourceValue, state, fragment, options = {}) {
+    const normalizedSequence = sequence(sequenceValue);
+    if (normalizedSequence === 0 || !SOURCE_STATES.has(state)) {
+        throw new TypeError('Lookup source event sequence or state is invalid.');
+    }
+    return Object.freeze({
+        schema: LOOKUP_PROGRESS_SCHEMA,
+        version: LOOKUP_PROGRESS_VERSION,
+        event: 'source',
+        sequence: normalizedSequence,
+        source: source(sourceValue),
+        state,
+        complete: options.complete === true,
+        truncated: options.truncated === true,
+        fragment: cloneBounded(fragment, MAX_LOOKUP_PROGRESS_FRAGMENT_BYTES, 'Lookup progress fragment'),
+        persistence: 'prohibited',
+    });
+}
+export function createLookupProgressFinal(sequenceValue, rawSources, result) {
+    const normalizedSequence = sequence(sequenceValue);
+    if (normalizedSequence < 2)
+        throw new TypeError('Lookup final event sequence is invalid.');
+    return Object.freeze({
+        schema: LOOKUP_PROGRESS_SCHEMA,
+        version: LOOKUP_PROGRESS_VERSION,
+        event: 'final',
+        sequence: normalizedSequence,
+        sources: Object.freeze(sources(rawSources)),
+        result: cloneBounded(result, MAX_LOOKUP_PROGRESS_FINAL_BYTES, 'Final Lookup result'),
+        persistence: 'authoritative_after_validation',
+    });
+}
+export function parseLookupProgressEvent(raw) {
+    const value = record(raw);
+    if (!value
+        || value.schema !== LOOKUP_PROGRESS_SCHEMA
+        || value.version !== LOOKUP_PROGRESS_VERSION) {
+        throw new TypeError('Lookup progress event uses an unsupported contract.');
+    }
+    if (value.event === 'start') {
+        exactKeys(value, ['schema', 'version', 'event', 'sequence', 'mode', 'sources', 'persistence'], 'Lookup start event');
+        if (sequence(value.sequence) !== 0
+            || !['fast', 'deep'].includes(String(value.mode))
+            || value.persistence !== 'prohibited') {
+            throw new TypeError('Lookup start event is invalid.');
+        }
+        return createLookupProgressStart(value.mode, sources(value.sources));
+    }
+    if (value.event === 'source') {
+        exactKeys(value, [
+            'schema', 'version', 'event', 'sequence', 'source', 'state',
+            'complete', 'truncated', 'fragment', 'persistence',
+        ], 'Lookup source event');
+        if (typeof value.complete !== 'boolean'
+            || typeof value.truncated !== 'boolean'
+            || value.persistence !== 'prohibited'
+            || typeof value.state !== 'string'
+            || !SOURCE_STATES.has(value.state)) {
+            throw new TypeError('Lookup source event is invalid.');
+        }
+        return createLookupProgressSource(sequence(value.sequence), source(value.source), value.state, value.fragment, { complete: value.complete, truncated: value.truncated });
+    }
+    if (value.event === 'final') {
+        exactKeys(value, ['schema', 'version', 'event', 'sequence', 'sources', 'result', 'persistence'], 'Lookup final event');
+        if (value.persistence !== 'authoritative_after_validation') {
+            throw new TypeError('Lookup final event is invalid.');
+        }
+        return createLookupProgressFinal(sequence(value.sequence), sources(value.sources), value.result);
+    }
+    throw new TypeError('Lookup progress event type is invalid.');
+}
+export function encodeLookupProgressEvent(event) {
+    return `${JSON.stringify(parseLookupProgressEvent(event))}\n`;
+}
+export function createLookupProgressReducer(options) {
+    if (typeof options.validateFinalResult !== 'function') {
+        throw new TypeError('Lookup progress reducer requires a final-result validator.');
+    }
+    let planned = [];
+    let mode = null;
+    let expectedSequence = 0;
+    let finalResult = null;
+    let finalReceived = false;
+    const settled = new Map();
+    function apply(rawEvent) {
+        const event = parseLookupProgressEvent(rawEvent);
+        if (event.sequence !== expectedSequence) {
+            throw new TypeError(`Lookup progress sequence ${event.sequence} arrived while ${expectedSequence} was required.`);
+        }
+        if (event.event === 'start') {
+            if (mode !== null)
+                throw new TypeError('Lookup progress stream started more than once.');
+            mode = event.mode;
+            planned = [...event.sources];
+        }
+        else if (event.event === 'source') {
+            if (mode === null || finalReceived || !planned.includes(event.source) || settled.has(event.source)) {
+                throw new TypeError('Lookup source event is duplicate, unplanned, or out of order.');
+            }
+            settled.set(event.source, Object.freeze({
+                source: event.source,
+                state: event.state,
+                complete: event.complete,
+                truncated: event.truncated,
+                fragment: cloneBounded(event.fragment, MAX_LOOKUP_PROGRESS_FRAGMENT_BYTES, 'Lookup progress fragment'),
+            }));
+        }
+        else {
+            if (mode === null || finalReceived
+                || event.sources.length !== planned.length
+                || event.sources.some((item, index) => item !== planned[index])
+                || settled.size !== planned.length) {
+                throw new TypeError('Final Lookup event arrived before every planned source settled.');
+            }
+            const settledSources = planned.map((item) => settled.get(item)).filter((item) => Boolean(item));
+            if (!options.validateFinalResult(event.result, settledSources)) {
+                throw new TypeError('Final Lookup result failed its ordinary response validator.');
+            }
+            finalResult = cloneBounded(event.result, MAX_LOOKUP_PROGRESS_FINAL_BYTES, 'Final Lookup result');
+            finalReceived = true;
+        }
+        expectedSequence += 1;
+        return snapshot();
+    }
+    function snapshot() {
+        return Object.freeze({
+            started: mode !== null,
+            mode,
+            plannedSources: Object.freeze([...planned]),
+            settledSources: Object.freeze(planned.map((item) => settled.get(item)).filter((item) => Boolean(item))),
+            finalReceived,
+            persistable: false,
+        });
+    }
+    function finish() {
+        if (!finalReceived) {
+            throw new TypeError('Lookup progress stream ended before a validated final result.');
+        }
+        return cloneBounded(finalResult, MAX_LOOKUP_PROGRESS_FINAL_BYTES, 'Final Lookup result');
+    }
+    return Object.freeze({ apply, snapshot, finish });
+}
+export function createLookupProgressNdjsonDecoder(onEvent) {
+    if (typeof onEvent !== 'function')
+        throw new TypeError('NDJSON decoder requires an event consumer.');
+    const decoder = new TextDecoder('utf-8', { fatal: true });
+    let pending = '';
+    let totalBytes = 0;
+    let eventCount = 0;
+    function consumeLine(line) {
+        if (!line)
+            return;
+        if (byteLength(line) > MAX_LOOKUP_PROGRESS_FINAL_BYTES + MAX_LOOKUP_PROGRESS_FRAGMENT_BYTES) {
+            throw new TypeError('Lookup progress line exceeds its byte bound.');
+        }
+        eventCount += 1;
+        if (eventCount > MAX_LOOKUP_PROGRESS_EVENTS) {
+            throw new TypeError('Lookup progress stream contains too many events.');
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            throw new TypeError('Lookup progress stream contains malformed JSON.');
+        }
+        onEvent(parseLookupProgressEvent(parsed));
+    }
+    function push(chunk) {
+        if (!(chunk instanceof Uint8Array))
+            throw new TypeError('NDJSON chunks must be byte arrays.');
+        totalBytes += chunk.byteLength;
+        if (totalBytes > MAX_LOOKUP_PROGRESS_STREAM_BYTES) {
+            throw new TypeError('Lookup progress stream exceeds its total byte bound.');
+        }
+        pending += decoder.decode(chunk, { stream: true });
+        let newline = pending.indexOf('\n');
+        while (newline >= 0) {
+            const line = pending.slice(0, newline).replace(/\r$/u, '');
+            pending = pending.slice(newline + 1);
+            consumeLine(line);
+            newline = pending.indexOf('\n');
+        }
+        if (byteLength(pending) > MAX_LOOKUP_PROGRESS_FINAL_BYTES + MAX_LOOKUP_PROGRESS_FRAGMENT_BYTES) {
+            throw new TypeError('Lookup progress line exceeds its byte bound.');
+        }
+    }
+    function finish() {
+        pending += decoder.decode();
+        if (pending)
+            consumeLine(pending.replace(/\r$/u, ''));
+        pending = '';
+    }
+    return Object.freeze({ push, finish });
+}

package/lib/lookup-source-progress.mjs

@@ -0,0 +1,109 @@
+// Minimal presentation-only source settlement summaries for incremental Deep
+// Lookup. These deliberately exclude raw RDAP/WHOIS bodies, contacts, URLs,
+// provider payloads, and request errors. The ordinary final Lookup response
+// remains the only authoritative and persistable result.
+const RESULT_STATES = new Set([
+    'available',
+    'expiring',
+    'for_sale',
+    'registered',
+    'unknown',
+]);
+const DIRECT_STATES = new Set([
+    'success',
+    'partial',
+    'not_found',
+    'skipped',
+    'error',
+    'unsupported',
+    'unavailable',
+    'rate_limited',
+]);
+function record(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : {};
+}
+function plannedLookupProgressSources(classified, options = {}) {
+    const sources = ['rdap', 'whois'];
+    if (classified.type === 'domain') {
+        sources.push('domain_evidence', 'registrar_rdap', 'network_context');
+        if (options.securityTxt)
+            sources.push('security_txt');
+        if (options.externalIntelligence)
+            sources.push('external_intelligence');
+        if (options.malwareHostIntelligence)
+            sources.push('malware_host_intelligence');
+        if (options.malwareIocIntelligence)
+            sources.push('malware_ioc_intelligence');
+    }
+    else if (classified.type === 'ipv4' || classified.type === 'ipv6') {
+        sources.push('reverse_dns');
+    }
+    return Object.freeze(sources);
+}
+function normalizedState(source, outcome, value) {
+    if (outcome === 'rejected')
+        return 'error';
+    if (value === null || value === undefined)
+        return 'skipped';
+    if (source === 'rdap') {
+        const rdap = record(value);
+        if (rdap.upstreamStatus === 404)
+            return 'not_found';
+        return rdap.upstreamStatus === 200 ? 'success' : 'unsupported';
+    }
+    if (source === 'whois') {
+        if (!Array.isArray(value))
+            return 'error';
+        if (value.length <= 1)
+            return 'unsupported';
+        return 'success';
+    }
+    if (source === 'domain_evidence') {
+        return record(value).deepScanComplete === false ? 'partial' : 'success';
+    }
+    const status = record(value).status;
+    if (typeof status === 'string' && DIRECT_STATES.has(status)) {
+        return status;
+    }
+    if (status === 'complete' || status === 'observed' || status === 'found')
+        return 'success';
+    if (status === 'not_applicable' || status === 'disabled')
+        return 'skipped';
+    return 'success';
+}
+function normalizeLookupSourceSettlement(source, outcome, value) {
+    const state = normalizedState(source, outcome, value);
+    const sourceRecord = record(value);
+    const resultState = source === 'domain_evidence'
+        && typeof sourceRecord.state === 'string'
+        && RESULT_STATES.has(sourceRecord.state)
+        ? sourceRecord.state
+        : null;
+    const complete = !['partial', 'error', 'unavailable', 'rate_limited'].includes(state);
+    const truncated = sourceRecord.truncated === true
+        || sourceRecord.recordsTruncated === true
+        || sourceRecord.bodyTruncated === true;
+    const limitation = outcome === 'rejected'
+        ? 'This source did not complete. No absence or safety conclusion was inferred.'
+        : state === 'skipped'
+            ? 'This source was not requested or was not applicable to the target.'
+            : state === 'unsupported'
+                ? 'This source could not provide authoritative evidence for this target.'
+                : state === 'partial'
+                    ? 'This source returned incomplete evidence; missing fields remain unknown.'
+                    : null;
+    return Object.freeze({
+        source,
+        state,
+        complete,
+        truncated,
+        fragment: Object.freeze({
+            status: state,
+            ...(resultState ? { resultState } : {}),
+            ...(limitation ? { limitation } : {}),
+        }),
+    });
+}
+export { normalizeLookupSourceSettlement, plannedLookupProgressSources, };

package/lib/lookup-threat-provider-inventory.mjs

@@ -0,0 +1,13 @@
+// One explicit inventory for optional threat-intelligence providers that can
+// contribute to Deep Lookup. Runtime orchestration and the offline conformance
+// suite both consume these exact definitions so adding a provider cannot omit
+// the shared timeout, truncation, neutral-miss, and provenance checks silently.
+import { THREATFOX_PROVIDER } from "./threatfox-intelligence.mjs";
+import { URLHAUS_PROVIDER } from "./urlhaus-intelligence.mjs";
+import { URLSCAN_PROVIDER } from "./urlscan-intelligence.mjs";
+const LOOKUP_THREAT_INTELLIGENCE_PROVIDERS = Object.freeze([
+    URLSCAN_PROVIDER,
+    URLHAUS_PROVIDER,
+    THREATFOX_PROVIDER,
+]);
+export { LOOKUP_THREAT_INTELLIGENCE_PROVIDERS, THREATFOX_PROVIDER, URLHAUS_PROVIDER, URLSCAN_PROVIDER, };