@slicedearth/whoisleuth-cli 1.32.0

package/cli/discovery-observation-snapshot.mjs

@@ -0,0 +1,315 @@
+import { createHash } from 'node:crypto';
+import { open } from 'node:fs/promises';
+import { availabilityState, bulkDnsSummary } from "./bulk-output.mjs";
+import { CliUsageError } from "./errors.mjs";
+import { writePrivateFile } from "./output-file.mjs";
+import { isValidAsciiDomainName } from "../lib/hostname.mjs";
+export const CLI_DISCOVERY_OBSERVATION_SCHEMA = 'whoisleuth.cli.discovery-observation-snapshot';
+export const CLI_DISCOVERY_OBSERVATION_VERSION = 2;
+export const MAX_DISCOVERY_OBSERVATION_BYTES = 8 * 1024 * 1024;
+const MAX_DISCOVERY_OBSERVATIONS = 500;
+const SUPPORTED_DISCOVERY_OBSERVATION_VERSIONS = new Set([1, CLI_DISCOVERY_OBSERVATION_VERSION]);
+function record(value) {
+    return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+}
+function domainValue(value) {
+    const domain = typeof value === 'string' ? value.trim().toLowerCase() : '';
+    if (!isValidAsciiDomainName(domain, { requireDot: true, requireLowercase: true })) {
+        throw new CliUsageError('Discovery observation snapshot contains an invalid domain.');
+    }
+    return domain;
+}
+function normalizedTimestamp(value, label, optional = false) {
+    if (optional && value === null)
+        return null;
+    if (typeof value !== 'string' || value.length > 64 || /[\u0000-\u001f\u007f]/u.test(value)) {
+        throw new CliUsageError(`${label} is invalid.`);
+    }
+    const parsed = Date.parse(value);
+    if (!Number.isFinite(parsed))
+        throw new CliUsageError(`${label} is invalid.`);
+    return new Date(parsed).toISOString();
+}
+function latestObservedAt(registrationObservedAt, dnsObservedAt) {
+    if (!registrationObservedAt)
+        return dnsObservedAt;
+    if (!dnsObservedAt)
+        return registrationObservedAt;
+    return registrationObservedAt >= dnsObservedAt ? registrationObservedAt : dnsObservedAt;
+}
+function snapshotConfigurationDigest(configuration) {
+    return createHash('sha256').update(JSON.stringify({
+        domains: [...configuration.domains],
+        mode: configuration.deep ? 'deep' : 'fast',
+        resolverServers: [...configuration.resolverServers],
+    })).digest('hex');
+}
+function normalizeDns(value) {
+    const dns = record(value);
+    const stringList = (candidate, maximum) => Array.isArray(candidate)
+        ? candidate.flatMap((item) => typeof item === 'string' && item.length <= 500 ? [item] : []).slice(0, maximum)
+        : [];
+    const nullableBoolean = (candidate) => typeof candidate === 'boolean' ? candidate : null;
+    if (typeof dns.status !== 'string' || !dns.status || dns.status.length > 40
+        || /[\u0000-\u001f\u007f]/u.test(dns.status))
+        return null;
+    return {
+        status: dns.status,
+        a: stringList(dns.a, 100),
+        aaaa: stringList(dns.aaaa, 100),
+        ns: stringList(dns.ns, 100),
+        mx: stringList(dns.mx, 100),
+        hasNullMx: nullableBoolean(dns.hasNullMx),
+        hasSpf: nullableBoolean(dns.hasSpf),
+        hasDmarc: nullableBoolean(dns.hasDmarc),
+    };
+}
+function normalizeObservation(value, version) {
+    const item = record(value);
+    const dns = normalizeDns(item.dns);
+    try {
+        const validAttemptStates = version === 1 ? ['error', 'success'] : ['error', 'partial', 'success'];
+        if (!dns
+            || !validAttemptStates.includes(String(item.latestAttemptState))
+            || typeof item.availabilityState !== 'string' || !item.availabilityState || item.availabilityState.length > 40
+            || /[\u0000-\u001f\u007f]/u.test(item.availabilityState)
+            || typeof item.confidence !== 'string' || !item.confidence || item.confidence.length > 40
+            || /[\u0000-\u001f\u007f]/u.test(item.confidence))
+            return null;
+        const legacyObservedAt = normalizedTimestamp(item.observedAt, 'Discovery observation time', version >= 2);
+        const registrationObservedAt = version === 1
+            ? item.availabilityState === 'unknown' ? null : legacyObservedAt
+            : normalizedTimestamp(item.registrationObservedAt, 'Discovery registration observation time', true);
+        const dnsObservedAt = version === 1
+            ? ['partial', 'success'].includes(dns.status) ? legacyObservedAt : null
+            : normalizedTimestamp(item.dnsObservedAt, 'Discovery DNS observation time', true);
+        const observedAt = latestObservedAt(registrationObservedAt, dnsObservedAt);
+        if (version >= 2 && legacyObservedAt !== observedAt)
+            return null;
+        const latestRegistrationState = version === 1
+            ? registrationObservedAt ? 'observed' : 'unavailable'
+            : item.latestRegistrationState;
+        const latestDnsState = version === 1 ? dns.status : item.latestDnsState;
+        if (!['observed', 'unavailable'].includes(String(latestRegistrationState))
+            || typeof latestDnsState !== 'string' || !latestDnsState || latestDnsState.length > 40
+            || /[\u0000-\u001f\u007f]/u.test(latestDnsState))
+            return null;
+        if (latestRegistrationState === 'observed' && registrationObservedAt === null)
+            return null;
+        return {
+            domain: domainValue(item.domain),
+            observedAt,
+            registrationObservedAt,
+            dnsObservedAt,
+            latestAttemptAt: normalizedTimestamp(item.latestAttemptAt, 'Discovery latest-attempt time') ?? '',
+            latestAttemptState: item.latestAttemptState,
+            latestRegistrationState: latestRegistrationState,
+            latestDnsState,
+            availabilityState: item.availabilityState,
+            confidence: item.confidence,
+            dns,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+function parseObservationSnapshot(text, expectedDigest) {
+    if (Buffer.byteLength(text, 'utf8') > MAX_DISCOVERY_OBSERVATION_BYTES) {
+        throw new CliUsageError(`Discovery observation snapshots are limited to ${MAX_DISCOVERY_OBSERVATION_BYTES} bytes.`);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(text.replace(/^\uFEFF/u, ''));
+    }
+    catch {
+        throw new CliUsageError('Discovery observation snapshot must be valid JSON.');
+    }
+    const document = record(parsed);
+    const version = typeof document.version === 'number' && Number.isSafeInteger(document.version)
+        ? document.version
+        : 0;
+    if (document.schema !== CLI_DISCOVERY_OBSERVATION_SCHEMA || !SUPPORTED_DISCOVERY_OBSERVATION_VERSIONS.has(version)) {
+        throw new CliUsageError(`Discovery observation snapshot must use ${CLI_DISCOVERY_OBSERVATION_SCHEMA} version 1 or ${CLI_DISCOVERY_OBSERVATION_VERSION}.`);
+    }
+    if (document.configurationDigestSha256 !== expectedDigest) {
+        throw new CliUsageError('Discovery observation snapshot does not match this candidate set, scan mode, or resolver selection.');
+    }
+    if (typeof document.generatedAt !== 'string' || !Number.isFinite(Date.parse(document.generatedAt))
+        || !Array.isArray(document.observations) || document.observations.length > MAX_DISCOVERY_OBSERVATIONS) {
+        throw new CliUsageError('Discovery observation snapshot metadata is invalid.');
+    }
+    const observations = document.observations.map((item) => normalizeObservation(item, version));
+    if (observations.some((item) => item === null) || document.observationCount !== observations.length) {
+        throw new CliUsageError('Discovery observation snapshot contains invalid evidence.');
+    }
+    const normalized = observations;
+    if (new Set(normalized.map((item) => item.domain)).size !== normalized.length) {
+        throw new CliUsageError('Discovery observation snapshot contains duplicate domains.');
+    }
+    return {
+        schema: CLI_DISCOVERY_OBSERVATION_SCHEMA,
+        version: CLI_DISCOVERY_OBSERVATION_VERSION,
+        generatedAt: normalizedTimestamp(document.generatedAt, 'Discovery snapshot generation time') ?? '',
+        configurationDigestSha256: expectedDigest,
+        observationCount: normalized.length,
+        observations: normalized,
+    };
+}
+async function readExisting(path) {
+    try {
+        const handle = await open(path, 'r');
+        try {
+            const metadata = await handle.stat();
+            if (!metadata.isFile() || metadata.size > MAX_DISCOVERY_OBSERVATION_BYTES) {
+                throw new CliUsageError(`Discovery observation snapshots are limited to ${MAX_DISCOVERY_OBSERVATION_BYTES} bytes.`);
+            }
+            return await handle.readFile({ encoding: 'utf8' });
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    catch (error) {
+        if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT')
+            return null;
+        throw error;
+    }
+}
+function currentAttempt(domain, item, previous, generatedAt, deep) {
+    const availability = item.ok ? record(record(item.result).availability) : {};
+    const currentAvailabilityState = availabilityState(item) || 'unknown';
+    const registrationObserved = item.ok && currentAvailabilityState !== 'unknown';
+    const currentDns = bulkDnsSummary(item);
+    const dnsComplete = deep && currentDns.status === 'success';
+    const dnsRetainable = deep && currentDns.status === 'partial';
+    const previousDnsRetainable = previous && ['partial', 'success'].includes(previous.dns.status);
+    const dns = dnsComplete || (dnsRetainable && !previousDnsRetainable)
+        ? currentDns
+        : previousDnsRetainable ? previous.dns : currentDns;
+    const registrationObservedAt = registrationObserved ? generatedAt : previous?.registrationObservedAt ?? null;
+    const dnsObservedAt = dnsComplete || (dnsRetainable && !previousDnsRetainable)
+        ? generatedAt
+        : previous?.dnsObservedAt ?? null;
+    const unavailableComponents = [];
+    if (!registrationObserved)
+        unavailableComponents.push('registration');
+    if (deep && !dnsComplete)
+        unavailableComponents.push('dns');
+    const attemptState = !item.ok
+        ? 'error'
+        : unavailableComponents.length ? 'partial' : 'success';
+    const confidence = registrationObserved && typeof availability.confidence === 'string'
+        ? availability.confidence.replace(/[\u0000-\u001f\u007f]+/gu, ' ').trim().slice(0, 40) || 'unknown'
+        : previous?.confidence ?? 'unknown';
+    const retainedAvailabilityState = registrationObserved
+        ? currentAvailabilityState
+        : previous?.availabilityState ?? 'unknown';
+    return {
+        registrationObserved,
+        dnsComplete,
+        unavailableComponents,
+        observation: {
+            domain,
+            observedAt: latestObservedAt(registrationObservedAt, dnsObservedAt),
+            registrationObservedAt,
+            dnsObservedAt,
+            latestAttemptAt: generatedAt,
+            latestAttemptState: attemptState,
+            latestRegistrationState: registrationObserved ? 'observed' : 'unavailable',
+            latestDnsState: deep ? currentDns.status : 'not_requested',
+            availabilityState: retainedAvailabilityState,
+            confidence,
+            dns,
+        },
+    };
+}
+function materialChanges(previous, current, attempt = {
+    dnsComplete: current.latestDnsState === 'success',
+    registrationObserved: current.latestRegistrationState === 'observed',
+}) {
+    const changes = [];
+    const compare = (field, before, after) => {
+        if (JSON.stringify(before) !== JSON.stringify(after))
+            changes.push({ field, before, after });
+    };
+    if (attempt.registrationObserved) {
+        compare('availabilityState', previous.availabilityState, current.availabilityState);
+        compare('confidence', previous.confidence, current.confidence);
+    }
+    if (attempt.dnsComplete) {
+        for (const field of ['a', 'aaaa', 'ns', 'mx', 'hasNullMx', 'hasSpf', 'hasDmarc']) {
+            compare(`dns.${field}`, previous.dns[field], current.dns[field]);
+        }
+        compare('dns.status', previous.dns.status, current.dns.status);
+    }
+    return changes;
+}
+async function updateDiscoveryObservationSnapshot(path, candidates, items, configuration, generatedAt) {
+    if (!path || path.length > 1024 || /[\u0000-\u001f\u007f]/u.test(path)) {
+        throw new CliUsageError('Discovery observation snapshot path must be bounded text without control characters.');
+    }
+    if (candidates.length !== items.length || candidates.length > MAX_DISCOVERY_OBSERVATIONS) {
+        throw new CliUsageError(`Discovery observation snapshots support at most ${MAX_DISCOVERY_OBSERVATIONS} aligned results.`);
+    }
+    const currentGeneratedAt = normalizedTimestamp(generatedAt, 'Discovery observation generation time') ?? '';
+    const domains = candidates.map((candidate) => domainValue(candidate.domain));
+    const digest = snapshotConfigurationDigest({ domains, ...configuration });
+    const priorText = await readExisting(path);
+    const prior = priorText === null ? null : parseObservationSnapshot(priorText, digest);
+    const priorByDomain = new Map((prior?.observations || []).map((item) => [item.domain, item]));
+    const observations = [];
+    const changed = [];
+    const unavailable = [];
+    const unavailableComponents = [];
+    for (let index = 0; index < domains.length; index++) {
+        const domain = domains[index];
+        const item = items[index];
+        if (!domain || !item)
+            throw new CliUsageError('Discovery observation snapshot alignment failed.');
+        const previous = priorByDomain.get(domain);
+        const attempt = currentAttempt(domain, item, previous, currentGeneratedAt, configuration.deep);
+        const current = attempt.observation;
+        if (attempt.unavailableComponents.length) {
+            unavailable.push(domain);
+            unavailableComponents.push({ domain, components: attempt.unavailableComponents });
+        }
+        observations.push(current);
+        if (previous) {
+            const changes = materialChanges(previous, current, attempt);
+            if (changes.length)
+                changed.push({ domain, changes });
+        }
+    }
+    const document = {
+        schema: CLI_DISCOVERY_OBSERVATION_SCHEMA,
+        version: CLI_DISCOVERY_OBSERVATION_VERSION,
+        generatedAt: currentGeneratedAt,
+        configurationDigestSha256: digest,
+        observationCount: observations.length,
+        observations,
+    };
+    const content = `${JSON.stringify(document, null, 2)}\n`;
+    if (Buffer.byteLength(content, 'utf8') > MAX_DISCOVERY_OBSERVATION_BYTES) {
+        throw new CliUsageError(`Discovery observation snapshots are limited to ${MAX_DISCOVERY_OBSERVATION_BYTES} bytes.`);
+    }
+    await writePrivateFile(path, content, {
+        force: prior !== null,
+        existingFileMessage: 'Discovery observation snapshot already exists and could not be replaced.',
+    });
+    return {
+        baselineCreated: prior === null,
+        previousGeneratedAt: prior?.generatedAt ?? null,
+        currentGeneratedAt,
+        changed,
+        unavailable,
+        unavailableComponents,
+        unchanged: prior === null ? 0 : domains.filter((domain) => !changed.some((item) => item.domain === domain) && !unavailable.includes(domain)).length,
+        limitations: [
+            'Only bounded registration-state and DNS summaries are retained; raw registry records, contacts, page contents, and request details are excluded.',
+            'Registration and DNS retain separate observation times. A failed or partial component preserves its previous usable evidence and is reported as unavailable rather than as a removal or negative finding.',
+            'Material differences are observation changes for analyst review and do not establish ownership, control, intent, safety, or maliciousness.',
+        ],
+    };
+}
+export { materialChanges, parseObservationSnapshot, snapshotConfigurationDigest, updateDiscoveryObservationSnapshot, };

package/cli/discovery-scan-command-runner.mjs

@@ -0,0 +1,141 @@
+import { createReadStream } from 'node:fs';
+import { classifyQuery } from "../lib/classify.mjs";
+import { normalizeSelectedDnsResolvers } from "../lib/dns-resolver-selection.mjs";
+import { runUnifiedLookup } from "../lib/lookup.mjs";
+import { createBulkCheckpointWriter } from "./bulk-checkpoint.mjs";
+import { buildDiscoveryScanDocument, formatDiscoveryScanCsv, formatDiscoveryScanDomains, formatDiscoveryScanJsonLines, formatTerminalDiscoveryScan, parseDiscoveryScanAllowlist, readDiscoveryScanListBounded, runDiscoveryScanChunks, } from "./discovery-scan.mjs";
+import { updateDiscoveryObservationSnapshot } from "./discovery-observation-snapshot.mjs";
+import { generateDiscoveryCandidates } from "./discovery-workflow.mjs";
+import { boundedCliErrorMessage, CliUsageError } from "./errors.mjs";
+import EXIT_CODES from "./exit-codes.mjs";
+import { formatJsonDocument } from "./formatters/json.mjs";
+import { createCliProgressEvents } from "./progress-events.mjs";
+async function readAllowlist(source, dependencies, classify) {
+    if (!source)
+        return new Set();
+    try {
+        const text = dependencies.readDiscoveryAllowlist
+            ? await dependencies.readDiscoveryAllowlist(source)
+            : await readDiscoveryScanListBounded(createReadStream(source, { highWaterMark: 64 * 1024 }));
+        return parseDiscoveryScanAllowlist(text, classify);
+    }
+    catch (error) {
+        if (error instanceof CliUsageError)
+            throw error;
+        throw new CliUsageError(`Could not read discovery scan allowlist: ${boundedCliErrorMessage(error, 'Input could not be read')}`);
+    }
+}
+async function runDiscoveryScanCommand(args, dependencies, context) {
+    const eventProgress = createCliProgressEvents(context.stderr, {
+        command: 'discover-scan',
+        enabled: args.events,
+        ...(dependencies.now ? { now: dependencies.now } : {}),
+    });
+    context.setEventProgress(eventProgress);
+    eventProgress.emit({ event: 'started' });
+    const { metadata: generationMetadata, result } = await generateDiscoveryCandidates(args, dependencies, context);
+    const candidates = result.candidates
+        .filter((candidate) => typeof candidate.domain === 'string' && candidate.domain.length > 0)
+        .slice(0, args.scanLimit);
+    if (!candidates.length)
+        throw new CliUsageError('discover-scan did not generate any valid candidates to collect.');
+    const queries = candidates.map((candidate) => String(candidate.domain));
+    const classify = dependencies.classifyQuery || classifyQuery;
+    let resolverServers = [];
+    if (args.resolverText) {
+        try {
+            resolverServers = normalizeSelectedDnsResolvers(args.resolverText);
+        }
+        catch (error) {
+            throw new CliUsageError(boundedCliErrorMessage(error, 'Invalid DNS resolver selection'));
+        }
+    }
+    const allowlist = await readAllowlist(args.allowlistSource, dependencies, classify);
+    const checkpointWriter = dependencies.createBulkCheckpointWriter || createBulkCheckpointWriter;
+    const checkpoint = args.checkpoint
+        ? await checkpointWriter({
+            path: args.checkpoint,
+            queries,
+            deep: args.deep,
+            resume: args.resume,
+            classifyQuery: classify,
+            ...(dependencies.now ? { now: dependencies.now } : {}),
+        })
+        : null;
+    const indicator = context.beginProgress(`Collecting 0 of ${queries.length} generated candidates`);
+    let completed = checkpoint?.initialResults.length || 0;
+    if (completed)
+        indicator.update(`Resumed ${completed} of ${queries.length} generated candidates`);
+    let checkpointFailure = null;
+    let items;
+    try {
+        items = await runDiscoveryScanChunks(queries, {
+            deep: args.deep,
+            chunkSize: args.chunkSize,
+            concurrency: args.concurrency,
+            classifyQuery: classify,
+            runUnifiedLookup: dependencies.runUnifiedLookup || runUnifiedLookup,
+            ...(resolverServers.length ? { dnsResolverServers: resolverServers } : {}),
+            ...(checkpoint ? { initialResults: checkpoint.initialResults } : {}),
+            ...(dependencies.signal ? { signal: dependencies.signal } : {}),
+            onItemSettled: (item) => {
+                completed += 1;
+                indicator.update(`Collected ${completed} of ${queries.length} generated candidates`);
+                eventProgress.emit({ event: 'item_settled', index: item.index, ok: item.ok });
+                checkpoint?.record(item);
+            },
+        });
+    }
+    finally {
+        context.endProgress();
+        try {
+            await checkpoint?.flush();
+        }
+        catch (error) {
+            checkpointFailure = error;
+        }
+    }
+    const snapshot = args.observationSnapshot
+        ? await updateDiscoveryObservationSnapshot(args.observationSnapshot, candidates, items, { deep: args.deep, resolverServers }, generationMetadata.generatedAt)
+        : null;
+    const document = buildDiscoveryScanDocument(candidates, items, {
+        generatedAt: generationMetadata.generatedAt,
+        seed: generationMetadata.seed,
+        preset: generationMetadata.preset,
+        keyboardLayout: generationMetadata.keyboardLayout,
+        tlds: generationMetadata.tlds,
+        mutationFamilies: generationMetadata.mutationFamilies,
+        generatedCandidateCount: result.candidates.length,
+        selectedCandidateCount: candidates.length,
+        scanLimit: args.scanLimit,
+        chunkSize: args.chunkSize,
+        concurrency: args.concurrency,
+        deep: args.deep,
+        filter: args.filter,
+        resolverServers,
+    }, allowlist, snapshot);
+    if (!args.quiet) {
+        if (args.output === 'json')
+            context.writeStdout(formatJsonDocument(document));
+        else if (args.output === 'jsonl')
+            context.writeStdout(formatDiscoveryScanJsonLines(document));
+        else if (args.output === 'csv')
+            context.writeStdout(formatDiscoveryScanCsv(document));
+        else if (args.output === 'domains')
+            context.writeStdout(formatDiscoveryScanDomains(document));
+        else
+            context.writeStdout(context.terminal(formatTerminalDiscoveryScan(document), args.color));
+    }
+    if (checkpointFailure) {
+        eventProgress.emit({ event: 'warning', state: 'checkpoint_unavailable' });
+        if (!eventProgress.enabled) {
+            context.writeStderr(`Checkpoint warning: ${boundedCliErrorMessage(checkpointFailure, 'Checkpoint could not be written')}. Completed output is still available.\n`);
+        }
+    }
+    const exitCode = checkpointFailure || items.some((item) => !item.ok)
+        ? EXIT_CODES.PARTIAL_FAILURE
+        : EXIT_CODES.SUCCESS;
+    eventProgress.emit({ event: 'completed', exitCode });
+    return exitCode;
+}
+export { runDiscoveryScanCommand };