@slicedearth/whoisleuth-cli 2.3.0 → 2.5.1

package/cli/config-profile.mjs

@@ -2,9 +2,9 @@ import { Buffer } from 'node:buffer';
 import { homedir } from 'node:os';
 import { join, resolve } from 'node:path';
 import { CliUsageError } from "./errors.mjs";
-import { cliMetaActionForInvocation } from "./command-reference.mjs";
+import { cliInvocationOptionIndices, cliMetaActionForInvocation } from "./command-reference.mjs";
 import { readBoundedRegularTextFile } from "../lib/bounded-file.mjs";
-import { scanBoundedJson } from "../lib/bounded-json.mjs";
+import { scanBoundedJson } from "../packages/analysis/bounded-json.mjs";
 export const CLI_CONFIG_SCHEMA = 'whoisleuth.cli.config';
 export const CLI_CONFIG_VERSION = 1;
 export const MAX_CLI_CONFIG_BYTES = 64 * 1024;
@@ -107,7 +107,8 @@ async function readConfigFile(path) {
 }
 function suppliedGroups(argumentsList) {
     const groups = new Set();
-    for (const argument of argumentsList) {
+    for (const index of cliInvocationOptionIndices(argumentsList)) {
+        const argument = argumentsList[index];
         const group = OPTION_GROUPS[argument];
         if (group)
             groups.add(group);
@@ -119,10 +120,11 @@ function suppliedGroups(argumentsList) {
 export async function resolveCliProfileArguments(argv, options = {}) {
     if (cliMetaActionForInvocation(argv))
         return [...argv];
+    const optionIndices = cliInvocationOptionIndices(argv);
     // registry-scaffold owns --profile as the capability template to generate.
     // Do not reinterpret it as a global CLI-default profile.
     if (argv[0] === 'registry-scaffold') {
-        if (argv.includes('--config')) {
+        if (argv.some((argument, index) => argument === '--config' && optionIndices.has(index))) {
             throw new CliUsageError('registry-scaffold does not accept global CLI configuration options.');
         }
         return [...argv];
@@ -134,7 +136,7 @@ export async function resolveCliProfileArguments(argv, options = {}) {
         const argument = argv[index];
         if (argument === undefined)
             break;
-        if (argument === '--config' || argument === '--profile') {
+        if (optionIndices.has(index) && (argument === '--config' || argument === '--profile')) {
             const next = argv[++index];
             if (!next || next.startsWith('-'))
                 throw new CliUsageError(`${argument} requires one value.`);
@@ -177,7 +179,7 @@ export async function resolveCliProfileArguments(argv, options = {}) {
         throw new CliUsageError(`CLI profile "${selectedName}" was not found.`);
     if (!retained.length)
         throw new CliUsageError('A CLI profile must accompany a command.');
-    const explicitGroups = suppliedGroups(retained.slice(1));
+    const explicitGroups = suppliedGroups(retained);
     const defaults = [];
     for (let index = 0; index < selected.arguments.length; index += 1) {
         const argument = selected.arguments[index];

package/cli/ct-event-intake.mjs

@@ -1,14 +1,15 @@
 import { createHash } from 'node:crypto';
 import { domainToASCII } from 'node:url';
-import { exactKeys, requireBoundedString, requireIsoTimestamp, requireRecord, } from "../lib/bounded-contract-normalizers.mjs";
+import { compareCodeUnits, exactKeys, requireBoundedString, requireIsoTimestamp, requireRecord, } from "../lib/bounded-contract-normalizers.mjs";
+import { EXTERNAL_FINDINGS_SCHEMA, EXTERNAL_FINDINGS_VERSION, MAX_EXTERNAL_FINDINGS, MAX_EXTERNAL_FINDING_DOMAINS, MAX_EXTERNAL_FINDINGS_PER_DOMAIN, MAX_EXTERNAL_FINDING_LIMITATIONS, MAX_EXTERNAL_FINDING_LIMITATION_LENGTH, retainExternalFindingLimitations, } from "../packages/interchange/external-findings-import.mjs";
 export const CT_EVENT_BATCH_SCHEMA = 'whoisleuth.ct-event-batch';
 export const CT_EVENT_BATCH_VERSION = 1;
 export const MAX_CT_EVENT_INPUT_BYTES = 4 * 1024 * 1024;
 export const MAX_CT_EVENTS = 500;
 export const MAX_CT_EVENT_NAMES = 100;
-export const MAX_CT_EXTERNAL_FINDINGS = 100;
-export const MAX_CT_EXTERNAL_DOMAINS = 25;
-export const MAX_CT_EXTERNAL_FINDINGS_PER_DOMAIN = 20;
+export const MAX_CT_EXTERNAL_FINDINGS = MAX_EXTERNAL_FINDINGS;
+export const MAX_CT_EXTERNAL_DOMAINS = MAX_EXTERNAL_FINDING_DOMAINS;
+export const MAX_CT_EXTERNAL_FINDINGS_PER_DOMAIN = MAX_EXTERNAL_FINDINGS_PER_DOMAIN;
 const ROOT_KEYS = new Set(['schema', 'version', 'source', 'events']);
 const SOURCE_KEYS = new Set(['name', 'reference', 'collectedAt']);
 const EVENT_KEYS = new Set([
@@ -47,9 +48,9 @@ function domain(value, label) {
 function boundedLimitations(value, label) {
     if (value === undefined)
         return Object.freeze([]);
-    if (!Array.isArray(value) || value.length > 8)
-        throw new TypeError(`${label} must contain no more than 8 entries.`);
-    return Object.freeze([...new Set(value.map((item, index) => requireBoundedString(item, `${label}[${index}]`, 240)))]);
+    if (!Array.isArray(value) || value.length > MAX_EXTERNAL_FINDING_LIMITATIONS)
+        throw new TypeError(`${label} must contain no more than ${MAX_EXTERNAL_FINDING_LIMITATIONS} entries.`);
+    return Object.freeze([...new Set(value.map((item, index) => requireBoundedString(item, `${label}[${index}]`, MAX_EXTERNAL_FINDING_LIMITATION_LENGTH)))]);
 }
 export function buildCtEventFindings(inputRaw) {
     const input = requireRecord(inputRaw, 'Certificate event batch');
@@ -117,9 +118,9 @@ export function buildCtEventFindings(inputRaw) {
     });
     const seen = new Set();
     const unique = candidates
-        .sort((left, right) => left.domain.localeCompare(right.domain)
-        || left.observedAt.localeCompare(right.observedAt)
-        || left.structuredObservation.value.localeCompare(right.structuredObservation.value))
+        .sort((left, right) => compareCodeUnits(left.domain, right.domain)
+        || compareCodeUnits(left.observedAt, right.observedAt)
+        || compareCodeUnits(left.structuredObservation.value, right.structuredObservation.value))
         .filter((item) => {
         const key = `${item.domain}\u0000${item.observedAt}\u0000${item.structuredObservation.value}\u0000${item.structuredObservation.eventId}`;
         if (seen.has(key))
@@ -155,19 +156,18 @@ export function buildCtEventFindings(inputRaw) {
             namesComplete: finding.structuredObservation.namesComplete
                 && selectedNamesByEvent.get(finding.structuredObservation.eventId) === finding.structuredObservation.dnsNameCount,
         }),
-        limitations: Object.freeze([
-            ...finding.limitations,
+        limitations: retainExternalFindingLimitations(finding.limitations, [
             'The supplied event is an observation, not proof that the certificate was served by the named domain or that the domain operator requested it.',
             ...(finding.structuredObservation.namesComplete
                 && selectedNamesByEvent.get(finding.structuredObservation.eventId) !== finding.structuredObservation.dnsNameCount
                 ? ['The bounded import did not retain every DNS name from this certificate event.']
                 : []),
-            ...(truncated ? [`The batch contained ${unique.length} unique domain observations; this import preserves the first ${MAX_CT_EXTERNAL_FINDINGS} in deterministic order.`] : []),
-        ].slice(0, 8)),
+            ...(truncated ? [`The batch contained ${unique.length} unique domain observations; this import preserves the first ${selected.length} in deterministic order.`] : []),
+        ]),
     }));
     return Object.freeze({
-        schema: 'whoisleuth.external-findings',
-        schemaVersion: 4,
+        schema: EXTERNAL_FINDINGS_SCHEMA,
+        schemaVersion: EXTERNAL_FINDINGS_VERSION,
         source,
         findings: Object.freeze(findings),
     });

package/cli/discover.mjs

@@ -1,6 +1,7 @@
 import { Buffer } from 'node:buffer';
 import { decodeBoundedUtf8 } from "../lib/bounded-file.mjs";
-import { CliUsageError } from "./arguments.mjs";
+import { CliUsageError } from "./errors.mjs";
+import { normalizeDiscoverySuffix } from "../packages/analysis/registrable-domain.mjs";
 const DEFAULT_DISCOVERY_TLDS = Object.freeze(['com', 'net', 'org']);
 const MAX_DISCOVERY_TLD_TEXT_LENGTH = 1024;
 const MAX_DISCOVERY_TLD_TOKENS_INSPECTED = 80;
@@ -40,9 +41,9 @@ function normalizeDiscoveryTlds(raw, maximum) {
     const values = [];
     const seen = new Set();
     for (const token of tokens) {
-        const value = token.toLowerCase().replace(/^\./, '');
-        if (!/^[a-z]{2,63}$/.test(value))
-            throw new CliUsageError(`Invalid TLD "${token}".`);
+        const value = normalizeDiscoverySuffix(token);
+        if (!value)
+            throw new CliUsageError(`Invalid TLD or public suffix "${token}".`);
         if (seen.has(value))
             continue;
         seen.add(value);

package/cli/discovery-observation-snapshot.mjs

@@ -2,9 +2,9 @@ import { createHash } from 'node:crypto';
 import { availabilityState, bulkDnsSummary } from "./bulk-output.mjs";
 import { boundedCliInputError, CliUsageError } from "./errors.mjs";
 import { writePrivateFile } from "./output-file.mjs";
-import { isValidAsciiDomainName } from "../lib/hostname.mjs";
+import { isValidAsciiDomainName } from "../packages/contracts/domain-name.mjs";
 import { readBoundedRegularTextFile } from "../lib/bounded-file.mjs";
-import { scanBoundedJson } from "../lib/bounded-json.mjs";
+import { scanBoundedJson } from "../packages/analysis/bounded-json.mjs";
 import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
 export const CLI_DISCOVERY_OBSERVATION_SCHEMA = 'whoisleuth.cli.discovery-observation-snapshot';
 export const CLI_DISCOVERY_OBSERVATION_VERSION = 2;
@@ -171,6 +171,20 @@ async function readExisting(path) {
         throw boundedCliInputError(error, 'Discovery observation snapshot');
     }
 }
+const DNS_MATERIAL_FIELDS = ['a', 'aaaa', 'ns', 'mx', 'hasNullMx', 'hasSpf', 'hasDmarc', 'status'];
+function comparableValue(value) {
+    return Array.isArray(value) ? [...new Set(value.map((item) => String(item).toLowerCase()))].sort() : value;
+}
+function dnsMaterial(dns) {
+    return JSON.stringify(DNS_MATERIAL_FIELDS.map((field) => comparableValue(dns[field])));
+}
+function componentRecency(previousAt, previousMaterial, currentAt, currentMaterial) {
+    if (!previousAt || Date.parse(currentAt) > Date.parse(previousAt))
+        return 'newer';
+    if (Date.parse(currentAt) < Date.parse(previousAt))
+        return 'older';
+    return previousMaterial === currentMaterial ? 'equal' : 'conflicting';
+}
 function currentAttempt(domain, item, previous, generatedAt, deep) {
     const suppliedObservedAt = item.observedAt === null || item.observedAt === undefined
         ? null
@@ -182,35 +196,46 @@ function currentAttempt(domain, item, previous, generatedAt, deep) {
     const availability = item.ok ? record(record(item.result).availability) : {};
     const currentAvailabilityState = availabilityState(item) || 'unknown';
     const registrationObserved = item.ok && currentAvailabilityState !== 'unknown';
+    const currentConfidence = registrationObserved && typeof availability.confidence === 'string'
+        ? availability.confidence.replace(/[\u0000-\u001f\u007f]+/gu, ' ').trim().slice(0, 40) || 'unknown'
+        : 'unknown';
     const currentDns = bulkDnsSummary(item);
     const dnsComplete = deep && currentDns.status === 'success';
     const dnsRetainable = deep && currentDns.status === 'partial';
     const previousDnsRetainable = previous && ['partial', 'success'].includes(previous.dns.status);
-    const dns = dnsComplete || (dnsRetainable && !previousDnsRetainable)
-        ? currentDns
+    const registrationRecency = componentRecency(previous?.registrationObservedAt, JSON.stringify([previous?.availabilityState, previous?.confidence]), attemptAt, JSON.stringify([currentAvailabilityState, currentConfidence]));
+    const dnsRecency = componentRecency(previous?.dnsObservedAt, previous ? dnsMaterial(previous.dns) : '', attemptAt, dnsMaterial(currentDns));
+    const registrationAdmitted = registrationObserved && ['newer', 'equal'].includes(registrationRecency);
+    const dnsAdmitted = (dnsComplete || (dnsRetainable && !previousDnsRetainable)) && ['newer', 'equal'].includes(dnsRecency);
+    const dns = dnsAdmitted
+        ? dnsRecency === 'equal' && previous ? previous.dns : currentDns
         : previousDnsRetainable ? previous.dns : currentDns;
-    const registrationObservedAt = registrationObserved ? attemptAt : previous?.registrationObservedAt ?? null;
-    const dnsObservedAt = dnsComplete || (dnsRetainable && !previousDnsRetainable)
+    const registrationObservedAt = registrationAdmitted ? attemptAt : previous?.registrationObservedAt ?? null;
+    const dnsObservedAt = dnsAdmitted
         ? attemptAt
         : previous?.dnsObservedAt ?? null;
     const unavailableComponents = [];
-    if (!registrationObserved)
+    if (!registrationAdmitted)
         unavailableComponents.push('registration');
-    if (deep && !dnsComplete)
+    if (deep && !(dnsComplete && dnsAdmitted))
         unavailableComponents.push('dns');
     const attemptState = !item.ok
         ? 'error'
         : unavailableComponents.length ? 'partial' : 'success';
-    const confidence = registrationObserved && typeof availability.confidence === 'string'
-        ? availability.confidence.replace(/[\u0000-\u001f\u007f]+/gu, ' ').trim().slice(0, 40) || 'unknown'
+    const confidence = registrationAdmitted
+        ? currentConfidence
         : previous?.confidence ?? 'unknown';
-    const retainedAvailabilityState = registrationObserved
+    const retainedAvailabilityState = registrationAdmitted
         ? currentAvailabilityState
         : previous?.availabilityState ?? 'unknown';
     return {
-        registrationObserved,
-        dnsComplete,
+        registrationObserved: registrationAdmitted,
+        dnsComplete: dnsComplete && dnsAdmitted,
         unavailableComponents,
+        withheld: [
+            ...(registrationObserved && !registrationAdmitted ? [`${registrationRecency === 'older' ? 'Older' : 'Conflicting equal-time'} registration observations`] : []),
+            ...((dnsComplete || dnsRetainable) && ['older', 'conflicting'].includes(dnsRecency) ? [`${dnsRecency === 'older' ? 'Older' : 'Conflicting equal-time'} DNS observations`] : []),
+        ],
         observation: {
             domain,
             observedAt: latestObservedAt(registrationObservedAt, dnsObservedAt),
@@ -219,7 +244,7 @@ function currentAttempt(domain, item, previous, generatedAt, deep) {
             latestAttemptAt: attemptAt,
             collectionOrigin: item.collectionOrigin ?? 'current_run',
             latestAttemptState: attemptState,
-            latestRegistrationState: registrationObserved ? 'observed' : 'unavailable',
+            latestRegistrationState: registrationAdmitted ? 'observed' : 'unavailable',
             latestDnsState: deep ? currentDns.status : 'not_requested',
             availabilityState: retainedAvailabilityState,
             confidence,
@@ -233,7 +258,7 @@ function materialChanges(previous, current, attempt = {
 }) {
     const changes = [];
     const compare = (field, before, after) => {
-        if (JSON.stringify(before) !== JSON.stringify(after))
+        if (JSON.stringify(comparableValue(before)) !== JSON.stringify(comparableValue(after)))
             changes.push({ field, before, after });
     };
     if (attempt.registrationObserved) {
@@ -241,10 +266,9 @@ function materialChanges(previous, current, attempt = {
         compare('confidence', previous.confidence, current.confidence);
     }
     if (attempt.dnsComplete) {
-        for (const field of ['a', 'aaaa', 'ns', 'mx', 'hasNullMx', 'hasSpf', 'hasDmarc']) {
+        for (const field of DNS_MATERIAL_FIELDS) {
             compare(`dns.${field}`, previous.dns[field], current.dns[field]);
         }
-        compare('dns.status', previous.dns.status, current.dns.status);
     }
     return changes;
 }
@@ -265,6 +289,7 @@ async function updateDiscoveryObservationSnapshot(path, candidates, items, confi
     const changed = [];
     const unavailable = [];
     const unavailableComponents = [];
+    const withheld = new Map();
     for (let index = 0; index < domains.length; index++) {
         const domain = domains[index];
         const item = items[index];
@@ -273,6 +298,8 @@ async function updateDiscoveryObservationSnapshot(path, candidates, items, confi
         const previous = priorByDomain.get(domain);
         const attempt = currentAttempt(domain, item, previous, currentGeneratedAt, configuration.deep);
         const current = attempt.observation;
+        for (const reason of attempt.withheld)
+            withheld.set(reason, (withheld.get(reason) ?? 0) + 1);
         if (attempt.unavailableComponents.length) {
             unavailable.push(domain);
             unavailableComponents.push({ domain, components: attempt.unavailableComponents });
@@ -310,7 +337,8 @@ async function updateDiscoveryObservationSnapshot(path, candidates, items, confi
         unchanged: prior === null ? 0 : domains.filter((domain) => !changed.some((item) => item.domain === domain) && !unavailable.includes(domain)).length,
         limitations: [
             'Only bounded registration-state and DNS summaries are retained; raw registry records, contacts, page contents, and request details are excluded.',
-            'Registration and DNS retain separate observation times. A failed or partial component preserves its previous usable evidence and is reported as unavailable rather than as a removal or negative finding.',
+            'Registration and DNS retain separate observation times. Failed, incomplete, older or conflicting equal-time components preserve the prior usable baseline and are unavailable for a new comparison, not evidence of removal.',
+            ...[...withheld].map(([reason, count]) => `${reason}: ${count}. The retained component baseline was not replaced.`),
             'Material differences are observation changes for analyst review and do not establish ownership, control, intent, safety, or maliciousness.',
         ],
     };

package/cli/discovery-scan-command-runner.mjs

@@ -23,7 +23,7 @@ async function readAllowlist(source, dependencies, context, classify) {
     catch (error) {
         if (error instanceof CliUsageError)
             throw error;
-        throw new CliUsageError(`Could not read discovery scan allowlist: ${boundedCliErrorMessage(error, 'Input could not be read')}`);
+        throw new CliUsageError(`Could not read discovery scan allowlist: ${boundedCliErrorMessage(error, 'Input could not be read')}`, 'input_unavailable');
     }
 }
 async function runDiscoveryScanCommand(args, dependencies, context) {
@@ -52,7 +52,7 @@ async function runDiscoveryScanCommand(args, dependencies, context) {
     const queries = candidates.map((candidate) => String(candidate.domain));
     if (args.plan) {
         const document = buildCollectionPreflight({
-            command: 'discover-scan', targetCount: queries.length, targetLimit: args.deep ? 50 : 500,
+            command: 'discover-scan', targetCount: queries.length,
             deep: args.deep, concurrency: args.concurrency, output: args.output, checkpoint: false,
             customResolvers: Boolean(args.resolverText), allowlist: Boolean(args.allowlistSource),
         });
@@ -86,7 +86,7 @@ async function runDiscoveryScanCommand(args, dependencies, context) {
             throw new CliUsageError('Bulk checkpoint collection context does not match the current DNS resolver selection.');
         }
         if (item.observedAt === null || item.observedAt === undefined) {
-            throw new CliUsageError('Discovery resume requires an observation time for every retained checkpoint result.');
+            throw new CliUsageError('Discovery resume requires an observation time for every retained checkpoint result.', 'missing_input');
         }
     }
     const indicator = context.beginProgress(`Collecting 0 of ${queries.length} generated candidates`);
@@ -163,6 +163,8 @@ async function runDiscoveryScanCommand(args, dependencies, context) {
             context.writeStdout(formatDiscoveryScanJsonLines(document));
         else if (args.output === 'csv')
             context.writeStdout(formatDiscoveryScanCsv(document));
+        else if (args.output === 'csv_metadata')
+            context.writeStdout(formatDiscoveryScanCsv(document, true));
         else if (args.output === 'domains')
             context.writeStdout(formatDiscoveryScanDomains(document));
         else

package/cli/discovery-scan.mjs

@@ -1,8 +1,9 @@
 import { createHash } from 'node:crypto';
 import { Buffer } from 'node:buffer';
 import { decodeBoundedUtf8 } from "../lib/bounded-file.mjs";
+import { MAX_DISCOVERY_SUFFIX_LENGTH } from "../packages/analysis/registrable-domain.mjs";
 import { runBulkLookups } from "./bulk.mjs";
-import { REGISTERED_STATES, availabilityState, bulkDnsSummary } from "./bulk-output.mjs";
+import { REGISTERED_STATES, BULK_CSV_METADATA_COLUMNS, availabilityState, bulkCsvMetadataValues, bulkDnsSummary } from "./bulk-output.mjs";
 import { cliCsvCell } from "./csv.mjs";
 import { CliUsageError } from "./errors.mjs";
 import { terminalSafeJson } from "./formatters/json.mjs";
@@ -70,7 +71,7 @@ function parseDiscoveryScanAllowlist(text, classifyQuery) {
 function normalizedCandidate(candidate) {
     const domain = boundedText(candidate.domain, 253).toLowerCase();
     const source = boundedText(candidate.source, 253).toLowerCase();
-    const tld = boundedText(candidate.tld, 63).toLowerCase();
+    const tld = boundedText(candidate.tld, MAX_DISCOVERY_SUFFIX_LENGTH).toLowerCase();
     const mutationTypes = Array.isArray(candidate.mutationTypes)
         ? [...new Set(candidate.mutationTypes.flatMap((value) => {
                 const normalized = boundedText(value, 80);
@@ -308,12 +309,14 @@ function formatDiscoveryScanJsonLines(document) {
         ? `${document.results.map((item) => terminalSafeJson({ ...item, generatedAt: document.generatedAt })).join('\n')}\n`
         : '';
 }
-function formatDiscoveryScanCsv(document) {
-    const header = ['domain', 'availability', 'confidence', 'review_lane', 'mutation_types', 'a', 'aaaa', 'ns', 'mx', 'relationship_ids', 'error'];
+function formatDiscoveryScanCsv(document, includeMetadata = false) {
+    const header = ['domain', 'availability', 'confidence', 'review_lane', 'mutation_types', 'a', 'aaaa', 'ns', 'mx', 'relationship_ids', 'error',
+        ...(includeMetadata ? BULK_CSV_METADATA_COLUMNS : [])];
     const rows = document.results.map((item) => [
         item.domain, item.availabilityState, item.confidence, item.review.lane, item.mutationTypes,
         item.dnsSummary.a, item.dnsSummary.aaaa, item.dnsSummary.ns, item.dnsSummary.mx,
         item.relationshipIds, 'error' in item ? item.error : '',
+        ...(includeMetadata ? bulkCsvMetadataValues(item, document) : []),
     ].map(cliCsvCell).join(','));
     return `${[header.join(','), ...rows].join('\n')}\n`;
 }

package/cli/discovery-snapshot.mjs

@@ -1,9 +1,9 @@
 import { createHash } from 'node:crypto';
 import { boundedCliInputError, CliUsageError } from "./errors.mjs";
 import { writePrivateFile } from "./output-file.mjs";
-import { isValidAsciiDomainName } from "../lib/hostname.mjs";
+import { isValidAsciiDomainName } from "../packages/contracts/domain-name.mjs";
 import { readBoundedRegularTextFile } from "../lib/bounded-file.mjs";
-import { scanBoundedJson } from "../lib/bounded-json.mjs";
+import { scanBoundedJson } from "../packages/analysis/bounded-json.mjs";
 import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
 export const CLI_DISCOVERY_SNAPSHOT_SCHEMA = 'whoisleuth.cli.discovery-snapshot';
 export const CLI_DISCOVERY_SNAPSHOT_VERSION = 1;

package/cli/discovery-workflow.mjs

@@ -4,7 +4,7 @@ import { boundedCliErrorMessage, CliUsageError } from "./errors.mjs";
 async function generateDiscoveryCandidates(args, dependencies, context) {
     const seed = args.seed || await context.readSingleInput();
     if (!seed)
-        throw new CliUsageError('discover requires one brand label or domain as an argument or on stdin.');
+        throw new CliUsageError('discover requires one brand label or domain as an argument or on stdin.', 'missing_input');
     const loadGenerator = dependencies.loadTyposquatGenerator || (() => import("../lib/typosquat-generator.mjs"));
     const generator = await loadGenerator();
     const tlds = normalizeDiscoveryTlds(args.tldText || DEFAULT_DISCOVERY_TLDS.join(','), generator.MAX_GENERATION_TLDS);
@@ -23,7 +23,7 @@ async function generateDiscoveryCandidates(args, dependencies, context) {
         if (args.preset === 'custom'
             && !mutationFamilies.includes('dictionary')
             && !mutationFamilies.includes('dictionary_token_replacement')) {
-            throw new CliUsageError('--dictionary requires a dictionary mutation family.');
+            throw new CliUsageError('--dictionary requires a dictionary mutation family.', 'conflicting_options');
         }
         try {
             dictionaryText = dependencies.readDiscoveryDictionary
@@ -33,11 +33,11 @@ async function generateDiscoveryCandidates(args, dependencies, context) {
         catch (error) {
             if (error instanceof CliUsageError)
                 throw error;
-            throw new CliUsageError(`Could not read discovery dictionary: ${boundedCliErrorMessage(error, 'Input could not be read')}`);
+            throw new CliUsageError(`Could not read discovery dictionary: ${boundedCliErrorMessage(error, 'Input could not be read')}`, 'input_unavailable');
         }
         const normalizedDictionary = generator.normalizeCustomDictionaryTerms(dictionaryText);
         if (!normalizedDictionary.values.length) {
-            throw new CliUsageError('The discovery dictionary did not contain any valid terms.');
+            throw new CliUsageError('The discovery dictionary did not contain any valid terms.', 'missing_input');
         }
     }
     const result = generator.generateTyposquatCandidateSet(seed, tlds, {
@@ -47,7 +47,7 @@ async function generateDiscoveryCandidates(args, dependencies, context) {
         ...(args.preset === 'custom' ? { mutationTypes: mutationFamilies } : {}),
     });
     if (!result.inputValid) {
-        throw new CliUsageError('discover requires a valid brand label or domain with one suffix label.');
+        throw new CliUsageError('discover requires a valid brand label or registrable domain, without a subdomain or URL.');
     }
     const normalizedDictionary = generator.normalizeCustomDictionaryTerms(dictionaryText);
     const metadata = {

package/cli/domain-control-monitor.mjs

@@ -1,13 +1,13 @@
 import { classifyQuery } from "../lib/classify.mjs";
-import { scanBoundedJson } from "../lib/bounded-json.mjs";
-import { buildDomainControlFlightRecorder, validateDomainControlFlightRecorderDocument, } from "../lib/domain-control-flight-recorder.mjs";
+import { scanBoundedJson } from "../packages/analysis/bounded-json.mjs";
+import { buildDomainControlFlightRecorder, validateDomainControlFlightRecorderDocument, normalizeDomainControlFlightRecorderObservations, } from "../lib/domain-control-flight-recorder.mjs";
 import { DOMAIN_CONTROL_FLIGHT_RECORDER_SCHEMA, DOMAIN_CONTROL_FLIGHT_RECORDER_INPUT_SCHEMA, DOMAIN_CONTROL_FLIGHT_RECORDER_VERSION, } from "../packages/contracts/domain-control-flight-recorder.mjs";
-import { DOMAIN_CONTROL_REVIEW_SCHEMA, DOMAIN_CONTROL_REVIEW_VERSION, CLI_DOMAIN_CONTROL_REVIEW_INPUT_SCHEMA, CLI_DOMAIN_CONTROL_REVIEW_VERSION, } from "../packages/contracts/domain-control-review.mjs";
-import { CLI_DOMAIN_CONTROL_MONITOR_SCHEMA, CLI_DOMAIN_CONTROL_MONITOR_VERSION, DOMAIN_CONTROL_MONITOR_COLLECTION_KEYS, DOMAIN_CONTROL_MONITOR_FAILURE_CATEGORIES, DOMAIN_CONTROL_MONITOR_FAILURE_KEYS, DOMAIN_CONTROL_MONITOR_LIMITATIONS, DOMAIN_CONTROL_MONITOR_MANIFEST_KEYS, DOMAIN_CONTROL_MONITOR_ROOT_KEYS, MAX_DOMAIN_CONTROL_MONITOR_CONCURRENCY, MAX_DOMAIN_CONTROL_MONITOR_DOMAINS as CONTRACT_MAX_DOMAIN_CONTROL_MONITOR_DOMAINS, MAX_DOMAIN_CONTROL_MONITOR_ERROR_LENGTH, MAX_DOMAIN_CONTROL_MONITOR_FAILURES, MAX_DOMAIN_CONTROL_MONITOR_INPUT_BYTES, MAX_DOMAIN_CONTROL_MONITOR_JSON_CONTAINER_ITEMS, MAX_DOMAIN_CONTROL_MONITOR_JSON_DEPTH, MAX_DOMAIN_CONTROL_MONITOR_JSON_KEYS, MAX_DOMAIN_CONTROL_MONITOR_JSON_VALUES, MIN_DOMAIN_CONTROL_MONITOR_CONCURRENCY, MIN_DOMAIN_CONTROL_MONITOR_DOMAINS, } from "../packages/contracts/domain-control-monitor.mjs";
+import { DOMAIN_CONTROL_REVIEW_SCHEMA, CLI_DOMAIN_CONTROL_REVIEW_INPUT_SCHEMA, CLI_DOMAIN_CONTROL_REVIEW_VERSION, } from "../packages/contracts/domain-control-review.mjs";
+import { CLI_DOMAIN_CONTROL_MONITOR_SCHEMA, CLI_DOMAIN_CONTROL_MONITOR_VERSION, PUBLIC_CLI_DOMAIN_CONTROL_MONITOR_VERSION, SUPPORTED_CLI_DOMAIN_CONTROL_MONITOR_VERSIONS, DOMAIN_CONTROL_MONITOR_COLLECTION_KEYS, DOMAIN_CONTROL_MONITOR_EMBEDDED_VERSIONS, DOMAIN_CONTROL_MONITOR_FAILURE_CATEGORIES, DOMAIN_CONTROL_MONITOR_FAILURE_KEYS, DOMAIN_CONTROL_MONITOR_LIMITATIONS, DOMAIN_CONTROL_MONITOR_MANIFEST_KEYS, DOMAIN_CONTROL_MONITOR_ROOT_KEYS, MAX_DOMAIN_CONTROL_MONITOR_CONCURRENCY, MAX_DOMAIN_CONTROL_MONITOR_DOMAINS as CONTRACT_MAX_DOMAIN_CONTROL_MONITOR_DOMAINS, MAX_DOMAIN_CONTROL_MONITOR_ERROR_LENGTH, MAX_DOMAIN_CONTROL_MONITOR_FAILURES, MAX_DOMAIN_CONTROL_MONITOR_INPUT_BYTES, MAX_DOMAIN_CONTROL_MONITOR_JSON_CONTAINER_ITEMS, MAX_DOMAIN_CONTROL_MONITOR_JSON_DEPTH, MAX_DOMAIN_CONTROL_MONITOR_JSON_KEYS, MAX_DOMAIN_CONTROL_MONITOR_JSON_VALUES, MIN_DOMAIN_CONTROL_MONITOR_CONCURRENCY, MIN_DOMAIN_CONTROL_MONITOR_DOMAINS, } from "../packages/contracts/domain-control-monitor.mjs";
 import { validateDomainControlReviewDocument, verifyDomainControlManifest, } from "../lib/domain-control-manifest.mjs";
 import { requireDomainName, requireIsoTimestamp } from "../lib/bounded-contract-normalizers.mjs";
 import { buildCliLookupDocument } from "./formatters/json.mjs";
-import { buildCliDomainControlReview, } from "./domain-control-observations.mjs";
+import { buildCliDomainControlReview, DOMAIN_CONTROL_REVIEW_SOURCE_FIELDS, } from "./domain-control-observations.mjs";
 import { CliUsageError } from "./errors.mjs";
 export { CLI_DOMAIN_CONTROL_MONITOR_SCHEMA, CLI_DOMAIN_CONTROL_MONITOR_VERSION };
 export const MAX_DOMAIN_CONTROL_MONITOR_DOMAINS = CONTRACT_MAX_DOMAIN_CONTROL_MONITOR_DOMAINS;
@@ -58,9 +58,11 @@ function previousSnapshot(input, currentGeneratedAt) {
         return { observations: [] };
     const parsed = parseMonitorJson(input, 'Previous monitor snapshot');
     const root = exactMonitorRecord(parsed, DOMAIN_CONTROL_MONITOR_ROOT_KEYS, 'Previous monitor snapshot');
-    if (root.schema !== CLI_DOMAIN_CONTROL_MONITOR_SCHEMA || root.version !== CLI_DOMAIN_CONTROL_MONITOR_VERSION) {
-        throw new CliUsageError(`Previous monitor snapshot must use ${CLI_DOMAIN_CONTROL_MONITOR_SCHEMA} version ${CLI_DOMAIN_CONTROL_MONITOR_VERSION}.`);
+    if (root.schema !== CLI_DOMAIN_CONTROL_MONITOR_SCHEMA || !SUPPORTED_CLI_DOMAIN_CONTROL_MONITOR_VERSIONS.some((version) => version === root.version)) {
+        throw new CliUsageError(`Previous monitor snapshot must use ${CLI_DOMAIN_CONTROL_MONITOR_SCHEMA} supported version ${SUPPORTED_CLI_DOMAIN_CONTROL_MONITOR_VERSIONS.join(' or ')}.`);
     }
+    const legacy = root.version === PUBLIC_CLI_DOMAIN_CONTROL_MONITOR_VERSION;
+    const { review: reviewVersion, flightRecorder: flightVersion } = DOMAIN_CONTROL_MONITOR_EMBEDDED_VERSIONS[root.version];
     let previousGeneratedAt;
     try {
         previousGeneratedAt = requireIsoTimestamp(root.generatedAt, 'Previous monitor snapshot generatedAt');
@@ -83,7 +85,7 @@ function previousSnapshot(input, currentGeneratedAt) {
     catch {
         throw new CliUsageError('Previous monitor snapshot manifest expiresAt must be a valid ISO 8601 timestamp.');
     }
-    if (Date.parse(manifest.expiresAt) <= Date.parse(previousGeneratedAt)) {
+    if (legacy && Date.parse(manifest.expiresAt) <= Date.parse(previousGeneratedAt)) {
         throw new CliUsageError('Previous monitor snapshot manifest must have been unexpired when the checkpoint was generated.');
     }
     const collection = exactMonitorRecord(root.collection, DOMAIN_CONTROL_MONITOR_COLLECTION_KEYS, 'Previous monitor snapshot collection');
@@ -124,25 +126,31 @@ function previousSnapshot(input, currentGeneratedAt) {
     let flightRecorder;
     try {
         review = validateDomainControlReviewDocument(root.review);
+        if (review.version !== reviewVersion)
+            throw new TypeError('Mismatched review version.');
     }
     catch {
-        throw new CliUsageError(`Previous monitor snapshot review must use the exact ${DOMAIN_CONTROL_REVIEW_SCHEMA} version ${DOMAIN_CONTROL_REVIEW_VERSION} contract.`);
+        throw new CliUsageError(`Previous monitor snapshot review must use the exact ${DOMAIN_CONTROL_REVIEW_SCHEMA} version ${reviewVersion} contract.`);
     }
     try {
         flightRecorder = validateDomainControlFlightRecorderDocument(root.flightRecorder);
+        if (flightRecorder.version !== flightVersion)
+            throw new TypeError('Mismatched flight-recorder version.');
     }
     catch {
-        throw new CliUsageError(`Previous monitor snapshot flight recorder must use the exact ${DOMAIN_CONTROL_FLIGHT_RECORDER_SCHEMA} version ${DOMAIN_CONTROL_FLIGHT_RECORDER_VERSION} contract.`);
+        throw new CliUsageError(`Previous monitor snapshot flight recorder must use the exact ${DOMAIN_CONTROL_FLIGHT_RECORDER_SCHEMA} version ${flightVersion} contract.`);
     }
     if (!Array.isArray(root.limitations)
         || root.limitations.length !== DOMAIN_CONTROL_MONITOR_LIMITATIONS.length
         || root.limitations.some((value, index) => value !== DOMAIN_CONTROL_MONITOR_LIMITATIONS[index])) {
         throw new CliUsageError('Previous monitor snapshot limitations are invalid.');
     }
+    let observations;
     try {
+        observations = normalizeDomainControlFlightRecorderObservations(root.observations, flightVersion);
         buildDomainControlFlightRecorder({
             schema: DOMAIN_CONTROL_FLIGHT_RECORDER_INPUT_SCHEMA,
-            version: DOMAIN_CONTROL_FLIGHT_RECORDER_VERSION,
+            version: flightVersion,
             observations: root.observations,
             approvedWindows: [],
         }, previousGeneratedAt);
@@ -150,20 +158,30 @@ function previousSnapshot(input, currentGeneratedAt) {
     catch {
         throw new CliUsageError('Previous monitor snapshot observations must use the exact bounded flight-recorder input contract.');
     }
-    const observations = root.observations;
     const observationDomainOrder = observations.map((observation) => observation.domain);
     const observationDomains = new Set(observationDomainOrder);
     const requestedDomains = review.domains.slice(0, requested).map((item) => item.domain);
     const expectedObservationDomains = requestedDomains.filter((domain) => !failureDomains.has(domain));
     const expectedFailureDomains = requestedDomains.filter((domain) => !observationDomains.has(domain));
+    const mismatchedSource = !legacy && review.domains.some((item) => {
+        const observation = observations.find((candidate) => candidate.domain === item.domain);
+        return item.comparisons.some((comparison) => {
+            if (comparison.source === null)
+                return false;
+            const field = observation?.fields.find((candidate) => DOMAIN_CONTROL_REVIEW_SOURCE_FIELDS[candidate.id] === comparison.field);
+            return !field || field.source !== comparison.source || field.observedAt !== comparison.observedAt;
+        });
+    });
     if (review.domains.length < requested
+        || mismatchedSource
         || observationDomains.size !== observations.length
         || observationDomainOrder.some((domain, index) => domain !== expectedObservationDomains[index])
         || observationDomainOrder.length !== expectedObservationDomains.length
         || failureDomainOrder.some((domain, index) => domain !== expectedFailureDomains[index])
         || failureDomainOrder.length !== expectedFailureDomains.length
-        || observations.some((observation) => observation.observedAt !== previousGeneratedAt)
-        || review.domains.some((item) => item.comparisons.some((comparison) => comparison.observedAt !== null && comparison.observedAt !== previousGeneratedAt))
+        || observations.some((observation) => legacy ? observation.capturedAt !== previousGeneratedAt
+            : Date.parse(observation.capturedAt) > Date.parse(previousGeneratedAt))
+        || review.domains.some((item) => item.comparisons.some((comparison) => legacy && comparison.observedAt !== null && comparison.observedAt !== previousGeneratedAt))
         || review.generatedAt !== previousGeneratedAt
         || review.manifest.digestSha256 !== manifest.digestSha256
         || review.manifest.expiresAt !== manifest.expiresAt
@@ -173,7 +191,11 @@ function previousSnapshot(input, currentGeneratedAt) {
         || [...observationDomains].some((domain) => !flightRecorder.domains.includes(domain))) {
         throw new CliUsageError('Previous monitor snapshot embedded documents are inconsistent with its observation projection.');
     }
-    return { observations };
+    // Published checkpoints recorded run time, not independently retained source
+    // clocks. Their values remain available without upgrading that timestamp.
+    return { observations: legacy ? observations.map((item) => Object.freeze({
+            ...item, fields: Object.freeze(item.fields.map((field) => Object.freeze({ ...field, observedAt: null }))),
+        })) : observations };
 }
 function monitorFailureMessage() {
     return DOMAIN_CONTROL_MONITOR_FAILURE_CATEGORIES[0];
@@ -189,9 +211,9 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
         || options.concurrency > MAX_DOMAIN_CONTROL_MONITOR_CONCURRENCY) {
         throw new CliUsageError(`Domain-control monitor concurrency must be from ${MIN_DOMAIN_CONTROL_MONITOR_CONCURRENCY} to ${MAX_DOMAIN_CONTROL_MONITOR_CONCURRENCY}.`);
     }
-    let generatedAt;
+    let startedAt;
     try {
-        generatedAt = requireIsoTimestamp(options.now(), 'Domain-control monitor time');
+        startedAt = requireIsoTimestamp(options.now(), 'Domain-control monitor time');
     }
     catch {
         throw new CliUsageError('Domain-control monitor time must be a valid ISO 8601 timestamp.');
@@ -204,11 +226,11 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
     catch {
         throw new CliUsageError('Domain-control manifest must satisfy its supported bounded integrity contract.');
     }
-    const checkedAt = Date.parse(generatedAt);
+    const checkedAt = Date.parse(startedAt);
     if (Date.parse(manifest.expiresAt) <= checkedAt) {
         throw new CliUsageError('Domain-control monitor requires an unexpired manifest.');
     }
-    const previous = previousSnapshot(previousInput, generatedAt);
+    const previous = previousSnapshot(previousInput, startedAt);
     const entries = manifest.entries.slice(0, Math.min(options.limit, MAX_DOMAIN_CONTROL_MONITOR_DOMAINS));
     const lookups = new Array(entries.length);
     const failuresByIndex = new Array(entries.length);
@@ -225,7 +247,10 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
                 const classified = classifyQuery(entry.domain);
                 const result = await options.executeLookup(classified, { fast: false, compact: false, ...(options.signal ? { signal: options.signal } : {}) });
                 options.signal?.throwIfAborted();
-                lookups[index] = buildCliLookupDocument(entry.domain, classified, result, generatedAt, 'deep');
+                const capturedAt = requireIsoTimestamp(options.now(), 'Domain-control capture time');
+                if (Date.parse(capturedAt) < checkedAt)
+                    throw new CliUsageError('Domain-control capture time must not precede the run.');
+                lookups[index] = buildCliLookupDocument(entry.domain, classified, result, capturedAt, 'deep');
             }
             catch {
                 if (options.signal?.aborted) {
@@ -245,6 +270,10 @@ export async function runDomainControlMonitor(manifestInput, previousInput, opti
     const successful = lookups.filter(Boolean);
     if (!successful.length)
         throw new Error('No domain-control monitor lookup completed successfully.');
+    const generatedAt = requireIsoTimestamp(options.now(), 'Domain-control completion time');
+    if (Date.parse(generatedAt) < checkedAt || successful.some((lookup) => Date.parse(String(lookup.generatedAt)) > Date.parse(generatedAt))) {
+        throw new CliUsageError('Domain-control completion time must not precede its captures.');
+    }
     const review = buildCliDomainControlReview(JSON.stringify({
         schema: CLI_DOMAIN_CONTROL_REVIEW_INPUT_SCHEMA,
         version: CLI_DOMAIN_CONTROL_REVIEW_VERSION,
@@ -277,6 +306,7 @@ export function formatDomainControlMonitor(document) {
         `Succeeded    ${document.collection.succeeded}`,
         `Failed       ${document.collection.failed}`,
         `Review       ${document.review.state}`,
+        `Due          ${document.review.counts.due ?? 0}`,
         `Unexpected   ${document.flightRecorder.summary.unexpectedChanges}`,
         '',
     ].join('\n');