@slicedearth/whoisleuth-cli 1.36.1 → 1.43.0

package/cli/registry-cohort.mjs

@@ -0,0 +1,121 @@
+import { Buffer } from 'node:buffer';
+import { CliUsageError } from "./errors.mjs";
+import { buildRegistryDoctorReport } from "./registry-doctor.mjs";
+export const REGISTRY_COHORT_SCHEMA = 'whoisleuth.cli.registry-cohort';
+export const REGISTRY_COHORT_VERSION = 1;
+export const MAX_REGISTRY_COHORT_INPUT_BYTES = 16 * 1024 * 1024;
+export const MAX_REGISTRY_COHORT_SAMPLES = 500;
+export const MIN_REGISTRY_COHORT_SAMPLE = 5;
+function record(value) {
+    return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+}
+function documents(text) {
+    if (Buffer.byteLength(text, 'utf8') > MAX_REGISTRY_COHORT_INPUT_BYTES) {
+        throw new CliUsageError(`Registry cohort input is limited to ${MAX_REGISTRY_COHORT_INPUT_BYTES} bytes.`);
+    }
+    const trimmed = text.replace(/^\uFEFF/u, '').trim();
+    if (!trimmed)
+        throw new CliUsageError('Registry cohort input is empty.');
+    let values;
+    if (trimmed.startsWith('[')) {
+        let parsed;
+        try {
+            parsed = JSON.parse(trimmed);
+        }
+        catch {
+            throw new CliUsageError('Registry cohort input must be valid JSON or JSONL.');
+        }
+        if (!Array.isArray(parsed))
+            throw new CliUsageError('Registry cohort JSON input must be an array.');
+        values = parsed;
+    }
+    else {
+        const lines = trimmed.split(/\r?\n/u).filter((line) => line.trim());
+        values = lines.map((line, index) => {
+            try {
+                return JSON.parse(line);
+            }
+            catch {
+                throw new CliUsageError(`Registry cohort JSONL line ${index + 1} is invalid.`);
+            }
+        });
+    }
+    if (!values.length || values.length > MAX_REGISTRY_COHORT_SAMPLES) {
+        throw new CliUsageError(`Registry cohort input must contain from 1 to ${MAX_REGISTRY_COHORT_SAMPLES} saved Lookups.`);
+    }
+    const output = values.map(record);
+    if (output.some((item) => !Object.keys(item).length))
+        throw new CliUsageError('Every registry cohort item must be one saved Lookup object.');
+    return output;
+}
+export function buildRegistryCohortReport(text, generatedAt = new Date().toISOString()) {
+    const reports = documents(text).map((item) => buildRegistryDoctorReport(JSON.stringify(item), generatedAt));
+    const groups = new Map();
+    for (const report of reports) {
+        const key = `${report.suffix}\u0000${report.profile.id}`;
+        const group = groups.get(key) ?? [];
+        group.push(report);
+        groups.set(key, group);
+    }
+    const cohorts = [...groups.values()].map((group) => {
+        const sample = group[0];
+        if (!sample)
+            throw new CliUsageError('Registry cohort grouping failed.');
+        const sampleCount = group.length;
+        const investigate = group.reduce((total, item) => total + item.summary.investigate, 0);
+        const publicationReviewItems = group.reduce((total, item) => total + item.publication.reviewItems, 0);
+        return Object.freeze({
+            suffix: sample.suffix,
+            profileId: sample.profile.id,
+            sampleCount,
+            state: sampleCount < MIN_REGISTRY_COHORT_SAMPLE
+                ? 'insufficient_sample'
+                : investigate || publicationReviewItems
+                    ? 'review'
+                    : 'consistent',
+            sourceAlignment: Object.freeze({
+                investigate,
+                expectedConstraints: group.reduce((total, item) => total + item.summary.expectedConstraints, 0),
+                observed: group.reduce((total, item) => total + item.summary.observed, 0),
+            }),
+            publication: Object.freeze({
+                reviewItems: publicationReviewItems,
+                objectIdentifiersObserved: group.filter((item) => item.publication.objectIdentifier.state === 'observed').length,
+                baseConformanceObserved: group.filter((item) => item.publication.baseConformance.state === 'observed').length,
+                selfLinksObserved: group.filter((item) => item.publication.selfLink.state === 'observed').length,
+                redactionMetadataObserved: group.filter((item) => item.publication.redactionMetadata.state === 'observed').length,
+            }),
+        });
+    }).sort((left, right) => left.suffix.localeCompare(right.suffix) || left.profileId.localeCompare(right.profileId));
+    return Object.freeze({
+        schema: REGISTRY_COHORT_SCHEMA,
+        version: REGISTRY_COHORT_VERSION,
+        generatedAt,
+        sampleCount: reports.length,
+        minimumCohortSample: MIN_REGISTRY_COHORT_SAMPLE,
+        cohorts: Object.freeze(cohorts),
+        limitations: Object.freeze([
+            'The report intentionally omits domains, queries and raw evidence. It groups saved observations only by suffix and capability profile.',
+            `Cohorts with fewer than ${MIN_REGISTRY_COHORT_SAMPLE} observations remain insufficient sample and must not drive catalogue changes.`,
+            'Repeated observations from one environment are not representative by themselves. Review fixture provenance and source health before changing a parser or access profile.',
+        ]),
+    });
+}
+export function formatRegistryCohortReport(report) {
+    const output = [
+        'Registry quality cohort',
+        `Saved observations  ${report.sampleCount}`,
+        `Minimum sample      ${report.minimumCohortSample}`,
+        '',
+    ];
+    for (const cohort of report.cohorts) {
+        output.push(`.${cohort.suffix} · ${cohort.profileId} [${cohort.state.replaceAll('_', ' ')}]`);
+        output.push(`  Samples: ${cohort.sampleCount}`);
+        output.push(`  Source review items: ${cohort.sourceAlignment.investigate}`);
+        output.push(`  Publication review items: ${cohort.publication.reviewItems}`);
+    }
+    output.push('', 'Limitations:');
+    for (const limitation of report.limitations)
+        output.push(`  - ${limitation}`);
+    return `${output.join('\n')}\n`;
+}

package/cli/registry-doctor.mjs

@@ -0,0 +1,220 @@
+import { registryCapabilityFor, REGISTRY_CAPABILITIES_VERSION } from "../lib/registry-capabilities.mjs";
+import { recordOrEmpty } from "../lib/bounded-contract-normalizers.mjs";
+import { CliUsageError } from "./errors.mjs";
+import { parseSavedLookupDocument } from "./saved-lookup.mjs";
+const REGISTRY_DOCTOR_SCHEMA = 'whoisleuth.cli.registry-doctor';
+const REGISTRY_DOCTOR_VERSION = 2;
+function status(value) {
+    return typeof value === 'string'
+        ? value.replace(/[\u0000-\u001f\u007f]+/gu, ' ').trim().toLowerCase().slice(0, 40) || 'unavailable'
+        : 'unavailable';
+}
+function expectedWhois(profile) {
+    if (profile === 'iana-referral')
+        return 'allowed';
+    if (profile === 'registry-policy-restricted' || profile === 'source-ip-authorization-required')
+        return 'permission_required';
+    return 'unsupported';
+}
+function sourceAlignment(expected, observed) {
+    const successful = ['complete', 'partial', 'success'].includes(observed);
+    const constrained = ['disabled', 'skipped', 'unsupported'].includes(observed);
+    if (expected === 'allowed')
+        return successful ? 'observed' : 'investigate';
+    if (successful)
+        return 'unexpected_observation';
+    return constrained ? 'expected_constraint' : 'investigate';
+}
+function normalizedObjectIdentifier(parsed) {
+    const candidate = parsed.handle ?? parsed.objectIdentifier ?? parsed.registryObjectId ?? parsed.id;
+    return typeof candidate === 'string' && candidate.trim() ? 'observed' : 'not_observed';
+}
+function stringArray(value, limit = 100) {
+    return Array.isArray(value)
+        ? [...new Set(value.slice(0, limit).filter((item) => typeof item === 'string').map((item) => item.trim().toLowerCase()).filter(Boolean))]
+        : [];
+}
+function publicationQuality(parsed, rdapStatus) {
+    if (!['success', 'partial'].includes(rdapStatus)) {
+        const unavailable = { state: 'unavailable', value: null };
+        return {
+            objectClass: unavailable,
+            objectIdentifier: unavailable,
+            mediaType: unavailable,
+            baseConformance: { state: 'unavailable', declarations: [] },
+            redactionMetadata: { state: 'unavailable', count: 0, truncated: false },
+            selfLink: { state: 'unavailable', observedRelations: [] },
+            events: { state: 'unavailable', count: 0, conflictingActions: [] },
+            reviewItems: 0,
+        };
+    }
+    const objectClassName = typeof parsed.objectClassName === 'string' ? parsed.objectClassName.trim().toLowerCase() : '';
+    const identifier = [parsed.handle, parsed.objectIdentifier, parsed.registryObjectId, parsed.id]
+        .find((value) => typeof value === 'string' && Boolean(value.trim()))?.trim() ?? null;
+    const declarations = stringArray(parsed.conformance);
+    const redactions = Array.isArray(parsed.redactions) ? parsed.redactions.slice(0, 100) : [];
+    const links = Array.isArray(parsed.links) ? parsed.links.slice(0, 100).map(recordOrEmpty) : [];
+    const relations = [...new Set(links.flatMap((link) => stringArray(link.rel, 10).length
+            ? stringArray(link.rel, 10)
+            : typeof link.rel === 'string' ? [link.rel.trim().toLowerCase()] : []).filter(Boolean))];
+    const events = Array.isArray(parsed.events) ? parsed.events.slice(0, 100).map(recordOrEmpty) : [];
+    const actionDates = new Map();
+    for (const event of events) {
+        const action = typeof event.action === 'string' ? event.action.trim().toLowerCase() : '';
+        const date = typeof event.date === 'string' ? event.date.trim() : '';
+        if (!action || !date)
+            continue;
+        const dates = actionDates.get(action) ?? [];
+        dates.push(date);
+        actionDates.set(action, dates);
+    }
+    const lifecycle = recordOrEmpty(parsed.lifecycle);
+    const lifecycleMappings = [
+        ['registration', 'createdDateIso', false],
+        ['reregistration', 'reregistrationDateIso', true],
+        ['expiration', 'expiryDateIso', true],
+        ['last changed', 'updatedDateIso', true],
+        ['transfer', 'transferDateIso', true],
+        ['deletion', 'deletionDateIso', true],
+        ['reinstantiation', 'reinstantiationDateIso', true],
+        ['last update of rdap database', 'databaseUpdatedDateIso', true],
+    ];
+    const conflictingActions = lifecycleMappings.flatMap(([action, field, newest]) => {
+        const published = typeof lifecycle[field] === 'string' ? Date.parse(lifecycle[field]) : Number.NaN;
+        if (!Number.isFinite(published))
+            return [];
+        const dates = (actionDates.get(action) ?? []).map(Date.parse).filter(Number.isFinite);
+        if (!dates.length)
+            return [action];
+        const selected = newest ? Math.max(...dates) : Math.min(...dates);
+        return selected === published ? [] : [action];
+    }).sort();
+    const objectClassState = !objectClassName ? 'not_observed' : objectClassName === 'domain' ? 'observed' : 'inconsistent';
+    const baseConformanceState = declarations.includes('rdap_level_0') ? 'observed' : 'not_observed';
+    const selfLinkState = relations.includes('self') ? 'observed' : 'not_observed';
+    const hasLifecycle = lifecycleMappings.some(([, field]) => typeof lifecycle[field] === 'string');
+    const eventsState = conflictingActions.length
+        ? 'inconsistent'
+        : events.length && hasLifecycle
+            ? 'observed'
+            : events.length
+                ? 'unavailable'
+                : 'not_observed';
+    return {
+        objectClass: { state: objectClassState, value: objectClassName || null },
+        objectIdentifier: { state: identifier ? 'observed' : 'not_observed', value: identifier },
+        mediaType: { state: 'unavailable', value: null },
+        baseConformance: { state: baseConformanceState, declarations },
+        redactionMetadata: {
+            state: redactions.length ? 'observed' : 'not_observed',
+            count: redactions.length,
+            truncated: parsed.redactionsTruncated === true,
+        },
+        selfLink: { state: selfLinkState, observedRelations: relations.sort() },
+        events: { state: eventsState, count: events.length, conflictingActions },
+        reviewItems: [objectClassState, baseConformanceState, selfLinkState, eventsState]
+            .filter((state) => state === 'inconsistent' || state === 'not_observed').length,
+    };
+}
+function buildRegistryDoctorReport(raw, generatedAt = new Date().toISOString()) {
+    const document = parseSavedLookupDocument(raw, { label: 'Registry doctor input' });
+    const capability = registryCapabilityFor(document.registrableDomain);
+    if (!capability)
+        throw new CliUsageError('Registry doctor could not resolve a capability profile for this domain.');
+    const diagnostics = recordOrEmpty(document.diagnostics);
+    const rdap = recordOrEmpty(document.rdap);
+    const whois = recordOrEmpty(document.whois);
+    const rdapParsed = recordOrEmpty(rdap.parsed);
+    const whoisParsed = recordOrEmpty(whois.parsed);
+    const sourceInputs = [
+        {
+            source: 'rdap',
+            expected: capability.rdapAccessProfile === 'iana-bootstrap' ? 'allowed' : 'unsupported',
+            observed: status(recordOrEmpty(diagnostics.rdap).status),
+            parsed: rdapParsed,
+        },
+        {
+            source: 'whois',
+            expected: expectedWhois(capability.whoisAccessProfile),
+            observed: status(recordOrEmpty(diagnostics.whois).status),
+            parsed: whoisParsed,
+        },
+    ];
+    const sources = sourceInputs.map((source) => ({
+        source: source.source,
+        expected: source.expected,
+        observed: source.observed,
+        alignment: sourceAlignment(source.expected, source.observed),
+        normalizedFields: Object.keys(source.parsed).length,
+        objectIdentifier: normalizedObjectIdentifier(source.parsed),
+        limitation: capability.limitation.slice(0, 500),
+    }));
+    const publication = publicationQuality(rdapParsed, sourceInputs[0]?.observed ?? 'unavailable');
+    const recommendations = [
+        ...sources.filter((source) => source.alignment === 'investigate').map((source) => (source.expected === 'allowed'
+            ? `Review ${source.source.toUpperCase()} source health, transport attempts, and fixture coverage; the catalogue expected collection to be available.`
+            : `Review the ${source.source.toUpperCase()} result and capability profile; the observed state did not match the declared access constraint.`)),
+        ...sources.filter((source) => source.alignment === 'unexpected_observation').map((source) => (`Review why ${source.source.toUpperCase()} produced normalized evidence despite the declared ${source.expected.replaceAll('_', ' ')} access profile.`)),
+        ...sources.filter((source) => source.alignment === 'observed' && source.objectIdentifier === 'not_observed').map((source) => (`${source.source.toUpperCase()} succeeded without a retained registry object identifier; this can be a valid publication omission and should remain unknown rather than an error.`)),
+    ].slice(0, 12);
+    return {
+        schema: REGISTRY_DOCTOR_SCHEMA,
+        version: REGISTRY_DOCTOR_VERSION,
+        generatedAt,
+        domain: document.registrableDomain,
+        suffix: capability.suffixes[0] || document.registrableDomain.split('.').at(-1) || '',
+        catalogueVersion: REGISTRY_CAPABILITIES_VERSION,
+        profile: {
+            id: capability.id,
+            explicitSuffixProfile: capability.explicitSuffixProfile,
+            coverageState: capability.coverageState,
+        },
+        publication,
+        sources,
+        summary: {
+            investigate: sources.filter((source) => source.alignment === 'investigate' || source.alignment === 'unexpected_observation').length,
+            expectedConstraints: sources.filter((source) => source.alignment === 'expected_constraint').length,
+            observed: sources.filter((source) => source.alignment === 'observed').length,
+        },
+        recommendations,
+        limitations: [
+            'This diagnostic compares one saved observation with the local reviewed capability catalogue and makes no registry, registrar, RDAP, or WHOIS request.',
+            'A catalogue alignment does not prove live reachability, parser completeness, current registry policy, registration, availability, ownership, safety, or maliciousness.',
+            'A missing registry object identifier can be a publication characteristic. It remains not observed and is not converted into a collection failure.',
+            'Publication-quality observations describe the saved normalised RDAP object. The saved Lookup format does not retain the response media type, so that field remains unavailable.',
+            'Use fixture-based maintenance checks before changing a registry parser or access profile; automated tests must not contact live registries.',
+        ],
+    };
+}
+function formatRegistryDoctorReport(report) {
+    const output = [
+        'Registry compatibility diagnostic',
+        `Domain             ${report.domain}`,
+        `Profile            ${report.profile.id}${report.profile.explicitSuffixProfile ? ' (explicit)' : ' (default)'}`,
+        `Review items       ${report.summary.investigate}`,
+        '',
+    ];
+    for (const source of report.sources) {
+        output.push(`${source.source.toUpperCase()} [${source.alignment.replaceAll('_', ' ')}]`);
+        output.push(`  Expected: ${source.expected.replaceAll('_', ' ')}`);
+        output.push(`  Observed: ${source.observed.replaceAll('_', ' ')}`);
+        output.push(`  Object identifier: ${source.objectIdentifier.replaceAll('_', ' ')}`);
+    }
+    output.push('', 'RDAP publication quality');
+    output.push(`  Object class: ${report.publication.objectClass.state.replaceAll('_', ' ')}`);
+    output.push(`  Object identifier: ${report.publication.objectIdentifier.state.replaceAll('_', ' ')}`);
+    output.push(`  Base conformance: ${report.publication.baseConformance.state.replaceAll('_', ' ')}`);
+    output.push(`  Self link: ${report.publication.selfLink.state.replaceAll('_', ' ')}`);
+    output.push(`  Events: ${report.publication.events.state.replaceAll('_', ' ')}`);
+    output.push('  Media type: unavailable in the saved Lookup contract');
+    if (report.recommendations.length) {
+        output.push('', 'Recommendations:');
+        for (const item of report.recommendations)
+            output.push(`  - ${item}`);
+    }
+    output.push('', 'Limitations:');
+    for (const limitation of report.limitations)
+        output.push(`  - ${limitation}`);
+    return `${output.join('\n')}\n`;
+}
+export { REGISTRY_DOCTOR_SCHEMA, REGISTRY_DOCTOR_VERSION, buildRegistryDoctorReport, formatRegistryDoctorReport, };

package/cli/registry-fixture-scaffold.mjs

@@ -0,0 +1,73 @@
+import { registryCompatibilityMatrix } from "../lib/registry-capabilities.mjs";
+const SAFE_ID_RE = /^[a-z0-9](?:[a-z0-9-]{0,126}[a-z0-9])?$/u;
+const SAFE_SUFFIX_RE = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/u;
+const SCENARIOS = new Set(['registered', 'not_found', 'inconclusive']);
+export function buildRegistryFixtureScaffold(profileIdInput, suffixInput, scenarioInput) {
+    const profileId = typeof profileIdInput === 'string' ? profileIdInput.toLowerCase() : '';
+    const suffix = typeof suffixInput === 'string' ? suffixInput.toLowerCase().replace(/^\./u, '') : '';
+    const scenario = typeof scenarioInput === 'string' ? scenarioInput.toLowerCase() : '';
+    if (!SAFE_ID_RE.test(profileId))
+        throw new TypeError('The capability profile is invalid.');
+    if (!SAFE_SUFFIX_RE.test(suffix))
+        throw new TypeError('The suffix is invalid.');
+    if (!SCENARIOS.has(scenario))
+        throw new TypeError('Scenario must be registered, not_found, or inconclusive.');
+    const profile = registryCompatibilityMatrix().find((item) => item.id === profileId);
+    if (!profile)
+        throw new TypeError('The capability profile is not present in the registry catalogue.');
+    if (!profile.suffixes.includes(suffix))
+        throw new TypeError(`The capability profile does not cover .${suffix}.`);
+    const domain = `EXAMPLE.${suffix.toUpperCase()}`;
+    const response = scenario === 'registered'
+        ? [
+            `Domain Name: ${domain}`,
+            'Registry Domain ID: EXAMPLE-SYNTHETIC-ID',
+            'Registrar: Example Registrar',
+            'Creation Date: 2020-01-02T03:04:05Z',
+            'Updated Date: 2024-05-06T07:08:09Z',
+            'Registry Expiry Date: 2030-01-02T03:04:05Z',
+            'Domain Status: active',
+            'Registrant Organization: Example Organisation',
+            'Registrant Email: registrant@example.invalid',
+            'Name Server: NS1.EXAMPLE.INVALID',
+            'DNSSEC: unsigned',
+        ]
+        : scenario === 'not_found'
+            ? ['REPLACE WITH A SYNTHETIC AUTHORITATIVE NO-RECORD DIALECT']
+            : ['REPLACE WITH A SYNTHETIC RESTRICTED, RATE-LIMITED, OR TEMPORARY-FAILURE DIALECT'];
+    const expected = scenario === 'registered'
+        ? {
+            registrationStatus: 'registered',
+            domainName: domain,
+            registrar: 'Example Registrar',
+            registrantOrg: 'Example Organisation',
+            registrantEmail: 'registrant@example.invalid',
+            nameservers: ['NS1.EXAMPLE.INVALID'],
+        }
+        : scenario === 'not_found'
+            ? { registrationStatus: 'not_found', notFound: true }
+            : { registrationStatus: 'inconclusive', notFound: false };
+    return [
+        '// Synthetic scaffold only. Do not paste a live WHOIS response or personal registration data.',
+        '// Preserve the registry dialect while replacing domains, contacts, handles, addresses, phones,',
+        '// nameservers, dates, identifiers, and free text with reserved or fictional values.',
+        '{',
+        `  name: ${JSON.stringify(`.${suffix} ${scenario} synthetic response`)},`,
+        `  capabilityProfile: ${JSON.stringify(profileId)},`,
+        `  scenario: ${JSON.stringify(scenario)},`,
+        '  chain: [',
+        `    rootHop(${JSON.stringify(suffix.toUpperCase())}, ${JSON.stringify(`whois.${suffix}.invalid`)}),`,
+        `    registryHop(${JSON.stringify(`whois.${suffix}.invalid`)}, ${JSON.stringify(response, null, 2).replace(/\n/gu, '\n    ')}),`,
+        '  ],',
+        `  expected: ${JSON.stringify(expected, null, 2).replace(/\n/gu, '\n  ')},`,
+        '},',
+        '',
+        '// Review checklist:',
+        '// - every domain and hostname uses the requested suffix or .invalid;',
+        '// - every email ends in .invalid; IP examples use documentation ranges;',
+        '// - no real name, address, phone, handle, identifier, or free-text notice remains;',
+        '// - authoritative absence is asserted only for a documented registry response;',
+        '// - add or refresh the tracked provenance digest after review.',
+        '',
+    ].join('\n');
+}

package/cli/risk-calibration.mjs

@@ -1,11 +1,13 @@
 import { Buffer } from 'node:buffer';
 import { isValidAsciiDomainName } from "../lib/hostname.mjs";
+import { ANALYST_REVIEW_REASON_VALUES, analystInteroperabilityTags } from "../lib/analyst-taxonomy.mjs";
 import { CliUsageError } from "./arguments.mjs";
 import { RISK_MUTATION_TYPES, RISK_REVIEW_THRESHOLD } from "../lib/risk-scoring.mjs";
 export const RISK_CALIBRATION_DATASET_SCHEMA = 'whoisleuth.risk-calibration-dataset';
-export const RISK_CALIBRATION_DATASET_VERSION = 1;
+export const RISK_CALIBRATION_DATASET_VERSION = 2;
+export const SUPPORTED_RISK_CALIBRATION_DATASET_VERSIONS = Object.freeze([1, RISK_CALIBRATION_DATASET_VERSION]);
 export const RISK_CALIBRATION_REPORT_SCHEMA = 'whoisleuth.cli.risk-calibration';
-export const RISK_CALIBRATION_REPORT_VERSION = 1;
+export const RISK_CALIBRATION_REPORT_VERSION = 2;
 export const MAX_RISK_CALIBRATION_INPUT_BYTES = 2 * 1024 * 1024;
 export const MAX_RISK_CALIBRATION_RECORDS = 500;
 export const MAX_RISK_CALIBRATION_STRING_LENGTH = 256;
@@ -24,8 +26,10 @@ const AVAILABILITY_STATES = new Set(['registered', 'for_sale', 'expiring', 'avai
 const ACTIVITY_STATES = new Set(['active', 'parked', 'unreachable', 'no_site']);
 const BOOLEAN_FIELDS = [
     'faviconMatch', 'faviconNearMatch', 'reusesOfficialAssets', 'hasPasswordField',
-    'hasMx', 'hasSpf', 'hasDmarc', 'privacyProtected',
+    'hasMx', 'hasSpf', 'hasDmarc', 'privacyProtected', 'hasExternalFormAction',
+    'idnReferenceMatch', 'pageBaselineMatch', 'hasActiveBrandProfile',
 ];
+const REVIEW_REASON_CODES = ANALYST_REVIEW_REASON_VALUES;
 const MUTATION_TYPES = new Set(RISK_MUTATION_TYPES);
 function object(value, field) {
     if (!value || typeof value !== 'object' || Array.isArray(value)) {
@@ -109,6 +113,17 @@ function projectEvidence(value, field) {
             throw new CliUsageError(`${field}.activityStatus is unsupported.`);
         result.activityStatus = activity;
     }
+    if (source.scanDepth !== null && source.scanDepth !== undefined) {
+        const depth = boundedString(source.scanDepth, `${field}.scanDepth`, 16);
+        if (depth !== 'fast' && depth !== 'deep')
+            throw new CliUsageError(`${field}.scanDepth must be fast or deep.`);
+        Object.assign(result, { scanDepth: depth });
+    }
+    if (source.observedAt !== null && source.observedAt !== undefined) {
+        const observedAt = optionalTimestamp(source.observedAt, `${field}.observedAt`);
+        if (observedAt)
+            Object.assign(result, { observedAt });
+    }
     if (source.phishingLanguageMatch !== null && source.phishingLanguageMatch !== undefined) {
         result.phishingLanguageMatch = boundedString(source.phishingLanguageMatch, `${field}.phishingLanguageMatch`);
     }
@@ -164,8 +179,13 @@ export function parseRiskCalibrationDataset(text) {
         throw new CliUsageError('Risk calibration input must be valid JSON.');
     }
     const document = object(parsed, 'Risk calibration input');
-    if (document.schema !== RISK_CALIBRATION_DATASET_SCHEMA || document.version !== RISK_CALIBRATION_DATASET_VERSION) {
-        throw new CliUsageError(`Risk calibration input must use ${RISK_CALIBRATION_DATASET_SCHEMA} version ${RISK_CALIBRATION_DATASET_VERSION}.`);
+    const documentVersion = typeof document.version === 'number' && Number.isInteger(document.version)
+        ? document.version
+        : null;
+    if (document.schema !== RISK_CALIBRATION_DATASET_SCHEMA
+        || documentVersion === null
+        || !SUPPORTED_RISK_CALIBRATION_DATASET_VERSIONS.includes(documentVersion)) {
+        throw new CliUsageError(`Risk calibration input must use ${RISK_CALIBRATION_DATASET_SCHEMA} version ${SUPPORTED_RISK_CALIBRATION_DATASET_VERSIONS.join(' or ')}.`);
     }
     if (!Array.isArray(document.records) || !document.records.length) {
         throw new CliUsageError('Risk calibration input must contain a non-empty records array.');
@@ -187,22 +207,39 @@ export function parseRiskCalibrationDataset(text) {
         const analystDisposition = boundedString(record.analystDisposition, `${prefix}.analystDisposition`, 32);
         if (!DISPOSITIONS.has(analystDisposition))
             throw new CliUsageError(`${prefix}.analystDisposition is unsupported.`);
+        let reviewReasonCode;
+        if (record.reviewReasonCode !== null && record.reviewReasonCode !== undefined) {
+            reviewReasonCode = boundedString(record.reviewReasonCode, `${prefix}.reviewReasonCode`, 64);
+            if (!REVIEW_REASON_CODES.has(reviewReasonCode))
+                throw new CliUsageError(`${prefix}.reviewReasonCode is unsupported.`);
+        }
         return {
             id,
             domain,
             analystDisposition: analystDisposition,
+            ...(reviewReasonCode ? { reviewReasonCode } : {}),
             evidence: projectEvidence(record.evidence, `${prefix}.evidence`),
         };
     });
     return {
         schema: RISK_CALIBRATION_DATASET_SCHEMA,
-        version: RISK_CALIBRATION_DATASET_VERSION,
+        version: documentVersion,
         records,
     };
 }
 function ratio(numerator, denominator) {
     return denominator ? Number((numerator / denominator).toFixed(4)) : null;
 }
+function wilson95(successes, total) {
+    if (!total)
+        return null;
+    const z = 1.96;
+    const proportion = successes / total;
+    const denominator = 1 + (z * z) / total;
+    const center = (proportion + (z * z) / (2 * total)) / denominator;
+    const margin = (z * Math.sqrt((proportion * (1 - proportion) + (z * z) / (4 * total)) / total)) / denominator;
+    return { lower: Number(Math.max(0, center - margin).toFixed(4)), upper: Number(Math.min(1, center + margin).toFixed(4)) };
+}
 function metricsForThreshold(records, threshold) {
     let truePositive = 0;
     let falsePositive = 0;
@@ -217,16 +254,30 @@ function metricsForThreshold(records, threshold) {
         else
             flagged ? falsePositive += 1 : trueNegative += 1;
     }
+    const precision = ratio(truePositive, truePositive + falsePositive);
+    const recall = ratio(truePositive, truePositive + falseNegative);
+    const specificity = ratio(trueNegative, trueNegative + falsePositive);
     return {
         threshold,
         truePositive,
         falsePositive,
         trueNegative,
         falseNegative,
-        precision: ratio(truePositive, truePositive + falsePositive),
-        recall: ratio(truePositive, truePositive + falseNegative),
-        specificity: ratio(trueNegative, trueNegative + falsePositive),
+        precision,
+        recall,
+        specificity,
         falsePositiveRate: ratio(falsePositive, falsePositive + trueNegative),
+        f1: precision === null || recall === null || precision + recall === 0
+            ? null
+            : Number(((2 * precision * recall) / (precision + recall)).toFixed(4)),
+        balancedAccuracy: recall === null || specificity === null
+            ? null
+            : Number(((recall + specificity) / 2).toFixed(4)),
+        confidence95: {
+            precision: wilson95(truePositive, truePositive + falsePositive),
+            recall: wilson95(truePositive, truePositive + falseNegative),
+            specificity: wilson95(trueNegative, trueNegative + falsePositive),
+        },
     };
 }
 function metricClass(disposition) {
@@ -254,6 +305,8 @@ export function buildRiskCalibrationReport(dataset, explainRiskScore, options) {
             id: record.id,
             domain: record.domain,
             analystDisposition: record.analystDisposition,
+            reviewReasonCode: record.reviewReasonCode ?? null,
+            interoperabilityTags: analystInteroperabilityTags(record.analystDisposition, record.reviewReasonCode),
             metricClass: classification,
             includedInMetrics,
             exclusionReason: includedInMetrics
@@ -273,6 +326,52 @@ export function buildRiskCalibrationReport(dataset, explainRiskScore, options) {
     const positive = records.filter((record) => record.metricClass === 'positive' && record.score !== null).length;
     const negative = records.filter((record) => record.metricClass === 'negative' && record.score !== null).length;
     const excluded = records.length - positive - negative;
+    const strata = [];
+    const appendStrata = (dimension, values) => {
+        for (const [value, members] of [...values].sort(([left], [right]) => left.localeCompare(right))) {
+            const sampleCount = members.filter((member) => member.includedInMetrics).length;
+            strata.push({
+                dimension,
+                value,
+                sampleCount,
+                insufficientSample: sampleCount < 20,
+                metrics: metricsForThreshold(members, options.reviewThreshold),
+            });
+        }
+    };
+    const byDepth = new Map();
+    const byReason = new Map();
+    for (let index = 0; index < dataset.records.length; index += 1) {
+        const source = dataset.records[index];
+        const scored = records[index];
+        if (!source || !scored)
+            continue;
+        const depth = source.evidence.scanDepth === 'deep' || source.evidence.scanDepth === 'fast' ? source.evidence.scanDepth : 'unknown';
+        byDepth.set(depth, [...(byDepth.get(depth) ?? []), scored]);
+        const reason = source.reviewReasonCode ?? 'not_recorded';
+        byReason.set(reason, [...(byReason.get(reason) ?? []), scored]);
+    }
+    appendStrata('scan_depth', byDepth);
+    appendStrata('review_reason', byReason);
+    let scoresChanged = 0;
+    let bandsChanged = 0;
+    let thresholdClassificationsChanged = 0;
+    if (options.explainPreviousRiskScore) {
+        for (let index = 0; index < dataset.records.length; index += 1) {
+            const source = dataset.records[index];
+            const current = records[index];
+            if (!source || !current)
+                continue;
+            const previous = options.explainPreviousRiskScore(source.evidence);
+            if (previous?.score !== current.score)
+                scoresChanged += 1;
+            if (scoreBand(previous?.score ?? null) !== current.band)
+                bandsChanged += 1;
+            if (((previous?.score ?? -1) >= options.reviewThreshold) !== ((current.score ?? -1) >= options.reviewThreshold)) {
+                thresholdClassificationsChanged += 1;
+            }
+        }
+    }
     return {
         schema: RISK_CALIBRATION_REPORT_SCHEMA,
         version: RISK_CALIBRATION_REPORT_VERSION,
@@ -282,6 +381,15 @@ export function buildRiskCalibrationReport(dataset, explainRiskScore, options) {
         currentReviewThreshold: options.reviewThreshold,
         summary: { total: records.length, positive, negative, excluded, scoreBands: bands },
         thresholds: RISK_CALIBRATION_THRESHOLDS.map((threshold) => metricsForThreshold(records, threshold)),
+        strata,
+        modelComparison: {
+            available: Boolean(options.explainPreviousRiskScore),
+            previousModelVersion: options.explainPreviousRiskScore ? options.previousModelVersion ?? null : null,
+            currentModelVersion: options.modelVersion,
+            scoresChanged,
+            bandsChanged,
+            thresholdClassificationsChanged,
+        },
         records,
         interpretation: {
             authority: 'analyst_context_only',