@slicedearth/whoisleuth-cli 2.4.0 → 2.6.0

package/lib/interchange-fidelity-registry.mjs

@@ -1,4 +1,5 @@
 import { DOMAIN_CONTROL_MANIFEST_COMPATIBILITY } from "../packages/contracts/domain-control-manifest.mjs";
+import { MANAGED_INDICATOR_SET_COMPATIBILITY } from "../packages/contracts/analyst-interchange.mjs";
 import { CLI_CASE_PACK_COMPATIBILITY, ENCRYPTED_WORKSPACE_ARCHIVE_COMPATIBILITY, WORKSPACE_ARCHIVE_COMPATIBILITY, } from "../packages/contracts/case-portability.mjs";
 import { LOOKUP_CLAIM_PASSPORT_COMPATIBILITY } from "../packages/contracts/investigation-portability.mjs";
 import { BRAND_PROFILE_EXPORT_COMPATIBILITY } from "../packages/contracts/workspace-portability.mjs";
@@ -16,6 +17,15 @@ function compatibilityReference(descriptor) {
     });
 }
 export const INTERCHANGE_ARTIFACT_CONTRACTS = Object.freeze([
+    Object.freeze({
+        id: 'managed_indicators', ...compatibilityReference(MANAGED_INDICATOR_SET_COMPATIBILITY),
+        versionField: 'version', nestedSchemaPath: Object.freeze([]),
+        browser: Object.freeze({ import: 'supported', export: 'supported' }),
+        cli: Object.freeze({ read: 'supported', write: 'supported', verify: 'supported' }),
+        fidelity: 'semantic_exact_after_normalisation', requiredAssurance: 'whole_integrity',
+        preservedFieldGroups: Object.freeze(['stable_indicator_identities', 'original_observations', 'review_windows', 'withdrawal_records', 'preceding_revision_digest']),
+        excludedFieldGroups: Object.freeze(['raw_responses', 'browser_local_records', 'credentials', 'remote_publication', 'author_authentication', 'unavailable_revision_history']),
+    }),
     Object.freeze({
         id: 'lookup_claim_passport',
         ...compatibilityReference(LOOKUP_CLAIM_PASSPORT_COMPATIBILITY),

package/lib/lookup-child-profile-contract.mjs

@@ -8,6 +8,7 @@ import { MAX_LOOKUP_TLS_ALT_NAMES, MAX_LOOKUP_TLS_CERTIFICATE_POLICIES, MAX_LOOK
 import { isJsonObject, normalizedDomain, validBoundedString, validHttpProvenanceUrl, validObservationDiagnostics, validOptionalNullableText, validStringArray, validUint, } from "./lookup-contract-primitives.mjs";
 import { TECHNOLOGY_EVIDENCE_ROLE_ORDER } from "./technology-evidence-role.mjs";
 import { MAX_HOMEPAGE_BYTES } from "./outbound-request-bounds.mjs";
+import { CREDENTIAL_CATEGORY_NAMES, MAX_CREDENTIAL_FORMS, MAX_CREDENTIAL_INPUTS, externalPasswordFormObservation, validCredentialFormAttribution } from "../packages/evidence/credential-form-attribution.mjs";
 export const TECHNOLOGY_PROFILE_VERSION = 12;
 export const SUPPORTED_TECHNOLOGY_PROFILE_VERSIONS = Object.freeze([10, 11, TECHNOLOGY_PROFILE_VERSION]);
 export const MAX_TECHNOLOGY_FINDINGS = 24;
@@ -17,7 +18,7 @@ export const BROWSER_LIBRARY_PROFILE_VERSION = 2;
 export const MAX_LIBRARY_FINDINGS = 16;
 export const WEBSITE_SECURITY_POSTURE_VERSION = 3;
 export const MAX_SECURITY_POSTURE_FINDINGS = 32;
-export const CREDENTIAL_SURFACE_PROFILE_VERSION = 1;
+export const CREDENTIAL_SURFACE_PROFILE_VERSION = 2;
 export const STRUCTURED_DATA_IDENTITY_VERSION = 1;
 export const MAX_STRUCTURED_DATA_ENTITIES = 16;
 export const MAX_STRUCTURED_DATA_SAME_AS_HOSTS = 12;
@@ -66,7 +67,7 @@ const SIMHASH64_RE = /^[a-f0-9]{16}$/u;
 const OID_RE = /^\d{1,10}(?:\.\d{1,10}){1,31}$/u;
 const CREDENTIAL_METHOD_KEYS = Object.freeze(['missing', 'get', 'post', 'dialog', 'other']);
 const CREDENTIAL_ACTION_KEYS = Object.freeze(['sameOrigin', 'external', 'missing', 'cleartext', 'unclassified']);
-const CREDENTIAL_CATEGORY_KEYS = Object.freeze(['password', 'email', 'username', 'one_time_code', 'payment']);
+const CREDENTIAL_CATEGORY_KEYS = CREDENTIAL_CATEGORY_NAMES;
 const TLS_NAME_KEYS = Object.freeze([
     'commonNames', 'organizations', 'organizationalUnits', 'countries', 'localities', 'states',
 ]);
@@ -683,28 +684,37 @@ function securityPostureContractState(value) {
     return 'supported';
 }
 function credentialSurfaceContractState(value) {
-    const versionState = childVersionState(value, 'credentialSurfaceVersion', [CREDENTIAL_SURFACE_PROFILE_VERSION]);
+    const versionState = childVersionState(value, 'credentialSurfaceVersion', [1, CREDENTIAL_SURFACE_PROFILE_VERSION]);
     if (versionState !== 'supported')
         return versionState;
     const profile = value;
     const forms = profile.forms;
     const inputs = profile.inputs;
-    if (!hasExactKeys(profile, [...OBSERVATION_FIELDS, 'credentialSurfaceVersion', 'forms', 'inputs'])
+    if (!hasExactKeys(profile, [...OBSERVATION_FIELDS, 'credentialSurfaceVersion', 'forms', 'inputs', ...(profile.credentialSurfaceVersion === 2 ? ['formAttribution'] : [])])
         || !validProfileObservation(profile, new Set(['success', 'partial']), new Set(['html']))
         || !isJsonObject(forms)
         || !hasExactKeys(forms, ['count', 'methods', 'actions'])
-        || !validUint(forms.count, 50)
-        || !validCredentialCountRecord(forms.methods, CREDENTIAL_METHOD_KEYS, 50)
-        || !validCredentialCountRecord(forms.actions, CREDENTIAL_ACTION_KEYS, 50)
+        || !validUint(forms.count, MAX_CREDENTIAL_FORMS)
+        || !validCredentialCountRecord(forms.methods, CREDENTIAL_METHOD_KEYS, MAX_CREDENTIAL_FORMS)
+        || !validCredentialCountRecord(forms.actions, CREDENTIAL_ACTION_KEYS, MAX_CREDENTIAL_FORMS)
         || CREDENTIAL_METHOD_KEYS.reduce((sum, key) => sum + Number(forms.methods[key]), 0) !== forms.count
         || CREDENTIAL_ACTION_KEYS.reduce((sum, key) => sum + Number(forms.actions[key]), 0) !== forms.count
         || !isJsonObject(inputs)
         || !hasExactKeys(inputs, ['count', 'classifiedCount', 'categories'])
-        || !validUint(inputs.count, 500)
-        || !validUint(inputs.classifiedCount, 500)
+        || !validUint(inputs.count, MAX_CREDENTIAL_INPUTS)
+        || !validUint(inputs.classifiedCount, MAX_CREDENTIAL_INPUTS)
         || Number(inputs.classifiedCount) > Number(inputs.count)
-        || !validCredentialCountRecord(inputs.categories, CREDENTIAL_CATEGORY_KEYS, 500))
+        || !validCredentialCountRecord(inputs.categories, CREDENTIAL_CATEGORY_KEYS, MAX_CREDENTIAL_INPUTS))
         return 'invalid';
+    if (profile.credentialSurfaceVersion === 2 && (!validCredentialFormAttribution(profile.formAttribution)
+        || profile.formAttribution.forms.length !== forms.count
+        || (profile.complete === true && !profile.formAttribution.complete)))
+        return 'invalid';
+    if (profile.credentialSurfaceVersion === 2 && validCredentialFormAttribution(profile.formAttribution)) {
+        const attribution = profile.formAttribution;
+        if (CREDENTIAL_CATEGORY_KEYS.some(key => attribution.forms.reduce((sum, form) => sum + form.categories[key], 0) > Number(inputs.categories[key])))
+            return 'invalid';
+    }
     return 'supported';
 }
 function structuredDataContractState(value) {
@@ -819,6 +829,13 @@ function sanitizeLookupChildProfiles(value) {
     replaceChild('tls', 'TLS profile', 'tls', tlsProfileContractState);
     replaceChild('securityPosture', 'Security-posture profile', 'derived', securityPostureContractState);
     replaceChild('credentialSurfaceProfile', 'Credential-surface profile', 'html', credentialSurfaceContractState);
+    if (availability.hasExternalPasswordForm !== undefined && availability.credentialSurfaceProfile != null) {
+        const credential = availability.credentialSurfaceProfile;
+        if (!isJsonObject(credential) || credential.credentialSurfaceVersion !== 2
+            || availability.hasExternalPasswordForm !== externalPasswordFormObservation(credential.formAttribution)) {
+            mutableAvailability().hasExternalPasswordForm = null;
+        }
+    }
     replaceChild('structuredDataIdentity', 'Structured-data identity profile', 'html', structuredDataContractState);
     replaceChild('pageRoleProfile', 'Page-role profile', 'derived', pageRoleContractState);
     replaceChild('clientBehaviorProfile', 'Client-behaviour profile', 'derived', clientBehaviorContractState);
@@ -864,4 +881,4 @@ function sanitizeLookupChildProfiles(value) {
     }
     return output;
 }
-export { sanitizeLookupChildProfiles, technologyProfileContractState, validSecurityPostureFinding, validTlsChainCertificate, validTlsFinding, };
+export { sanitizeLookupChildProfiles, credentialSurfaceContractState, technologyProfileContractState, validSecurityPostureFinding, validTlsChainCertificate, validTlsFinding, };

package/lib/lookup-contract-primitives.mjs

@@ -2,7 +2,7 @@
 // child-profile contracts. These checks validate without normalising or
 // copying untrusted values.
 import { MAX_OBSERVATION_DIAGNOSTICS } from "../packages/evidence/observation.mjs";
-import { MAX_HTTP_PROVENANCE_URL } from "./http-evidence-bounds.mjs";
+import { MAX_HTTP_PROVENANCE_URL } from "../packages/analysis/http-evidence-bounds.mjs";
 const LOOKUP_CONTROL_CHAR_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/u;
 const MAX_LOOKUP_CONTRACT_HOST_LENGTH = 253;
 function isJsonObject(value) {

package/lib/lookup-diagnostics.mjs

@@ -1,31 +1,9 @@
 // Lookup diagnostics, bounded source errors and optional deep-lookup timing.
 import { OPERATION_BUDGET_ERROR_CODE } from "./operation-budget.mjs";
 import { FEATURE_DISABLED_ERROR_CODE } from "./feature-policy.mjs";
+import { LOOKUP_SOURCE_STATES, lookupDiagnosticStates } from "./lookup-progress.mjs";
 const LOOKUP_DIAGNOSTICS_VERSION = 8;
 const LOOKUP_LEGACY_DIAGNOSTICS_VERSION = 7;
-const LOOKUP_SOURCE_STATES = Object.freeze([
-    'success', 'partial', 'not_found', 'skipped', 'error', 'unsupported',
-    'not_applicable', 'unavailable', 'rate_limited', 'complete', 'disabled', 'stale',
-]);
-const SOURCE_STATES = new Set(LOOKUP_SOURCE_STATES);
-function lookupDiagnosticStates(value) {
-    const record = (input) => input && typeof input === 'object' && !Array.isArray(input)
-        ? input : {};
-    const diagnostics = record(value);
-    const candidates = {
-        rdap: record(diagnostics.rdap).status,
-        registrar_rdap: record(record(diagnostics.rdap).registrar).status,
-        whois: record(diagnostics.whois).status,
-        availability: record(diagnostics.availability).status,
-        reverse_dns: record(diagnostics.reverseDns).status,
-        network_context: record(diagnostics.network).status,
-        security_txt: record(diagnostics.securityTxt).status,
-        sslbl: record(diagnostics.sslbl).status,
-    };
-    return Object.freeze(Object.fromEntries(Object.entries(candidates).map(([source, state]) => [
-        source, typeof state === 'string' && SOURCE_STATES.has(state) ? state : null,
-    ])));
-}
 const LOOKUP_TIMING_VERSION = 1;
 const MAX_LOOKUP_TIMING_MS = 120_000;
 const LOOKUP_TIMING_SOURCE_ORDER = Object.freeze([

package/lib/lookup-network-evidence-bounds.mjs

@@ -1,10 +1 @@
-// Browser-safe collection bounds shared by the network evidence producers,
-// the Lookup response trust boundary, and the display projection.
-export const MAX_LOOKUP_DNS_RECORDS_PER_TYPE = 16;
-export const MAX_LOOKUP_REVERSE_DNS_PTR_RECORDS = 8;
-export const TLS_PROFILE_VERSION = 3;
-export const MAX_LOOKUP_TLS_CHAIN_CERTIFICATES = 8;
-export const MAX_LOOKUP_TLS_ALT_NAMES = 50;
-export const MAX_LOOKUP_TLS_NAME_VALUES = 4;
-export const MAX_LOOKUP_TLS_FINDINGS = 5;
-export const MAX_LOOKUP_TLS_CERTIFICATE_POLICIES = 16;
+export * from "../packages/analysis/lookup-network-evidence-bounds.mjs";

package/lib/lookup-progress.mjs

@@ -1,7 +1,7 @@
 // Portable bounded NDJSON Lookup progress contract. Partial events are
 // presentation-only; only a validated final ordinary Lookup result may cross
 // the persistence boundary.
-import { scanBoundedJson, MAX_BOUNDED_JSON_DEPTH, MAX_BOUNDED_JSON_KEYS, MAX_BOUNDED_JSON_VALUES } from "./bounded-json.mjs";
+import { scanBoundedJson, MAX_BOUNDED_JSON_DEPTH, MAX_BOUNDED_JSON_KEYS, MAX_BOUNDED_JSON_VALUES } from "../packages/analysis/bounded-json.mjs";
 export const LOOKUP_PROGRESS_SCHEMA = 'whoisleuth.lookup-progress';
 export const LOOKUP_PROGRESS_VERSION = 1;
 export const MAX_LOOKUP_PROGRESS_SOURCES = 16;
@@ -32,7 +32,56 @@ const SOURCE_STATES = new Set([
     'unavailable',
     'rate_limited',
 ]);
+// Final diagnostic states are distinct from the provisional progress subset.
+export const LOOKUP_SOURCE_STATES = Object.freeze([
+    'success', 'partial', 'not_found', 'skipped', 'error', 'unsupported',
+    'not_applicable', 'unavailable', 'rate_limited', 'complete', 'disabled', 'stale',
+]);
+const FINAL_SOURCE_STATES = new Set(LOOKUP_SOURCE_STATES);
+export function lookupDiagnosticStates(value) {
+    const record = (input) => input && typeof input === 'object' && !Array.isArray(input)
+        ? input : {};
+    const diagnostics = record(value);
+    const candidates = {
+        rdap: record(diagnostics.rdap).status,
+        registrar_rdap: record(record(diagnostics.rdap).registrar).status,
+        whois: record(diagnostics.whois).status,
+        availability: record(diagnostics.availability).status,
+        reverse_dns: record(diagnostics.reverseDns).status,
+        network_context: record(diagnostics.network).status,
+        security_txt: record(diagnostics.securityTxt).status,
+        sslbl: record(diagnostics.sslbl).status,
+    };
+    return Object.freeze(Object.fromEntries(Object.entries(candidates).map(([source, state]) => [
+        source, typeof state === 'string' && FINAL_SOURCE_STATES.has(state) ? state : null,
+    ])));
+}
 const SOURCE_ID_RE = /^[a-z][a-z0-9_]{0,39}$/u;
+/** The existing Lookup recipe, shared by offline plans and source progress.
+ * This describes admitted families, not provider health or request counts. */
+export function plannedLookupSources(targetType, mode, options = {}) {
+    const sources = ['rdap'];
+    if (mode === 'fast') {
+        if (targetType === 'domain')
+            sources.push('domain_evidence');
+        return Object.freeze(sources);
+    }
+    sources.push('whois');
+    if (targetType === 'domain') {
+        sources.push('domain_evidence', 'registrar_rdap', 'network_context');
+        if (options.securityTxt)
+            sources.push('security_txt');
+        if (options.externalIntelligence)
+            sources.push('external_intelligence');
+        if (options.malwareHostIntelligence)
+            sources.push('malware_host_intelligence');
+        if (options.malwareIocIntelligence)
+            sources.push('malware_ioc_intelligence');
+    }
+    else if (targetType === 'ipv4' || targetType === 'ipv6')
+        sources.push('reverse_dns');
+    return Object.freeze(sources);
+}
 const TEXT_ENCODER = new TextEncoder();
 function byteLength(value) {
     return TEXT_ENCODER.encode(value).byteLength;

package/lib/lookup-response-contract.mjs

@@ -6,12 +6,13 @@
 // relies on before any evidence is derived or rendered. Nested source records
 // stay separately attributed and additive; their own normalizers remain the
 // authority for source-specific fields.
-import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_CATEGORIES, THREAT_INTELLIGENCE_CONFIDENCES, THREAT_INTELLIGENCE_ENVELOPE_VERSION, THREAT_INTELLIGENCE_RESULT_STATES, THREAT_INTELLIGENCE_SCHEMA, THREAT_INTELLIGENCE_SEVERITIES, } from "./threat-intelligence-types.mjs";
-import { MAX_HTTP_ATTEMPTS, MAX_HTTP_ERROR_LENGTH, MAX_HTTP_EVIDENCE_REDIRECTS, } from "./http-evidence-bounds.mjs";
+import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_CATEGORIES, THREAT_INTELLIGENCE_CONFIDENCES, THREAT_INTELLIGENCE_ENVELOPE_VERSION, THREAT_INTELLIGENCE_RESULT_STATES, THREAT_INTELLIGENCE_SCHEMA, THREAT_INTELLIGENCE_SEVERITIES, } from "../packages/analysis/threat-intelligence-types.mjs";
+import { MAX_HTTP_ATTEMPTS, MAX_HTTP_ERROR_LENGTH, MAX_HTTP_EVIDENCE_REDIRECTS, } from "../packages/analysis/http-evidence-bounds.mjs";
 import { MAX_OBSERVATION_LIMITATIONS, MAX_OBSERVATION_LIMITATION_LENGTH, normalizeExplicitIsoTimestamp, } from "../packages/evidence/observation.mjs";
-import { assertBoundedJsonStructure } from "./bounded-json.mjs";
-import { canonicalRegistrableDomain } from "./registrable-domain.mjs";
+import { assertBoundedJsonStructure } from "../packages/analysis/bounded-json.mjs";
+import { canonicalRegistrableDomain } from "../packages/analysis/registrable-domain.mjs";
 import { validLookupObservationScope } from "../packages/evidence/lookup-target.mjs";
+import { isWebCollectionQuality } from "../packages/evidence/collection-quality.mjs";
 import { MAX_LOOKUP_DNS_RECORDS_PER_TYPE, MAX_LOOKUP_REVERSE_DNS_PTR_RECORDS, MAX_LOOKUP_TLS_ALT_NAMES, MAX_LOOKUP_TLS_CERTIFICATE_POLICIES, MAX_LOOKUP_TLS_CHAIN_CERTIFICATES, MAX_LOOKUP_TLS_FINDINGS, MAX_LOOKUP_TLS_NAME_VALUES, } from "./lookup-network-evidence-bounds.mjs";
 import { MAX_SECURITY_POSTURE_FINDINGS, sanitizeLookupChildProfiles, validSecurityPostureFinding, validTlsChainCertificate, validTlsFinding, } from "./lookup-child-profile-contract.mjs";
 import { validHttpDeliveryMetadata, validPagePublicationMetadata, } from "./homepage-metadata-contract.mjs";
@@ -136,22 +137,28 @@ function normalizeThreatFinding(value, providerId) {
 }
 function normalizeThreatProvider(value, expectedDomain) {
     const input = record(value);
+    const identity = record(input.provider);
+    const providerId = boundedThreatText(identity.id, 80);
+    const providerDefinition = providerId && Object.hasOwn(THREAT_INTELLIGENCE_PROVIDERS, providerId)
+        ? THREAT_INTELLIGENCE_PROVIDERS[providerId] : null;
+    const withheld = (reason) => ({ provider: null, withheld: {
+            providerId: providerDefinition ? providerId : null,
+            label: providerDefinition ? `${providerDefinition.label} record` : 'Unrecognised provider record',
+            count: 1, reason,
+        } });
     if (input.schema !== THREAT_INTELLIGENCE_SCHEMA
         || input.version !== THREAT_INTELLIGENCE_CONTRACT_VERSION)
-        return null;
+        return withheld('Unsupported provider format; evidence was withheld.');
     const target = record(input.target);
     if (target.type !== 'domain'
         || target.exposure !== 'registrable_domain'
         || target.value !== expectedDomain)
-        return null;
-    const identity = record(input.provider);
-    const providerId = boundedThreatText(identity.id, 80);
-    const providerDefinition = providerId ? THREAT_INTELLIGENCE_PROVIDERS[providerId] : null;
+        return withheld('The provider target did not match this Lookup; evidence was withheld.');
     if (!providerId || !providerDefinition)
-        return null;
+        return withheld('The provider identity was not recognised; evidence was withheld.');
     const state = boundedThreatText(input.state, 32);
     if (!state || !THREAT_INTELLIGENCE_STATES.has(state))
-        return null;
+        return withheld('The provider result state was not supported; evidence was withheld.');
     const observationInput = record(input.observation);
     const limitations = Array.isArray(observationInput.limitations)
         ? observationInput.limitations
@@ -177,35 +184,40 @@ function normalizeThreatProvider(value, expectedDomain) {
             ? `${omitted} invalid or over-limit provider finding${omitted === 1 ? ' was' : 's were'} omitted from this view.`
             : 'Provider findings were not a supported array and were withheld from this view.');
     }
-    return {
-        schema: THREAT_INTELLIGENCE_SCHEMA,
-        version: THREAT_INTELLIGENCE_CONTRACT_VERSION,
-        provider: {
-            id: providerId,
-            label: providerDefinition.label,
-        },
-        target: {
-            type: 'domain',
-            value: expectedDomain,
-            exposure: 'registrable_domain',
-        },
-        state: projectionIncomplete && ['success', 'not_found'].includes(state) ? 'partial' : state,
-        detail: boundedThreatText(input.detail),
-        findings,
-        observation: {
-            observedAt: threatTimestamp(observationInput.observedAt),
-            limitations: limitations.slice(0, MAX_THREAT_INTELLIGENCE_LIMITATIONS),
-            complete: projectionIncomplete ? false : typeof observationInput.complete === 'boolean' ? observationInput.complete : null,
-            truncated: projectionTruncated ? true : typeof observationInput.truncated === 'boolean' ? observationInput.truncated : null,
-        },
-    };
+    return { withheld: null, provider: {
+            schema: THREAT_INTELLIGENCE_SCHEMA,
+            version: THREAT_INTELLIGENCE_CONTRACT_VERSION,
+            provider: {
+                id: providerId,
+                label: providerDefinition.label,
+            },
+            target: {
+                type: 'domain',
+                value: expectedDomain,
+                exposure: 'registrable_domain',
+            },
+            state: projectionIncomplete && ['success', 'not_found'].includes(state) ? 'partial' : state,
+            detail: boundedThreatText(input.detail),
+            findings,
+            observation: {
+                observedAt: threatTimestamp(observationInput.observedAt),
+                limitations: limitations.slice(0, MAX_THREAT_INTELLIGENCE_LIMITATIONS),
+                complete: projectionIncomplete ? false : typeof observationInput.complete === 'boolean' ? observationInput.complete : null,
+                truncated: projectionTruncated ? true : typeof observationInput.truncated === 'boolean' ? observationInput.truncated : null,
+            },
+        } };
 }
 function projectThreatProviders(values, expectedDomain) {
     const grouped = new Map();
+    const withheld = [];
+    const unexamined = Math.max(0, values.length - MAX_THREAT_INTELLIGENCE_PROVIDERS * 2);
     for (const value of values.slice(0, MAX_THREAT_INTELLIGENCE_PROVIDERS * 2)) {
-        const provider = normalizeThreatProvider(value, expectedDomain);
-        if (!provider)
+        const projection = normalizeThreatProvider(value, expectedDomain);
+        if (projection.withheld) {
+            withheld.push(projection.withheld);
             continue;
+        }
+        const provider = projection.provider;
         const providerId = String(record(provider.provider).id || '');
         if (!providerId)
             continue;
@@ -218,20 +230,25 @@ function projectThreatProviders(values, expectedDomain) {
             grouped.set(providerId, { first: provider, signatures: new Set([signature]) });
         }
     }
-    return [...grouped.values()].slice(0, MAX_THREAT_INTELLIGENCE_PROVIDERS).map(({ first, signatures }) => (signatures.size === 1
+    if (unexamined)
+        withheld.push({ providerId: null, label: 'Additional provider records', count: unexamined,
+            reason: 'The provider-record review bound was exceeded; these records were not examined.' });
+    const withheldIds = new Set(withheld.map((item) => item.providerId).filter(Boolean));
+    const providers = [...grouped.values()].slice(0, MAX_THREAT_INTELLIGENCE_PROVIDERS).map(({ first, signatures }) => (signatures.size === 1 && !withheldIds.has(String(record(first.provider).id)) && unexamined === 0
         ? first
         : {
             ...first,
             state: 'partial',
-            detail: 'Conflicting records were returned for this provider; no conclusive provider projection is available.',
+            detail: 'Conflicting or unvalidated provider records prevent a conclusive provider projection.',
             findings: [],
             observation: {
                 observedAt: null,
-                limitations: ['Conflicting records for the same provider identity were withheld rather than resolved by array order.'],
+                limitations: ['Conflicting or unvalidated records were withheld rather than resolved by array order.'],
                 complete: false,
                 truncated: null,
             },
         }));
+    return { providers, withheld };
 }
 function optionalBoundedText(value, maxLength) {
     return value === undefined || (typeof value === 'string'
@@ -624,10 +641,12 @@ function validAvailabilityScalars(value) {
         && validOptionalNullableText(value.pageTitle, MAX_LOOKUP_RESPONSE_PAGE_TITLE_LENGTH)
         && validOptionalNullableText(value.dnssec, 40)
         && (value.deepScanComplete === undefined || typeof value.deepScanComplete === 'boolean')
+        && (value.webCollectionQuality === undefined || isWebCollectionQuality(value.webCollectionQuality))
         && (value.hasMx === undefined || value.hasMx === null || typeof value.hasMx === 'boolean')
         && (value.hasNullMx === undefined || value.hasNullMx === null || typeof value.hasNullMx === 'boolean')
         && (value.hasSpf === undefined || value.hasSpf === null || typeof value.hasSpf === 'boolean')
         && (value.hasDmarc === undefined || value.hasDmarc === null || typeof value.hasDmarc === 'boolean')
+        && (value.hasExternalPasswordForm === undefined || value.hasExternalPasswordForm === null || typeof value.hasExternalPasswordForm === 'boolean')
         && validOptionalStringArray(value.mxHosts, MAX_LOOKUP_DNS_RECORDS_PER_TYPE, 253);
 }
 function validLookupDomainIdentity(value) {
@@ -860,6 +879,7 @@ function parseCompactLookupHttpResponse(value, expectedDomain) {
         || typeof availability.confidence !== 'string'
         || !COMPACT_CONFIDENCE_LEVELS.has(availability.confidence)
         || (availability.deepScanComplete !== undefined && typeof availability.deepScanComplete !== 'boolean')
+        || (availability.webCollectionQuality !== undefined && !isWebCollectionQuality(availability.webCollectionQuality))
         || (availability.bulkComparison !== undefined
             && !validCompactBulkComparison(availability.bulkComparison))
         || compactPageIdentity !== null && Object.hasOwn(compactPageIdentity, 'publicationMetadata')
@@ -968,11 +988,20 @@ function createLookupViewModel(response) {
         ? canonicalRegistrableDomain(response.registrableDomain)
             ?? canonicalRegistrableDomain(availability.domain)
         : null;
-    const providers = expectedThreatDomain
+    const threatProjection = expectedThreatDomain
         && rawThreatIntelligence.version === THREAT_INTELLIGENCE_ENVELOPE_VERSION
         && Array.isArray(rawThreatIntelligence.providers)
         ? projectThreatProviders(rawThreatIntelligence.providers, expectedThreatDomain)
-        : [];
+        : { providers: [], withheld: Object.keys(rawThreatIntelligence).length ? [{
+                    providerId: null,
+                    label: 'External-intelligence data',
+                    count: Array.isArray(rawThreatIntelligence.providers) ? rawThreatIntelligence.providers.length : null,
+                    reason: !expectedThreatDomain ? 'External intelligence was not bound to a domain Lookup and was withheld.'
+                        : rawThreatIntelligence.version !== THREAT_INTELLIGENCE_ENVELOPE_VERSION
+                            ? 'The external-intelligence envelope uses an unsupported version; its records were withheld.'
+                            : 'External-intelligence records were not a supported array and were withheld.',
+                }] : [] };
+    const { providers, withheld } = threatProjection;
     const threatIntelligence = providers.length
         ? { version: THREAT_INTELLIGENCE_ENVELOPE_VERSION, providers }
         : {};
@@ -1008,6 +1037,7 @@ function createLookupViewModel(response) {
         sslbl,
         threatIntelligence,
         threatIntelligenceProviders: providers,
+        threatIntelligenceWithheld: withheld,
         dnsEvidence,
         dnsRecords: record(dnsEvidence.records),
         httpEvidence,

package/lib/lookup-response.mjs

@@ -1,11 +1,12 @@
 // Lookup result shaping after the source plan has settled. This module starts no
 // collection and preserves compact, diagnostics and enrichment response shapes.
+import { withoutHttpDeliveryMetadata } from "./http-intelligence.mjs";
 import { buildBulkComparisonEvidence } from "./bulk-comparison-evidence.mjs";
 import { failedReverseDnsIntelligence, skippedReverseDnsIntelligence, } from "./dns-intelligence.mjs";
 import { LOOKUP_DIAGNOSTICS_VERSION, LOOKUP_ERROR_CODES, LOOKUP_LEGACY_DIAGNOSTICS_VERSION, boundedSourceDetail, } from "./lookup-diagnostics.mjs";
 import { createThreatIntelligenceResult } from "./threat-intelligence-contract.mjs";
 import { THREATFOX_PROVIDER, URLHAUS_PROVIDER, URLSCAN_PROVIDER, } from "./lookup-threat-provider-inventory.mjs";
-import { THREAT_INTELLIGENCE_ENVELOPE_VERSION } from "./threat-intelligence-types.mjs";
+import { THREAT_INTELLIGENCE_ENVELOPE_VERSION } from "../packages/analysis/threat-intelligence-types.mjs";
 import { registryAccessDiagnosticFor } from "./registry-capabilities.mjs";
 import { buildRegistryInsights } from "./registry-insights.mjs";
 import { buildRegistrarStanding } from "./registrar-standing.mjs";
@@ -20,17 +21,6 @@ function withoutNestedPublicationMetadata(value) {
     delete output.publicationMetadata;
     return output;
 }
-function withoutNestedDeliveryMetadata(value) {
-    if (!value || typeof value !== 'object' || Array.isArray(value))
-        return value;
-    const output = { ...value };
-    if (output.response && typeof output.response === 'object' && !Array.isArray(output.response)) {
-        const response = { ...output.response };
-        delete response.deliveryMetadata;
-        output.response = response;
-    }
-    return output;
-}
 async function buildUnifiedLookupResponse(context) {
     const { availabilityEnabled, availabilityPromise, classified, compact, dnsIntelligenceEnabled, fast, inspectCertificateWarningData, networkContextPromise, options, rdapEnabled, rdapPromise, registrarRdapPromise, reverseDnsEligible, reverseDnsPromise, securityTxtPromise, skipWhois, threatfoxIntelligencePromise, timing, urlhausIntelligencePromise, urlscanIntelligencePromise, whoisEnabled, whoisPromise, } = context;
     const [rdapResult, whoisResult, availabilityResult, reverseDnsResult, registrarRdapResult, networkContextResult, securityTxtResult, urlscanIntelligenceResult, urlhausIntelligenceResult, threatfoxIntelligenceResult] = await Promise.allSettled([
@@ -100,7 +90,7 @@ async function buildUnifiedLookupResponse(context) {
         whois = { skipped: true, detail: 'WHOIS is omitted in fast RDAP-only mode.' };
     }
     else if (Array.isArray(whoisChain)) {
-        whois = { chain: whoisChain, parsed: parseWhoisChain(whoisChain) };
+        whois = { chain: whoisChain, parsed: parseWhoisChain(whoisChain, classified.type === 'domain' ? classified.registrableDomain ?? classified.value : undefined) };
     }
     else {
         whois = {
@@ -154,7 +144,7 @@ async function buildUnifiedLookupResponse(context) {
             ? 'skipped'
             : whoisResult.status === 'rejected'
                 ? 'error'
-                : whoisCollectionStatus(whoisChain);
+                : whoisCollectionStatus(whoisChain, classified.type === 'domain' ? classified.registrableDomain ?? classified.value : undefined);
     const availabilityStatus = classified.type !== 'domain'
         ? 'not_applicable'
         : !availabilityEnabled ? 'disabled'
@@ -298,7 +288,7 @@ async function buildUnifiedLookupResponse(context) {
             availability: {
                 ...compactAvailability,
                 ...(richPageIdentity !== undefined ? { pageIdentity: withoutNestedPublicationMetadata(richPageIdentity) } : {}),
-                ...(richHttp !== undefined ? { http: withoutNestedDeliveryMetadata(richHttp) } : {}),
+                ...(richHttp !== undefined ? { http: withoutHttpDeliveryMetadata(richHttp) } : {}),
                 ...(bulkComparison ? { bulkComparison } : {}),
             },
             diagnostics,

package/lib/lookup-source-progress.mjs

@@ -3,7 +3,8 @@
 // provider payloads, and request errors. The ordinary final Lookup response
 // remains the only authoritative and persistable result.
 import { whoisCollectionStatus } from "./whois-authority.mjs";
-import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_RESULT_STATES, THREAT_INTELLIGENCE_SCHEMA, } from "./threat-intelligence-types.mjs";
+import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_RESULT_STATES, THREAT_INTELLIGENCE_SCHEMA, } from "../packages/analysis/threat-intelligence-types.mjs";
+import { plannedLookupSources } from "./lookup-progress.mjs";
 const RESULT_STATES = new Set([
     'available',
     'expiring',
@@ -38,28 +39,13 @@ function sourceTruncated(value) {
         .some(([key, field]) => field === true && (key === 'truncated' || key.endsWith('Truncated'))));
 }
 function plannedLookupProgressSources(classified, options = {}) {
-    const sources = ['rdap', 'whois'];
-    if (classified.type === 'domain') {
-        sources.push('domain_evidence', 'registrar_rdap', 'network_context');
-        if (options.securityTxt)
-            sources.push('security_txt');
-        if (options.externalIntelligence)
-            sources.push('external_intelligence');
-        if (options.malwareHostIntelligence)
-            sources.push('malware_host_intelligence');
-        if (options.malwareIocIntelligence)
-            sources.push('malware_ioc_intelligence');
-    }
-    else if (classified.type === 'ipv4' || classified.type === 'ipv6') {
-        sources.push('reverse_dns');
-    }
-    return Object.freeze(sources);
+    return plannedLookupSources(classified.type, 'deep', options);
 }
-function normalizedState(source, outcome, value) {
+function normalizedState(source, outcome, value, requestedDomain, requested = true) {
     if (outcome === 'rejected')
         return 'error';
     if (value === null || value === undefined)
-        return 'skipped';
+        return source === 'rdap' && requested ? 'unsupported' : 'skipped';
     if (source === 'rdap') {
         const rdap = record(value);
         if (rdap.upstreamStatus === 404)
@@ -69,7 +55,7 @@ function normalizedState(source, outcome, value) {
         return sourceTruncated(rdap) ? 'partial' : 'success';
     }
     if (source === 'whois') {
-        const status = whoisCollectionStatus(value);
+        const status = whoisCollectionStatus(value, requestedDomain);
         return status === 'complete' ? 'success' : status;
     }
     if (source === 'domain_evidence') {
@@ -94,8 +80,8 @@ function normalizedState(source, outcome, value) {
         return 'skipped';
     return 'error';
 }
-function normalizeLookupSourceSettlement(source, outcome, value) {
-    const state = normalizedState(source, outcome, value);
+function normalizeLookupSourceSettlement(source, outcome, value, requestedDomain, requested = true) {
+    const state = normalizedState(source, outcome, value, requestedDomain, requested);
     const sourceRecord = record(value);
     const threatObservation = THREAT_INTELLIGENCE_SOURCES.has(source)
         ? record(sourceRecord.observation)

package/lib/lookup.mjs

@@ -170,7 +170,7 @@ async function runUnifiedLookup(classified, options = {}) {
         ]);
         const notify = (source, outcome, value) => {
             try {
-                options.onSourceSettled?.(normalizeLookupSourceSettlement(source, outcome, value));
+                options.onSourceSettled?.(normalizeLookupSourceSettlement(source, outcome, value, classified.type === 'domain' ? classified.registrableDomain ?? classified.value : undefined, source !== 'rdap' || rdapEnabled));
             }
             catch {
                 // Presentation callbacks must never change evidence collection.

package/lib/operation-budget.mjs

@@ -34,6 +34,7 @@ const OPERATION_FEATURES = Object.freeze({
     AVAILABILITY_FAST: 'availability_fast',
     AVAILABILITY_DEEP: 'availability_deep',
     CERTIFICATE_TRANSPARENCY: 'certificate_transparency',
+    DOMAIN_FEED_SEARCH: 'domain_feed_search',
     DOMAIN_POSTURE: 'domain_posture',
 });
 const OPERATION_FEATURE_CLASSES = Object.freeze({
@@ -47,6 +48,7 @@ const OPERATION_FEATURE_CLASSES = Object.freeze({
     [OPERATION_FEATURES.AVAILABILITY_FAST]: OPERATION_CLASSES.REGISTRY_LIGHT,
     [OPERATION_FEATURES.AVAILABILITY_DEEP]: OPERATION_CLASSES.REGISTRY_DEEP,
     [OPERATION_FEATURES.CERTIFICATE_TRANSPARENCY]: OPERATION_CLASSES.CERTIFICATE_SEARCH,
+    [OPERATION_FEATURES.DOMAIN_FEED_SEARCH]: OPERATION_CLASSES.CERTIFICATE_SEARCH,
     [OPERATION_FEATURES.DOMAIN_POSTURE]: OPERATION_CLASSES.POSTURE_AUDIT,
 });
 // The browser currently runs at most 12 fast Bulk workers, 4 deep Bulk
@@ -87,6 +89,8 @@ function operationFeatureFor(feature, { fast = false, compact = false } = {}) {
         return OPERATION_FEATURES.WHOIS;
     if (feature === 'certificate_transparency')
         return OPERATION_FEATURES.CERTIFICATE_TRANSPARENCY;
+    if (feature === 'domain_feed_search')
+        return OPERATION_FEATURES.DOMAIN_FEED_SEARCH;
     if (feature === 'domain_posture')
         return OPERATION_FEATURES.DOMAIN_POSTURE;
     return null;