@slicedearth/whoisleuth-cli 2.4.0 → 2.6.0

package/lib/domain-registration.mjs

@@ -0,0 +1,392 @@
+// Registration authority, lifecycle dates and positive-only delegation evidence.
+import { promises as dns } from 'node:dns';
+import { abortable } from "./abort.mjs";
+import { fetchRdapRecord } from "./rdap.mjs";
+import { buildWhoisChain, parseWhoisChain } from "./whois.mjs";
+import { featureDecision, networkFeaturePolicy } from "./feature-policy.mjs";
+import { parseRegistryDate, registryDateIso } from "./registry-dates.mjs";
+import { nonEmptyErrorMessage } from "./error-detail.mjs";
+import { registryServiceAdmissionFor } from "./registry-capabilities.mjs";
+import { recordOrEmpty as errorRecord } from "./json-record.mjs";
+const DNS_DELEGATION_TIMEOUT_MS = 4000;
+const MAX_DELEGATION_NAMESERVERS = 50;
+const MISSING_DNS_CODES = new Set(['ENODATA', 'ENOTFOUND', 'ENONAME', 'NXDOMAIN']);
+function registryStatusToken(value) {
+    return value.toLowerCase().replace(/[^a-z0-9]+/gu, '');
+}
+function rdapEventDate(events, action) {
+    const date = events.find((event) => event.action === action)?.date;
+    return typeof date === 'string' ? date : null;
+}
+function computeAgeDays(dateStr) {
+    const d = parseRegistryDate(dateStr);
+    return d ? Math.floor((Date.now() - d.getTime()) / 86400000) : null;
+}
+function computeDaysUntil(dateStr) {
+    const d = parseRegistryDate(dateStr);
+    return d ? Math.ceil((d.getTime() - Date.now()) / 86400000) : null;
+}
+const PRIVACY_MARKERS = [
+    /redacted for privacy/i,
+    /data protected/i,
+    /privacy\s*protect/i,
+    /whoisguard/i,
+    /domains by proxy/i,
+    /perfect privacy/i,
+    /contact privacy/i,
+    /private registration/i,
+    /identity protect/i,
+    /not disclosed/i,
+    /withheld for privacy/i,
+];
+// Explicit privacy/redaction markers establish a positive privacy signal.
+// Usable public contact data establishes a negative one. A missing or blank
+// contact can instead reflect registry policy, access tier, parser coverage,
+// or ordinary omission, so it remains inconclusive rather than being promoted
+// to an affirmative privacy claim.
+function isPrivacyProtected(registrant) {
+    if (!registrant)
+        return null;
+    const blob = [registrant.name, registrant.org, registrant.email].filter(Boolean).join(' ');
+    if (!blob)
+        return null;
+    return PRIVACY_MARKERS.some((re) => re.test(blob));
+}
+// RDAP Lookup retains a bounded multi-value contact inventory, but Bulk and
+// watchlist availability records deliberately keep the historical compact
+// shape. This prevents repeated contact arrays and registry links from
+// expanding browser-local stores or flowing into case evidence implicitly.
+function compactContact(contact) {
+    if (!contact || typeof contact !== 'object' || Array.isArray(contact))
+        return null;
+    const value = contact;
+    const compact = {
+        handle: value.handle || null,
+        name: value.name || null,
+        org: value.org || null,
+        email: value.email || null,
+        phone: value.phone || null,
+        address: value.address || null,
+    };
+    return Object.values(compact).some(Boolean) ? compact : null;
+}
+// A number of ccTLD registries publish neither an RDAP bootstrap entry nor a
+// generally reachable port-43 WHOIS service. A positive NS answer cannot
+// replace registry registration data, but it does prove that the registrable
+// domain has an active DNS delegation. Use that as a bounded, positive-only
+// fallback: no answer is never interpreted as availability because registered
+// domains can legitimately be undelegated.
+async function checkDnsDelegation(domain, options = {}) {
+    options.signal?.throwIfAborted();
+    // Cancel only this lookup's c-ares requests, never the shared DNS resolver.
+    const ownedResolver = options.resolver ? null : new dns.Resolver();
+    const resolve = options.resolver || ((name) => ownedResolver.resolveNs(name));
+    const controller = new AbortController();
+    const signal = options.signal ? AbortSignal.any([options.signal, controller.signal]) : controller.signal;
+    const cancel = () => ownedResolver?.cancel();
+    signal.addEventListener('abort', cancel, { once: true });
+    const timer = setTimeout(() => controller.abort(new Error('DNS delegation lookup timed out')), DNS_DELEGATION_TIMEOUT_MS);
+    try {
+        const records = await abortable(() => resolve(domain), signal);
+        const validNameservers = [...new Set((Array.isArray(records) ? records : [])
+                .filter((value) => typeof value === 'string')
+                .map((value) => value.trim().replace(/\.+$/, '').toLowerCase())
+                .filter((value) => value.length > 0 && value.length <= 253)
+                .filter((value) => value.split('.').every((label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i.test(label))))]
+            .sort();
+        const normalized = validNameservers.slice(0, MAX_DELEGATION_NAMESERVERS);
+        return {
+            delegated: normalized.length > 0,
+            nameservers: normalized,
+            nameserversTruncated: validNameservers.length > MAX_DELEGATION_NAMESERVERS,
+            error: null,
+        };
+    }
+    catch (err) {
+        options.signal?.throwIfAborted();
+        const error = errorRecord(err);
+        if (typeof error.code === 'string' && MISSING_DNS_CODES.has(error.code)) {
+            return { delegated: false, nameservers: [], nameserversTruncated: false, error: null };
+        }
+        return {
+            delegated: false,
+            nameservers: [],
+            nameserversTruncated: false,
+            error: nonEmptyErrorMessage(err, String(err)).slice(0, 180),
+        };
+    }
+    finally {
+        clearTimeout(timer);
+        signal.removeEventListener('abort', cancel);
+    }
+}
+function registryPolicyDetail(domain, fast) {
+    const details = [];
+    const rdapAdmission = registryServiceAdmissionFor(domain, 'rdap');
+    if (rdapAdmission && !rdapAdmission.allowed) {
+        details.push('RDAP was not queried because the registry access policy does not permit collection.');
+    }
+    const whoisAdmission = fast ? null : registryServiceAdmissionFor(domain, 'whois');
+    if (whoisAdmission && !whoisAdmission.allowed) {
+        details.push(whoisAdmission.state === 'permission_required'
+            ? 'WHOIS was not queried because registry permission or source authorisation is required.'
+            : 'WHOIS was not queried because IANA publishes no domain WHOIS service for this suffix.');
+    }
+    return details.length ? ` ${details.join(' ')}` : '';
+}
+// fast: true skips the WHOIS fallback (no TCP:43 chain) and the homepage
+// fetch (no for-sale/parking detection) - just RDAP plus the signals
+// derivable from it (age, expiry proximity, privacy). Meant for scanning
+// large sourcing candidate lists quickly and gently on registry rate
+// limits; anything it can't resolve (state "unknown") is meant to get a
+// follow-up deep check (fast: false, the default) on the shortlist only.
+async function resolveDomainRegistration(domain, options, featurePolicy) {
+    options.signal?.throwIfAborted();
+    const fast = options.fast === true;
+    const rdapEnabled = featureDecision('rdap', featurePolicy).enabled;
+    const whoisEnabled = featureDecision('whois', featurePolicy).enabled;
+    const dnsIntelligenceEnabled = featureDecision('dns_intelligence', featurePolicy).enabled;
+    const hasPreloadedRdap = Object.prototype.hasOwnProperty.call(options, 'rdapRecord');
+    const hasPreloadedWhois = Object.prototype.hasOwnProperty.call(options, 'whoisChain');
+    const hasPreloadedRdapPromise = Object.prototype.hasOwnProperty.call(options, 'rdapRecordPromise');
+    const hasPreloadedWhoisPromise = Object.prototype.hasOwnProperty.call(options, 'whoisChainPromise');
+    const hasPreloadedDnsDelegation = Object.prototype.hasOwnProperty.call(options, 'dnsDelegation');
+    let nameservers = [];
+    let statuses = [];
+    let rdapServer = null;
+    let rdapFound = false;
+    let registrar = null;
+    let registrant = null;
+    let abuse = null;
+    let createdDate = null;
+    let expiryDate = null;
+    let createdDateIso = null;
+    let expiryDateIso = null;
+    let registrationSource = null;
+    let registrationConfidence = 'high';
+    let dnssec = null;
+    let registryDnsEvidence = null;
+    const assessment = (registration) => ({
+        registration, registryDnsEvidence, nameservers, dnssec,
+    });
+    if (rdapEnabled) {
+        try {
+            // Shared with /api/rdap (lib/rdap.mts's fetchRdapRecord) rather than a
+            // separate fetch+parse here - same registry data either way, and this
+            // also picks up that function's short-TTL cache (lib/lookup-cache.mts)
+            // and upstream timeout for free.
+            const recordValue = hasPreloadedRdapPromise
+                ? await abortable(() => options.rdapRecordPromise, options.signal)
+                : hasPreloadedRdap
+                    ? options.rdapRecord
+                    : await fetchRdapRecord('domain', domain, options.signal ? { signal: options.signal } : {});
+            options.signal?.throwIfAborted();
+            const record = errorRecord(recordValue);
+            const recordRdapServer = typeof record.rdapServer === 'string' ? record.rdapServer : null;
+            const upstreamStatus = typeof record.upstreamStatus === 'number' ? record.upstreamStatus : null;
+            if (Object.keys(record).length) {
+                rdapServer = recordRdapServer;
+                if (upstreamStatus === 404) {
+                    const cleartext = record.transportSecurity === 'http'
+                        || (recordRdapServer !== null && /^http:\/\//iu.test(recordRdapServer));
+                    return assessment({
+                        state: 'available',
+                        confidence: cleartext ? 'medium' : 'high',
+                        detail: `The registry's RDAP service has no record for this domain.${cleartext ? ' This answer arrived over unencrypted HTTP, so its transport was not authenticated.' : ''}`,
+                        source: 'rdap',
+                        rdapServer: recordRdapServer,
+                    });
+                }
+                const parsed = errorRecord(record.parsed);
+                if (Object.keys(parsed).length) {
+                    registryDnsEvidence = parsed;
+                    statuses = Array.isArray(parsed.statuses)
+                        ? parsed.statuses.filter((status) => typeof status === 'string').map((status) => status.toLowerCase())
+                        : [];
+                    nameservers = Array.isArray(parsed.nameservers)
+                        ? parsed.nameservers.filter((nameserver) => typeof nameserver === 'string')
+                        : [];
+                    registrar = compactContact(parsed.registrar);
+                    registrant = compactContact(parsed.registrant);
+                    abuse = compactContact(parsed.abuse);
+                    const events = Array.isArray(parsed.events)
+                        ? parsed.events.map(errorRecord)
+                        : [];
+                    const lifecycle = errorRecord(parsed.lifecycle);
+                    createdDate = typeof lifecycle.createdDate === 'string'
+                        ? lifecycle.createdDate
+                        : rdapEventDate(events, 'registration');
+                    expiryDate = typeof lifecycle.expiryDate === 'string'
+                        ? lifecycle.expiryDate
+                        : rdapEventDate(events, 'expiration');
+                    createdDateIso = typeof lifecycle.createdDateIso === 'string'
+                        ? lifecycle.createdDateIso
+                        : registryDateIso(createdDate);
+                    expiryDateIso = typeof lifecycle.expiryDateIso === 'string'
+                        ? lifecycle.expiryDateIso
+                        : registryDateIso(expiryDate);
+                    dnssec = typeof parsed.dnssec === 'string' ? parsed.dnssec : null;
+                    rdapFound = true;
+                    registrationSource = 'rdap';
+                }
+            }
+        }
+        catch {
+            options.signal?.throwIfAborted();
+            /* fall through to WHOIS-based detection (deep mode only) */
+        }
+    }
+    const dnsDelegationPromise = !rdapFound && dnsIntelligenceEnabled
+        ? hasPreloadedDnsDelegation
+            ? Promise.resolve(options.dnsDelegation)
+            : checkDnsDelegation(domain, {
+                ...(options.resolveNs ? { resolver: options.resolveNs } : {}),
+                ...(options.signal ? { signal: options.signal } : {}),
+            })
+        : null;
+    // DNS may settle while the independent WHOIS source is still pending. Keep
+    // its rejection observed even if WHOIS supplies an early conclusive return.
+    void dnsDelegationPromise?.catch(() => { });
+    let whoisChain = null;
+    let whoisParsed = null;
+    if (!rdapFound && !fast && whoisEnabled) {
+        try {
+            whoisChain = (hasPreloadedWhoisPromise
+                ? await abortable(() => options.whoisChainPromise, options.signal)
+                : hasPreloadedWhois
+                    ? options.whoisChain
+                    : await buildWhoisChain(domain, options.signal ? { signal: options.signal } : {})) ?? null;
+            options.signal?.throwIfAborted();
+            if (!Array.isArray(whoisChain))
+                throw new Error('WHOIS chain unavailable');
+            const parsed = parseWhoisChain(whoisChain, domain);
+            whoisParsed = parsed;
+            if (parsed.notFound) {
+                return assessment({
+                    state: 'available',
+                    confidence: 'medium',
+                    detail: `WHOIS reports no matching record for this domain${parsed.notFoundSource ? ` (per ${parsed.notFoundSource})` : ''}.`,
+                    source: 'whois',
+                });
+            }
+            if (parsed.registrationStatus === 'registered') {
+                if (parsed.nameservers.length)
+                    nameservers = parsed.nameservers;
+                if (parsed.statuses.length)
+                    statuses = parsed.statuses.map((status) => status.toLowerCase());
+                if (parsed.registrar) {
+                    registrar = {
+                        handle: null,
+                        ianaId: parsed.registrarIanaId || null,
+                        name: parsed.registrar,
+                        org: null,
+                        email: parsed.abuseEmail || null,
+                        phone: parsed.abusePhone || null,
+                    };
+                }
+                if (parsed.registrantName || parsed.registrantOrg || parsed.registrantEmail || parsed.registrantPhone) {
+                    registrant = {
+                        handle: null,
+                        name: parsed.registrantName || null,
+                        org: parsed.registrantOrg || null,
+                        email: parsed.registrantEmail || null,
+                        phone: parsed.registrantPhone || null,
+                    };
+                }
+                if (parsed.abuseEmail || parsed.abusePhone) {
+                    abuse = { handle: null, name: null, org: parsed.registrar || null, email: parsed.abuseEmail || null, phone: parsed.abusePhone || null };
+                }
+                createdDate = parsed.createdDate || null;
+                expiryDate = parsed.expiryDate || null;
+                createdDateIso = parsed.createdDateIso || parsed.lifecycle?.createdDateIso || registryDateIso(createdDate);
+                expiryDateIso = parsed.expiryDateIso || parsed.lifecycle?.expiryDateIso || registryDateIso(expiryDate);
+                if (!dnssec)
+                    dnssec = parsed.dnssec || null;
+                registrationSource = 'whois';
+            }
+        }
+        catch {
+            options.signal?.throwIfAborted();
+            /* if both RDAP and WHOIS fail, we simply can't determine availability */
+        }
+    }
+    // WHOIS ran and was not a confirmed not-found, but produced no positive
+    // registration evidence either - e.g. the registry answered inconclusively
+    // or every referral hop failed/rate-limited. Report "unknown" rather than
+    // fabricating "registered" from an empty record.
+    const hasWhoisRegistrationData = whoisParsed?.registrationStatus === 'registered';
+    let dnsDelegated = false;
+    if (!rdapFound && !hasWhoisRegistrationData && dnsDelegationPromise) {
+        const delegation = await dnsDelegationPromise;
+        if (delegation && delegation.delegated === true && Array.isArray(delegation.nameservers) && delegation.nameservers.length) {
+            dnsDelegated = true;
+            nameservers = delegation.nameservers;
+            registrationSource = 'dns';
+            registrationConfidence = 'medium';
+        }
+    }
+    options.signal?.throwIfAborted();
+    if (!rdapFound && !hasWhoisRegistrationData && !dnsDelegated) {
+        const disabledSources = [
+            !rdapEnabled ? 'RDAP' : null,
+            !fast && !whoisEnabled ? 'WHOIS' : null,
+            !dnsIntelligenceEnabled ? 'DNS intelligence' : null,
+        ].filter(Boolean);
+        const disabledDetail = disabledSources.length
+            ? ` ${disabledSources.join(', ')} ${disabledSources.length === 1 ? 'is' : 'are'} disabled by deployment policy.`
+            : '';
+        const policyDetail = registryPolicyDetail(domain, fast);
+        return assessment({
+            state: 'unknown',
+            confidence: 'low',
+            detail: fast
+                ? `No enabled registration source produced a record or authoritative delegation. A fast scan cannot determine registration status.${disabledDetail}${policyDetail}`
+                : whoisParsed && whoisParsed.failedHop
+                    ? `WHOIS was inconclusive - a referral hop did not answer conclusively (${whoisParsed.failedHop}).${policyDetail}`
+                    : `No enabled registration source returned conclusive data or an authoritative DNS delegation.${disabledDetail}${policyDetail}`,
+            ...(!fast && whoisEnabled ? { source: 'whois' } : {}),
+        });
+    }
+    const domainAgeDays = computeAgeDays(createdDateIso || createdDate);
+    const expiresInDays = computeDaysUntil(expiryDateIso || expiryDate);
+    // DNS proves delegation only; it says nothing about whether registry
+    // contact data is privacy-protected or merely unavailable.
+    const privacyProtected = registrationSource === 'dns' ? null : isPrivacyProtected(registrant);
+    const baseInfo = {
+        nameservers,
+        statuses,
+        registrar,
+        registrant,
+        abuse,
+        createdDate,
+        expiryDate,
+        createdDateIso,
+        expiryDateIso,
+        rdapServer,
+        domainAgeDays,
+        expiresInDays,
+        privacyProtected,
+        dnssec,
+        source: registrationSource,
+    };
+    if (statuses.some((status) => {
+        const normalizedStatus = registryStatusToken(status);
+        return normalizedStatus.includes('pendingdelete') || normalizedStatus.includes('redemptionperiod');
+    })) {
+        return assessment({
+            state: 'expiring',
+            confidence: 'medium',
+            detail: 'Domain is in redemption/pending-delete status and may become available soon.',
+            ...baseInfo,
+        });
+    }
+    return assessment({
+        state: 'registered',
+        confidence: registrationConfidence,
+        detail: registrationSource === 'dns'
+            ? 'Authoritative DNS delegation confirms the domain is registered, but RDAP/WHOIS registration details were unavailable.'
+            : 'Domain is registered. Run a deep check for parking/for-sale detection.',
+        ...baseInfo,
+    });
+}
+export { resolveDomainRegistration, checkDnsDelegation, isPrivacyProtected };

package/lib/evidence-export-privacy.mjs

@@ -1,9 +1,10 @@
 // Portable tree bounds, private-field exclusion and availability projection.
 // Source-specific publication assembly remains in evidence-export.mts.
-import { isSafeJsonObjectKey } from "./bounded-json.mjs";
+import { isSafeJsonObjectKey } from "../packages/analysis/bounded-json.mjs";
 import { recordOrNull } from "./json-record.mjs";
-import { isUriShapedLabel } from "./portable-generator.mjs";
+import { isUriShapedLabel } from "../packages/analysis/portable-generator.mjs";
 import { validHttpDeliveryMetadata, validPagePublicationMetadata } from "./homepage-metadata-contract.mjs";
+import { normalizeWebCollectionQuality } from "../packages/evidence/collection-quality.mjs";
 import { LOOKUP_EVIDENCE_PORTABLE_MAX_BYTES, LOOKUP_EVIDENCE_PORTABLE_MAX_ENTRIES, LOOKUP_EVIDENCE_PORTABLE_MAX_DEPTH, LOOKUP_EVIDENCE_PORTABLE_MAX_ARRAY_ITEMS, LOOKUP_EVIDENCE_PORTABLE_MAX_KEY_LENGTH, LOOKUP_EVIDENCE_PORTABLE_MAX_STRING_LENGTH, } from "../packages/contracts/lookup-evidence.mjs";
 const PUBLIC_LOOKUP_AVAILABILITY_ANALYSIS_KEYS = new Set([
     'applicable', 'type', 'domain', 'state', 'confidence', 'detail', 'source',
@@ -12,7 +13,7 @@ const PUBLIC_LOOKUP_AVAILABILITY_ANALYSIS_KEYS = new Set([
     'expiresInDays', 'privacyProtected', 'dnssec', 'activityStatus',
     'websiteProbeStatus', 'websiteProbeDetail', 'http', 'deepScanComplete',
     'faviconHash', 'faviconPHash', 'pageTitle', 'hasPasswordField',
-    'phishingLanguageMatch', 'hasExternalFormAction', 'externalAssetHosts',
+    'phishingLanguageMatch', 'hasExternalFormAction', 'hasExternalPasswordForm', 'externalAssetHosts',
     'pageIdentity', 'credentialSurfaceProfile', 'structuredDataIdentity',
     'technologyProfile', 'pageRoleProfile', 'clientBehaviorProfile',
     'securityPosture', 'dns', 'tls', 'hasMx', 'hasNullMx', 'mxHosts', 'hasSpf',
@@ -22,7 +23,9 @@ const LOOKUP_AVAILABILITY_ANALYSIS_KEYS = new Set([
     ...[...PUBLIC_LOOKUP_AVAILABILITY_ANALYSIS_KEYS]
         .filter((key) => !['registrar', 'registrant', 'abuse'].includes(key)),
     'registryContactsExcluded', 'observationHostname', 'webObservationMode',
+    'webCollectionQuality',
 ]);
+const HISTORICAL_LOOKUP_AVAILABILITY_ANALYSIS_KEYS = new Set([...LOOKUP_AVAILABILITY_ANALYSIS_KEYS].filter(key => key !== 'webCollectionQuality'));
 const PRIVATE_EVIDENCE_KEYS = new Set([
     'authorization', 'proxyauthorization', 'cookie', 'cookies', 'setcookie',
     'session', 'sessionid', 'sessiontoken', 'token', 'accesstoken', 'refreshtoken',
@@ -102,6 +105,7 @@ const LOOKUP_AVAILABILITY_PORTABLE_NESTED_KEYS = new Set([
     // Credential, structured identity, role, behaviour, technology, and posture
     // projections. Only fixed categories/counts and bounded descriptions enter.
     'credentialSurfaceVersion', 'methods', 'actions', 'missing', 'get', 'post',
+    'formAttribution', 'unassociatedInputs', 'index', 'destinations', 'relationship', 'origin',
     'dialog', 'other', 'sameOrigin', 'external', 'cleartext', 'unclassified',
     'inputs', 'classifiedCount', 'categories', 'password', 'email', 'username',
     'one_time_code', 'payment', 'inputsObserved', 'classifiedInputs',
@@ -174,7 +178,7 @@ function privateEvidenceKey(value, item) {
         || /(?:request|response|authorization|proxy)headers?$/u.test(normalized)
         || (normalized.includes('credential') && !normalized.startsWith('credentialsurface'));
 }
-export function portableUri(value) {
+export function portableUri(value, legacyUris = false) {
     const text = boundedString(value, 2048);
     if (!text)
         return null;
@@ -184,7 +188,7 @@ export function portableUri(value) {
             || url.username
             || url.password)
             return null;
-        if (url.protocol === 'http:' || url.protocol === 'https:') {
+        if (!legacyUris || url.protocol === 'http:' || url.protocol === 'https:') {
             url.search = '';
             url.hash = '';
         }
@@ -268,7 +272,7 @@ function projectLookupEvidenceAvailabilityPublicValue(value, path, state, depth)
     if (typeof value === 'string') {
         const normalizedParent = normalizedEvidenceKey(path.at(-1) || '');
         if (PORTABLE_URL_KEYS.has(normalizedParent))
-            return portableUri(value);
+            return portableUri(value, true);
         if (PORTABLE_ORIGIN_COLLECTION_KEYS.has(normalizedParent))
             return portableOrigin(value);
         return projectPortableString(value);
@@ -313,21 +317,23 @@ function projectLookupEvidenceAvailabilityPublicValue(value, path, state, depth)
     }
     return output;
 }
-function projectLookupEvidenceAvailabilityValue(value, path, state, depth) {
+function projectLookupEvidenceAvailabilityValue(value, path, state, depth, legacyUris) {
     consumePortableProjectionEntry(state, depth);
     if (value === null || typeof value === 'boolean' || typeof value === 'number')
         return value;
     if (typeof value === 'string') {
         const normalizedParent = normalizedEvidenceKey(path.at(-1) || '');
+        if (path.join('.') === 'credentialSurfaceProfile.formAttribution.forms.destinations.origin')
+            return portableOrigin(value);
         if (PORTABLE_URL_KEYS.has(normalizedParent))
-            return portableUri(value);
+            return portableUri(value, legacyUris);
         if (PORTABLE_ORIGIN_COLLECTION_KEYS.has(normalizedParent))
             return portableOrigin(value);
         return projectLookupEvidenceAvailabilityString(value);
     }
     if (Array.isArray(value)) {
         return value.slice(0, LOOKUP_EVIDENCE_PORTABLE_MAX_ARRAY_ITEMS)
-            .map((item) => projectLookupEvidenceAvailabilityValue(item, path, state, depth + 1));
+            .map((item) => projectLookupEvidenceAvailabilityValue(item, path, state, depth + 1, legacyUris));
     }
     const source = recordOrNull(value);
     if (!source)
@@ -341,7 +347,7 @@ function projectLookupEvidenceAvailabilityValue(value, path, state, depth) {
         const boundedCount = Number.isSafeInteger(item)
             && Number(item) >= 0
             && Number(item) <= LOOKUP_EVIDENCE_PORTABLE_MAX_ARRAY_ITEMS;
-        const credentialCategoryCount = parentPath === 'credentialSurfaceProfile.inputs.categories'
+        const credentialCategoryCount = ['credentialSurfaceProfile.inputs.categories', 'credentialSurfaceProfile.formAttribution.forms.categories'].includes(parentPath)
             && boundedCount;
         const sanClassCount = parentPath === 'tls.certificate.subjectAltNames.classes'
             && LOOKUP_AVAILABILITY_TLS_SAN_CLASS_KEYS.has(key)
@@ -371,7 +377,7 @@ function projectLookupEvidenceAvailabilityValue(value, path, state, depth) {
             && (!LOOKUP_AVAILABILITY_VALUE_PATHS.has(parentPath)
                 || (item !== null && typeof item !== 'string')))
             continue;
-        output[key] = projectLookupEvidenceAvailabilityValue(item, [...path, key], state, depth + 1);
+        output[key] = projectLookupEvidenceAvailabilityValue(item, [...path, key], state, depth + 1, legacyUris);
     }
     return output;
 }
@@ -381,7 +387,7 @@ function projectLookupEvidenceAvailabilityValue(value, path, state, depth) {
  * extension cannot become authenticated evidence merely by avoiding a
  * credential-key denylist.
  */
-function projectLookupEvidenceAvailabilityWithKeys(value, rootKeys, registryContactsExcluded, currentPrivacyRules) {
+function projectLookupEvidenceAvailabilityWithKeys(value, rootKeys, registryContactsExcluded, currentPrivacyRules, legacyUris = false) {
     const source = recordOrNull(value);
     if (!source)
         return null;
@@ -394,8 +400,12 @@ function projectLookupEvidenceAvailabilityWithKeys(value, rootKeys, registryCont
         const item = source[key];
         if (privateEvidenceKey(key, item))
             continue;
+        if (key === 'webCollectionQuality') {
+            output[key] = normalizeWebCollectionQuality(item);
+            continue;
+        }
         output[key] = currentPrivacyRules
-            ? projectLookupEvidenceAvailabilityValue(item, [key], state, 1)
+            ? projectLookupEvidenceAvailabilityValue(item, [key], state, 1, legacyUris)
             : projectLookupEvidenceAvailabilityPublicValue(item, [key], state, 1);
     }
     if (registryContactsExcluded)
@@ -437,10 +447,10 @@ export function projectLookupEvidenceAvailabilityPublic(value) {
  * and abuse contact routes are deliberately excluded at the root boundary;
  * the marker prevents that privacy omission from being read as source absence.
  */
-export function projectLookupEvidenceAvailability(value) {
-    return projectLookupEvidenceAvailabilityWithKeys(value, LOOKUP_AVAILABILITY_ANALYSIS_KEYS, true, true);
+export function projectLookupEvidenceAvailability(value, { legacyUris = false } = {}) {
+    return projectLookupEvidenceAvailabilityWithKeys(value, legacyUris ? HISTORICAL_LOOKUP_AVAILABILITY_ANALYSIS_KEYS : LOOKUP_AVAILABILITY_ANALYSIS_KEYS, true, true, legacyUris);
 }
-export function assertLookupEvidencePrivacySafeTree(value) {
+export function assertLookupEvidencePrivacySafeTree(value, { legacyUris = false } = {}) {
     assertLookupEvidencePortableTree(value);
     const pending = [value];
     while (pending.length) {
@@ -466,7 +476,7 @@ export function assertLookupEvidencePrivacySafeTree(value) {
             if (privateEvidenceKey(normalized, item)) {
                 throw new TypeError('Lookup evidence contains excluded private request or session material.');
             }
-            if (typeof item === 'string' && PORTABLE_URL_KEYS.has(normalized) && portableUri(item) !== item) {
+            if (typeof item === 'string' && PORTABLE_URL_KEYS.has(normalized) && portableUri(item, legacyUris) !== item) {
                 throw new TypeError('Lookup evidence contains a credential-bearing or non-portable URL.');
             }
             pending.push(item);

package/lib/evidence-export.mjs

@@ -3,12 +3,12 @@ import { projectLookupEvidencePrivacySafeTree, projectLookupEvidenceAvailability
 export { projectLookupEvidencePrivacySafeTree, projectLookupEvidenceAvailabilityPublic, projectLookupEvidenceAvailability, assertLookupEvidencePrivacySafeTree, assertLookupEvidencePortableTree, serializeLookupEvidence, } from "./evidence-export-privacy.mjs";
 import { compareRdapPublications, compareRegistrySources } from "./registry-comparison.mjs";
 import { buildRegistryInsights } from "./registry-insights.mjs";
-import { buildPortableGeneratorMetadata } from "./portable-generator.mjs";
-import { assertBoundedJsonStructure } from "./bounded-json.mjs";
+import { buildPortableGeneratorMetadata } from "../packages/analysis/portable-generator.mjs";
+import { assertBoundedJsonStructure } from "../packages/analysis/bounded-json.mjs";
 import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
 import { LOOKUP_EVIDENCE_SCHEMA, LOOKUP_EVIDENCE_SCHEMA_VERSION, } from "../packages/contracts/lookup-evidence.mjs";
 export * from "../packages/contracts/lookup-evidence.mjs";
-import { isValidAsciiHostname } from "./hostname.mjs";
+import { isValidAsciiHostname } from "../packages/contracts/domain-name.mjs";
 import { validLookupObservationScope } from "../packages/evidence/lookup-target.mjs";
 import { REGISTRAR_STANDING_SCHEMA, REGISTRAR_STANDING_VERSION, registrarIanaIds, registrarStandingObservedBy, resolveRegistrarIanaId, validRegistrarStanding, } from "./registrar-standing-contract.mjs";
 const REGISTRAR_RDAP_STATUSES = new Set([

package/lib/evidence-report.mjs

@@ -1,5 +1,6 @@
 import { registryAccessProfileLabel } from "./registry-access.mjs";
-import { buildPortableGeneratorMetadata, portableGeneratorAttribution, } from "./portable-generator.mjs";
+import { dnsAddressFamilyEvidence } from "../packages/evidence/dns-query-outcome.mjs";
+import { buildPortableGeneratorMetadata, portableGeneratorAttribution, } from "../packages/analysis/portable-generator.mjs";
 import { technologyEvidenceRoles, } from "./technology-evidence-role.mjs";
 const MAX_REPORT_VALUE_LENGTH = 300;
 const MAX_REPORT_LIST_ITEMS = 50;
@@ -241,6 +242,8 @@ function buildLookupEvidenceReport(document, options = {}) {
             title: 'DNS and mail',
             fields: [
                 reportField('DNS status', displayLabel(dns.status)),
+                reportField('A address result', dnsAddressFamilyEvidence(dns, 'a').value),
+                reportField('AAAA address result', dnsAddressFamilyEvidence(dns, 'aaaa').value),
                 reportField('Observed', dns.observedAt),
                 reportField('Name servers', listText(availability.nameservers)),
                 reportField('MX hosts', listText(availability.mxHosts)),
@@ -264,6 +267,7 @@ function buildLookupEvidenceReport(document, options = {}) {
                 reportField('Redirects', http.redirectCount),
                 reportField('Page title', availability.pageTitle),
                 reportField('Password field observed', yesNoUnknown(availability.hasPasswordField)),
+                reportField('Password form declares external destination', yesNoUnknown(availability.hasExternalPasswordForm)),
                 reportField('Authoritative nameserver evidence', listText(availability.nameservers)),
                 reportField('Observed edge, CDN, reverse proxy or WAF', reportTechnologyNames(technologyFindings, 'observed_edge')),
                 reportField('Application-platform indicators', reportTechnologyNames(technologyFindings, 'application_platform')),