@slicedearth/whoisleuth-cli 2.3.0 → 2.4.0

package/lib/lookup-response.mjs

@@ -2,7 +2,7 @@
 // collection and preserves compact, diagnostics and enrichment response shapes.
 import { buildBulkComparisonEvidence } from "./bulk-comparison-evidence.mjs";
 import { failedReverseDnsIntelligence, skippedReverseDnsIntelligence, } from "./dns-intelligence.mjs";
-import { LOOKUP_DIAGNOSTICS_VERSION, LOOKUP_ERROR_CODES, LOOKUP_LEGACY_DIAGNOSTICS_VERSION, boundedSourceDetail, errorMessage, } from "./lookup-diagnostics.mjs";
+import { LOOKUP_DIAGNOSTICS_VERSION, LOOKUP_ERROR_CODES, LOOKUP_LEGACY_DIAGNOSTICS_VERSION, boundedSourceDetail, } from "./lookup-diagnostics.mjs";
 import { createThreatIntelligenceResult } from "./threat-intelligence-contract.mjs";
 import { THREATFOX_PROVIDER, URLHAUS_PROVIDER, URLSCAN_PROVIDER, } from "./lookup-threat-provider-inventory.mjs";
 import { THREAT_INTELLIGENCE_ENVELOPE_VERSION } from "./threat-intelligence-types.mjs";
@@ -12,6 +12,7 @@ import { buildRegistrarStanding } from "./registrar-standing.mjs";
 import { resolveRegistrarIanaId } from "./registrar-standing-contract.mjs";
 import { securityTxtUnavailable } from "./security-txt.mjs";
 import { parseWhoisChain } from "./whois.mjs";
+import { whoisCollectionStatus } from "./whois-authority.mjs";
 function withoutNestedPublicationMetadata(value) {
     if (!value || typeof value !== 'object' || Array.isArray(value))
         return value;
@@ -125,10 +126,8 @@ async function buildUnifiedLookupResponse(context) {
             availability = {
                 applicable: true,
                 ...result,
-                // Every observation nested in the availability envelope is collected
-                // against this registrable target. Request context remains on the
-                // top-level response; exact-host sources such as security.txt retain
-                // their own requested/final URL attribution.
+                // Registration authority retains its registrable target. Deep source
+                // observations declare their independently collected hostname.
                 domain: classified.value,
             };
         }
@@ -138,7 +137,7 @@ async function buildUnifiedLookupResponse(context) {
                 domain: classified.value,
                 state: 'unknown',
                 confidence: 'low',
-                detail: errorMessage(availabilityResult.reason, 'Availability lookup failed'),
+                detail: boundedSourceDetail(availabilityResult.reason, 'Availability lookup failed'),
             };
         }
     }
@@ -153,11 +152,9 @@ async function buildUnifiedLookupResponse(context) {
         ? 'disabled'
         : skipWhois
             ? 'skipped'
-            : whoisResult.status === 'rejected' || !Array.isArray(whoisChain)
+            : whoisResult.status === 'rejected'
                 ? 'error'
-                : whoisChain.length === 1
-                    ? 'unsupported'
-                    : whois.parsed && whois.parsed.chainStatus === 'complete' ? 'complete' : 'partial';
+                : whoisCollectionStatus(whoisChain);
     const availabilityStatus = classified.type !== 'domain'
         ? 'not_applicable'
         : !availabilityEnabled ? 'disabled'

package/lib/lookup-source-progress.mjs

@@ -2,6 +2,7 @@
 // Lookup. These deliberately exclude raw RDAP/WHOIS bodies, contacts, URLs,
 // provider payloads, and request errors. The ordinary final Lookup response
 // remains the only authoritative and persistable result.
+import { whoisCollectionStatus } from "./whois-authority.mjs";
 import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_RESULT_STATES, THREAT_INTELLIGENCE_SCHEMA, } from "./threat-intelligence-types.mjs";
 const RESULT_STATES = new Set([
     'available',
@@ -31,6 +32,11 @@ function record(value) {
         ? value
         : {};
 }
+function sourceTruncated(value) {
+    // RDAP's normalised *Truncated fields describe loss independently of HTTP success.
+    return [record(value), record(record(value).parsed)].some(part => Object.entries(part)
+        .some(([key, field]) => field === true && (key === 'truncated' || key.endsWith('Truncated'))));
+}
 function plannedLookupProgressSources(classified, options = {}) {
     const sources = ['rdap', 'whois'];
     if (classified.type === 'domain') {
@@ -58,17 +64,16 @@ function normalizedState(source, outcome, value) {
         const rdap = record(value);
         if (rdap.upstreamStatus === 404)
             return 'not_found';
-        return rdap.upstreamStatus === 200 ? 'success' : 'unsupported';
+        if (rdap.upstreamStatus !== 200 || !Object.keys(record(rdap.parsed)).length)
+            return 'error';
+        return sourceTruncated(rdap) ? 'partial' : 'success';
     }
     if (source === 'whois') {
-        if (!Array.isArray(value))
-            return 'error';
-        if (value.length <= 1)
-            return 'unsupported';
-        return 'success';
+        const status = whoisCollectionStatus(value);
+        return status === 'complete' ? 'success' : status;
     }
     if (source === 'domain_evidence') {
-        return record(value).deepScanComplete === false ? 'partial' : 'success';
+        return record(value).deepScanComplete === true ? 'success' : 'partial';
     }
     if (THREAT_INTELLIGENCE_SOURCES.has(source)) {
         const intelligence = record(value);
@@ -81,13 +86,13 @@ function normalizedState(source, outcome, value) {
     }
     const status = record(value).status;
     if (typeof status === 'string' && DIRECT_STATES.has(status)) {
+        if (status === 'success' && (sourceTruncated(value) || record(value).complete === false))
+            return 'partial';
         return status;
     }
-    if (status === 'complete' || status === 'observed' || status === 'found')
-        return 'success';
     if (status === 'not_applicable' || status === 'disabled')
         return 'skipped';
-    return 'success';
+    return 'error';
 }
 function normalizeLookupSourceSettlement(source, outcome, value) {
     const state = normalizedState(source, outcome, value);
@@ -102,12 +107,10 @@ function normalizeLookupSourceSettlement(source, outcome, value) {
         : null;
     const complete = threatObservation
         ? state !== 'error' && threatObservation.complete === true
-        : !['partial', 'error', 'unavailable', 'rate_limited'].includes(state);
+        : ['success', 'not_found'].includes(state) && sourceRecord.complete !== false;
     const truncated = threatObservation
         ? threatObservation.truncated === true
-        : sourceRecord.truncated === true
-            || sourceRecord.recordsTruncated === true
-            || sourceRecord.bodyTruncated === true;
+        : sourceTruncated(value);
     const limitation = outcome === 'rejected'
         ? 'This source did not complete. No absence or safety conclusion was inferred.'
         : state === 'skipped'

package/lib/lookup.mjs

@@ -21,6 +21,7 @@ import { inspectSslblCertificate } from "./sslbl-intelligence.mjs";
 import { normalizeLookupSourceSettlement, plannedLookupProgressSources, } from "./lookup-source-progress.mjs";
 import { buildUnifiedLookupResponse } from "./lookup-response.mjs";
 import { createLookupTimingTracker } from "./lookup-diagnostics.mjs";
+import { prepareSelectedLookupUrl } from "../packages/evidence/lookup-target.mjs";
 async function runUnifiedLookup(classified, options = {}) {
     const fetchRdap = options.fetchRdapRecord || fetchRdapRecord;
     const fetchRegistrarRdap = options.fetchRegistrarRdapRecord || fetchRegistrarRdapRecord;
@@ -44,19 +45,30 @@ async function runUnifiedLookup(classified, options = {}) {
     const securityTxtRequested = options.securityTxt === true;
     const featurePolicy = options.featurePolicy || networkFeaturePolicy();
     const timing = createLookupTimingTracker(!fast && !compact, options.now || Date.now);
-    const measure = (source, operation) => (timing.measure(source, () => abortable(operation, options.signal)));
+    const startedCollectors = [];
+    const measure = (source, operation) => (timing.measure(source, () => abortable(() => {
+        const running = Promise.resolve(operation());
+        if (options.waitForStartedCollectors)
+            startedCollectors.push(running.catch(() => { }));
+        return running;
+    }, options.signal)));
     const rdapEnabled = featureDecision('rdap', featurePolicy).enabled;
     const whoisEnabled = featureDecision('whois', featurePolicy).enabled;
     const availabilityEnabled = featureDecision('availability', featurePolicy).enabled;
     const websiteProbeEnabled = featureDecision('website_probe', featurePolicy).enabled;
     const dnsIntelligenceEnabled = featureDecision('dns_intelligence', featurePolicy).enabled;
     const skipWhois = fast || !whoisEnabled;
+    const selectedUrl = options.selectedUrl === undefined ? undefined
+        : prepareSelectedLookupUrl(options.selectedUrl, classified.inputHostname ?? classified.value);
+    if (selectedUrl && (classified.type !== 'domain' || fast || compact || !availabilityEnabled || !websiteProbeEnabled)) {
+        throw new TypeError('Selected URL collection requires an enabled full Deep domain lookup.');
+    }
     const rdapPromise = rdapEnabled
-        ? measure('rdap', () => fetchRdap(classified.type, classified.value))
+        ? measure('rdap', () => fetchRdap(classified.type, classified.value, options.signal ? { signal: options.signal } : {}))
         : Promise.resolve(null);
     const whoisPromise = skipWhois
         ? Promise.resolve(null)
-        : measure('whois', () => fetchWhois(classified.value));
+        : measure('whois', () => fetchWhois(classified.value, options.signal ? { signal: options.signal } : {}));
     // Registrar RDAP is a separately attributed deep-lookup enrichment. It may
     // overlap the WHOIS chain, but it never joins the promises used to decide
     // availability and can add up to its own bounded timeout to a deep lookup.
@@ -76,7 +88,10 @@ async function runUnifiedLookup(classified, options = {}) {
             includeStructuredDataIdentity: !fast && !compact,
             includeTechnologyProfile: !fast,
             includeSecurityPosture: !compact,
+            ...(!fast && !compact ? { observationHostname: classified.inputHostname } : {}),
+            ...(selectedUrl ? { selectedUrl } : {}),
             featurePolicy,
+            ...(options.signal ? { signal: options.signal } : {}),
             rdapRecordPromise: rdapPromise,
             whoisChainPromise: whoisPromise,
             ...(selectedDnsResolvers ? {
@@ -171,30 +186,36 @@ async function runUnifiedLookup(classified, options = {}) {
             void Promise.resolve(sourcePromise).then((value) => notify(source, 'fulfilled', value), (error) => notify(source, 'rejected', error));
         }
     }
-    return buildUnifiedLookupResponse({
-        availabilityEnabled,
-        availabilityPromise,
-        classified,
-        compact,
-        dnsIntelligenceEnabled,
-        fast,
-        inspectCertificateWarningData,
-        networkContextPromise,
-        options,
-        rdapEnabled,
-        rdapPromise,
-        registrarRdapPromise,
-        reverseDnsEligible,
-        reverseDnsPromise,
-        securityTxtPromise,
-        skipWhois,
-        threatfoxIntelligencePromise,
-        timing,
-        urlhausIntelligencePromise,
-        urlscanIntelligencePromise,
-        whoisEnabled,
-        whoisPromise,
-    });
+    try {
+        return await buildUnifiedLookupResponse({
+            availabilityEnabled,
+            availabilityPromise,
+            classified,
+            compact,
+            dnsIntelligenceEnabled,
+            fast,
+            inspectCertificateWarningData,
+            networkContextPromise,
+            options,
+            rdapEnabled,
+            rdapPromise,
+            registrarRdapPromise,
+            reverseDnsEligible,
+            reverseDnsPromise,
+            securityTxtPromise,
+            skipWhois,
+            threatfoxIntelligencePromise,
+            timing,
+            urlhausIntelligencePromise,
+            urlscanIntelligencePromise,
+            whoisEnabled,
+            whoisPromise,
+        });
+    }
+    finally {
+        if (options.waitForStartedCollectors)
+            await Promise.all(startedCollectors);
+    }
 }
 export { runUnifiedLookup };
 export { LOOKUP_DIAGNOSTICS_VERSION, LOOKUP_LEGACY_DIAGNOSTICS_VERSION, LOOKUP_TIMING_VERSION, MAX_LOOKUP_TIMING_MS, LOOKUP_ERROR_CODES, } from "./lookup-diagnostics.mjs";

package/lib/observed-network-context.mjs

@@ -11,6 +11,7 @@ const OBSERVED_NETWORK_CONTEXT_VERSION = 1;
 const MAX_NETWORK_CIDRS = 16;
 const MAX_NETWORK_ATTEMPTS = 3;
 const MAX_NETWORK_ABUSE_ROUTES = 6;
+const NETWORK_ABUSE_ROUTE_LIMITATION = 'The network abuse-route projection reached its local retention limit; additional published routes may exist.';
 const MAX_ENDPOINT_LENGTH = 2048;
 const MAX_DETAIL_LENGTH = 240;
 const MAX_NAME_LENGTH = 300;
@@ -144,20 +145,25 @@ function networkAbuseEntities(parsed) {
     const fallback = parsed.abuse && typeof parsed.abuse === 'object' && !Array.isArray(parsed.abuse)
         ? [parsed.abuse]
         : [];
-    return [...roleEntities, ...fallback]
-        .filter((item) => Boolean(item) && typeof item === 'object' && !Array.isArray(item))
-        .slice(0, MAX_NETWORK_ABUSE_ROUTES);
+    return {
+        entities: [...roleEntities.slice(0, MAX_NETWORK_ABUSE_ROUTES + 1), ...fallback]
+            .filter((item) => Boolean(item) && typeof item === 'object' && !Array.isArray(item)),
+        truncated: roleEntities.length > MAX_NETWORK_ABUSE_ROUTES + 1,
+    };
 }
 function projectNetworkAbuseRoutes(parsed, provenance) {
     const routes = new Map();
-    for (const entity of networkAbuseEntities(parsed)) {
+    const inventory = networkAbuseEntities(parsed);
+    let truncated = inventory.truncated;
+    for (const entity of inventory.entities) {
         const emails = Array.isArray(entity.emails) ? entity.emails : [];
         const phones = Array.isArray(entity.phones) ? entity.phones : [];
+        truncated ||= emails.length > MAX_NETWORK_ABUSE_ROUTES + 1 || phones.length > MAX_NETWORK_ABUSE_ROUTES + 1;
         const candidates = [
             { channel: 'email', value: entity.email },
-            ...emails.slice(0, MAX_NETWORK_ABUSE_ROUTES).map((value) => ({ channel: 'email', value })),
+            ...emails.slice(0, MAX_NETWORK_ABUSE_ROUTES + 1).map((value) => ({ channel: 'email', value })),
             { channel: 'phone', value: entity.phone },
-            ...phones.slice(0, MAX_NETWORK_ABUSE_ROUTES).map((value) => ({ channel: 'phone', value })),
+            ...phones.slice(0, MAX_NETWORK_ABUSE_ROUTES + 1).map((value) => ({ channel: 'phone', value })),
         ];
         for (const candidate of candidates) {
             const contact = normalizedAbuseContact(candidate.value, candidate.channel);
@@ -165,6 +171,10 @@ function projectNetworkAbuseRoutes(parsed, provenance) {
                 continue;
             const key = `${candidate.channel}:${contact.toLowerCase()}`;
             if (!routes.has(key)) {
+                if (routes.size >= MAX_NETWORK_ABUSE_ROUTES) {
+                    truncated = true;
+                    continue;
+                }
                 routes.set(key, {
                     kind: 'network_hosting',
                     channel: candidate.channel,
@@ -186,11 +196,14 @@ function projectNetworkAbuseRoutes(parsed, provenance) {
                     ],
                 });
             }
-            if (routes.size >= MAX_NETWORK_ABUSE_ROUTES)
-                return [...routes.values()];
         }
     }
-    return [...routes.values()];
+    return {
+        routes: [...routes.values()].map((route) => truncated
+            ? { ...route, complete: false, truncated: true, limitations: [...route.limitations, NETWORK_ABUSE_ROUTE_LIMITATION] }
+            : route),
+        truncated,
+    };
 }
 function networkSummary(parsedValue, family) {
     const parsed = record(parsedValue);
@@ -204,10 +217,12 @@ function networkSummary(parsedValue, family) {
     const startAddress = publicAddress(parsed.startAddress, family);
     const endAddress = publicAddress(parsed.endAddress, family);
     const country = boundedString(parsed.country, 2);
-    const truncated = parsed.serverTruncated === true
-        || parsed.cidrsTruncated === true
-        || parsed.entitiesTruncated === true
-        || sourceCidrs.length > MAX_NETWORK_CIDRS;
+    const limitations = [
+        ...(parsed.serverTruncated === true ? ['The RDAP server declared that part of its response was truncated.'] : []),
+        ...(parsed.entitiesTruncated === true ? ['Some IP RDAP contact records or fields were omitted during bounded normalisation.'] : []),
+        ...(parsed.cidrsTruncated === true ? ['Some IP RDAP CIDR entries were invalid or exceeded the normalisation limit.'] : []),
+        ...(sourceCidrs.length > MAX_NETWORK_CIDRS ? ['The network CIDR summary reached its local retention limit.'] : []),
+    ];
     return {
         value: {
             handle: boundedString(parsed.handle, MAX_NAME_LENGTH),
@@ -220,8 +235,8 @@ function networkSummary(parsedValue, family) {
             networkType: boundedString(parsed.networkType, 160),
             databaseUpdatedAt: isoTimestamp(lifecycle.databaseUpdatedDateIso || lifecycle.databaseUpdatedDate),
         },
-        truncated,
-        serverTruncated: parsed.serverTruncated === true,
+        truncated: limitations.length > 0,
+        limitations,
     };
 }
 function baseContext(selection, input) {
@@ -299,28 +314,33 @@ async function collectObservedNetworkContext(availabilityValue, options = {}) {
         if (rdap.httpStatus !== 200 || !Object.keys(parsed).length)
             throw new Error('IP RDAP returned no usable normalised object');
         const summary = networkSummary(parsed, selection.family);
-        const partial = summary.truncated;
         const sourceObservedAt = rdap.fetchedAt || observedAt();
-        const abuseRouting = projectNetworkAbuseRoutes(parsed, {
+        const routeProjection = projectNetworkAbuseRoutes(parsed, {
             endpoint: rdap.endpoint,
             observedAt: sourceObservedAt,
             selection,
-            complete: !partial,
-            truncated: partial,
+            complete: !summary.truncated,
+            truncated: summary.truncated,
         });
+        const incompleteReasons = [
+            ...summary.limitations,
+            ...(routeProjection.truncated ? [NETWORK_ABUSE_ROUTE_LIMITATION] : []),
+        ];
+        const partial = incompleteReasons.length > 0;
         return baseContext(selection, {
             status: partial ? 'partial' : 'success',
             observedAt: sourceObservedAt,
             durationMs,
             complete: !partial,
             truncated: partial,
-            detail: 'The selected public endpoint address was mapped to its separately attributed IP RDAP registration.',
+            detail: partial
+                ? `Network registration was retained, but the source record is incomplete. ${incompleteReasons[0]}`
+                : 'The selected public endpoint address was mapped to its separately attributed IP RDAP registration.',
             limitations: [
+                ...incompleteReasons,
                 'This identifies the registered network for one point-in-time endpoint address, not a definitive origin host or hosting provider.',
                 'CDNs, reverse proxies, load balancers, shared hosting, and location-dependent DNS can present different networks.',
                 'Network registration is an investigative lead and does not prove control, ownership, intent, or maliciousness.',
-                ...(summary.serverTruncated ? ['The RDAP server declared that part of its response was truncated.'] : []),
-                ...(sourceCidrsWereCapped(parsed) ? ['The network CIDR summary reached its local retention limit.'] : []),
             ],
             diagnostics: {
                 requestCount: 1,
@@ -330,7 +350,7 @@ async function collectObservedNetworkContext(availabilityValue, options = {}) {
             },
             rdap,
             network: summary.value,
-            abuseRouting,
+            abuseRouting: routeProjection.routes,
         });
     }
     catch (error) {
@@ -348,8 +368,4 @@ async function collectObservedNetworkContext(availabilityValue, options = {}) {
         });
     }
 }
-function sourceCidrsWereCapped(parsed) {
-    return parsed.cidrsTruncated === true
-        || (Array.isArray(parsed.cidrs) && parsed.cidrs.length > MAX_NETWORK_CIDRS);
-}
 export { MAX_NETWORK_ATTEMPTS, MAX_NETWORK_ABUSE_ROUTES, MAX_NETWORK_CIDRS, OBSERVED_NETWORK_CONTEXT_VERSION, collectObservedNetworkContext, selectObservedNetworkAddress, };

package/lib/outbound-request-bounds.mjs

@@ -3,7 +3,10 @@
 // disclosures cannot drift from the enforced request boundaries.
 export const MAX_OUTBOUND_REDIRECTS = 5;
 export const HOMEPAGE_FETCH_TIMEOUT_MS = 6_000;
-export const MAX_HOMEPAGE_BYTES = 300_000;
+// One MiB admits large static application pages while bounding bytes retained
+// from each response. Parsing, derived outputs and request concurrency retain
+// separate work limits; this is not a claim that larger pages lack evidence.
+export const MAX_HOMEPAGE_BYTES = 1024 * 1024;
 export const FAVICON_FETCH_TIMEOUT_MS = 5_000;
 export const MAX_FAVICON_BYTES = 200_000;
 export const MAX_FAVICON_CANDIDATES = 4;