@slicedearth/whoisleuth-cli 2.2.0 → 2.4.0

package/lib/availability.mjs

@@ -8,6 +8,7 @@
 //
 // Shared by the Express server and the Netlify Functions.
 import { promises as dns } from 'node:dns';
+import { abortable } from "./abort.mjs";
 import { fetchRdapRecord } from "./rdap.mjs";
 import { buildWhoisChain, parseWhoisChain } from "./whois.mjs";
 import { safeFetchDetailed, readTextCapped } from "./safe-fetch.mjs";
@@ -15,6 +16,7 @@ import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
 import { collectDnsIntelligence, skippedDnsIntelligence } from "./dns-intelligence.mjs";
 import { fetchFaviconHash } from "./favicon.mjs";
 import { extractHtmlSignals } from "./html-signals.mjs";
+import { analyzeStaticHtml } from "./static-html-analysis.mjs";
 import { featureDecision, networkFeaturePolicy } from "./feature-policy.mjs";
 import { buildHttpObservation, failedHttpObservation, skippedHttpObservation } from "./http-intelligence.mjs";
 import { collectTlsIntelligence, skippedTlsObservation } from "./tls-intelligence.mjs";
@@ -24,6 +26,9 @@ import { analyzeWebsiteTechnology, captureTechnologyResponseHeaders, } from "./w
 import { analyzeResponsePolicyHeaders, qualifyResponsePolicyWithCspMeta, } from "./response-policy.mjs";
 import { nonEmptyErrorMessage } from "./error-detail.mjs";
 import { registryServiceAdmissionFor } from "./registry-capabilities.mjs";
+import { canonicalRegistrableDomain } from "./registrable-domain.mjs";
+import { isValidAsciiHostname } from "./hostname.mjs";
+import { prepareSelectedLookupUrl } from "../packages/evidence/lookup-target.mjs";
 import { HOMEPAGE_FETCH_TIMEOUT_MS, MAX_HOMEPAGE_BYTES, } from "./outbound-request-bounds.mjs";
 const DNS_DELEGATION_TIMEOUT_MS = 4000;
 const MAX_DELEGATION_NAMESERVERS = 50;
@@ -76,7 +81,6 @@ const PARKING_NS_PATTERNS = [
     /trafficz\.com$/i,
     /dsredirection\.com$/i,
 ];
-const FOR_SALE_TEXT_RE = /(this domain (?:name )?(?:may be|is) for sale|buy this domain|domain(?: name)? for sale|make (?:an|your) offer|inquire about (?:this|the) domain|purchase this domain|this domain is available for purchase|backorder this domain|bid on this domain|premium domain for sale|own this domain|click here to buy this domain|would you like to buy this domain|this domain is (?:available for lease|listed for sale))/i;
 const FOR_SALE_PATH_RE = /\/(?:premium-)?domains?-for-sale(?:\/|$)/i;
 function forSaleRedirectSignal(httpObservation) {
     if (!httpObservation || typeof httpObservation !== 'object')
@@ -167,15 +171,18 @@ function compactContact(contact) {
 // domain has an active DNS delegation. Use that as a bounded, positive-only
 // fallback: no answer is never interpreted as availability because registered
 // domains can legitimately be undelegated.
-async function checkDnsDelegation(domain, { resolver = dns.resolveNs } = {}) {
-    let timer;
+async function checkDnsDelegation(domain, options = {}) {
+    options.signal?.throwIfAborted();
+    // Cancel only this lookup's c-ares requests, never the shared DNS resolver.
+    const ownedResolver = options.resolver ? null : new dns.Resolver();
+    const resolve = options.resolver || ((name) => ownedResolver.resolveNs(name));
+    const controller = new AbortController();
+    const signal = options.signal ? AbortSignal.any([options.signal, controller.signal]) : controller.signal;
+    const cancel = () => ownedResolver?.cancel();
+    signal.addEventListener('abort', cancel, { once: true });
+    const timer = setTimeout(() => controller.abort(new Error('DNS delegation lookup timed out')), DNS_DELEGATION_TIMEOUT_MS);
     try {
-        const records = await Promise.race([
-            Promise.resolve().then(() => resolver(domain)),
-            new Promise((_, reject) => {
-                timer = setTimeout(() => reject(new Error('DNS delegation lookup timed out')), DNS_DELEGATION_TIMEOUT_MS);
-            }),
-        ]);
+        const records = await abortable(() => resolve(domain), signal);
         const validNameservers = [...new Set((Array.isArray(records) ? records : [])
                 .filter((value) => typeof value === 'string')
                 .map((value) => value.trim().replace(/\.+$/, '').toLowerCase())
@@ -191,6 +198,7 @@ async function checkDnsDelegation(domain, { resolver = dns.resolveNs } = {}) {
         };
     }
     catch (err) {
+        options.signal?.throwIfAborted();
         const error = errorRecord(err);
         if (typeof error.code === 'string' && MISSING_DNS_CODES.has(error.code)) {
             return { delegated: false, nameservers: [], nameserversTruncated: false, error: null };
@@ -204,6 +212,7 @@ async function checkDnsDelegation(domain, { resolver = dns.resolveNs } = {}) {
     }
     finally {
         clearTimeout(timer);
+        signal.removeEventListener('abort', cancel);
     }
 }
 // Fetches enough of the homepage for the lightweight HTML signals below and
@@ -211,20 +220,22 @@ async function checkDnsDelegation(domain, { resolver = dns.resolveNs } = {}) {
 // domain has "no site": transient DNS/TLS/network failures, a slow origin, or
 // an HTTP error can all produce the same null body. Keeping that distinction
 // avoids turning an inconclusive probe into a false inactivity claim.
-async function fetchHomepage(domain, { fetcher = safeFetchDetailed, timeoutMs = HOMEPAGE_FETCH_TIMEOUT_MS } = {}) {
+async function fetchHomepage(domain, { fetcher = safeFetchDetailed, timeoutMs = HOMEPAGE_FETCH_TIMEOUT_MS, selectedUrl, signal } = {}) {
+    const selected = selectedUrl === undefined ? null : prepareSelectedLookupUrl(selectedUrl, domain);
     const requestTimeoutMs = Number.isInteger(timeoutMs) && timeoutMs >= 10 && timeoutMs <= HOMEPAGE_FETCH_TIMEOUT_MS
         ? timeoutMs
         : HOMEPAGE_FETCH_TIMEOUT_MS;
     const headers = whoisleuthRequestHeaders();
     const failures = [];
     const probeStartedAt = Date.now();
-    for (const scheme of ['https', 'http']) {
-        const requestUrl = `${scheme}://${domain}`;
+    for (const requestUrl of selected ? [selected] : [`https://${domain}`, `http://${domain}`]) {
+        signal?.throwIfAborted();
+        const scheme = new URL(requestUrl).protocol.slice(0, -1);
         const attemptStartedAt = Date.now();
         const controller = new AbortController();
         const timeout = setTimeout(() => controller.abort(), requestTimeoutMs);
         try {
-            const fetched = await fetcher(requestUrl, { signal: controller.signal, headers });
+            const fetched = await fetcher(requestUrl, { signal: signal ? AbortSignal.any([signal, controller.signal]) : controller.signal, headers });
             const fetchedResponse = fetched instanceof Response ? fetched : fetched.response;
             const detail = fetched && typeof fetched === 'object' && 'response' in fetched
                 ? fetched
@@ -238,8 +249,11 @@ async function fetchHomepage(domain, { fetcher = safeFetchDetailed, timeoutMs =
                     durationMs: Date.now() - attemptStartedAt,
                 };
             const res = detail.response;
-            // A truncated homepage is still fine here - only used to scan for a
-            // for-sale text match, not parsed as well-formed content. The
+            const analysisBase = new URL(detail.finalUrl);
+            analysisBase.search = '';
+            analysisBase.hash = '';
+            // Retain an explicitly partial prefix when the body exceeds its bound;
+            // downstream analyses must not treat it as a complete page. The
             // timeout stays armed through this read (cleared in `finally` below,
             // not here) - a malicious site could otherwise send headers
             // immediately and then trickle or stall the body forever, hanging
@@ -249,9 +263,10 @@ async function fetchHomepage(domain, { fetcher = safeFetchDetailed, timeoutMs =
                 return {
                     text: body.text,
                     status: 'fetched',
-                    detail: `Homepage responded over ${scheme.toUpperCase()} (HTTP ${res.status}).`,
+                    detail: `${selected ? 'Selected URL' : 'Homepage'} responded over ${scheme.toUpperCase()} (HTTP ${res.status}).`,
                     responsePolicy: analyzeResponsePolicyHeaders(res.headers),
                     technologyHeaders: captureTechnologyResponseHeaders(res.headers),
+                    ...(selected ? { analysisBaseUrl: analysisBase.toString() } : {}),
                     http: buildHttpObservation({ ...detail, durationMs: Date.now() - attemptStartedAt }, {
                         previousAttempts: failures,
                         capturedBodyBytes: body.bytesRead,
@@ -271,7 +286,7 @@ async function fetchHomepage(domain, { fetcher = safeFetchDetailed, timeoutMs =
             return {
                 text: null,
                 status: 'responded',
-                detail: `Web server responded over ${scheme.toUpperCase()} (HTTP ${res.status}); homepage content was not available for inspection.`,
+                detail: `Web server responded over ${scheme.toUpperCase()} (HTTP ${res.status}); ${selected ? 'selected page' : 'homepage'} content was not available for inspection.`,
                 responsePolicy: analyzeResponsePolicyHeaders(res.headers),
                 technologyHeaders: captureTechnologyResponseHeaders(res.headers),
                 http: buildHttpObservation({ ...detail, durationMs: Date.now() - attemptStartedAt }, {
@@ -282,12 +297,14 @@ async function fetchHomepage(domain, { fetcher = safeFetchDetailed, timeoutMs =
             };
         }
         catch (err) {
+            signal?.throwIfAborted();
             const error = errorRecord(err);
             const reason = error.name === 'AbortError'
                 ? `timed out after ${requestTimeoutMs} milliseconds`
-                : nonEmptyErrorMessage(err, 'request failed')
-                    .replace(/[\u0000-\u001f\u007f]+/g, ' ')
-                    .slice(0, 180);
+                : selected ? 'request failed; the selected URL was not retried at another path or scheme'
+                    : nonEmptyErrorMessage(err, 'request failed')
+                        .replace(/[\u0000-\u001f\u007f]+/g, ' ')
+                        .slice(0, 180);
             failures.push({ url: requestUrl, error: `${scheme.toUpperCase()} ${reason}` });
         }
         finally {
@@ -298,7 +315,7 @@ async function fetchHomepage(domain, { fetcher = safeFetchDetailed, timeoutMs =
         text: null,
         status: 'inconclusive',
         detail: failures.length
-            ? `Could not confirm homepage activity: ${failures.map((attempt) => attempt.error).join('; ')}.`
+            ? `Could not confirm ${selected ? 'selected URL' : 'homepage'} activity: ${failures.map((attempt) => attempt.error).join('; ')}.`
             : 'Could not confirm homepage activity.',
         http: failedHttpObservation(failures, { durationMs: Date.now() - probeStartedAt }),
     };
@@ -314,7 +331,7 @@ function registryPolicyDetail(domain, fast) {
     const details = [];
     const rdapAdmission = registryServiceAdmissionFor(domain, 'rdap');
     if (rdapAdmission && !rdapAdmission.allowed) {
-        details.push('RDAP was not queried because the registry capability profile records no IANA-published RDAP service for this suffix.');
+        details.push('RDAP was not queried because the registry access policy does not permit collection.');
     }
     const whoisAdmission = fast ? null : registryServiceAdmissionFor(domain, 'whois');
     if (whoisAdmission && !whoisAdmission.allowed) {
@@ -330,23 +347,12 @@ function registryPolicyDetail(domain, fast) {
 // large sourcing candidate lists quickly and gently on registry rate
 // limits; anything it can't resolve (state "unknown") is meant to get a
 // follow-up deep check (fast: false, the default) on the shortlist only.
-async function checkDomainAvailability(domain, options = {}) {
+async function resolveDomainRegistration(domain, options, featurePolicy) {
+    options.signal?.throwIfAborted();
     const fast = options.fast === true;
-    const collectDns = options.collectDnsIntelligence || collectDnsIntelligence;
-    const collectTls = options.collectTlsIntelligence || collectTlsIntelligence;
-    const fetchHomepageForDomain = options.fetchHomepage || fetchHomepage;
-    const fetchFaviconForDomain = options.fetchFaviconHash || fetchFaviconHash;
-    const featurePolicy = options.featurePolicy || networkFeaturePolicy();
     const rdapEnabled = featureDecision('rdap', featurePolicy).enabled;
     const whoisEnabled = featureDecision('whois', featurePolicy).enabled;
     const dnsIntelligenceEnabled = featureDecision('dns_intelligence', featurePolicy).enabled;
-    const websiteProbeEnabled = featureDecision('website_probe', featurePolicy).enabled;
-    const tlsIntelligenceEnabled = featureDecision('tls_intelligence', featurePolicy).enabled;
-    const deepScanComplete = rdapEnabled
-        && whoisEnabled
-        && dnsIntelligenceEnabled
-        && websiteProbeEnabled
-        && tlsIntelligenceEnabled;
     const hasPreloadedRdap = Object.prototype.hasOwnProperty.call(options, 'rdapRecord');
     const hasPreloadedWhois = Object.prototype.hasOwnProperty.call(options, 'whoisChain');
     const hasPreloadedRdapPromise = Object.prototype.hasOwnProperty.call(options, 'rdapRecordPromise');
@@ -367,6 +373,9 @@ async function checkDomainAvailability(domain, options = {}) {
     let registrationConfidence = 'high';
     let dnssec = null;
     let registryDnsEvidence = null;
+    const assessment = (registration) => ({
+        registration, registryDnsEvidence, nameservers, dnssec,
+    });
     if (rdapEnabled) {
         try {
             // Shared with /api/rdap (lib/rdap.mts's fetchRdapRecord) rather than a
@@ -374,23 +383,24 @@ async function checkDomainAvailability(domain, options = {}) {
             // also picks up that function's short-TTL cache (lib/lookup-cache.mts)
             // and upstream timeout for free.
             const recordValue = hasPreloadedRdapPromise
-                ? await options.rdapRecordPromise
+                ? await abortable(() => options.rdapRecordPromise, options.signal)
                 : hasPreloadedRdap
                     ? options.rdapRecord
-                    : await fetchRdapRecord('domain', domain);
+                    : await fetchRdapRecord('domain', domain, options.signal ? { signal: options.signal } : {});
+            options.signal?.throwIfAborted();
             const record = errorRecord(recordValue);
             const recordRdapServer = typeof record.rdapServer === 'string' ? record.rdapServer : null;
             const upstreamStatus = typeof record.upstreamStatus === 'number' ? record.upstreamStatus : null;
             if (Object.keys(record).length) {
                 rdapServer = recordRdapServer;
                 if (upstreamStatus === 404) {
-                    return {
+                    return assessment({
                         state: 'available',
                         confidence: 'high',
                         detail: 'The registry\'s RDAP service has no record for this domain.',
                         source: 'rdap',
                         rdapServer: recordRdapServer,
-                    };
+                    });
                 }
                 const parsed = errorRecord(record.parsed);
                 if (Object.keys(parsed).length) {
@@ -427,34 +437,42 @@ async function checkDomainAvailability(domain, options = {}) {
             }
         }
         catch {
+            options.signal?.throwIfAborted();
             /* fall through to WHOIS-based detection (deep mode only) */
         }
     }
     const dnsDelegationPromise = !rdapFound && dnsIntelligenceEnabled
         ? hasPreloadedDnsDelegation
             ? Promise.resolve(options.dnsDelegation)
-            : checkDnsDelegation(domain, { resolver: options.resolveNs || dns.resolveNs })
+            : checkDnsDelegation(domain, {
+                ...(options.resolveNs ? { resolver: options.resolveNs } : {}),
+                ...(options.signal ? { signal: options.signal } : {}),
+            })
         : null;
+    // DNS may settle while the independent WHOIS source is still pending. Keep
+    // its rejection observed even if WHOIS supplies an early conclusive return.
+    void dnsDelegationPromise?.catch(() => { });
     let whoisChain = null;
     let whoisParsed = null;
     if (!rdapFound && !fast && whoisEnabled) {
         try {
             whoisChain = (hasPreloadedWhoisPromise
-                ? await options.whoisChainPromise
+                ? await abortable(() => options.whoisChainPromise, options.signal)
                 : hasPreloadedWhois
                     ? options.whoisChain
-                    : await buildWhoisChain(domain)) ?? null;
+                    : await buildWhoisChain(domain, options.signal ? { signal: options.signal } : {})) ?? null;
+            options.signal?.throwIfAborted();
             if (!Array.isArray(whoisChain))
                 throw new Error('WHOIS chain unavailable');
             const parsed = parseWhoisChain(whoisChain);
             whoisParsed = parsed;
             if (parsed.notFound) {
-                return {
+                return assessment({
                     state: 'available',
                     confidence: 'medium',
                     detail: `WHOIS reports no matching record for this domain${parsed.notFoundSource ? ` (per ${parsed.notFoundSource})` : ''}.`,
                     source: 'whois',
-                };
+                });
             }
             if (parsed.registrationStatus === 'registered') {
                 if (parsed.nameservers.length)
@@ -493,6 +511,7 @@ async function checkDomainAvailability(domain, options = {}) {
             }
         }
         catch {
+            options.signal?.throwIfAborted();
             /* if both RDAP and WHOIS fail, we simply can't determine availability */
         }
     }
@@ -511,6 +530,7 @@ async function checkDomainAvailability(domain, options = {}) {
             registrationConfidence = 'medium';
         }
     }
+    options.signal?.throwIfAborted();
     if (!rdapFound && !hasWhoisRegistrationData && !dnsDelegated) {
         const disabledSources = [
             !rdapEnabled ? 'RDAP' : null,
@@ -521,7 +541,7 @@ async function checkDomainAvailability(domain, options = {}) {
             ? ` ${disabledSources.join(', ')} ${disabledSources.length === 1 ? 'is' : 'are'} disabled by deployment policy.`
             : '';
         const policyDetail = registryPolicyDetail(domain, fast);
-        return {
+        return assessment({
             state: 'unknown',
             confidence: 'low',
             detail: fast
@@ -530,7 +550,7 @@ async function checkDomainAvailability(domain, options = {}) {
                     ? `WHOIS was inconclusive - a referral hop did not answer conclusively (${whoisParsed.failedHop}).${policyDetail}`
                     : `No enabled registration source returned conclusive data or an authoritative DNS delegation.${disabledDetail}${policyDetail}`,
             ...(!fast && whoisEnabled ? { source: 'whois' } : {}),
-        };
+        });
     }
     const domainAgeDays = computeAgeDays(createdDateIso || createdDate);
     const expiresInDays = computeDaysUntil(expiryDateIso || expiryDate);
@@ -558,42 +578,69 @@ async function checkDomainAvailability(domain, options = {}) {
         const normalizedStatus = registryStatusToken(status);
         return normalizedStatus.includes('pendingdelete') || normalizedStatus.includes('redemptionperiod');
     })) {
-        return {
+        return assessment({
             state: 'expiring',
             confidence: 'medium',
             detail: 'Domain is in redemption/pending-delete status and may become available soon.',
             ...baseInfo,
-        };
+        });
     }
-    if (fast) {
-        return {
-            state: 'registered',
-            confidence: registrationConfidence,
-            detail: registrationSource === 'dns'
-                ? 'Authoritative DNS delegation confirms the domain is registered, but RDAP/WHOIS registration details were unavailable.'
-                : 'Domain is registered. Run a deep check for parking/for-sale detection.',
-            ...baseInfo,
-        };
+    return assessment({
+        state: 'registered',
+        confidence: registrationConfidence,
+        detail: registrationSource === 'dns'
+            ? 'Authoritative DNS delegation confirms the domain is registered, but RDAP/WHOIS registration details were unavailable.'
+            : 'Domain is registered. Run a deep check for parking/for-sale detection.',
+        ...baseInfo,
+    });
+}
+async function checkDomainAvailability(domain, options = {}) {
+    options.signal?.throwIfAborted();
+    const fast = options.fast === true;
+    const observationHostname = fast ? domain : options.observationHostname ?? domain;
+    if (options.observationHostname !== undefined && (!isValidAsciiHostname(options.observationHostname)
+        || options.observationHostname !== options.observationHostname.toLowerCase()
+        || canonicalRegistrableDomain(options.observationHostname) !== domain)) {
+        throw new TypeError('The observation hostname must belong to the registration target.');
     }
+    const selectedUrl = options.selectedUrl === undefined ? undefined : prepareSelectedLookupUrl(options.selectedUrl, observationHostname);
+    const featurePolicy = options.featurePolicy || networkFeaturePolicy();
+    const websiteProbeEnabled = featureDecision('website_probe', featurePolicy).enabled;
+    if (selectedUrl && (fast || !websiteProbeEnabled))
+        throw new TypeError('Selected URL collection requires an enabled Deep website observation.');
+    const { registration, registryDnsEvidence, nameservers, dnssec } = await resolveDomainRegistration(domain, options, featurePolicy);
+    if (fast || registration.state !== 'registered' && !selectedUrl)
+        return registration;
+    const collectDns = options.collectDnsIntelligence || collectDnsIntelligence;
+    const collectTls = options.collectTlsIntelligence || collectTlsIntelligence;
+    const fetchHomepageForDomain = options.fetchHomepage || fetchHomepage;
+    const fetchFaviconForDomain = options.fetchFaviconHash || fetchFaviconHash;
+    const dnsIntelligenceEnabled = featureDecision('dns_intelligence', featurePolicy).enabled;
+    const tlsIntelligenceEnabled = featureDecision('tls_intelligence', featurePolicy).enabled;
+    const deepScanComplete = ['rdap', 'whois', 'dns_intelligence', 'website_probe', 'tls_intelligence']
+        .every((feature) => featureDecision(feature, featurePolicy).enabled);
     // Registered - look for for-sale/parked/website signals (homepage fetch,
     // deep mode only), and check for a configured mail exchanger as a
     // phishing-risk signal (a lookalike domain that can receive/send mail is
     // capable of running credential-harvesting or BEC campaigns).
-    const nsSignal = nameservers.find((ns) => PARKING_NS_PATTERNS.some((re) => re.test(ns)));
+    const nsSignal = registration.state === 'registered' ? nameservers.find((ns) => PARKING_NS_PATTERNS.some((re) => re.test(ns))) : undefined;
     let forSaleSignal = nsSignal ? `parking nameserver (${nsSignal})` : null;
-    let activityStatus = nsSignal ? 'parked' : 'unknown';
+    let activityStatus = nsSignal && observationHostname === domain ? 'parked' : 'unknown';
     // Homepage + bounded DNS evidence resolve in parallel; the favicon fetch is sequenced after
     // the homepage so it can use any <link rel="icon"> the page declares (many
     // sites serve no /favicon.ico and only point to a CDN PNG this way). One
     // extra round-trip on the already-slow deep path, in exchange for finding
     // favicons the bare /favicon.ico probe would miss.
     const [homepage, dnsIntelligence, tlsIntelligence] = await Promise.all([
-        websiteProbeEnabled ? fetchHomepageForDomain(domain).catch((err) => ({
+        websiteProbeEnabled ? fetchHomepageForDomain(observationHostname, {
+            ...(selectedUrl ? { selectedUrl } : {}), ...(options.signal ? { signal: options.signal } : {}),
+        }).catch((err) => ({
             text: null,
             status: 'inconclusive',
-            detail: `Could not confirm homepage activity: ${String(err && err.message ? err.message : 'request failed').slice(0, 180)}.`,
+            detail: selectedUrl ? 'Could not confirm selected URL activity.'
+                : `Could not confirm homepage activity: ${String(err && err.message ? err.message : 'request failed').slice(0, 180)}.`,
             http: failedHttpObservation([
-                { url: `https://${domain}`, error: String(err && err.message ? err.message : 'request failed') },
+                { url: selectedUrl ?? `https://${observationHostname}`, error: selectedUrl ? 'request failed' : String(err && err.message ? err.message : 'request failed') },
             ]),
         })) : Promise.resolve({
             text: null,
@@ -602,27 +649,27 @@ async function checkDomainAvailability(domain, options = {}) {
             http: skippedHttpObservation(),
         }),
         dnsIntelligenceEnabled
-            ? collectDns(domain, {
+            ? collectDns(observationHostname, {
                 ...(options.dnsResolvers ? { resolvers: options.dnsResolvers } : {}),
                 includeExtendedContext: options.includeExtendedDnsContext === true,
                 includeInheritedCaa: options.includeInheritedCaa === true,
                 registryEvidence: registryDnsEvidence,
+                ...(observationHostname !== domain ? { registrationDomain: domain } : {}),
             })
             : Promise.resolve(skippedDnsIntelligence('DNS intelligence is disabled by deployment policy.', {
                 includeExtendedContext: options.includeExtendedDnsContext === true,
                 includeInheritedCaa: options.includeInheritedCaa === true,
             })),
         tlsIntelligenceEnabled
-            ? collectTls(domain)
+            ? collectTls(observationHostname)
             : Promise.resolve(skippedTlsObservation()),
     ]);
+    options.signal?.throwIfAborted();
     const page = homepage.text;
-    const favicon = websiteProbeEnabled
-        ? await fetchFaviconForDomain(domain, { html: page || '' }).catch(() => null)
-        : null;
-    const faviconHash = favicon ? favicon.hash : null;
-    const faviconPHash = favicon ? favicon.phash : null;
+    const pageBaseUrl = homepage.analysisBaseUrl ?? (typeof homepage.http?.finalUrl === 'string' ? homepage.http.finalUrl : `https://${observationHostname}/`);
+    let pageAnalysis = page ? analyzeStaticHtml(page, { baseUrl: pageBaseUrl, includeVisibleText: true }) : undefined;
     let htmlSignals = {
+        domainSaleSignal: null,
         pageTitle: null,
         hasPasswordField: false,
         phishingLanguageMatch: null,
@@ -638,17 +685,14 @@ async function checkDomainAvailability(domain, options = {}) {
     };
     if (homepage.status === 'fetched') {
         if (page) {
-            const saleMatch = page.match(FOR_SALE_TEXT_RE);
-            if (saleMatch) {
-                forSaleSignal = forSaleSignal || `homepage text ("${saleMatch[0]}")`;
-                activityStatus = 'parked';
-            }
             const responseContentType = homepage.http?.response?.contentType;
             const pageIdentityEligible = typeof responseContentType !== 'string'
                 || responseContentType.trim() === ''
                 || /^(?:text\/html|application\/xhtml\+xml)(?:\s*;|$)/i.test(responseContentType.trim());
-            htmlSignals = extractHtmlSignals(page, domain, {
-                ...(typeof homepage.http?.finalUrl === 'string' ? { baseUrl: homepage.http.finalUrl } : {}),
+            htmlSignals = await extractHtmlSignals(page, observationHostname, {
+                ...(options.signal ? { signal: options.signal } : {}),
+                ...(pageAnalysis ? { htmlAnalysis: pageAnalysis } : {}),
+                baseUrl: pageBaseUrl,
                 ...(typeof homepage.http?.observedAt === 'string' ? { observedAt: homepage.http.observedAt } : {}),
                 sourceTruncated: homepage.http?.response?.bodyTruncated === true,
                 exactBodyHash: homepage.http?.response?.bodyHash,
@@ -667,22 +711,41 @@ async function checkDomainAvailability(domain, options = {}) {
                     ? { includeTechnologyProfile: options.includeTechnologyProfile }
                     : {}),
             });
+            if (pageIdentityEligible && htmlSignals.domainSaleSignal) {
+                if (!selectedUrl && observationHostname === domain)
+                    forSaleSignal = forSaleSignal || htmlSignals.domainSaleSignal;
+                activityStatus = 'parked';
+            }
         }
     }
     if (options.includeTechnologyProfile !== false && htmlSignals.technologyProfile === null
         && ['fetched', 'responded'].includes(homepage.status)) {
-        htmlSignals.technologyProfile = analyzeWebsiteTechnology({
+        htmlSignals.technologyProfile = await analyzeWebsiteTechnology({
+            ...(options.signal ? { signal: options.signal } : {}),
             htmlAvailable: false,
             httpServer: homepage.http?.response?.server,
             responseHeaders: homepage.technologyHeaders,
             observedAt: homepage.http?.observedAt,
         });
     }
+    // Complete document consumers before an optional network wait. Only the
+    // small icon projection, not every parsed element/token, crosses that wait.
+    const faviconEvidence = pageAnalysis ? {
+        iconLinks: pageAnalysis.iconLinks, effectiveBaseUrl: pageAnalysis.effectiveBaseUrl,
+    } : undefined;
+    pageAnalysis = undefined;
+    const favicon = websiteProbeEnabled
+        ? await fetchFaviconForDomain(observationHostname, {
+            baseUrl: pageBaseUrl, ...(faviconEvidence ? { htmlAnalysis: faviconEvidence } : {}),
+        }).catch(() => null)
+        : null;
+    const faviconHash = favicon ? favicon.hash : null;
+    const faviconPHash = favicon ? favicon.phash : null;
     const responsePolicy = qualifyResponsePolicyWithCspMeta(homepage.responsePolicy, htmlSignals.cspMetaPolicy);
     const retainedHttp = options.includeDeliveryMetadata === false
         ? withoutHttpDeliveryMetadata(homepage.http)
         : homepage.http;
-    const { cspMetaPolicy: _cspMetaPolicy, ...retainedHtmlSignals } = htmlSignals;
+    const { cspMetaPolicy: _cspMetaPolicy, domainSaleSignal: _domainSaleSignal, ...retainedHtmlSignals } = htmlSignals;
     const securityPosture = options.includeSecurityPosture === false ? null : analyzeWebsiteSecurityPosture({
         http: homepage.http,
         responsePolicy,
@@ -693,60 +756,46 @@ async function checkDomainAvailability(domain, options = {}) {
         observedAt: homepage.http?.observedAt,
     });
     const redirectSaleSignal = forSaleRedirectSignal(homepage.http);
-    if (!forSaleSignal && redirectSaleSignal) {
+    if (!selectedUrl && observationHostname === domain && !forSaleSignal && redirectSaleSignal) {
         forSaleSignal = redirectSaleSignal;
         activityStatus = 'parked';
     }
-    const faviconProvedActive = homepage.status === 'inconclusive' && Boolean(favicon);
+    const faviconProvedActive = !selectedUrl && homepage.status === 'inconclusive' && Boolean(favicon);
     const websiteProbeStatus = faviconProvedActive ? 'responded' : homepage.status;
     const websiteProbeDetail = faviconProvedActive
         ? `${homepage.detail} A favicon responded successfully, confirming an active web service.`
         : homepage.detail;
     if (websiteProbeEnabled) {
-        activityStatus = deriveWebsiteActivity(homepage.status, Boolean(favicon), activityStatus === 'parked');
+        activityStatus = deriveWebsiteActivity(homepage.status, !selectedUrl && Boolean(favicon), activityStatus === 'parked');
     }
-    if (!forSaleSignal) {
-        return {
-            state: 'registered',
-            confidence: registrationConfidence,
-            detail: registrationSource === 'dns'
-                ? 'Authoritative DNS delegation confirms the domain is registered, but RDAP/WHOIS registration details were unavailable. No for-sale signals were observed.'
-                : 'Domain is registered and shows no for-sale signals.',
-            activityStatus,
-            websiteProbeStatus,
-            websiteProbeDetail,
-            http: retainedHttp,
-            deepScanComplete,
-            faviconHash,
-            faviconPHash,
-            ...retainedHtmlSignals,
-            securityPosture,
-            ...baseInfo,
-            nameservers: nameservers.length ? nameservers : dnsIntelligence.records.ns,
-            dns: dnsIntelligence,
-            tls: tlsIntelligence,
-            hasMx: dnsIntelligence.hasMx,
-            hasNullMx: dnsIntelligence.hasNullMx,
-            mxHosts: dnsIntelligence.mxHosts,
-            hasSpf: dnsIntelligence.hasSpf,
-            hasDmarc: dnsIntelligence.hasDmarc,
+    const registrationResult = registration.state !== 'registered' ? registration
+        : !forSaleSignal ? {
+            ...registration,
+            detail: 'source' in registration && registration.source === 'dns'
+                ? 'Authoritative DNS delegation confirms the domain is registered, but RDAP/WHOIS registration details were unavailable.'
+                : selectedUrl ? 'The domain is registered. Website observations concern the explicitly selected URL.'
+                    : observationHostname === domain ? 'Domain is registered and shows no for-sale signals.'
+                        : 'The domain is registered. Website observations concern the separately identified hostname.',
+        } : {
+            ...registration,
+            state: 'for_sale',
+            confidence: 'medium',
+            detail: `Detected a for-sale listing (${forSaleSignal}).`,
         };
-    }
     return {
-        state: 'for_sale',
-        confidence: 'medium',
-        detail: `Detected a for-sale listing (${forSaleSignal}).`,
+        ...registrationResult,
         activityStatus,
         websiteProbeStatus,
         websiteProbeDetail,
         http: retainedHttp,
+        ...(options.observationHostname !== undefined || selectedUrl ? { observationHostname } : {}),
+        ...(selectedUrl ? { webObservationMode: 'selected_url' } : {}),
         deepScanComplete,
         faviconHash,
         faviconPHash,
         ...retainedHtmlSignals,
         securityPosture,
-        ...baseInfo,
-        nameservers: nameservers.length ? nameservers : dnsIntelligence.records.ns,
+        nameservers: nameservers.length || observationHostname !== domain ? nameservers : dnsIntelligence.records.ns,
         dns: dnsIntelligence,
         tls: tlsIntelligence,
         hasMx: dnsIntelligence.hasMx,

package/lib/bounded-file.mjs

@@ -1,7 +1,9 @@
 import { constants } from 'node:fs';
 import { lstat, open, realpath } from 'node:fs/promises';
 import path from 'node:path';
-const MAX_BOUNDED_FILE_BYTES = 64 * 1024 * 1024;
+// Caller-specific payload limits remain authoritative. The emergency ceiling
+// also admits a 64-MiB evidence package plus its bounded manifest/ZIP headers.
+const MAX_BOUNDED_FILE_BYTES = 128 * 1024 * 1024;
 function boundedSize(value, fallback) {
     return Number.isSafeInteger(value) && Number(value) >= 0 && Number(value) <= MAX_BOUNDED_FILE_BYTES
         ? Number(value)