@slicedearth/whoisleuth-cli 2.2.0 → 2.4.0

package/lib/hostname.mjs

@@ -1,38 +1,2 @@
-// Linear-time validation for already-normalized ASCII DNS hostnames. Keep this
-// helper deliberately independent of public-suffix policy: callers decide
-// whether a syntactically valid hostname must also be registrable.
-const ASCII_LABEL_RE = /^[A-Za-z0-9-]+$/u;
-function isValidAsciiHostname(value, options = {}) {
-    if (typeof value !== 'string'
-        || value.length < 1
-        || value.length > 253
-        || (options.requireLowercase === true && value !== value.toLowerCase()))
-        return false;
-    const labels = value.split('.');
-    if (options.requireDot !== false && labels.length < 2)
-        return false;
-    for (const label of labels) {
-        if (!label
-            || label.length > 63
-            || label.startsWith('-')
-            || label.endsWith('-')
-            || !ASCII_LABEL_RE.test(label))
-            return false;
-    }
-    return true;
-}
-function isIpv4Literal(value) {
-    const parts = value.split('.');
-    return parts.length === 4 && parts.every((part) => {
-        if (!/^(?:0|[1-9]\d{0,2})$/u.test(part))
-            return false;
-        return Number(part) <= 255;
-    });
-}
-// Domain-only records must not accept an IPv4 literal merely because its
-// dotted decimal labels also satisfy hostname syntax. IPv6 literals already
-// fail the ASCII hostname grammar because they contain colons.
-function isValidAsciiDomainName(value, options = {}) {
-    return isValidAsciiHostname(value, options) && !isIpv4Literal(value);
-}
-export { isValidAsciiDomainName, isValidAsciiHostname };
+// Preserve the existing public import while sharing one dependency-neutral grammar.
+export { isValidAsciiDomainName, isValidAsciiHostname } from "../packages/contracts/domain-name.mjs";

package/lib/html-signals.mjs

@@ -16,7 +16,6 @@ import { analyzeStaticHtml } from "./static-html-analysis.mjs";
 import { analyzeStructuredDataIdentity } from "./structured-data-identity.mjs";
 import { analyzeWebsiteTechnology } from "./website-technology.mjs";
 import { PAGE_PUBLICATION_LIMITATIONS, PAGE_PUBLICATION_METADATA_VERSION, } from "./homepage-metadata-contract.mjs";
-const MAX_TITLE_LENGTH = 200;
 const MAX_EXTERNAL_ASSET_HOSTS = 20;
 const MAX_IDENTITY_TAGS = 512;
 const MAX_IDENTITY_TAG_LENGTH = 4096;
@@ -37,10 +36,6 @@ const MAX_URLS_PER_TAG = 20;
 const PAGE_IDENTITY_VERSION = 3;
 const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/gu;
 const HAS_CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/u;
-const TITLE_RE = /<title[^>]*>([\s\S]*?)<\/title>/i;
-// A password input is the single strongest static-HTML tell that a page is
-// asking for credentials, regardless of what it claims to be.
-const PASSWORD_FIELD_RE = /<input\b[^>]*\btype\s*=\s*["']?password["']?/i;
 // <img>/<script>/<link> tags loading a resource from an absolute, external
 // URL - a common phishing-kit tell is hotlinking the real brand's own logo/
 // CSS/JS instead of copying it. Relative URLs (the common case, same
@@ -48,13 +43,8 @@ const PASSWORD_FIELD_RE = /<input\b[^>]*\btype\s*=\s*["']?password["']?/i;
 // scoped to resource tags, not every <a href> - an outbound link to the
 // real site is normal on all sorts of pages; a resource pulled live from it
 // during page load is not.
-const ASSET_TAG_RE = /<(?:img|script|link)\b[^>]*?\b(?:src|href)\s*=\s*["']([^"']+)["'][^>]*>/gi;
-const ABSOLUTE_URL_HOST_RE = /^(?:https?:)?\/\/([^/]+)/i;
-const IDENTITY_TAG_RE = /<(html|link|meta|form)\b[^>]{0,4096}>/gi;
-const OVERSIZED_IDENTITY_TAG_RE = /<(?:html|link|meta|form)\b[^>]{4097}/i;
-const RELATIONSHIP_TAG_RE = /<(a|img|script|link|iframe|frame|source|video|audio|object|embed)\b[^>]{0,4096}>/gi;
-const OVERSIZED_RELATIONSHIP_TAG_RE = /<(?:a|img|script|link|iframe|frame|source|video|audio|object|embed)\b[^>]{4097}/i;
-const ATTRIBUTE_RE = /([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
+const IDENTITY_TAGS = new Set(['html', 'link', 'meta', 'form']);
+const RELATIONSHIP_TAGS = new Set(['a', 'img', 'script', 'link', 'iframe', 'frame', 'source', 'video', 'audio', 'object', 'embed']);
 const LANGUAGE_TAG_RE = /^[a-z]{2,8}(?:-[a-z0-9]{1,8})*$/i;
 const HOSTNAME_RE = /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i;
 const RESOURCE_LINK_RELS = new Set(['stylesheet', 'icon', 'preload', 'prefetch', 'modulepreload', 'manifest']);
@@ -62,9 +52,6 @@ const RISKY_DOWNLOAD_EXTENSIONS = new Set([
     '7z', 'apk', 'bat', 'cmd', 'dmg', 'docm', 'exe', 'img', 'iso', 'jar', 'js',
     'msi', 'pkg', 'ps1', 'rar', 'scr', 'vbs', 'xlsm', 'zip',
 ]);
-const HTML_COMMENT_RE = /<!--[\s\S]*?(?:-->|$)/g;
-const RAW_TEXT_BLOCK_RE = /<(script|style|textarea|template)\b([^>]*)>[\s\S]*?<\/\1\s*>/gi;
-const NON_EXECUTING_RAW_TEXT_BLOCK_RE = /<(style|textarea|template)\b[^>]*>[\s\S]*?<\/\1\s*>/gi;
 function stripWwwPrefix(host) {
     return host.toLowerCase().replace(/^www\./, '');
 }
@@ -77,24 +64,19 @@ function boundedHtmlText(value, maxLength, ellipsis = false) {
         return null;
     if (text.length <= maxLength)
         return text;
-    return ellipsis ? `${text.slice(0, maxLength)}…` : text.slice(0, maxLength);
+    return ellipsis ? `${text.slice(0, Math.max(0, maxLength - 1))}…` : text.slice(0, maxLength);
 }
-function extractExternalAssetHosts(html, ownDomain) {
+function extractExternalAssetHosts(analysis, ownDomain, documentUrl) {
     const ownHost = stripWwwPrefix(ownDomain);
     const hosts = new Set();
-    let match;
-    ASSET_TAG_RE.lastIndex = 0;
-    while ((match = ASSET_TAG_RE.exec(html))) {
-        const resourceUrl = match[1];
-        if (!resourceUrl)
+    for (const element of analysis.elements) {
+        if (!element.html || !['img', 'script', 'link'].includes(element.name))
             continue;
-        const hostMatch = resourceUrl.match(ABSOLUTE_URL_HOST_RE);
-        if (!hostMatch)
+        const attribute = element.attributes.find((attribute) => attribute.name === (element.name === 'link' ? 'href' : 'src'));
+        const resource = normalizeIdentityUrl(attribute?.value, analysis.effectiveBaseUrl ?? documentUrl);
+        if (!resource)
             continue;
-        const matchedHost = hostMatch[1];
-        if (!matchedHost || HAS_CONTROL_CHARACTER_RE.test(matchedHost))
-            continue;
-        const host = stripWwwPrefix(matchedHost);
+        const host = stripWwwPrefix(new URL(resource.url).hostname);
         if (host && host !== ownHost)
             hosts.add(host);
         if (hosts.size >= MAX_EXTERNAL_ASSET_HOSTS)
@@ -102,38 +84,6 @@ function extractExternalAssetHosts(html, ownDomain) {
     }
     return [...hosts];
 }
-function extractPageTitle(html) {
-    const match = html.match(TITLE_RE);
-    if (!match)
-        return null;
-    return boundedHtmlText(match[1], MAX_TITLE_LENGTH, true);
-}
-function parseAttributes(tag) {
-    const start = tag.search(/\s/);
-    if (start === -1)
-        return new Map();
-    const attributes = new Map();
-    ATTRIBUTE_RE.lastIndex = 0;
-    let match;
-    while ((match = ATTRIBUTE_RE.exec(tag.slice(start)))) {
-        const name = (match[1] ?? '').toLowerCase();
-        if (!name)
-            continue;
-        if (!attributes.has(name))
-            attributes.set(name, match[2] ?? match[3] ?? match[4] ?? '');
-    }
-    return attributes;
-}
-function markupForTagParsing(html) {
-    return html
-        .replace(HTML_COMMENT_RE, ' ')
-        .replace(RAW_TEXT_BLOCK_RE, '<$1$2>');
-}
-function markupForTrackingIdentifiers(html) {
-    return html
-        .replace(HTML_COMMENT_RE, ' ')
-        .replace(NON_EXECUTING_RAW_TEXT_BLOCK_RE, ' ');
-}
 function normalizeLanguage(value) {
     if (typeof value !== 'string' || value.length > 35 || HAS_CONTROL_CHARACTER_RE.test(value))
         return null;
@@ -306,10 +256,10 @@ function trackingIdentifiers(html) {
         truncated,
     };
 }
-function extractPageRelationships(html, domain, options = {}) {
-    const baseUrl = resolvedBaseUrl(domain, options.baseUrl);
-    const markup = typeof options.tagMarkup === 'string' ? options.tagMarkup : markupForTagParsing(html);
-    const baseOrigin = new URL(baseUrl).origin;
+function extractPageRelationships(analysis, domain, options = {}) {
+    const documentUrl = resolvedBaseUrl(domain, options.baseUrl);
+    const baseUrl = resolvedBaseUrl(domain, options.effectiveBaseUrl ?? documentUrl);
+    const baseOrigin = options.documentOrigin ?? new URL(documentUrl).origin;
     const resourceKeys = new Set();
     const resourceOrigins = new Set();
     const embeddedOrigins = new Set();
@@ -322,25 +272,26 @@ function extractPageRelationships(html, domain, options = {}) {
     let riskyDownloadCount = 0;
     let tagsExamined = 0;
     let discardedUrls = 0;
-    let tagLimitReached = OVERSIZED_RELATIONSHIP_TAG_RE.test(markup);
+    let tagLimitReached = analysis.inputLimitReached || analysis.tagLimitReached;
     let resourceOriginLimitReached = false;
     let embeddedOriginLimitReached = false;
     let contactDomainLimitReached = false;
     let downloadOriginLimitReached = false;
     let downloadFileTypeLimitReached = false;
     let perTagLimitReached = false;
-    let match;
-    RELATIONSHIP_TAG_RE.lastIndex = 0;
-    while ((match = RELATIONSHIP_TAG_RE.exec(markup))) {
+    let resourceCountLimitReached = false;
+    for (const element of analysis.elements) {
+        if (!element.html || !RELATIONSHIP_TAGS.has(element.name))
+            continue;
         if (tagsExamined >= MAX_RESOURCE_TAGS) {
             tagLimitReached = true;
             break;
         }
         tagsExamined += 1;
-        const tagName = (match[1] ?? '').toLowerCase();
-        if (!tagName)
-            continue;
-        const attributes = parseAttributes(match[0]);
+        const tagName = element.name;
+        const attributes = new Map(element.attributes.map(({ name, value }) => [name, value]));
+        if (element.attributesTruncated)
+            tagLimitReached = true;
         if (tagName === 'a') {
             const href = attributes.get('href');
             const contacts = mailtoDomains(href);
@@ -388,6 +339,10 @@ function extractPageRelationships(html, domain, options = {}) {
             }
             const key = `${reference.type}:${normalized.url}`;
             if (!resourceKeys.has(key)) {
+                if (resourceKeys.size >= MAX_RESOURCE_TAGS) {
+                    resourceCountLimitReached = true;
+                    continue;
+                }
                 resourceKeys.add(key);
                 byType[reference.type] += 1;
             }
@@ -399,10 +354,15 @@ function extractPageRelationships(html, domain, options = {}) {
                 embeddedOriginLimitReached = true;
         }
     }
-    const tracking = trackingIdentifiers(markupForTrackingIdentifiers(html));
+    const tracking = trackingIdentifiers([
+        ...analysis.elements.filter((element) => element.html).flatMap((element) => element.attributes.map((attribute) => attribute.value)),
+        ...analysis.scripts.map((script) => script.inlineContent),
+    ].join('\n'));
+    if (analysis.scriptLimitReached || analysis.inlineLimitReached)
+        tracking.truncated = true;
     const truncated = tagLimitReached || resourceOriginLimitReached || embeddedOriginLimitReached
         || contactDomainLimitReached || downloadOriginLimitReached || downloadFileTypeLimitReached
-        || perTagLimitReached || tracking.truncated;
+        || perTagLimitReached || resourceCountLimitReached || tracking.truncated;
     const limitations = [];
     if (tagLimitReached)
         limitations.push(`Resource parsing reached the ${MAX_RESOURCE_TAGS}-tag or ${MAX_IDENTITY_TAG_LENGTH}-character tag limit.`);
@@ -418,6 +378,8 @@ function extractPageRelationships(html, domain, options = {}) {
         limitations.push(`Only the first ${MAX_DOWNLOAD_FILE_TYPES} risky download file types were retained.`);
     if (perTagLimitReached)
         limitations.push(`Some srcset URL candidates could not be safely enumerated within the ${MAX_URLS_PER_TAG}-candidate per-tag boundary.`);
+    if (resourceCountLimitReached)
+        limitations.push(`Only the first ${MAX_RESOURCE_TAGS} distinct resource references were examined.`);
     if (tracking.truncated)
         limitations.push(`Only the first ${MAX_TRACKING_IDENTIFIERS} tracking identifiers were retained.`);
     return {
@@ -425,7 +387,7 @@ function extractPageRelationships(html, domain, options = {}) {
             count: resourceKeys.size,
             byType,
             externalOrigins: [...resourceOrigins].sort(),
-            truncated: tagLimitReached || resourceOriginLimitReached || perTagLimitReached,
+            truncated: tagLimitReached || resourceOriginLimitReached || perTagLimitReached || resourceCountLimitReached,
         },
         embeddedOrigins: [...embeddedOrigins].sort(),
         contactDomains: [...contactDomains].sort(),
@@ -454,11 +416,13 @@ function metaRefreshTarget(content) {
     return match ? (match[1] ?? match[2] ?? match[3] ?? '').trim() : null;
 }
 function extractPageIdentity(html, domain, options = {}) {
-    const baseUrl = resolvedBaseUrl(domain, options.baseUrl);
-    const tagMarkup = markupForTagParsing(html);
-    const parsedBase = new URL(baseUrl);
-    const baseOrigin = parsedBase.origin;
-    const baseUsesHttps = parsedBase.protocol === 'https:';
+    const documentUrl = resolvedBaseUrl(domain, options.baseUrl);
+    const analysis = options.htmlAnalysis ?? analyzeStaticHtml(html, { baseUrl: documentUrl, includeVisibleText: true });
+    const baseUrl = resolvedBaseUrl(domain, options.effectiveBaseUrl ?? analysis.effectiveBaseUrl ?? documentUrl);
+    const baseHrefState = options.baseHrefState ?? analysis.baseHrefState;
+    const parsedDocument = new URL(documentUrl);
+    const baseOrigin = options.documentOrigin ?? parsedDocument.origin;
+    const baseUsesHttps = parsedDocument.protocol === 'https:';
     const externalFormOrigins = new Set();
     let documentLanguage = null;
     let canonical = null;
@@ -474,19 +438,19 @@ function extractPageIdentity(html, domain, options = {}) {
     let discardedUrls = 0;
     let formLimitReached = false;
     let originLimitReached = false;
-    let tagLimitReached = OVERSIZED_IDENTITY_TAG_RE.test(tagMarkup);
-    let match;
-    IDENTITY_TAG_RE.lastIndex = 0;
-    while ((match = IDENTITY_TAG_RE.exec(tagMarkup))) {
+    let tagLimitReached = analysis.inputLimitReached || analysis.tagLimitReached;
+    for (const element of analysis.elements) {
+        if (!element.html || !IDENTITY_TAGS.has(element.name))
+            continue;
         if (tagsExamined >= MAX_IDENTITY_TAGS) {
             tagLimitReached = true;
             break;
         }
         tagsExamined += 1;
-        const tagName = (match[1] ?? '').toLowerCase();
-        if (!tagName)
-            continue;
-        const attributes = parseAttributes(match[0]);
+        const tagName = element.name;
+        const attributes = new Map(element.attributes.map(({ name, value }) => [name, value]));
+        if (element.attributesTruncated)
+            tagLimitReached = true;
         if (tagName === 'html' && documentLanguage === null) {
             documentLanguage = normalizeLanguage(attributes.get('lang'));
             continue;
@@ -556,20 +520,23 @@ function extractPageIdentity(html, domain, options = {}) {
     const queryOmitted = [canonical, metaRefresh, openGraphUrl].some((item) => item?.queryOmitted);
     const pathTruncated = [canonical, metaRefresh, openGraphUrl].some((item) => item?.pathTruncated);
     const sourceTruncated = options.sourceTruncated === true;
-    const relationships = extractPageRelationships(html, domain, { baseUrl, tagMarkup });
+    const relationships = extractPageRelationships(analysis, domain, { baseUrl: documentUrl, effectiveBaseUrl: baseUrl, documentOrigin: baseOrigin });
     const fingerprints = createPageFingerprints(html, {
-        baseUrl,
+        baseUrl: documentUrl,
+        htmlAnalysis: analysis,
         exactBodyHash: options.exactBodyHash,
         sourceTruncated,
         resources: relationships.resources,
         trackingIdentifiers: relationships.trackingIdentifiers,
         identifiersTruncated: relationships.diagnostics.trackingIdentifiersTruncated === true,
     });
-    const truncated = sourceTruncated || tagLimitReached || formLimitReached || originLimitReached
+    const truncated = sourceTruncated || baseHrefState === 'invalid' || tagLimitReached || formLimitReached || originLimitReached
         || pathTruncated || relationships.truncated || fingerprints.truncated;
     const limitations = ['Static HTML metadata only; JavaScript-rendered changes are not evaluated.'];
     if (sourceTruncated)
         limitations.push('Homepage body capture reached its byte limit; identity fields may be incomplete.');
+    if (baseHrefState === 'invalid')
+        limitations.push('The first bounded document base URL was invalid; relative URL relationships use the response URL and remain partial.');
     if (tagLimitReached)
         limitations.push(`Page identity parsing reached the ${MAX_IDENTITY_TAGS}-tag or ${MAX_IDENTITY_TAG_LENGTH}-character tag limit.`);
     if (formLimitReached)
@@ -696,34 +663,67 @@ function buildPagePublicationMetadata(analysis, sourceTruncated) {
         },
     };
 }
-function extractHtmlSignals(html, domain, options = {}) {
-    const pageIdentity = options.includePageIdentity === false ? null : extractPageIdentity(html, domain, options);
+function domainSaleLandingPage(analysis, domain) {
+    if (analysis.inputLimitReached || analysis.tagLimitReached || analysis.visibleTextLimitReached)
+        return false;
+    const title = analysis.title?.trim().toLowerCase() ?? '';
+    const content = analysis.visibleText.trim().toLowerCase();
+    const saleSuffix = /^\s+(?:(?:is|may be)\s+)?(?:for sale|available for purchase|available for lease)(?:[.!?:\s]|$)/u;
+    const explicitSale = (value) => {
+        const genericSubject = /^(?:(?:this|the) domain(?: name)?|domain(?: name)?)/u.exec(value)?.[0];
+        return (genericSubject !== undefined && saleSuffix.test(value.slice(genericSubject.length)))
+            || (value.startsWith(domain) && saleSuffix.test(value.slice(domain.length)));
+    };
+    const explicitPurchase = /^(?:buy|purchase|own|bid on|inquire about) (?:this|the) domain(?: name)?(?:[.!?:\s]|$)/u;
+    // A short, directly worded landing page is sufficient. Long articles need
+    // corroborating page-title and transaction-control evidence; a generic
+    // commerce CTA or quoted example cannot classify the domain for sale.
+    if (content.length <= 240 && (explicitSale(content) || explicitPurchase.test(content)))
+        return true;
+    if (!explicitSale(title) && !explicitPurchase.test(title))
+        return false;
+    if (!/(?:this domain(?: name)? (?:is )?for sale|buy this domain|purchase this domain|domain name for sale)/u.test(content))
+        return false;
+    return analysis.elements.some((element) => element.html
+        && ['form', 'button', 'a'].includes(element.name));
+}
+async function extractHtmlSignals(html, domain, options = {}) {
+    options.signal?.throwIfAborted();
+    const documentUrl = resolvedBaseUrl(domain, options.baseUrl);
+    const htmlAnalysis = options.htmlAnalysis ?? analyzeStaticHtml(html, { baseUrl: documentUrl, includeVisibleText: true });
+    const effectiveBaseUrl = htmlAnalysis.effectiveBaseUrl ?? documentUrl;
+    const documentOrigin = new URL(documentUrl).origin;
+    const pageIdentity = options.includePageIdentity === false ? null : extractPageIdentity(html, domain, {
+        ...options,
+        baseUrl: documentUrl,
+        effectiveBaseUrl,
+        documentOrigin,
+        baseHrefState: htmlAnalysis.baseHrefState,
+        htmlAnalysis,
+    });
     const includeCredentialSurfaceProfile = pageIdentity && options.includeCredentialSurfaceProfile === true;
     const includeStructuredDataIdentity = pageIdentity && options.includeStructuredDataIdentity !== false;
     const includeTechnologyProfile = pageIdentity && options.includeTechnologyProfile !== false;
     const includeDerivedPageProfiles = Boolean(pageIdentity);
-    const baseUrl = resolvedBaseUrl(domain, options.baseUrl);
-    const htmlAnalysis = includeCredentialSurfaceProfile || includeStructuredDataIdentity || includeTechnologyProfile || includeDerivedPageProfiles
-        ? analyzeStaticHtml(html, { baseUrl, includeVisibleText: true })
-        : null;
     const pageIdentityOutput = pageIdentity && htmlAnalysis && options.includePublicationMetadata !== false
         ? {
             ...pageIdentity,
             publicationMetadata: buildPagePublicationMetadata(htmlAnalysis.publicationMetadata, options.sourceTruncated === true),
         }
         : pageIdentity;
-    const pageLanguageSignal = detectPageLanguageSignal(html, pageIdentity?.documentLanguage, htmlAnalysis ?? undefined);
+    const pageLanguageSignal = detectPageLanguageSignal(html, pageIdentity?.documentLanguage, htmlAnalysis);
+    const domainSaleSignal = options.sourceTruncated !== true && domainSaleLandingPage(htmlAnalysis, domain)
+        ? 'explicit domain-sale landing-page content' : null;
     return {
-        pageTitle: extractPageTitle(html),
-        hasPasswordField: PASSWORD_FIELD_RE.test(html),
+        domainSaleSignal,
+        pageTitle: htmlAnalysis.title,
+        hasPasswordField: htmlAnalysis.forms.categories.password > 0,
         phishingLanguageMatch: pageLanguageSignal?.label ?? null,
         hasExternalFormAction: pageIdentity
             ? pageIdentity.forms.externalActionOrigins.length > 0
             : null,
-        externalAssetHosts: extractExternalAssetHosts(html, domain),
-        cspMetaPolicy: htmlAnalysis
-            ? analyzeCspMetaPolicies(htmlAnalysis.cspMetaPolicies, htmlAnalysis.cspMetaLimitReached)
-            : null,
+        externalAssetHosts: extractExternalAssetHosts(htmlAnalysis, domain, documentUrl),
+        cspMetaPolicy: analyzeCspMetaPolicies(htmlAnalysis.cspMetaPolicies, htmlAnalysis.cspMetaLimitReached),
         pageIdentity: pageIdentityOutput,
         credentialSurfaceProfile: includeCredentialSurfaceProfile && htmlAnalysis ? analyzeCredentialSurfaceProfile({
             htmlAnalysis,
@@ -732,23 +732,26 @@ function extractHtmlSignals(html, domain, options = {}) {
         }) : null,
         structuredDataIdentity: includeStructuredDataIdentity && htmlAnalysis ? analyzeStructuredDataIdentity({
             htmlAnalysis,
-            baseUrl,
+            baseUrl: effectiveBaseUrl,
             observedAt: options.observedAt,
             sourceTruncated: options.sourceTruncated,
         }) : null,
-        technologyProfile: includeTechnologyProfile && htmlAnalysis ? analyzeWebsiteTechnology({
+        technologyProfile: includeTechnologyProfile && htmlAnalysis ? await analyzeWebsiteTechnology({
             htmlAnalysis,
             generator: pageIdentity.generator,
+            ...(options.signal ? { signal: options.signal } : {}),
             httpServer: options.httpServer,
             responseHeaders: options.responseHeaders,
             resourceOrigins: pageIdentity.resources.externalOrigins,
+            effectiveBaseUrl,
+            documentOrigin,
             observedAt: options.observedAt,
             sourceTruncated: options.sourceTruncated,
         }) : null,
         pageRoleProfile: includeDerivedPageProfiles && htmlAnalysis ? analyzePageRole({
             htmlAnalysis,
-            pageTitle: extractPageTitle(html),
-            activityStatus: options.activityStatus,
+            pageTitle: htmlAnalysis.title,
+            activityStatus: domainSaleSignal ? 'parked' : options.activityStatus,
             observedAt: options.observedAt,
             sourceTruncated: options.sourceTruncated,
         }) : null,

package/lib/http-evidence-bounds.mjs

@@ -1,7 +1,8 @@
 // Browser-safe bounds for the rich HTTP evidence contract and its compact
 // browser-local projection. Both producers and consumers import these values
 // so valid retained evidence is not rejected by a stale projection.
+import { MAX_OUTBOUND_HTTP_URL_CHARACTERS } from "../packages/contracts/http-url.mjs";
 export const MAX_HTTP_EVIDENCE_REDIRECTS = 5;
 export const MAX_HTTP_ATTEMPTS = 2;
-export const MAX_HTTP_PROVENANCE_URL = 2_048;
+export const MAX_HTTP_PROVENANCE_URL = MAX_OUTBOUND_HTTP_URL_CHARACTERS;
 export const MAX_HTTP_ERROR_LENGTH = 180;

package/lib/http-intelligence.mjs

@@ -3,6 +3,7 @@
 // converts the shared safe-fetch trace and selected response metadata into an
 // additive evidence object suitable for API responses and exports.
 import { createObservation } from "../packages/evidence/observation.mjs";
+import { MAX_OUTBOUND_HTTP_URL_CHARACTERS } from "../packages/contracts/http-url.mjs";
 import { MAX_HTTP_ATTEMPTS, MAX_HTTP_ERROR_LENGTH, MAX_HTTP_EVIDENCE_REDIRECTS, MAX_HTTP_PROVENANCE_URL, } from "./http-evidence-bounds.mjs";
 import { HTTP_DELIVERY_LIMITATIONS, HTTP_DELIVERY_METADATA_VERSION, MAX_HTTP_CACHE_SECONDS, MAX_HTTP_DELIVERY_HEADER_BYTES, MAX_HTTP_DELIVERY_TOKENS, } from "./homepage-metadata-contract.mjs";
 const MAX_HTTP_REDIRECTS = MAX_HTTP_EVIDENCE_REDIRECTS;
@@ -25,7 +26,7 @@ function boundedString(value, maxLength) {
     return normalized ? normalized.slice(0, maxLength) : null;
 }
 function normalizeProvenanceUrl(value) {
-    if (typeof value !== 'string' || value.length === 0 || value.length > 4096)
+    if (typeof value !== 'string' || value.length === 0 || value.length > MAX_OUTBOUND_HTTP_URL_CHARACTERS)
         return null;
     try {
         const parsed = new URL(value);
@@ -165,14 +166,14 @@ function headerValidity(header, validator) {
         return { present: true, valid: null };
     return { present: true, valid: header.state === 'observed' && Boolean(header.value && validator(header.value)) };
 }
-function validHttpDate(value) {
+function validHttpDate(value, observedAt) {
     const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
     const weekdays = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
     const longWeekdays = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'];
-    const time = '([01]\\d|2[0-3]):([0-5]\\d):([0-5]\\d)';
+    const time = '([01]\\d|2[0-3]):([0-5]\\d):([0-5]\\d|60)';
     const imfFixdate = value.match(new RegExp(`^(${weekdays.join('|')}), (0[1-9]|[12]\\d|3[01]) (${months.join('|')}) (\\d{4}) ${time} GMT$`, 'u'));
     const rfc850 = value.match(new RegExp(`^(${longWeekdays.join('|')}), (0[1-9]|[12]\\d|3[01])-(${months.join('|')})-(\\d{2}) ${time} GMT$`, 'u'));
-    const asctime = value.match(new RegExp(`^(${weekdays.join('|')}) (${months.join('|')}) ( {2}[1-9]| [12]\\d| 3[01]) ${time} (\\d{4})$`, 'u'));
+    const asctime = value.match(new RegExp(`^(${weekdays.join('|')}) (${months.join('|')}) ( [1-9]|0[1-9]|[12]\\d|3[01]) ${time} (\\d{4})$`, 'u'));
     let weekday;
     let day;
     let month;
@@ -196,10 +197,19 @@ function validHttpDate(value) {
         day = Number(rfc850[2]);
         month = months.indexOf(rfc850[3]);
         const shortYear = Number(rfc850[4]);
-        year = shortYear >= 70 ? 1900 + shortYear : 2000 + shortYear;
+        // RFC 9110's rolling window is relative to receipt, not a later review
+        // or the host's timezone. The observation supplies that stable reference.
+        const futureBoundary = new Date(observedAt);
+        futureBoundary.setUTCFullYear(futureBoundary.getUTCFullYear() + 50);
+        year = Math.floor(futureBoundary.getUTCFullYear() / 100) * 100 + shortYear;
         hour = Number(rfc850[5]);
         minute = Number(rfc850[6]);
         second = Number(rfc850[7]);
+        const candidate = new Date(0);
+        candidate.setUTCFullYear(year, month, day);
+        candidate.setUTCHours(hour, minute, Math.min(second, 59), 0);
+        if (candidate.getTime() > futureBoundary.getTime())
+            year -= 100;
         expectedWeekdays = longWeekdays;
     }
     else if (asctime) {
@@ -218,13 +228,15 @@ function validHttpDate(value) {
         return false;
     const parsed = new Date(0);
     parsed.setUTCFullYear(year, month, day);
-    parsed.setUTCHours(hour, minute, second, 0);
+    // A leap-second spelling must not roll into the following calendar day
+    // while validating the declared date and weekday.
+    parsed.setUTCHours(hour, minute, Math.min(second, 59), 0);
     return parsed.getUTCFullYear() === year
         && parsed.getUTCMonth() === month
         && parsed.getUTCDate() === day
         && expectedWeekdays[parsed.getUTCDay()] === weekday;
 }
-function parseCachePolicy(headers) {
+function parseCachePolicy(headers, observedAt) {
     const cacheControl = deliveryHeader(headers, 'cache-control');
     const age = deliveryHeader(headers, 'age', 64);
     const etagHeader = deliveryHeader(headers, 'etag');
@@ -329,8 +341,8 @@ function parseCachePolicy(headers) {
     if (age.state === 'observed' && ageSeconds === null)
         malformed = true;
     const etag = headerValidity(etagHeader, (value) => /^(?:W\/)?"[\x21\x23-\x7e]*"$/u.test(value));
-    const lastModified = headerValidity(lastModifiedHeader, validHttpDate);
-    const expires = headerValidity(expiresHeader, validHttpDate);
+    const lastModified = headerValidity(lastModifiedHeader, (value) => validHttpDate(value, observedAt));
+    const expires = headerValidity(expiresHeader, (value) => validHttpDate(value, observedAt));
     malformed = malformed || [etag, lastModified, expires].some((item) => item.present && item.valid === false);
     const headersObserved = [cacheControl, age, etagHeader, lastModifiedHeader, expiresHeader]
         .some((header) => header.state !== 'not_observed');
@@ -362,9 +374,9 @@ function parseCachePolicy(headers) {
         truncated: partial,
     };
 }
-function buildDeliveryMetadata(headers) {
+function buildDeliveryMetadata(headers, observedAt) {
     const contentEncoding = parseContentEncoding(headers);
-    const cacheResult = parseCachePolicy(headers);
+    const cacheResult = parseCachePolicy(headers, observedAt);
     const cachePolicy = cacheResult.value;
     const incomplete = [contentEncoding.status, cachePolicy.status].some((status) => ['partial', 'malformed'].includes(status));
     const truncated = contentEncoding.status === 'partial' || cacheResult.truncated;
@@ -466,21 +478,22 @@ function buildHttpObservation(detail, options = {}) {
         ...(Array.isArray(options.previousAttempts) ? options.previousAttempts : []),
         { url: detail && detail.requestedUrl, httpStatus: status },
     ]);
+    const observation = createObservation({
+        status: incomplete ? 'partial' : 'success',
+        observedAt: options.observedAt,
+        scanMode: 'deep',
+        source: 'http',
+        durationMs: detail && detail.durationMs,
+        complete: !incomplete,
+        truncated: incomplete,
+        limitations,
+        diagnostics: {
+            redirectCount: redirects.length,
+            httpStatus: status,
+        },
+    });
     return {
-        ...createObservation({
-            status: incomplete ? 'partial' : 'success',
-            observedAt: options.observedAt,
-            scanMode: 'deep',
-            source: 'http',
-            durationMs: detail && detail.durationMs,
-            complete: !incomplete,
-            truncated: incomplete,
-            limitations,
-            diagnostics: {
-                redirectCount: redirects.length,
-                httpStatus: status,
-            },
-        }),
+        ...observation,
         requestUrl: requested ? requested.url : null,
         finalUrl: final ? final.url : null,
         transportSecurity: final ? final.url.split(':', 1)[0] : null,
@@ -506,7 +519,7 @@ function buildHttpObservation(detail, options = {}) {
                 bytes: capturedBodyBytes,
             } : null,
             securityHeaders: securityHeaders(response && response.headers),
-            deliveryMetadata: buildDeliveryMetadata(response && response.headers),
+            deliveryMetadata: buildDeliveryMetadata(response && response.headers, observation.observedAt),
         },
     };
 }

package/lib/local-geoip-evidence.mjs

@@ -45,14 +45,30 @@ function buildLocalGeoIpDatabase(value) {
     }
     const truncated = rows.length > MAX_GEOIP_RECORDS;
     const normalized = rows.slice(0, MAX_GEOIP_RECORDS).map(normalizeGeoIpEntry);
-    const records = normalized.filter((item) => item !== null)
-        .sort((left, right) => right.parsedNetwork.length - left.parsedNetwork.length);
+    const byNetwork = new Map();
+    const sameAttribution = (left, right) => (left.countryCode === right.countryCode
+        && left.region === right.region
+        && left.city === right.city
+        && left.asn === right.asn
+        && left.asName === right.asName);
+    for (const item of normalized) {
+        if (!item)
+            continue;
+        const existing = byNetwork.get(item.network);
+        if (existing && !sameAttribution(existing, item)) {
+            throw new TypeError(`A local GeoIP database must not assign conflicting evidence to canonical prefix ${item.network}.`);
+        }
+        if (!existing)
+            byNetwork.set(item.network, item);
+    }
+    const records = [...byNetwork.values()]
+        .sort((left, right) => right.parsedNetwork.length - left.parsedNetwork.length || left.network.localeCompare(right.network));
     return Object.freeze({
         sourceLabel,
         databaseVersion,
         license,
         records: Object.freeze(records),
-        rejectedCount: normalized.length - records.length + Math.max(0, rows.length - MAX_GEOIP_RECORDS),
+        rejectedCount: normalized.filter((item) => item === null).length + Math.max(0, rows.length - MAX_GEOIP_RECORDS),
         truncated,
     });
 }