@slicedearth/whoisleuth-cli 1.47.4 → 2.0.1

package/lib/external-intelligence-risk.mjs

@@ -13,6 +13,9 @@ const QUALIFYING_CATEGORIES = new Set(['phishing', 'malware']);
 // Only built-in provider IDs can affect the built-in score. The two community
 // malware datasets share one publisher family and therefore cannot corroborate
 // one another by themselves.
+import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
+import { canonicalRegistrableDomain } from "./registrable-domain.mjs";
+import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_ENVELOPE_VERSION, THREAT_INTELLIGENCE_SCHEMA, } from "./threat-intelligence-types.mjs";
 const PUBLISHER_FAMILIES = Object.freeze({
     urlscan_search: 'archived-scan-publisher',
     urlhaus_host: 'community-malware-publisher',
@@ -22,20 +25,24 @@ function record(value) {
     return value && typeof value === 'object' && !Array.isArray(value) ? value : null;
 }
 function timestamp(value) {
-    if (typeof value !== 'string' || value.length > 64 || /[\u0000-\u001f\u007f]/u.test(value))
-        return null;
-    const parsed = Date.parse(value);
-    return Number.isFinite(parsed) ? parsed : null;
+    const normalized = normalizeExplicitIsoTimestamp(value);
+    return normalized ? Date.parse(normalized) : null;
 }
-function providerEvidence(value) {
+function providerEvidence(value, expectedDomain) {
     const provider = record(value);
     const identity = record(provider?.provider);
     const observation = record(provider?.observation);
+    const target = record(provider?.target);
     const providerId = typeof identity?.id === 'string' ? identity.id : '';
     const publisherFamily = Object.hasOwn(PUBLISHER_FAMILIES, providerId)
         ? PUBLISHER_FAMILIES[providerId]
         : null;
-    if (!publisherFamily
+    if (provider?.schema !== THREAT_INTELLIGENCE_SCHEMA
+        || provider.version !== THREAT_INTELLIGENCE_CONTRACT_VERSION
+        || target?.type !== 'domain'
+        || target.exposure !== 'registrable_domain'
+        || target.value !== expectedDomain
+        || !publisherFamily
         || typeof provider?.state !== 'string'
         || !POSITIVE_STATES.has(provider.state)
         || !Array.isArray(provider.findings))
@@ -47,8 +54,12 @@ function providerEvidence(value) {
         const finding = record(item);
         if (!finding || typeof finding.category !== 'string' || !QUALIFYING_CATEGORIES.has(finding.category))
             continue;
+        const firstObservedAt = timestamp(finding.firstObservedAt);
+        const lastObservedAt = timestamp(finding.lastObservedAt);
+        if (firstObservedAt !== null && lastObservedAt !== null && firstObservedAt > lastObservedAt)
+            continue;
         qualifyingFindings += 1;
-        const candidate = timestamp(finding.lastObservedAt) ?? timestamp(finding.firstObservedAt);
+        const candidate = lastObservedAt ?? firstObservedAt;
         if (candidate !== null && (latestFindingAt === null || candidate > latestFindingAt))
             latestFindingAt = candidate;
     }
@@ -67,12 +78,17 @@ function providerEvidence(value) {
         recent: ageDays !== null && ageDays <= EXTERNAL_INTELLIGENCE_RECENT_DAYS,
     };
 }
-export function calibrateExternalIntelligenceRisk(value) {
+export function calibrateExternalIntelligenceRisk(value, expectedDomainValue) {
     const envelope = record(value);
-    const providers = Array.isArray(envelope?.providers) ? envelope.providers.slice(0, MAX_PROVIDERS) : [];
+    const expectedDomain = canonicalRegistrableDomain(expectedDomainValue);
+    const providers = expectedDomain
+        && envelope?.version === THREAT_INTELLIGENCE_ENVELOPE_VERSION
+        && Array.isArray(envelope.providers)
+        ? envelope.providers.slice(0, MAX_PROVIDERS)
+        : [];
     const byProvider = new Map();
     for (const item of providers) {
-        const evidence = providerEvidence(item);
+        const evidence = providerEvidence(item, expectedDomain ?? '');
         if (!evidence || byProvider.has(evidence.providerId))
             continue;
         byProvider.set(evidence.providerId, evidence);

package/lib/feature-policy.mjs

@@ -14,7 +14,7 @@ const NETWORK_FEATURE_DEFINITIONS = Object.freeze({
     availability: Object.freeze({ env: 'WHOISLEUTH_DISABLE_AVAILABILITY', label: 'availability analysis' }),
     dns_intelligence: Object.freeze({ env: 'WHOISLEUTH_DISABLE_DNS_INTELLIGENCE', label: 'DNS intelligence' }),
     website_probe: Object.freeze({ env: 'WHOISLEUTH_DISABLE_WEBSITE_PROBE', label: 'website probing' }),
-    tls_intelligence: Object.freeze({ env: 'WHOISLEUTH_DISABLE_TLS_INTELLIGENCE', label: 'TLS intelligence' }),
+    tls_intelligence: Object.freeze({ env: 'WHOISLEUTH_DISABLE_TLS_INTELLIGENCE', label: 'TLS evidence collection' }),
     certificate_transparency: Object.freeze({ env: 'WHOISLEUTH_DISABLE_CERTIFICATE_TRANSPARENCY', label: 'Certificate Transparency search' }),
     domain_posture: Object.freeze({ env: 'WHOISLEUTH_DISABLE_DOMAIN_POSTURE', label: 'domain-posture auditing' }),
 });

package/lib/homepage-metadata-contract.mjs

@@ -0,0 +1,290 @@
+// Exact bounded contracts for metadata derived from the already-captured Deep
+// homepage response. These validators perform no collection and accept only
+// fixed enums, booleans, counts, and reviewed limitation text.
+const PAGE_PUBLICATION_METADATA_VERSION = 1;
+const HTTP_DELIVERY_METADATA_VERSION = 1;
+const MAX_PAGE_PUBLICATION_TAG_COUNT = 8_192;
+const MAX_PAGE_PUBLICATION_META_ELEMENTS = 128;
+const MAX_PAGE_PUBLICATION_DECLARATIONS = 64;
+const MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES = 64;
+const MAX_HTTP_DELIVERY_TOKENS = 32;
+const MAX_HTTP_DELIVERY_HEADER_BYTES = 1_024;
+const MAX_HTTP_CACHE_SECONDS = 315_360_000;
+const MAX_HOMEPAGE_METADATA_LIMITATIONS = 4;
+const MAX_HOMEPAGE_METADATA_LIMITATION_LENGTH = 300;
+const PAGE_PUBLICATION_LIMITATIONS = Object.freeze({
+    scope: 'Counts and declarations describe only the captured static homepage HTML; they are not a full accessibility, indexing, or performance audit.',
+    body: 'Homepage body capture reached its byte limit; later declarations and elements may be absent.',
+    bounds: 'Static HTML token or attribute bounds were reached; publication metadata is partial.',
+    malformed: 'At least one publication declaration could not be interpreted; raw declaration values were not retained.',
+});
+const HTTP_DELIVERY_LIMITATIONS = Object.freeze({
+    scope: 'Selected-response headers are point-in-time declarations and do not prove intermediary caching, compression effectiveness, or page performance.',
+    bounds: 'At least one delivery header exceeded its byte or item bound and was not partially parsed.',
+    malformed: 'At least one delivery header could not be interpreted; raw header values were not retained.',
+});
+const COMPONENT_STATES = new Set(['observed', 'not_observed', 'partial', 'malformed']);
+const ROBOTS_DIRECTIVES = new Set([
+    'all', 'follow', 'index', 'max-image-preview', 'max-snippet', 'max-video-preview',
+    'noarchive', 'nocache', 'nofollow', 'noimageindex', 'noindex', 'none', 'nositelinkssearchbox',
+    'nosnippet', 'notranslate', 'unavailable_after',
+]);
+const TWITTER_CARD_TYPES = new Set(['summary', 'summary_large_image', 'player', 'app', 'other']);
+const CONTENT_CODINGS = new Set(['gzip', 'br', 'deflate', 'zstd', 'identity', 'other']);
+const CONTROL_RE = /[\u0000-\u001f\u007f-\u009f\u061c\u200b-\u200f\u202a-\u202e\u2060-\u2069\ufeff]/u;
+function record(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : null;
+}
+function exactKeys(value, keys) {
+    const actual = Object.keys(value).sort();
+    const expected = [...keys].sort();
+    return actual.length === expected.length && actual.every((key, index) => key === expected[index]);
+}
+function boundedCount(value, maximum) {
+    return Number.isSafeInteger(value) && Number(value) >= 0 && Number(value) <= maximum;
+}
+function fixedSortedStrings(value, allowed, maximum) {
+    if (!Array.isArray(value) || value.length > maximum || !value.every((item) => typeof item === 'string' && allowed.has(item)))
+        return false;
+    return value.every((item, index) => index === 0 || String(value[index - 1]) < item);
+}
+function fixedLimitations(value, allowed) {
+    if (!Array.isArray(value) || value.length < 1 || value.length > MAX_HOMEPAGE_METADATA_LIMITATIONS)
+        return false;
+    return value.every((item, index) => typeof item === 'string'
+        && item.length <= MAX_HOMEPAGE_METADATA_LIMITATION_LENGTH
+        && !CONTROL_RE.test(item)
+        && allowed.includes(item)
+        && value.indexOf(item) === index);
+}
+function validComponentState(value) {
+    return typeof value === 'string' && COMPONENT_STATES.has(value);
+}
+function validPagePublicationMetadata(value) {
+    const root = record(value);
+    if (!root || !exactKeys(root, [
+        'version', 'status', 'complete', 'truncated', 'limitations', 'robots', 'twitterCard',
+        'headings', 'images', 'renderBlockingCandidates',
+    ])
+        || root.version !== PAGE_PUBLICATION_METADATA_VERSION
+        || !['success', 'partial'].includes(String(root.status))
+        || typeof root.complete !== 'boolean'
+        || typeof root.truncated !== 'boolean'
+        || !fixedLimitations(root.limitations, Object.values(PAGE_PUBLICATION_LIMITATIONS)))
+        return false;
+    const robots = record(root.robots);
+    const twitter = record(root.twitterCard);
+    const headings = record(root.headings);
+    const images = record(root.images);
+    const blocking = record(root.renderBlockingCandidates);
+    if (!robots || !twitter || !headings || !images || !blocking)
+        return false;
+    if (!exactKeys(robots, [
+        'status', 'complete', 'truncated', 'directives', 'recognizedDirectiveCount', 'unknownDirectiveCount', 'conflicting',
+    ])
+        || !validComponentState(robots.status)
+        || typeof robots.complete !== 'boolean'
+        || typeof robots.truncated !== 'boolean'
+        || !fixedSortedStrings(robots.directives, ROBOTS_DIRECTIVES, MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES)
+        || !boundedCount(robots.recognizedDirectiveCount, MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES)
+        || !boundedCount(robots.unknownDirectiveCount, MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES)
+        || Number(robots.recognizedDirectiveCount) + Number(robots.unknownDirectiveCount) > MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES
+        || Number(robots.recognizedDirectiveCount) < robots.directives.length
+        || typeof robots.conflicting !== 'boolean')
+        return false;
+    if (robots.status === 'partial'
+        ? robots.complete !== false || robots.truncated !== true
+        : robots.status === 'malformed'
+            ? robots.complete !== false || robots.truncated !== false
+            : robots.complete !== true || robots.truncated !== false)
+        return false;
+    const robotValues = new Set(robots.directives);
+    const conflict = (robotValues.has('index') || robotValues.has('all'))
+        && (robotValues.has('noindex') || robotValues.has('none'))
+        || (robotValues.has('follow') || robotValues.has('all'))
+            && (robotValues.has('nofollow') || robotValues.has('none'));
+    if (robots.conflicting !== conflict)
+        return false;
+    if (robots.status === 'observed' && (Number(robots.recognizedDirectiveCount) + Number(robots.unknownDirectiveCount) === 0
+        || (Number(robots.recognizedDirectiveCount) === 0) !== (robots.directives.length === 0)))
+        return false;
+    if (robots.status === 'not_observed' && (robots.directives.length || robots.recognizedDirectiveCount !== 0
+        || robots.unknownDirectiveCount !== 0 || robots.conflicting !== false))
+        return false;
+    if (!exactKeys(twitter, [
+        'status', 'complete', 'truncated', 'cardType', 'declarationCount', 'titlePresent', 'descriptionPresent', 'imagePresent',
+        'imageAltPresent', 'sitePresent', 'creatorPresent', 'playerPresent', 'appPresent',
+    ])
+        || !validComponentState(twitter.status)
+        || typeof twitter.complete !== 'boolean'
+        || typeof twitter.truncated !== 'boolean'
+        || !(twitter.cardType === null || typeof twitter.cardType === 'string' && TWITTER_CARD_TYPES.has(twitter.cardType))
+        || !boundedCount(twitter.declarationCount, MAX_PAGE_PUBLICATION_DECLARATIONS)
+        || !['titlePresent', 'descriptionPresent', 'imagePresent', 'imageAltPresent', 'sitePresent', 'creatorPresent', 'playerPresent', 'appPresent']
+            .every((key) => typeof twitter[key] === 'boolean'))
+        return false;
+    if (twitter.status === 'partial'
+        ? twitter.complete !== false || twitter.truncated !== true
+        : twitter.status === 'malformed'
+            ? twitter.complete !== false || twitter.truncated !== false
+            : twitter.complete !== true || twitter.truncated !== false)
+        return false;
+    if (twitter.status === 'not_observed' && (twitter.cardType !== null || twitter.declarationCount !== 0
+        || ['titlePresent', 'descriptionPresent', 'imagePresent', 'imageAltPresent', 'sitePresent', 'creatorPresent', 'playerPresent', 'appPresent']
+            .some((key) => twitter[key] === true)))
+        return false;
+    const twitterValuePresent = twitter.cardType !== null
+        || ['titlePresent', 'descriptionPresent', 'imagePresent', 'imageAltPresent', 'sitePresent', 'creatorPresent', 'playerPresent', 'appPresent']
+            .some((key) => twitter[key] === true);
+    if (twitterValuePresent && twitter.declarationCount === 0)
+        return false;
+    if (twitter.status === 'observed' && (twitter.declarationCount === 0 || !twitterValuePresent))
+        return false;
+    if (!exactKeys(headings, ['complete', 'truncated', 'total', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'])
+        || typeof headings.complete !== 'boolean'
+        || typeof headings.truncated !== 'boolean'
+        || headings.complete === headings.truncated
+        || !['total', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'].every((key) => boundedCount(headings[key], MAX_PAGE_PUBLICATION_TAG_COUNT))
+        || headings.total !== ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].reduce((sum, key) => sum + Number(headings[key]), 0))
+        return false;
+    if (!exactKeys(images, [
+        'totalComplete', 'classificationComplete', 'truncated',
+        'total', 'altMissing', 'altEmpty', 'altNonEmpty', 'altUnclassified',
+    ])
+        || typeof images.totalComplete !== 'boolean'
+        || typeof images.classificationComplete !== 'boolean'
+        || typeof images.truncated !== 'boolean'
+        || images.truncated !== (images.totalComplete !== true || images.classificationComplete !== true)
+        || images.classificationComplete === true && (images.totalComplete !== true || images.altUnclassified !== 0)
+        || !['total', 'altMissing', 'altEmpty', 'altNonEmpty', 'altUnclassified'].every((key) => boundedCount(images[key], MAX_PAGE_PUBLICATION_TAG_COUNT))
+        || images.total !== ['altMissing', 'altEmpty', 'altNonEmpty', 'altUnclassified'].reduce((sum, key) => sum + Number(images[key]), 0))
+        return false;
+    if (!exactKeys(blocking, ['complete', 'truncated', 'script', 'stylesheet', 'total', 'scope'])
+        || typeof blocking.complete !== 'boolean'
+        || typeof blocking.truncated !== 'boolean'
+        || blocking.complete === blocking.truncated
+        || !['script', 'stylesheet', 'total'].every((key) => boundedCount(blocking[key], MAX_PAGE_PUBLICATION_TAG_COUNT))
+        || blocking.total !== Number(blocking.script) + Number(blocking.stylesheet)
+        || blocking.scope !== 'explicit-head-static-v1')
+        return false;
+    const componentTruncated = robots.truncated === true || twitter.truncated === true
+        || headings.truncated === true || images.truncated === true || blocking.truncated === true;
+    const componentIncomplete = robots.complete !== true || twitter.complete !== true
+        || headings.complete !== true || images.totalComplete !== true
+        || images.classificationComplete !== true || blocking.complete !== true;
+    if (root.truncated !== componentTruncated)
+        return false;
+    if (root.status === 'success' ? root.complete !== true || componentIncomplete : root.complete !== false || !componentIncomplete)
+        return false;
+    const limitations = root.limitations;
+    const hasTruncationLimitation = limitations.some((item) => (item === PAGE_PUBLICATION_LIMITATIONS.body || item === PAGE_PUBLICATION_LIMITATIONS.bounds));
+    const hasMalformedLimitation = limitations.includes(PAGE_PUBLICATION_LIMITATIONS.malformed);
+    const malformedComponent = [robots.status, twitter.status].includes('malformed');
+    if (root.truncated !== hasTruncationLimitation
+        || malformedComponent && !hasMalformedLimitation
+        || root.status === 'success' && hasMalformedLimitation)
+        return false;
+    return limitations.includes(PAGE_PUBLICATION_LIMITATIONS.scope);
+}
+function validPresenceValidity(value) {
+    const item = record(value);
+    return Boolean(item && exactKeys(item, ['present', 'valid'])
+        && typeof item.present === 'boolean'
+        && (item.valid === null || typeof item.valid === 'boolean')
+        && (item.present === true || item.valid === null));
+}
+function validHttpDeliveryMetadata(value) {
+    const root = record(value);
+    if (!root || !exactKeys(root, ['version', 'status', 'complete', 'truncated', 'limitations', 'contentEncoding', 'cachePolicy'])
+        || root.version !== HTTP_DELIVERY_METADATA_VERSION
+        || !['success', 'partial'].includes(String(root.status))
+        || typeof root.complete !== 'boolean'
+        || typeof root.truncated !== 'boolean'
+        || !fixedLimitations(root.limitations, Object.values(HTTP_DELIVERY_LIMITATIONS)))
+        return false;
+    const encoding = record(root.contentEncoding);
+    const cache = record(root.cachePolicy);
+    if (!encoding || !cache)
+        return false;
+    if (!exactKeys(encoding, ['status', 'codings', 'encoded', 'unknownCodingCount'])
+        || !validComponentState(encoding.status)
+        || !fixedSortedStrings(encoding.codings, CONTENT_CODINGS, MAX_HTTP_DELIVERY_TOKENS)
+        || !(encoding.encoded === null || typeof encoding.encoded === 'boolean')
+        || !boundedCount(encoding.unknownCodingCount, MAX_HTTP_DELIVERY_TOKENS))
+        return false;
+    const codings = encoding.codings;
+    if (encoding.status === 'observed') {
+        if (!codings.length || typeof encoding.encoded !== 'boolean')
+            return false;
+        if (encoding.encoded !== codings.some((coding) => coding !== 'identity'))
+            return false;
+        if ((encoding.unknownCodingCount === 0) !== !codings.includes('other'))
+            return false;
+    }
+    else if (codings.length || encoding.encoded !== null || encoding.unknownCodingCount !== 0)
+        return false;
+    if (!exactKeys(cache, [
+        'status', 'noStore', 'noCache', 'mustRevalidate', 'public', 'private', 'immutable',
+        'maxAgeSeconds', 'sMaxAgeSeconds', 'ageSeconds', 'maxAgePresent', 'sMaxAgePresent',
+        'agePresent', 'unknownDirectiveCount', 'etag',
+        'lastModified', 'expires',
+    ])
+        || !validComponentState(cache.status)
+        || !['noStore', 'noCache', 'mustRevalidate', 'public', 'private', 'immutable'].every((key) => typeof cache[key] === 'boolean')
+        || !['maxAgeSeconds', 'sMaxAgeSeconds', 'ageSeconds'].every((key) => cache[key] === null || boundedCount(cache[key], MAX_HTTP_CACHE_SECONDS))
+        || !['maxAgePresent', 'sMaxAgePresent', 'agePresent'].every((key) => typeof cache[key] === 'boolean')
+        || cache.maxAgePresent === false && cache.maxAgeSeconds !== null
+        || cache.sMaxAgePresent === false && cache.sMaxAgeSeconds !== null
+        || cache.agePresent === false && cache.ageSeconds !== null
+        || !boundedCount(cache.unknownDirectiveCount, MAX_HTTP_DELIVERY_TOKENS)
+        || !validPresenceValidity(cache.etag)
+        || !validPresenceValidity(cache.lastModified)
+        || !validPresenceValidity(cache.expires))
+        return false;
+    if (cache.status === 'not_observed' && (['noStore', 'noCache', 'mustRevalidate', 'public', 'private', 'immutable'].some((key) => cache[key] === true)
+        || ['maxAgeSeconds', 'sMaxAgeSeconds', 'ageSeconds'].some((key) => cache[key] !== null)
+        || ['maxAgePresent', 'sMaxAgePresent', 'agePresent'].some((key) => cache[key] === true)
+        || cache.unknownDirectiveCount !== 0
+        || [cache.etag, cache.lastModified, cache.expires].some((item) => item.present === true)))
+        return false;
+    const numericCacheEvidence = [
+        [cache.maxAgePresent, cache.maxAgeSeconds],
+        [cache.sMaxAgePresent, cache.sMaxAgeSeconds],
+        [cache.agePresent, cache.ageSeconds],
+    ];
+    const validatorEvidence = [cache.etag, cache.lastModified, cache.expires]
+        .map((item) => item);
+    const cacheEvidencePresent = ['noStore', 'noCache', 'mustRevalidate', 'public', 'private', 'immutable']
+        .some((key) => cache[key] === true)
+        || numericCacheEvidence.some(([present]) => present === true)
+        || cache.unknownDirectiveCount !== 0
+        || validatorEvidence.some((item) => item.present === true);
+    if (cache.status === 'observed' && (!cacheEvidencePresent
+        || numericCacheEvidence.some(([present, result]) => present === true && result === null)
+        || validatorEvidence.some((item) => item.present === true && item.valid !== true)))
+        return false;
+    const uncertainValidator = [cache.etag, cache.lastModified, cache.expires]
+        .some((item) => item.present === true
+        && item.valid === null);
+    if (uncertainValidator && !['partial', 'malformed'].includes(String(cache.status)))
+        return false;
+    const componentIncomplete = [encoding.status, cache.status].some((state) => state === 'partial' || state === 'malformed');
+    if (root.status === 'success' ? root.complete !== true || componentIncomplete : root.complete !== false || !componentIncomplete)
+        return false;
+    if (root.truncated !== [encoding.status, cache.status].includes('partial'))
+        return false;
+    const limitations = root.limitations;
+    const hasBoundsLimitation = limitations.includes(HTTP_DELIVERY_LIMITATIONS.bounds);
+    const hasMalformedLimitation = limitations.includes(HTTP_DELIVERY_LIMITATIONS.malformed);
+    const definitelyMalformed = [encoding.status, cache.status].includes('malformed')
+        || numericCacheEvidence.some(([present, result]) => present === true && result === null)
+        || validatorEvidence.some((item) => item.present === true && item.valid === false);
+    if (root.truncated !== hasBoundsLimitation
+        || definitelyMalformed && !hasMalformedLimitation
+        || root.status === 'success' && hasMalformedLimitation)
+        return false;
+    return limitations.includes(HTTP_DELIVERY_LIMITATIONS.scope);
+}
+export { HTTP_DELIVERY_LIMITATIONS, MAX_HTTP_DELIVERY_HEADER_BYTES, HTTP_DELIVERY_METADATA_VERSION, MAX_HTTP_CACHE_SECONDS, MAX_HTTP_DELIVERY_TOKENS, MAX_PAGE_PUBLICATION_DECLARATIONS, MAX_PAGE_PUBLICATION_META_ELEMENTS, MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES, MAX_PAGE_PUBLICATION_TAG_COUNT, PAGE_PUBLICATION_LIMITATIONS, PAGE_PUBLICATION_METADATA_VERSION, validHttpDeliveryMetadata, validPagePublicationMetadata, };

package/lib/hostname.mjs

@@ -1,7 +1,7 @@
 // Linear-time validation for already-normalized ASCII DNS hostnames. Keep this
 // helper deliberately independent of public-suffix policy: callers decide
 // whether a syntactically valid hostname must also be registrable.
-const ASCII_LABEL_RE = /^[a-z0-9-]+$/iu;
+const ASCII_LABEL_RE = /^[A-Za-z0-9-]+$/u;
 function isValidAsciiHostname(value, options = {}) {
     if (typeof value !== 'string'
         || value.length < 1

package/lib/html-signals.mjs

@@ -7,7 +7,7 @@ import { isIP } from 'node:net';
 import { domainToASCII } from 'node:url';
 import { analyzeCredentialSurfaceProfile } from "./credential-surface-profile.mjs";
 import { analyzeClientBehavior } from "./client-behavior-profile.mjs";
-import { createObservation } from "./observation.mjs";
+import { createObservation } from "../packages/evidence/observation.mjs";
 import { createPageFingerprints } from "./page-fingerprints.mjs";
 import { detectPageLanguageSignal } from "./page-language-signals.mjs";
 import { analyzePageRole } from "./page-role-profile.mjs";
@@ -15,6 +15,7 @@ import { analyzeCspMetaPolicies } from "./response-policy.mjs";
 import { analyzeStaticHtml } from "./static-html-analysis.mjs";
 import { analyzeStructuredDataIdentity } from "./structured-data-identity.mjs";
 import { analyzeWebsiteTechnology } from "./website-technology.mjs";
+import { PAGE_PUBLICATION_LIMITATIONS, PAGE_PUBLICATION_METADATA_VERSION, } from "./homepage-metadata-contract.mjs";
 const MAX_TITLE_LENGTH = 200;
 const MAX_EXTERNAL_ASSET_HOSTS = 20;
 const MAX_IDENTITY_TAGS = 512;
@@ -34,8 +35,8 @@ const MAX_DOWNLOAD_FILE_TYPES = 20;
 const MAX_TRACKING_IDENTIFIERS = 30;
 const MAX_URLS_PER_TAG = 20;
 const PAGE_IDENTITY_VERSION = 3;
-const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f]/g;
-const HAS_CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f]/;
+const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/gu;
+const HAS_CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/u;
 const TITLE_RE = /<title[^>]*>([\s\S]*?)<\/title>/i;
 // A password input is the single strongest static-HTML tell that a page is
 // asking for credentials, regardless of what it claims to be.
@@ -618,6 +619,83 @@ function extractPageIdentity(html, domain, options = {}) {
         fingerprints,
     };
 }
+function publicationSourceStatus(observed, malformed, truncated) {
+    if (truncated)
+        return 'partial';
+    if (malformed)
+        return 'malformed';
+    return observed ? 'observed' : 'not_observed';
+}
+function buildPagePublicationMetadata(analysis, sourceTruncated) {
+    const truncated = sourceTruncated || analysis.truncated;
+    const malformed = analysis.robots.malformed || analysis.twitterCard.malformed;
+    const incomplete = truncated || malformed;
+    const limitations = [
+        PAGE_PUBLICATION_LIMITATIONS.scope,
+    ];
+    if (sourceTruncated)
+        limitations.push(PAGE_PUBLICATION_LIMITATIONS.body);
+    if (analysis.truncated)
+        limitations.push(PAGE_PUBLICATION_LIMITATIONS.bounds);
+    if (malformed)
+        limitations.push(PAGE_PUBLICATION_LIMITATIONS.malformed);
+    const cardTypes = analysis.twitterCard.cardTypes;
+    const suppliedCardType = cardTypes.length === 1 ? cardTypes[0] : null;
+    const cardType = suppliedCardType && ['summary', 'summary_large_image', 'player', 'app', 'other'].includes(suppliedCardType)
+        ? suppliedCardType
+        : cardTypes.length > 1 ? 'other' : null;
+    const documentTruncated = sourceTruncated || analysis.documentTruncated;
+    const robotsStatus = publicationSourceStatus(analysis.robots.observed, analysis.robots.malformed, documentTruncated || analysis.robots.truncated);
+    const twitterStatus = publicationSourceStatus(analysis.twitterCard.observed, analysis.twitterCard.malformed, documentTruncated || analysis.twitterCard.truncated);
+    return {
+        version: PAGE_PUBLICATION_METADATA_VERSION,
+        status: incomplete ? 'partial' : 'success',
+        complete: !incomplete,
+        truncated,
+        limitations,
+        robots: {
+            status: robotsStatus,
+            complete: robotsStatus === 'observed' || robotsStatus === 'not_observed',
+            truncated: robotsStatus === 'partial',
+            directives: analysis.robots.directives,
+            recognizedDirectiveCount: analysis.robots.recognizedDirectiveCount,
+            unknownDirectiveCount: analysis.robots.unknownDirectiveCount,
+            conflicting: analysis.robots.conflicting,
+        },
+        twitterCard: {
+            status: twitterStatus,
+            complete: twitterStatus === 'observed' || twitterStatus === 'not_observed',
+            truncated: twitterStatus === 'partial',
+            cardType,
+            declarationCount: analysis.twitterCard.declarationCount,
+            titlePresent: analysis.twitterCard.titlePresent,
+            descriptionPresent: analysis.twitterCard.descriptionPresent,
+            imagePresent: analysis.twitterCard.imagePresent,
+            imageAltPresent: analysis.twitterCard.imageAltPresent,
+            sitePresent: analysis.twitterCard.sitePresent,
+            creatorPresent: analysis.twitterCard.creatorPresent,
+            playerPresent: analysis.twitterCard.playerPresent,
+            appPresent: analysis.twitterCard.appPresent,
+        },
+        headings: {
+            ...analysis.headings,
+            complete: !documentTruncated && !analysis.headings.truncated,
+            truncated: documentTruncated || analysis.headings.truncated,
+        },
+        images: {
+            ...analysis.images,
+            totalComplete: !documentTruncated && analysis.images.totalComplete,
+            classificationComplete: !documentTruncated && analysis.images.classificationComplete,
+            truncated: documentTruncated || analysis.images.truncated,
+        },
+        renderBlockingCandidates: {
+            ...analysis.renderBlockingCandidates,
+            complete: !documentTruncated && !analysis.renderBlockingCandidates.truncated,
+            truncated: documentTruncated || analysis.renderBlockingCandidates.truncated,
+            scope: 'explicit-head-static-v1',
+        },
+    };
+}
 function extractHtmlSignals(html, domain, options = {}) {
     const pageIdentity = options.includePageIdentity === false ? null : extractPageIdentity(html, domain, options);
     const includeCredentialSurfaceProfile = pageIdentity && options.includeCredentialSurfaceProfile === true;
@@ -628,6 +706,12 @@ function extractHtmlSignals(html, domain, options = {}) {
     const htmlAnalysis = includeCredentialSurfaceProfile || includeStructuredDataIdentity || includeTechnologyProfile || includeDerivedPageProfiles
         ? analyzeStaticHtml(html, { baseUrl, includeVisibleText: true })
         : null;
+    const pageIdentityOutput = pageIdentity && htmlAnalysis && options.includePublicationMetadata !== false
+        ? {
+            ...pageIdentity,
+            publicationMetadata: buildPagePublicationMetadata(htmlAnalysis.publicationMetadata, options.sourceTruncated === true),
+        }
+        : pageIdentity;
     const pageLanguageSignal = detectPageLanguageSignal(html, pageIdentity?.documentLanguage, htmlAnalysis ?? undefined);
     return {
         pageTitle: extractPageTitle(html),
@@ -640,7 +724,7 @@ function extractHtmlSignals(html, domain, options = {}) {
         cspMetaPolicy: htmlAnalysis
             ? analyzeCspMetaPolicies(htmlAnalysis.cspMetaPolicies, htmlAnalysis.cspMetaLimitReached)
             : null,
-        pageIdentity,
+        pageIdentity: pageIdentityOutput,
         credentialSurfaceProfile: includeCredentialSurfaceProfile && htmlAnalysis ? analyzeCredentialSurfaceProfile({
             htmlAnalysis,
             observedAt: options.observedAt,
@@ -675,4 +759,4 @@ function extractHtmlSignals(html, domain, options = {}) {
         }) : null,
     };
 }
-export { PAGE_IDENTITY_VERSION, MAX_IDENTITY_TAGS, MAX_FORMS, MAX_FORM_ACTION_ORIGINS, MAX_RESOURCE_TAGS, MAX_RESOURCE_ORIGINS, MAX_EMBEDDED_ORIGINS, MAX_CONTACT_DOMAINS, MAX_TRACKING_IDENTIFIERS, extractHtmlSignals, extractPageIdentity, };
+export { PAGE_IDENTITY_VERSION, PAGE_PUBLICATION_METADATA_VERSION, MAX_IDENTITY_TAGS, MAX_FORMS, MAX_FORM_ACTION_ORIGINS, MAX_RESOURCE_TAGS, MAX_RESOURCE_ORIGINS, MAX_EMBEDDED_ORIGINS, MAX_CONTACT_DOMAINS, MAX_TRACKING_IDENTIFIERS, extractHtmlSignals, extractPageIdentity, };