@slicedearth/whoisleuth-cli 2.0.1 → 2.2.0

package/lib/structured-data-identity.mjs

@@ -5,9 +5,9 @@
 // discarded immediately after analysis.
 import { domainToASCII } from 'node:url';
 import { createObservation } from "../packages/evidence/observation.mjs";
+import { MAX_STRUCTURED_DATA_ENTITIES, MAX_STRUCTURED_DATA_SAME_AS_HOSTS, STRUCTURED_DATA_IDENTITY_VERSION, } from "./lookup-child-profile-contract.mjs";
 import { isUriShapedLabel } from "./portable-generator.mjs";
 import { analyzeStaticHtml, } from "./static-html-analysis.mjs";
-const STRUCTURED_DATA_IDENTITY_VERSION = 1;
 const MAX_STRUCTURED_DATA_SCRIPTS = 8;
 const MAX_STRUCTURED_DATA_SCRIPT_CHARS = 16_384;
 const MAX_STRUCTURED_DATA_TOTAL_CHARS = 32_768;
@@ -15,11 +15,9 @@ const MAX_STRUCTURED_DATA_DEPTH = 16;
 const MAX_STRUCTURED_DATA_OBJECTS = 256;
 const MAX_STRUCTURED_DATA_ARRAY_ITEMS = 256;
 const MAX_STRUCTURED_DATA_PROPERTIES = 64;
-const MAX_STRUCTURED_DATA_ENTITIES = 16;
 const MAX_STRUCTURED_DATA_TYPES = 8;
 const MAX_STRUCTURED_DATA_NAME_LENGTH = 160;
 const MAX_STRUCTURED_DATA_URL_LENGTH = 2_048;
-const MAX_STRUCTURED_DATA_SAME_AS_HOSTS = 12;
 const CONTROL_AND_DIRECTIONAL_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/u;
 const CONTROL_AND_DIRECTIONAL_GLOBAL_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/gu;
 const HOSTNAME_RE = /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/iu;

package/lib/technology-evidence-role.mjs

@@ -0,0 +1,51 @@
+const TECHNOLOGY_EVIDENCE_ROLE_ORDER = Object.freeze([
+    'observed_edge',
+    'application_platform',
+    'framework_runtime',
+    'embedded_dependency',
+]);
+const TECHNOLOGY_EVIDENCE_ROLE_SET = new Set(TECHNOLOGY_EVIDENCE_ROLE_ORDER);
+function record(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : {};
+}
+function technologyEvidenceRoles(value) {
+    const finding = record(value);
+    const roles = new Set();
+    let hasExplicitRole = false;
+    for (const role of Array.isArray(finding.roles) ? finding.roles.slice(0, 4) : []) {
+        if (typeof role === 'string' && TECHNOLOGY_EVIDENCE_ROLE_SET.has(role)) {
+            roles.add(role);
+            hasExplicitRole = true;
+        }
+    }
+    const evidenceValues = Array.isArray(finding.evidence) ? finding.evidence.slice(0, 4) : [];
+    for (const evidenceValue of evidenceValues) {
+        const evidence = record(evidenceValue);
+        if (typeof evidence.role === 'string'
+            && TECHNOLOGY_EVIDENCE_ROLE_SET.has(evidence.role)) {
+            roles.add(evidence.role);
+            hasExplicitRole = true;
+        }
+    }
+    if (!hasExplicitRole) {
+        const resourceOriginOnly = evidenceValues.length > 0
+            && evidenceValues.every((evidence) => (String(record(evidence).source).toLowerCase() === 'resource origin'));
+        if (evidenceValues.some((evidence) => String(record(evidence).source).toLowerCase() === 'resource origin')) {
+            roles.add('embedded_dependency');
+        }
+        if (!resourceOriginOnly) {
+            const category = String(finding.category).toLowerCase();
+            if (category === 'delivery platform')
+                roles.add('observed_edge');
+            else if (['application runtime', 'web framework', 'static site generator', 'web server'].includes(category)) {
+                roles.add('framework_runtime');
+            }
+            else
+                roles.add('application_platform');
+        }
+    }
+    return TECHNOLOGY_EVIDENCE_ROLE_ORDER.filter((role) => roles.has(role));
+}
+export { TECHNOLOGY_EVIDENCE_ROLE_ORDER, technologyEvidenceRoles, };

package/lib/threat-intelligence-runtime.mjs

@@ -4,7 +4,7 @@
 // must declare exact input exposure, reviewed terms, and bounded request and
 // output budgets before producing separately attributed normalized evidence.
 import { createObservation } from "../packages/evidence/observation.mjs";
-import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_CATEGORIES, THREAT_INTELLIGENCE_ENVELOPE_VERSION, THREAT_INTELLIGENCE_RESULT_STATES, THREAT_INTELLIGENCE_SCHEMA, } from "./threat-intelligence-types.mjs";
+import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_CATEGORIES, THREAT_INTELLIGENCE_CONFIDENCES, THREAT_INTELLIGENCE_ENVELOPE_VERSION, THREAT_INTELLIGENCE_RESULT_STATES, THREAT_INTELLIGENCE_SCHEMA, THREAT_INTELLIGENCE_SEVERITIES, } from "./threat-intelligence-types.mjs";
 import { assertCuratedConnectorDefinition, assertThreatIntelligenceProvider, } from "./threat-intelligence-definition-registry.mjs";
 import { boundedHttpsUrl, boundedString, isRecord, isoTimestamp, strictBoundedString, } from "./bounded-contract-normalizers.mjs";
 import { CURATED_CONNECTOR_ENTITY_TYPES, MAX_THREAT_INTELLIGENCE_URL_LENGTH, normalizeCuratedConnectorTarget, normalizeThreatIntelligenceTarget, } from "./threat-intelligence-targets.mjs";
@@ -20,8 +20,8 @@ const TERMINAL_STATES_WITHOUT_FINDINGS = new Set([
     'error',
 ]);
 const CATEGORIES = new Set(THREAT_INTELLIGENCE_CATEGORIES);
-const SEVERITIES = new Set(['critical', 'high', 'medium', 'low', 'unknown']);
-const CONFIDENCES = new Set(['high', 'medium', 'low', 'unknown']);
+const SEVERITIES = new Set(THREAT_INTELLIGENCE_SEVERITIES);
+const CONFIDENCES = new Set(THREAT_INTELLIGENCE_CONFIDENCES);
 const CONNECTOR_RELATIONSHIP_TYPES = new Set([
     'domain_resolves_to_ip',
     'domain_uses_nameserver',

package/lib/threat-intelligence-types.mjs

@@ -11,6 +11,19 @@ export const THREAT_INTELLIGENCE_CATEGORIES = Object.freeze([
     'abuse',
     'unknown',
 ]);
+export const THREAT_INTELLIGENCE_SEVERITIES = Object.freeze([
+    'critical',
+    'high',
+    'medium',
+    'low',
+    'unknown',
+]);
+export const THREAT_INTELLIGENCE_CONFIDENCES = Object.freeze([
+    'high',
+    'medium',
+    'low',
+    'unknown',
+]);
 export const THREAT_INTELLIGENCE_RESULT_STATES = Object.freeze([
     'success',
     'partial',

package/lib/threatfox-intelligence.mjs

@@ -7,6 +7,7 @@ import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
 import { providerPolicyAdmission } from "./provider-policy-admission.mjs";
 import { createThreatIntelligenceResult, defineThreatIntelligenceProvider, normalizeThreatIntelligenceTarget, } from "./threat-intelligence-contract.mjs";
 import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
+import { isRecord } from "./json-record.mjs";
 const THREATFOX_SEARCH_ENDPOINT = 'https://threatfox-api.abuse.ch/api/v1/';
 const THREATFOX_MAX_RESULTS = 20;
 const THREATFOX_MAX_INPUT_RESULTS = 100;
@@ -39,9 +40,6 @@ const THREATFOX_PROVIDER = defineThreatIntelligenceProvider({
         monthlyRequests: THREATFOX_MONTHLY_REQUESTS,
     },
 });
-function isRecord(value) {
-    return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
-}
 function enabledValue(value) {
     return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase());
 }

package/lib/tls-intelligence.mjs

@@ -11,10 +11,9 @@ import * as net from 'node:net';
 import * as tls from 'node:tls';
 import { domainToASCII } from 'node:url';
 import { createObservation } from "../packages/evidence/observation.mjs";
-import { MAX_LOOKUP_TLS_ALT_NAMES, MAX_LOOKUP_TLS_CHAIN_CERTIFICATES, MAX_LOOKUP_TLS_FINDINGS, MAX_LOOKUP_TLS_NAME_VALUES, } from "./lookup-network-evidence-bounds.mjs";
+import { MAX_LOOKUP_TLS_ALT_NAMES, MAX_LOOKUP_TLS_CHAIN_CERTIFICATES, MAX_LOOKUP_TLS_FINDINGS, MAX_LOOKUP_TLS_NAME_VALUES, TLS_PROFILE_VERSION, } from "./lookup-network-evidence-bounds.mjs";
 import { isPrivateAddress, resolvePublicAddresses } from "./safe-fetch.mjs";
 import { parseCertificateExtensionProfile, } from "./certificate-extension-profile.mjs";
-const TLS_PROFILE_VERSION = 3;
 const TLS_PORT = 443;
 const TLS_TIMEOUT_MS = 5000;
 const MAX_RESOLVED_ADDRESSES = 64;

package/lib/urlhaus-intelligence.mjs

@@ -10,6 +10,7 @@ import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
 import { providerPolicyAdmission } from "./provider-policy-admission.mjs";
 import { createThreatIntelligenceResult, defineThreatIntelligenceProvider, normalizeThreatIntelligenceTarget, } from "./threat-intelligence-contract.mjs";
 import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
+import { isRecord } from "./json-record.mjs";
 const URLHAUS_HOST_ENDPOINT = 'https://urlhaus-api.abuse.ch/v1/host/';
 const URLHAUS_MAX_RESULTS = 20;
 const URLHAUS_MAX_INPUT_RESULTS = 100;
@@ -46,9 +47,6 @@ const URLHAUS_PROVIDER = defineThreatIntelligenceProvider({
         monthlyRequests: URLHAUS_MONTHLY_REQUESTS,
     },
 });
-function isRecord(value) {
-    return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
-}
 function enabledValue(value) {
     return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase());
 }

package/lib/urlscan-intelligence.mjs

@@ -8,6 +8,7 @@ import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
 import { providerPolicyAdmission } from "./provider-policy-admission.mjs";
 import { createThreatIntelligenceResult, defineThreatIntelligenceProvider, normalizeThreatIntelligenceTarget, } from "./threat-intelligence-contract.mjs";
 import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
+import { isRecord } from "./json-record.mjs";
 const URLSCAN_SEARCH_ENDPOINT = 'https://urlscan.io/api/v1/search/';
 const URLSCAN_SEARCH_DAYS = 90;
 const URLSCAN_MAX_RESULTS = 20;
@@ -42,9 +43,6 @@ const URLSCAN_PROVIDER = defineThreatIntelligenceProvider({
         monthlyRequests: URLSCAN_MONTHLY_REQUESTS,
     },
 });
-function isRecord(value) {
-    return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
-}
 function enabledValue(value) {
     return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase());
 }

package/lib/website-security-posture.mjs

@@ -3,8 +3,7 @@
 // Findings use fixed explanatory text and bounded counts; upstream header
 // values, certificate errors, URLs, and DNS record contents are not copied.
 import { createObservation } from "../packages/evidence/observation.mjs";
-const WEBSITE_SECURITY_POSTURE_VERSION = 2;
-const MAX_SECURITY_POSTURE_FINDINGS = 32;
+import { MAX_SECURITY_POSTURE_FINDINGS, WEBSITE_SECURITY_POSTURE_VERSION, } from "./lookup-child-profile-contract.mjs";
 const MAX_RETAINED_ORIGINS = 30;
 function record(value) {
     return value && typeof value === 'object' && !Array.isArray(value) ? value : {};

package/lib/website-technology.mjs

@@ -7,16 +7,14 @@
 import { analyzeBrowserLibraries } from "./browser-library-profile.mjs";
 import { createObservation } from "../packages/evidence/observation.mjs";
 import { MAX_STATIC_HTML_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, } from "./static-html-analysis.mjs";
-const TECHNOLOGY_PROFILE_VERSION = 10;
+import { TECHNOLOGY_EVIDENCE_ROLE_ORDER, } from "./technology-evidence-role.mjs";
+import { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, TECHNOLOGY_PROFILE_VERSION, } from "./lookup-child-profile-contract.mjs";
 const MAX_TECHNOLOGY_HTML_CHARS = MAX_STATIC_HTML_CHARS;
 const MAX_TECHNOLOGY_TAG_LENGTH = MAX_TAG_LENGTH;
-const MAX_TECHNOLOGY_FINDINGS = 24;
-const MAX_EVIDENCE_PER_TECHNOLOGY = 4;
 const MAX_RESOURCE_ORIGINS = 30;
 const MAX_GENERATOR_INPUT = 160;
 const MAX_SERVER_INPUT = 240;
 const MAX_PASSIVE_RESPONSE_HEADERS = 8;
-const MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH = 180;
 const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f]/;
 function boundedLowercase(value, maxLength) {
     if (typeof value !== 'string' || value.length > maxLength || CONTROL_CHARACTER_RE.test(value))
@@ -51,14 +49,37 @@ const PASSIVE_TECHNOLOGY_HEADER_NAMES = Object.freeze([
     'x-vercel-id',
 ]);
 const PASSIVE_HEADER_NAMES = new Set(PASSIVE_TECHNOLOGY_HEADER_NAMES);
-function normalizedResponseHeaders(value) {
-    const input = value && typeof value === 'object' && !Array.isArray(value)
+function technologyHeaderValue(value) {
+    if (typeof value !== 'string' || value.length > 240 || CONTROL_CHARACTER_RE.test(value))
+        return '';
+    return value.trim();
+}
+function captureTechnologyResponseHeaders(value) {
+    const output = {};
+    const getter = value && typeof value === 'object' && typeof value.get === 'function'
+        ? value.get.bind(value)
+        : null;
+    const record = value && typeof value === 'object' && !Array.isArray(value)
         ? value
-        : {};
+        : null;
+    for (const name of PASSIVE_TECHNOLOGY_HEADER_NAMES) {
+        let rawValue;
+        try {
+            rawValue = getter ? getter(name) : record?.[name];
+        }
+        catch {
+            rawValue = undefined;
+        }
+        const headerValue = technologyHeaderValue(rawValue);
+        if (headerValue)
+            output[name] = headerValue;
+    }
+    return output;
+}
+function normalizedResponseHeaders(value) {
     const output = new Map();
-    for (const [rawName, rawValue] of Object.entries(input).slice(0, MAX_PASSIVE_RESPONSE_HEADERS * 4)) {
-        const name = rawName.trim().toLowerCase();
-        if (!PASSIVE_HEADER_NAMES.has(name) || typeof rawValue !== 'string')
+    for (const [name, rawValue] of Object.entries(captureTechnologyResponseHeaders(value))) {
+        if (!PASSIVE_HEADER_NAMES.has(name))
             continue;
         const normalized = boundedLowercase(rawValue, 240);
         if (!normalized)
@@ -121,6 +142,18 @@ function responseHeaderEvidence(name, pattern, description, confidence = 'high')
         },
     };
 }
+function evidenceRole(signature, evidence) {
+    if (evidence.role)
+        return evidence.role;
+    if (evidence.source === 'resource origin')
+        return 'embedded_dependency';
+    if (signature.category === 'delivery platform')
+        return 'observed_edge';
+    if (['application runtime', 'web framework', 'static site generator', 'web server'].includes(signature.category)) {
+        return 'framework_runtime';
+    }
+    return 'application_platform';
+}
 const TECHNOLOGY_SIGNATURES = [
     {
         id: 'wordpress', name: 'WordPress', category: 'content management',
@@ -325,14 +358,14 @@ const TECHNOLOGY_SIGNATURES = [
         id: 'netlify', name: 'Netlify', category: 'delivery platform',
         evidence: [
             serverEvidence(/^netlify(?:\s|$|\/)/i, 'The selected response server header identifies Netlify.'),
-            responseHeaderEvidence('x-nf-request-id', null, 'A Netlify request identifier response header was observed.', 'medium'),
+            { ...responseHeaderEvidence('x-nf-request-id', null, 'A Netlify application-platform response header was observed.', 'medium'), role: 'application_platform' },
         ],
     },
     {
         id: 'vercel', name: 'Vercel', category: 'delivery platform',
         evidence: [
             serverEvidence(/^vercel(?:\s|$|\/)/i, 'The selected response server header identifies Vercel.'),
-            responseHeaderEvidence('x-vercel-id', null, 'A Vercel request-trace response header was observed.', 'medium'),
+            { ...responseHeaderEvidence('x-vercel-id', null, 'A Vercel application-platform response header was observed.', 'medium'), role: 'application_platform' },
         ],
     },
     {
@@ -366,19 +399,26 @@ const TECHNOLOGY_SIGNATURE_CATALOGUE = Object.freeze(TECHNOLOGY_SIGNATURES.map((
     category: signature.category,
     minimumEvidenceMatches: signature.minimumEvidenceMatches || 1,
     requiresNonResourceEvidence: signature.requiresNonResourceEvidence === true,
-    evidence: Object.freeze(signature.evidence.map(({ source, description, confidence }) => Object.freeze({
-        source,
-        description,
-        confidence,
+    evidence: Object.freeze(signature.evidence.map((evidence) => Object.freeze({
+        source: evidence.source,
+        role: evidenceRole(signature, evidence),
+        description: evidence.description,
+        confidence: evidence.confidence,
     }))),
 })));
 function analyzeWebsiteTechnology(input = {}) {
+    // Existing direct callers pass minimised, already-derived page evidence rather
+    // than the page body. The real collector declares false for header-only
+    // responses, while an explicit malformed declaration also fails closed.
+    const htmlAvailable = input.htmlAvailable === undefined
+        ? true
+        : input.htmlAvailable === true;
     const htmlAnalysis = input.htmlAnalysis ?? analyzeStaticHtml(input.html);
-    const browserLibraryProfile = analyzeBrowserLibraries({
+    const browserLibraryProfile = htmlAvailable ? analyzeBrowserLibraries({
         htmlAnalysis,
         observedAt: input.observedAt,
         sourceTruncated: input.sourceTruncated,
-    });
+    }) : null;
     const context = {
         html: htmlAnalysis.markup,
         generator: boundedLowercase(input.generator, MAX_GENERATOR_INPUT),
@@ -393,12 +433,18 @@ function analyzeWebsiteTechnology(input = {}) {
             continue;
         if (signature.requiresNonResourceEvidence && matched.every((evidence) => evidence.source === 'resource origin'))
             continue;
+        const matchedEvidence = matched.slice(0, MAX_EVIDENCE_PER_TECHNOLOGY).map((evidence) => ({
+            source: evidence.source,
+            role: evidenceRole(signature, evidence),
+            description: evidence.description,
+        }));
         findings.push({
             id: signature.id,
             name: signature.name,
             category: signature.category,
             confidence: matched.some((evidence) => evidence.confidence === 'high') ? 'high' : 'medium',
-            evidence: matched.slice(0, MAX_EVIDENCE_PER_TECHNOLOGY).map(({ source, description }) => ({ source, description })),
+            roles: TECHNOLOGY_EVIDENCE_ROLE_ORDER.filter((role) => matchedEvidence.some((evidence) => evidence.role === role)),
+            evidence: matchedEvidence,
         });
     }
     findings.sort((left, right) => left.category.localeCompare(right.category) || left.name.localeCompare(right.name));
@@ -407,11 +453,14 @@ function analyzeWebsiteTechnology(input = {}) {
         || htmlAnalysis.inputLimitReached
         || htmlAnalysis.tagLimitReached
         || findingLimitReached;
+    const partial = truncated || !htmlAvailable;
     const limitations = [
         'Curated signature matching is selective; an unmatched technology may still be present.',
         'Static response evidence cannot identify JavaScript-rendered or deliberately concealed technologies.',
         'Technology indicators describe observed implementation clues, not ownership, safety, or maliciousness.',
     ];
+    if (!htmlAvailable)
+        limitations.push('HTML page identity was unavailable; only bounded permitted response-header indicators were evaluated.');
     if (input.sourceTruncated === true)
         limitations.push('The captured homepage body was truncated, so technology indicators may be incomplete.');
     if (htmlAnalysis.inputLimitReached)
@@ -423,16 +472,16 @@ function analyzeWebsiteTechnology(input = {}) {
     return {
         profileVersion: TECHNOLOGY_PROFILE_VERSION,
         ...createObservation({
-            status: truncated ? 'partial' : 'success',
+            status: partial ? 'partial' : 'success',
             observedAt: input.observedAt,
             scanMode: 'deep',
             source: 'derived',
-            complete: !truncated,
+            complete: !partial,
             truncated,
             limitations,
             diagnostics: {
                 findings: findings.length,
-                htmlEvaluated: Boolean(context.html),
+                htmlEvaluated: htmlAvailable,
                 generatorEvaluated: Boolean(context.generator),
                 serverEvaluated: Boolean(context.httpServer),
                 resourceOriginsEvaluated: context.resourceHosts.size,
@@ -444,4 +493,4 @@ function analyzeWebsiteTechnology(input = {}) {
         browserLibraryProfile,
     };
 }
-export { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, MAX_TECHNOLOGY_HTML_CHARS, MAX_TECHNOLOGY_TAGS, PASSIVE_TECHNOLOGY_HEADER_NAMES, TECHNOLOGY_PROFILE_VERSION, TECHNOLOGY_SIGNATURE_CATALOGUE, analyzeWebsiteTechnology, };
+export { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, MAX_TECHNOLOGY_HTML_CHARS, MAX_TECHNOLOGY_TAGS, PASSIVE_TECHNOLOGY_HEADER_NAMES, TECHNOLOGY_PROFILE_VERSION, TECHNOLOGY_SIGNATURE_CATALOGUE, analyzeWebsiteTechnology, captureTechnologyResponseHeaders, };