@slicedearth/whoisleuth-cli 1.36.1 → 1.43.0

package/lib/risk-scoring.mjs

@@ -1,23 +1,36 @@
 import { calibrateExternalIntelligenceRisk } from "./external-intelligence-risk.mjs";
+import { buildScoreEvidenceQuality, } from "./scoring-evidence-quality.mjs";
 const RISK_STATES = new Set(['registered', 'for_sale', 'expiring']);
 const STATE_LABELS = Object.freeze({
     for_sale: 'for sale',
     expiring: 'expiring/pending delete',
     registered: 'registered',
 });
-// Version 6 preserves the grouped external-evidence factor and recognizes the
-// newly generated addition, plural, embedded-TLD, advanced Unicode, and
-// analyst-directed token-replacement families conservatively. Older scores
-// remain readable, while case and watchlist comparisons gate numeric changes
-// on matching model versions so formula changes are not mistaken for changes
-// in the observed domain.
-export const RISK_MODEL_VERSION = 6;
+// Version 7 caps each independent evidence family, treats generic operational
+// properties as supporting context, and reports collection quality separately
+// from the heuristic score. Missing evidence never contributes points.
+export const RISK_MODEL_VERSION = 7;
 export const RISK_REVIEW_THRESHOLD = 70;
 const RISK_STATE_BASE = Object.freeze({
-    registered: 10,
-    for_sale: 5,
-    expiring: 8,
+    registered: 6,
+    for_sale: 4,
+    expiring: 5,
 });
+const FAMILY_CAPS = Object.freeze({
+    'domain-resemblance': 20,
+    'brand-presentation': 24,
+    'credential-lure': 18,
+    'external-intelligence': 18,
+    'operational-support': 12,
+});
+const REMOVABLE_RISK_FAMILY_MAP = Object.freeze({
+    'brand-presentation': true,
+    'credential-lure': true,
+    'domain-resemblance': true,
+    'external-intelligence': true,
+    'operational-support': true,
+});
+const REMOVABLE_RISK_FAMILIES = Object.freeze(Object.keys(REMOVABLE_RISK_FAMILY_MAP));
 const HIGH_CONTEXT_MUTATIONS = new Set([
     'unicode_homoglyph',
     'unicode_homoglyph_depth_2',
@@ -49,7 +62,7 @@ export function normalizeRiskModelVersion(value) {
 }
 function formatAge(days) {
     if (days < 60)
-        return `${days}d old`;
+        return `${Math.round(days)}d old`;
     const years = days / 365.25;
     return years < 1 ? `${Math.round(days / 30)}mo old` : `${years.toFixed(1)}y old`;
 }
@@ -68,83 +81,234 @@ function mutationContext(mutationTypes) {
     }
     return null;
 }
+function hasBoolean(value) {
+    return typeof value === 'boolean';
+}
+function scoreQuality(input) {
+    const observedFamilies = ['registration'];
+    const expectedFamilies = ['registration', 'operational-support'];
+    if (Array.isArray(input.mutationTypes) || hasBoolean(input.idnReferenceMatch))
+        observedFamilies.push('domain-resemblance');
+    if (input.hasActiveBrandProfile === true)
+        expectedFamilies.push('domain-resemblance', 'brand-presentation');
+    if ([input.faviconMatch, input.faviconNearMatch, input.reusesOfficialAssets, input.pageBaselineMatch].some(hasBoolean)) {
+        observedFamilies.push('brand-presentation');
+    }
+    if ([input.hasPasswordField, input.hasExternalFormAction].some(hasBoolean) || typeof input.phishingLanguageMatch === 'string') {
+        observedFamilies.push('credential-lure');
+    }
+    if (input.threatIntelligence !== undefined && input.threatIntelligence !== null)
+        observedFamilies.push('external-intelligence');
+    if ([input.hasMx, input.hasSpf, input.hasDmarc].some(hasBoolean)
+        || typeof input.activityStatus === 'string'
+        || typeof input.domainAgeDays === 'number')
+        observedFamilies.push('operational-support');
+    if (input.scanDepth === 'deep')
+        expectedFamilies.push('credential-lure');
+    return buildScoreEvidenceQuality(input, {
+        expectedDepth: 'deep',
+        observedFamilies,
+        expectedFamilies,
+    });
+}
 // Risk prioritizes a registered lookalike/typosquat domain for analyst review.
 // It is a heuristic indicator, never a maliciousness or safety verdict.
-export function explainRiskScore(r) {
-    const state = r.availability ?? r.state;
+function explainRiskScoreInternal(input, excludedFamily) {
+    const state = input.availability ?? input.state;
     if (typeof state !== 'string' || !RISK_STATES.has(state))
         return null;
-    const base = RISK_STATE_BASE[state] ?? 0;
-    const factors = [{ label: `Base score for "${STATE_LABELS[state] || state}"`, delta: base }];
-    let score = base;
-    const contextualFamilies = new Set();
-    const addContextualFactor = (family, label, delta) => {
-        contextualFamilies.add(family);
-        factors.push({ label, delta });
-        score += delta;
+    const factors = [];
+    const familyTotals = new Map();
+    const add = (family, label, requestedDelta) => {
+        if (family === excludedFamily)
+            return 0;
+        const current = familyTotals.get(family) ?? 0;
+        const cap = FAMILY_CAPS[family] ?? Number.POSITIVE_INFINITY;
+        const delta = Math.max(0, Math.min(requestedDelta, Math.max(0, cap - current)));
+        const capped = delta !== requestedDelta;
+        factors.push({ family, label: capped ? `${label} (family cap reached)` : label, delta });
+        familyTotals.set(family, current + delta);
+        return delta;
+    };
+    add('registration', `Base context for “${STATE_LABELS[state] ?? state}”`, RISK_STATE_BASE[state] ?? 0);
+    const mutation = mutationContext(input.mutationTypes);
+    if (mutation)
+        add('domain-resemblance', mutation.label, mutation.delta);
+    if (input.idnReferenceMatch === true)
+        add('domain-resemblance', 'IDN skeleton matches an official Brand Profile domain', 20);
+    if (input.faviconMatch === true)
+        add('brand-presentation', 'Favicon matches an official Brand Profile site', 18);
+    else if (input.faviconNearMatch === true)
+        add('brand-presentation', 'Favicon resembles an official Brand Profile site', 14);
+    if (input.reusesOfficialAssets === true)
+        add('brand-presentation', 'Official asset-host relationship observed', 6);
+    if (input.pageBaselineMatch === true)
+        add('brand-presentation', 'Multiple independently compared page-identity components match the official baseline', 14);
+    if (typeof input.phishingLanguageMatch === 'string' && input.phishingLanguageMatch.trim()) {
+        add('credential-lure', 'Suspicious urgency language observed', 8);
+    }
+    if (input.hasPasswordField === true && input.hasExternalFormAction === true) {
+        add('credential-lure', 'Password form submits to an external origin', 10);
+    }
+    else if (input.hasPasswordField === true) {
+        add('credential-lure', 'Login/password form present', 5);
+    }
+    else if (input.hasExternalFormAction === true) {
+        add('credential-lure', 'External form destination observed without a password field', 0);
+    }
+    const external = calibrateExternalIntelligenceRisk(input.threatIntelligence);
+    if (external.factor)
+        add('external-intelligence', external.factor.label, external.contribution);
+    const primaryFamilies = ['domain-resemblance', 'brand-presentation', 'credential-lure', 'external-intelligence']
+        .filter((family) => (familyTotals.get(family) ?? 0) > 0);
+    if (primaryFamilies.length >= 2) {
+        add('corroboration', `Corroborating context across ${primaryFamilies.length} independent evidence families`, primaryFamilies.length >= 3 ? 18 : 10);
+    }
+    const hasIndependentContext = primaryFamilies.length > 0;
+    const supporting = (label, delta) => {
+        add('operational-support', hasIndependentContext ? label : `${label}; neutral without independent suspicious context`, hasIndependentContext ? delta : 0);
+    };
+    if (input.activityStatus === 'active')
+        supporting('Active website observed', 4);
+    if (input.hasMx === true)
+        supporting('Mail server configured', 4);
+    if (input.hasSpf === true && input.hasDmarc === true)
+        supporting('SPF and DMARC records present', 1);
+    else if (input.hasSpf === true || input.hasDmarc === true)
+        supporting('SPF or DMARC record present', 0);
+    if (typeof input.domainAgeDays === 'number' && Number.isFinite(input.domainAgeDays) && input.domainAgeDays >= 0) {
+        if (input.domainAgeDays < 90)
+            supporting(`Recently registered (${formatAge(input.domainAgeDays)})`, 6);
+        else if (input.domainAgeDays < 365)
+            supporting(`Registered under a year ago (${formatAge(input.domainAgeDays)})`, 2);
+    }
+    if (typeof input.privacyProtected === 'boolean') {
+        add('registration', 'Registration privacy is neutral context', 0);
+    }
+    const rawScore = Math.round(factors.reduce((total, factor) => total + factor.delta, 0));
+    const score = Math.max(0, Math.min(100, rawScore));
+    const familyOrder = [
+        'registration', 'domain-resemblance', 'brand-presentation', 'credential-lure',
+        'external-intelligence', 'corroboration', 'operational-support',
+    ];
+    return {
+        modelVersion: RISK_MODEL_VERSION,
+        score,
+        rawScore,
+        capped: score !== rawScore,
+        factors,
+        families: familyOrder
+            .filter((id) => familyTotals.has(id))
+            .map((id) => ({ id, contribution: familyTotals.get(id) ?? 0, cap: FAMILY_CAPS[id] ?? null })),
+        evidenceQuality: scoreQuality(input),
     };
-    const context = mutationContext(r.mutationTypes);
-    if (context)
-        addContextualFactor('domain-resemblance', context.label, context.delta);
-    if (r.faviconMatch === true) {
-        addContextualFactor('brand-presentation', 'Favicon matches your Brand Profile official site', 18);
-    }
-    else if (r.faviconNearMatch === true) {
-        addContextualFactor('brand-presentation', 'Favicon resembles your official site (perceptual match)', 14);
-    }
-    if (r.reusesOfficialAssets === true) {
-        addContextualFactor('brand-presentation', 'Official asset host relationship observed', 6);
-    }
-    if (typeof r.phishingLanguageMatch === 'string' && r.phishingLanguageMatch.trim()) {
-        addContextualFactor('credential-lure', 'Suspicious urgency language observed', 8);
-    }
-    if (r.hasPasswordField === true) {
-        addContextualFactor('credential-lure', 'Login/password form present', 5);
-    }
-    if (contextualFamilies.size >= 2) {
-        const bonus = contextualFamilies.size >= 3 ? 20 : 10;
-        factors.push({ label: `Corroborating context across ${contextualFamilies.size} distinct evidence families`, delta: bonus });
-        score += bonus;
-    }
-    const externalEvidence = calibrateExternalIntelligenceRisk(r.threatIntelligence);
-    if (externalEvidence.factor) {
-        factors.push(externalEvidence.factor);
-        score += externalEvidence.contribution;
-    }
-    if (r.activityStatus === 'active') {
-        factors.push({ label: 'Active site in use', delta: 8 });
-        score += 8;
-    }
-    if (r.hasMx === true) {
-        factors.push({ label: 'Mail server configured', delta: 8 });
-        score += 8;
-    }
-    if (r.hasSpf === true && r.hasDmarc === true) {
-        factors.push({ label: 'SPF + DMARC records present', delta: 3 });
-        score += 3;
-    }
-    else if (r.hasSpf === true || r.hasDmarc === true) {
-        factors.push({ label: 'SPF or DMARC record present', delta: 1 });
-        score += 1;
-    }
-    if (r.privacyProtected === true) {
-        factors.push({ label: 'WHOIS privacy protected', delta: 3 });
-        score += 3;
-    }
-    if (typeof r.domainAgeDays === 'number' && Number.isFinite(r.domainAgeDays) && r.domainAgeDays >= 0) {
-        if (r.domainAgeDays < 90) {
-            factors.push({ label: `Recently registered (${formatAge(r.domainAgeDays)})`, delta: 10 });
-            score += 10;
-        }
-        else if (r.domainAgeDays < 365) {
-            factors.push({ label: `Registered under a year ago (${formatAge(r.domainAgeDays)})`, delta: 4 });
-            score += 4;
-        }
-    }
-    return { modelVersion: RISK_MODEL_VERSION, score: Math.max(0, Math.min(100, Math.round(score))), factors };
 }
-export function computeRiskScore(r) {
-    return explainRiskScore(r)?.score ?? null;
+export function explainRiskScore(input) {
+    return explainRiskScoreInternal(input, null);
+}
+export function buildRiskScoreSensitivity(input) {
+    const baseline = explainRiskScoreInternal(input, null);
+    if (!baseline)
+        return null;
+    const removableFamilies = REMOVABLE_RISK_FAMILIES
+        .filter((family) => baseline.families.some((item) => item.id === family && item.contribution > 0));
+    const scenarios = removableFamilies.map((excludedFamily) => {
+        const result = explainRiskScoreInternal(input, excludedFamily);
+        const score = result?.score ?? baseline.score;
+        return Object.freeze({ excludedFamily, score, difference: score - baseline.score });
+    }).sort((left, right) => left.score - right.score || left.excludedFamily.localeCompare(right.excludedFamily));
+    const minimumScenarioScore = scenarios[0]?.score ?? baseline.score;
+    const thresholdState = baseline.score < RISK_REVIEW_THRESHOLD
+        ? 'below'
+        : minimumScenarioScore < RISK_REVIEW_THRESHOLD ? 'crosses' : 'stable_above';
+    return Object.freeze({
+        version: 1,
+        baselineScore: baseline.score,
+        reviewThreshold: RISK_REVIEW_THRESHOLD,
+        minimumScenarioScore,
+        thresholdState,
+        scenarios: Object.freeze(scenarios),
+        limitations: Object.freeze([
+            'Each scenario removes one observed evidence family and recalculates family caps and corroboration; it does not predict missing evidence.',
+            'The sensitivity review qualifies prioritisation only and does not determine maliciousness, safety, ownership, or intent.',
+        ]),
+    });
+}
+export function computeRiskScore(input) {
+    return explainRiskScore(input)?.score ?? null;
+}
+// Retained only for deterministic offline comparison of reviewed calibration
+// datasets. Runtime lookups always use the current model above.
+export function explainRiskScoreV6(input) {
+    const state = input.availability ?? input.state;
+    if (typeof state !== 'string' || !RISK_STATES.has(state))
+        return null;
+    const legacyBase = { registered: 10, for_sale: 5, expiring: 8 };
+    const factors = [];
+    const totals = new Map();
+    const add = (family, label, delta) => {
+        factors.push({ family, label, delta });
+        totals.set(family, (totals.get(family) ?? 0) + delta);
+    };
+    add('registration', `Base score for “${STATE_LABELS[state] ?? state}”`, legacyBase[state] ?? 0);
+    const contextual = new Set();
+    const context = mutationContext(input.mutationTypes);
+    if (context) {
+        add('domain-resemblance', context.label, context.delta);
+        contextual.add('domain-resemblance');
+    }
+    if (input.faviconMatch === true) {
+        add('brand-presentation', 'Favicon matches an official Brand Profile site', 18);
+        contextual.add('brand-presentation');
+    }
+    else if (input.faviconNearMatch === true) {
+        add('brand-presentation', 'Favicon resembles an official Brand Profile site', 14);
+        contextual.add('brand-presentation');
+    }
+    if (input.reusesOfficialAssets === true) {
+        add('brand-presentation', 'Official asset-host relationship observed', 6);
+        contextual.add('brand-presentation');
+    }
+    if (typeof input.phishingLanguageMatch === 'string' && input.phishingLanguageMatch.trim()) {
+        add('credential-lure', 'Suspicious urgency language observed', 8);
+        contextual.add('credential-lure');
+    }
+    if (input.hasPasswordField === true) {
+        add('credential-lure', 'Login/password form present', 5);
+        contextual.add('credential-lure');
+    }
+    if (contextual.size >= 2)
+        add('corroboration', `Corroborating context across ${contextual.size} distinct evidence families`, contextual.size >= 3 ? 20 : 10);
+    const external = calibrateExternalIntelligenceRisk(input.threatIntelligence);
+    if (external.factor)
+        add('external-intelligence', external.factor.label, external.contribution);
+    if (input.activityStatus === 'active')
+        add('operational-support', 'Active site in use', 8);
+    if (input.hasMx === true)
+        add('operational-support', 'Mail server configured', 8);
+    if (input.hasSpf === true && input.hasDmarc === true)
+        add('operational-support', 'SPF and DMARC records present', 3);
+    else if (input.hasSpf === true || input.hasDmarc === true)
+        add('operational-support', 'SPF or DMARC record present', 1);
+    if (input.privacyProtected === true)
+        add('registration', 'WHOIS privacy protected', 3);
+    if (typeof input.domainAgeDays === 'number' && Number.isFinite(input.domainAgeDays) && input.domainAgeDays >= 0) {
+        if (input.domainAgeDays < 90)
+            add('operational-support', `Recently registered (${formatAge(input.domainAgeDays)})`, 10);
+        else if (input.domainAgeDays < 365)
+            add('operational-support', `Registered under a year ago (${formatAge(input.domainAgeDays)})`, 4);
+    }
+    const rawScore = Math.round(factors.reduce((total, factor) => total + factor.delta, 0));
+    const score = Math.max(0, Math.min(100, rawScore));
+    return {
+        modelVersion: 6,
+        score,
+        rawScore,
+        capped: score !== rawScore,
+        factors,
+        families: [...totals].map(([id, contribution]) => ({ id, contribution, cap: null })),
+        evidenceQuality: scoreQuality(input),
+    };
 }
 export function riskTone(score) {
     if (score === null)
@@ -159,6 +323,6 @@ export function formatScoreBreakdown(explained, separator = '\n') {
     if (!explained)
         return '';
     const parts = explained.factors.map((factor) => `${factor.label} ${factor.delta >= 0 ? '+' : ''}${Math.round(factor.delta)}`);
-    parts.push(`Total ${explained.score}`);
+    parts.push(`Total ${explained.score} · Risk model v${explained.modelVersion} · evidence ${explained.evidenceQuality.state}`);
     return parts.join(separator);
 }

package/lib/scoring-evidence-quality.mjs

@@ -0,0 +1,107 @@
+export const SCORE_EVIDENCE_QUALITY_VERSION = 1;
+const MAX_SOURCES = 24;
+const MAX_FAMILIES = 12;
+const SOURCE_ID_RE = /^[a-z][a-z0-9_-]{0,63}$/u;
+const COMPLETE_STATES = new Set(['complete', 'not_found', 'success']);
+const LIMITED_STATES = new Set(['partial', 'unknown']);
+const UNAVAILABLE_STATES = new Set(['error', 'rate_limited', 'unavailable']);
+const SKIPPED_STATES = new Set(['skipped', 'unsupported']);
+const ACCEPTED_SOURCE_STATES = new Set([
+    ...COMPLETE_STATES,
+    ...LIMITED_STATES,
+    ...UNAVAILABLE_STATES,
+    ...SKIPPED_STATES,
+]);
+function boundedFamilyList(value) {
+    const output = new Set();
+    for (const item of value.slice(0, MAX_FAMILIES * 2)) {
+        const normalized = String(item).trim().toLowerCase().replace(/[^a-z0-9_-]+/gu, '-').slice(0, 64);
+        if (SOURCE_ID_RE.test(normalized))
+            output.add(normalized);
+        if (output.size >= MAX_FAMILIES)
+            break;
+    }
+    return [...output].sort();
+}
+function sourceCoverage(value) {
+    if (!Array.isArray(value))
+        return [];
+    const output = [];
+    const seen = new Set();
+    for (const candidate of value.slice(0, MAX_SOURCES * 4)) {
+        if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate))
+            continue;
+        const item = candidate;
+        const source = typeof item.source === 'string' ? item.source.trim().toLowerCase() : '';
+        const state = typeof item.state === 'string' ? item.state.trim().toLowerCase() : '';
+        if (!SOURCE_ID_RE.test(source) || seen.has(source))
+            continue;
+        if (!ACCEPTED_SOURCE_STATES.has(state))
+            continue;
+        seen.add(source);
+        output.push({ source, state });
+        if (output.length >= MAX_SOURCES)
+            break;
+    }
+    return output;
+}
+function scanDepth(value) {
+    return value === 'deep' || value === 'fast' ? value : 'unknown';
+}
+function observedFreshness(value) {
+    if (typeof value !== 'string' || value.length > 64)
+        return 'unknown';
+    return Number.isFinite(Date.parse(value)) ? 'observed' : 'unknown';
+}
+export function buildScoreEvidenceQuality(input, options) {
+    const depth = scanDepth(input.scanDepth);
+    const coverage = sourceCoverage(input.sourceCoverage);
+    const completeSources = coverage.filter((item) => COMPLETE_STATES.has(item.state)).length;
+    const limitedSources = coverage.filter((item) => LIMITED_STATES.has(item.state)).length;
+    const unavailableSources = coverage.filter((item) => UNAVAILABLE_STATES.has(item.state)).length;
+    const skippedSources = coverage.filter((item) => SKIPPED_STATES.has(item.state)).length;
+    const observedFamilies = boundedFamilyList(options.observedFamilies ?? []);
+    const expectedFamilies = boundedFamilyList(options.expectedFamilies ?? []);
+    const observedSet = new Set(observedFamilies);
+    const missingFamilies = expectedFamilies.filter((family) => !observedSet.has(family));
+    const limitations = [];
+    let state = 'complete';
+    if (!coverage.length || depth === 'unknown')
+        state = 'unknown';
+    else if (unavailableSources || limitedSources)
+        state = 'partial';
+    else if (options.expectedDepth === 'deep' && depth !== 'deep')
+        state = 'limited';
+    if (missingFamilies.length && state === 'complete')
+        state = 'limited';
+    if (options.expectedDepth === 'deep' && depth !== 'deep') {
+        limitations.push('A fast or unknown-depth observation does not evaluate every website and mail scoring family.');
+    }
+    if (limitedSources || unavailableSources) {
+        limitations.push('One or more score-relevant sources were partial, unavailable, or indeterminate.');
+    }
+    if (skippedSources) {
+        limitations.push('One or more reported sources were skipped or unsupported and did not contribute to the score.');
+    }
+    if (!coverage.length)
+        limitations.push('Source-level coverage was not supplied with this score.');
+    if (missingFamilies.length)
+        limitations.push(`No usable observation was supplied for: ${missingFamilies.join(', ')}.`);
+    const boundedLimitations = [
+        ...limitations.slice(0, 5),
+        'Evidence coverage qualifies the score; it never adds points or turns missing evidence into a negative finding.',
+    ];
+    return Object.freeze({
+        version: SCORE_EVIDENCE_QUALITY_VERSION,
+        state,
+        scanDepth: depth,
+        freshness: observedFreshness(input.observedAt),
+        completeSources,
+        limitedSources,
+        unavailableSources,
+        skippedSources,
+        observedFamilies: Object.freeze(observedFamilies),
+        missingFamilies: Object.freeze(missingFamilies),
+        limitations: Object.freeze(boundedLimitations),
+    });
+}

package/lib/security-txt.mjs

@@ -202,7 +202,7 @@ function parseSecurityTxt(text, options = {}) {
     const limitations = [
         ...(signedPayload.signed ? ['A cleartext signature was present but was not cryptographically verified.'] : []),
         ...(canonicalMatches === false ? ['The fetched URL was not listed by the file as a Canonical location.'] : []),
-        ...(valuesTruncated ? ['The response or normalized field set exceeded a collection bound.'] : []),
+        ...(valuesTruncated ? ['The response or normalised field set exceeded a collection bound.'] : []),
     ];
     const detail = state === 'malformed'
         ? 'The file did not contain at least one valid Contact field and exactly one valid Expires field.'
@@ -314,7 +314,7 @@ async function collectSecurityTxt(hostname, dependencies = {}) {
             }
             if (response.status === 404 || response.status === 410) {
                 await response.body?.cancel().catch(() => { });
-                return emptyResult('absent', 'No security.txt file was published at the standardized location for this hostname.', {
+                return emptyResult('absent', 'No security.txt file was published at the standardised location for this hostname.', {
                     observedAt, requestedUrl, finalUrl: result.finalUrl, httpStatus: response.status, durationMs: now() - startedAt,
                 });
             }
@@ -328,7 +328,7 @@ async function collectSecurityTxt(hostname, dependencies = {}) {
             const charset = contentType.match(/;\s*charset\s*=\s*"?([^;"\s]+)/iu)?.[1]?.toLowerCase() || null;
             if (!/^text\/plain(?:\s*;|$)/iu.test(contentType) || (charset !== null && charset !== 'utf-8')) {
                 await response.body?.cancel().catch(() => { });
-                return emptyResult('unsupported', 'The standardized endpoint did not return a UTF-8 text/plain security.txt document.', {
+                return emptyResult('unsupported', 'The standardised endpoint did not return a UTF-8 text/plain security.txt document.', {
                     observedAt, requestedUrl, finalUrl: result.finalUrl, httpStatus: response.status, durationMs: now() - startedAt,
                 });
             }

package/lib/static-html-analysis.mjs

@@ -15,6 +15,7 @@ const MAX_SCRIPT_REFERENCE_LENGTH = 2_048;
 const MAX_SCRIPT_MEDIA_TYPE_LENGTH = 120;
 const MAX_INLINE_SCRIPT_CHARS = 32_768;
 const MAX_INLINE_SCRIPT_TOTAL_CHARS = 65_536;
+const MAX_STATIC_VISIBLE_TEXT_CHARS = MAX_STATIC_HTML_CHARS;
 const MAX_STATIC_FORMS = 50;
 const MAX_STATIC_INPUTS = 500;
 const MAX_FORM_ATTRIBUTE_LENGTH = 2_048;
@@ -48,6 +49,7 @@ const VOID_TAGS = new Set([
     'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta',
     'param', 'source', 'track', 'wbr',
 ]);
+const NON_VISIBLE_TEXT_TAGS = new Set(['script', 'style', 'template']);
 function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LENGTH) {
     const attribute = attributes
         .slice(0, MAX_ATTRIBUTES_PER_TAG)
@@ -190,7 +192,9 @@ function analyzeStaticHtml(value, options = {}) {
     const inputLimitReached = supplied.length > MAX_STATIC_HTML_CHARS;
     const html = supplied.slice(0, MAX_STATIC_HTML_CHARS);
     const baseUrl = safeBaseUrl(options.baseUrl);
+    const includeVisibleText = options.includeVisibleText === true;
     const markup = [];
+    const visibleTextParts = [];
     const structureTokens = [];
     const scripts = [];
     const cspMetaPolicies = [];
@@ -211,9 +215,12 @@ function analyzeStaticHtml(value, options = {}) {
     let structureLimitReached = false;
     let scriptLimitReached = false;
     let inlineLimitReached = false;
+    let visibleTextLimitReached = false;
     let cspMetaLimitReached = false;
     let insideExplicitHead = false;
     let scriptElementSeen = false;
+    let nonVisibleTextDepth = 0;
+    let visibleTextCharacters = 0;
     function appendInlineScript(chars) {
         if (!activeInlineScript || !chars)
             return;
@@ -234,12 +241,26 @@ function analyzeStaticHtml(value, options = {}) {
         }
         structureTokens.push(value);
     }
+    function appendVisibleText(chars) {
+        if (!includeVisibleText || nonVisibleTextDepth > 0 || !chars)
+            return;
+        const remaining = MAX_STATIC_VISIBLE_TEXT_CHARS - visibleTextCharacters;
+        const retainedLength = Math.max(0, Math.min(chars.length, remaining));
+        if (retainedLength > 0) {
+            visibleTextParts.push(chars.slice(0, retainedLength));
+            visibleTextCharacters += retainedLength;
+        }
+        if (retainedLength < chars.length)
+            visibleTextLimitReached = true;
+    }
     const handler = {
         onStartTag(token) {
             const tagName = token.tagName.toLowerCase();
             const rawTextMode = RAW_TEXT_MODES[tagName];
             if (!token.selfClosing && rawTextMode !== undefined)
                 tokenizer.state = rawTextMode;
+            if (!token.selfClosing && NON_VISIBLE_TEXT_TAGS.has(tagName))
+                nonVisibleTextDepth += 1;
             if (tagsExamined >= MAX_STATIC_HTML_TAGS) {
                 tagLimitReached = true;
                 activeInlineScript = null;
@@ -332,15 +353,20 @@ function analyzeStaticHtml(value, options = {}) {
                 insideExplicitHead = false;
             if (!VOID_TAGS.has(tagName))
                 appendStructureToken(`/${tagName}`);
+            if (NON_VISIBLE_TEXT_TAGS.has(tagName) && nonVisibleTextDepth > 0)
+                nonVisibleTextDepth -= 1;
         },
         onCharacter(token) {
             appendInlineScript(token.chars);
+            appendVisibleText(token.chars);
         },
         onNullCharacter(token) {
             appendInlineScript(token.chars);
+            appendVisibleText(token.chars);
         },
         onWhitespaceCharacter(token) {
             appendInlineScript(token.chars);
+            appendVisibleText(token.chars);
         },
         onComment() { },
         onDoctype() { },
@@ -350,6 +376,7 @@ function analyzeStaticHtml(value, options = {}) {
     tokenizer.write(html, true);
     return {
         markup: markup.join('\n'),
+        visibleText: visibleTextParts.join(''),
         structureTokens,
         scripts,
         cspMetaPolicies,
@@ -360,8 +387,9 @@ function analyzeStaticHtml(value, options = {}) {
         structureLimitReached,
         scriptLimitReached,
         inlineLimitReached,
+        visibleTextLimitReached,
         tagsExamined,
         inlineCharactersExamined,
     };
 }
-export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, };
+export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, };

package/lib/threat-intelligence-manifests.mjs

@@ -352,7 +352,7 @@ function normalizeConnectorOutputs(value) {
         throw new TypeError('Connector outputs must be unique');
     }
     if (!entities.length) {
-        throw new TypeError('Connector outputs require at least one normalized entity type');
+        throw new TypeError('Connector outputs require at least one normalised entity type');
     }
     const entityTypes = new Set(entities);
     for (const relationship of relationships) {

package/lib/threatfox-intelligence.mjs

@@ -282,7 +282,7 @@ function createThreatfoxIntelligenceAdapter(dependencies = {}) {
                 truncated,
                 detail: findings.length
                     ? `Found ${findings.length} bounded malware-IOC record${findings.length === 1 ? '' : 's'} for this domain${truncated ? '; additional provider records may exist' : ''}.`
-                    : 'The provider reported a match but no returned record could be safely normalized.',
+                    : 'The provider reported a match but no returned record could be safely normalised.',
                 upstreamStatus,
                 findings,
                 limitations: [