@slicedearth/whoisleuth-cli 1.39.1 → 1.43.0

package/lib/static-html-analysis.mjs

@@ -15,6 +15,7 @@ const MAX_SCRIPT_REFERENCE_LENGTH = 2_048;
 const MAX_SCRIPT_MEDIA_TYPE_LENGTH = 120;
 const MAX_INLINE_SCRIPT_CHARS = 32_768;
 const MAX_INLINE_SCRIPT_TOTAL_CHARS = 65_536;
+const MAX_STATIC_VISIBLE_TEXT_CHARS = MAX_STATIC_HTML_CHARS;
 const MAX_STATIC_FORMS = 50;
 const MAX_STATIC_INPUTS = 500;
 const MAX_FORM_ATTRIBUTE_LENGTH = 2_048;
@@ -48,6 +49,7 @@ const VOID_TAGS = new Set([
     'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta',
     'param', 'source', 'track', 'wbr',
 ]);
+const NON_VISIBLE_TEXT_TAGS = new Set(['script', 'style', 'template']);
 function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LENGTH) {
     const attribute = attributes
         .slice(0, MAX_ATTRIBUTES_PER_TAG)
@@ -190,7 +192,9 @@ function analyzeStaticHtml(value, options = {}) {
     const inputLimitReached = supplied.length > MAX_STATIC_HTML_CHARS;
     const html = supplied.slice(0, MAX_STATIC_HTML_CHARS);
     const baseUrl = safeBaseUrl(options.baseUrl);
+    const includeVisibleText = options.includeVisibleText === true;
     const markup = [];
+    const visibleTextParts = [];
     const structureTokens = [];
     const scripts = [];
     const cspMetaPolicies = [];
@@ -211,9 +215,12 @@ function analyzeStaticHtml(value, options = {}) {
     let structureLimitReached = false;
     let scriptLimitReached = false;
     let inlineLimitReached = false;
+    let visibleTextLimitReached = false;
     let cspMetaLimitReached = false;
     let insideExplicitHead = false;
     let scriptElementSeen = false;
+    let nonVisibleTextDepth = 0;
+    let visibleTextCharacters = 0;
     function appendInlineScript(chars) {
         if (!activeInlineScript || !chars)
             return;
@@ -234,12 +241,26 @@ function analyzeStaticHtml(value, options = {}) {
         }
         structureTokens.push(value);
     }
+    function appendVisibleText(chars) {
+        if (!includeVisibleText || nonVisibleTextDepth > 0 || !chars)
+            return;
+        const remaining = MAX_STATIC_VISIBLE_TEXT_CHARS - visibleTextCharacters;
+        const retainedLength = Math.max(0, Math.min(chars.length, remaining));
+        if (retainedLength > 0) {
+            visibleTextParts.push(chars.slice(0, retainedLength));
+            visibleTextCharacters += retainedLength;
+        }
+        if (retainedLength < chars.length)
+            visibleTextLimitReached = true;
+    }
     const handler = {
         onStartTag(token) {
             const tagName = token.tagName.toLowerCase();
             const rawTextMode = RAW_TEXT_MODES[tagName];
             if (!token.selfClosing && rawTextMode !== undefined)
                 tokenizer.state = rawTextMode;
+            if (!token.selfClosing && NON_VISIBLE_TEXT_TAGS.has(tagName))
+                nonVisibleTextDepth += 1;
             if (tagsExamined >= MAX_STATIC_HTML_TAGS) {
                 tagLimitReached = true;
                 activeInlineScript = null;
@@ -332,15 +353,20 @@ function analyzeStaticHtml(value, options = {}) {
                 insideExplicitHead = false;
             if (!VOID_TAGS.has(tagName))
                 appendStructureToken(`/${tagName}`);
+            if (NON_VISIBLE_TEXT_TAGS.has(tagName) && nonVisibleTextDepth > 0)
+                nonVisibleTextDepth -= 1;
         },
         onCharacter(token) {
             appendInlineScript(token.chars);
+            appendVisibleText(token.chars);
         },
         onNullCharacter(token) {
             appendInlineScript(token.chars);
+            appendVisibleText(token.chars);
         },
         onWhitespaceCharacter(token) {
             appendInlineScript(token.chars);
+            appendVisibleText(token.chars);
         },
         onComment() { },
         onDoctype() { },
@@ -350,6 +376,7 @@ function analyzeStaticHtml(value, options = {}) {
     tokenizer.write(html, true);
     return {
         markup: markup.join('\n'),
+        visibleText: visibleTextParts.join(''),
         structureTokens,
         scripts,
         cspMetaPolicies,
@@ -360,8 +387,9 @@ function analyzeStaticHtml(value, options = {}) {
         structureLimitReached,
         scriptLimitReached,
         inlineLimitReached,
+        visibleTextLimitReached,
         tagsExamined,
         inlineCharactersExamined,
     };
 }
-export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, };
+export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, };

package/lib/trust-store-comparison.mjs

@@ -0,0 +1,146 @@
+import { enumValue, exactKeys, requireBoundedString, requireIsoTimestamp, requireRecord, } from "./bounded-contract-normalizers.mjs";
+export const TRUST_STORE_COMPARISON_INPUT_SCHEMA = 'whoisleuth.trust-store-comparison.input';
+export const TRUST_STORE_COMPARISON_REVIEW_SCHEMA = 'whoisleuth.trust-store-comparison.review';
+export const TRUST_STORE_COMPARISON_REVIEW_VERSION = 1;
+export const MAX_TRUST_STORE_COMPARISON_STORES = 16;
+export const MAX_TRUST_STORE_ANCHORS = 2_048;
+export const MAX_TRUST_STORE_CHAIN_CERTIFICATES = 16;
+const ROOT_KEYS = new Set(['schema', 'version', 'certificate', 'stores']);
+const CERTIFICATE_KEYS = new Set([
+    'source', 'observedAt', 'state', 'chainTruncated',
+    'fingerprintsSha256', 'runtimeAuthorisation',
+]);
+const STORE_KEYS = new Set(['name', 'version', 'source', 'reviewedAt', 'state', 'anchorsSha256']);
+const EVIDENCE_STATES = new Set(['observed', 'partial', 'unavailable']);
+const RUNTIME_AUTHORISATION = new Set(['authorised', 'unauthorised', 'unknown']);
+const SHA256_RE = /^[a-f0-9]{64}$/u;
+function digest(value, label) {
+    const normalised = requireBoundedString(value, label, 64).toLowerCase();
+    if (!SHA256_RE.test(normalised))
+        throw new TypeError(`${label} must be a SHA-256 hexadecimal digest.`);
+    return normalised;
+}
+function digests(value, label, maximum, sort = true) {
+    if (!Array.isArray(value) || value.length > maximum) {
+        throw new TypeError(`${label} must contain no more than ${maximum} SHA-256 digests.`);
+    }
+    const values = [...new Set(value.map((entry, index) => digest(entry, `${label}[${index}]`)))];
+    return Object.freeze(sort ? values.sort() : values);
+}
+export function compareTrustStoreEvidence(inputRaw, generatedAtValue = new Date().toISOString()) {
+    const input = requireRecord(inputRaw, 'Trust-store comparison input');
+    if (input.schema !== TRUST_STORE_COMPARISON_INPUT_SCHEMA || input.version !== 1) {
+        throw new TypeError(`Trust-store comparison input must use ${TRUST_STORE_COMPARISON_INPUT_SCHEMA} version 1.`);
+    }
+    exactKeys(input, ROOT_KEYS, 'Trust-store comparison input');
+    const certificateInput = requireRecord(input.certificate, 'certificate');
+    exactKeys(certificateInput, CERTIFICATE_KEYS, 'certificate');
+    const certificateState = enumValue(certificateInput.state, EVIDENCE_STATES, 'certificate.state');
+    if (typeof certificateInput.chainTruncated !== 'boolean')
+        throw new TypeError('certificate.chainTruncated must be a Boolean.');
+    const certificateFingerprints = digests(certificateInput.fingerprintsSha256, 'certificate.fingerprintsSha256', MAX_TRUST_STORE_CHAIN_CERTIFICATES, false);
+    if (certificateState === 'unavailable' && certificateFingerprints.length) {
+        throw new TypeError('An unavailable certificate observation cannot contain fingerprints.');
+    }
+    if (certificateState !== 'unavailable' && !certificateFingerprints.length) {
+        throw new TypeError('An observed or partial certificate observation requires at least one fingerprint.');
+    }
+    const certificate = Object.freeze({
+        source: requireBoundedString(certificateInput.source, 'certificate.source', 240),
+        observedAt: requireIsoTimestamp(certificateInput.observedAt, 'certificate.observedAt'),
+        state: certificateState,
+        chainTruncated: certificateInput.chainTruncated,
+        leafSha256: certificateFingerprints[0] ?? null,
+        fingerprintsSha256: certificateFingerprints,
+        runtimeAuthorisation: enumValue(certificateInput.runtimeAuthorisation, RUNTIME_AUTHORISATION, 'certificate.runtimeAuthorisation'),
+    });
+    if (!Array.isArray(input.stores) || input.stores.length < 1 || input.stores.length > MAX_TRUST_STORE_COMPARISON_STORES) {
+        throw new TypeError(`stores must contain between 1 and ${MAX_TRUST_STORE_COMPARISON_STORES} entries.`);
+    }
+    const storeKeys = new Set();
+    const stores = input.stores.map((raw, index) => {
+        const store = requireRecord(raw, `stores[${index}]`);
+        exactKeys(store, STORE_KEYS, `stores[${index}]`);
+        const name = requireBoundedString(store.name, `stores[${index}].name`, 120);
+        const version = requireBoundedString(store.version, `stores[${index}].version`, 120);
+        const key = `${name}\u0000${version}`;
+        if (storeKeys.has(key))
+            throw new TypeError(`stores[${index}] repeats a store name and version.`);
+        storeKeys.add(key);
+        const storeState = enumValue(store.state, EVIDENCE_STATES, `stores[${index}].state`);
+        const anchors = digests(store.anchorsSha256, `stores[${index}].anchorsSha256`, MAX_TRUST_STORE_ANCHORS);
+        if (storeState === 'unavailable' && anchors.length)
+            throw new TypeError(`stores[${index}] cannot contain anchors when unavailable.`);
+        if (storeState === 'observed' && !anchors.length)
+            throw new TypeError(`stores[${index}] requires at least one anchor when observed.`);
+        return Object.freeze({
+            name,
+            version,
+            source: requireBoundedString(store.source, `stores[${index}].source`, 240),
+            reviewedAt: requireIsoTimestamp(store.reviewedAt, `stores[${index}].reviewedAt`),
+            state: storeState,
+            anchors,
+        });
+    }).sort((left, right) => left.name.localeCompare(right.name) || left.version.localeCompare(right.version));
+    const chainSet = new Set(certificate.fingerprintsSha256);
+    const comparisons = stores.map((store) => {
+        const matchedAnchorSha256 = store.anchors.filter((anchor) => chainSet.has(anchor));
+        const completeInputs = certificate.state === 'observed'
+            && certificate.chainTruncated === false
+            && store.state === 'observed';
+        const state = matchedAnchorSha256.length
+            ? 'anchor_observed'
+            : completeInputs
+                ? 'not_observed'
+                : 'inconclusive';
+        return Object.freeze({
+            store: Object.freeze({
+                name: store.name,
+                version: store.version,
+                source: store.source,
+                reviewedAt: store.reviewedAt,
+                state: store.state,
+                anchorCount: store.anchors.length,
+            }),
+            state,
+            matchedAnchorSha256: Object.freeze(matchedAnchorSha256),
+            explanation: state === 'anchor_observed'
+                ? 'At least one supplied chain fingerprint exactly matched a supplied trust-anchor fingerprint.'
+                : state === 'not_observed'
+                    ? 'No supplied chain fingerprint matched this complete supplied trust-store snapshot.'
+                    : 'Certificate-chain or trust-store evidence was incomplete or unavailable, so non-observation cannot be established.',
+        });
+    });
+    const incomplete = certificate.state !== 'observed'
+        || certificate.chainTruncated
+        || stores.some((store) => store.state !== 'observed');
+    return Object.freeze({
+        schema: TRUST_STORE_COMPARISON_REVIEW_SCHEMA,
+        version: TRUST_STORE_COMPARISON_REVIEW_VERSION,
+        generatedAt: requireIsoTimestamp(generatedAtValue, 'generatedAt'),
+        state: incomplete ? 'partial' : 'complete',
+        certificate: Object.freeze({
+            source: certificate.source,
+            observedAt: certificate.observedAt,
+            state: certificate.state,
+            chainTruncated: certificate.chainTruncated,
+            leafSha256: certificate.leafSha256,
+            chainFingerprintCount: certificate.fingerprintsSha256.length,
+            runtimeAuthorisation: certificate.runtimeAuthorisation,
+        }),
+        comparisons: Object.freeze(comparisons),
+        counts: Object.freeze({
+            stores: comparisons.length,
+            anchorObserved: comparisons.filter((comparison) => comparison.state === 'anchor_observed').length,
+            notObserved: comparisons.filter((comparison) => comparison.state === 'not_observed').length,
+            inconclusive: comparisons.filter((comparison) => comparison.state === 'inconclusive').length,
+        }),
+        limitations: Object.freeze([
+            'This local comparison uses only supplied SHA-256 fingerprints and makes no TLS, certificate-authority, operating-system, or trust-store request.',
+            'An exact anchor fingerprint intersection does not build or validate a certificate path, check signatures, names, validity, constraints, revocation, policy, or application-specific trust.',
+            'A complete supplied chain can omit its root certificate. A not-observed result therefore does not mean the runtime, browser, operating system, or another client would reject the certificate.',
+            'Trust stores differ by product, platform, version, policy, update state, and local administration. The analyst-supplied source and version remain part of every comparison.',
+            'Runtime authorisation is preserved as separate context and is not overridden by this fingerprint comparison.',
+        ]),
+    });
+}

package/lib/website-technology.mjs

@@ -7,7 +7,7 @@
 import { analyzeBrowserLibraries } from "./browser-library-profile.mjs";
 import { createObservation } from "./observation.mjs";
 import { MAX_STATIC_HTML_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, } from "./static-html-analysis.mjs";
-const TECHNOLOGY_PROFILE_VERSION = 6;
+const TECHNOLOGY_PROFILE_VERSION = 8;
 const MAX_TECHNOLOGY_HTML_CHARS = MAX_STATIC_HTML_CHARS;
 const MAX_TECHNOLOGY_TAG_LENGTH = MAX_TAG_LENGTH;
 const MAX_TECHNOLOGY_FINDINGS = 24;
@@ -40,15 +40,17 @@ function normalizedResourceHosts(value) {
     }
     return hosts;
 }
-const PASSIVE_HEADER_NAMES = new Set([
+const PASSIVE_TECHNOLOGY_HEADER_NAMES = Object.freeze([
     'cf-ray',
     'x-drupal-cache',
-    'x-fastly-request-id',
+    'x-served-by',
+    'x-nf-request-id',
     'x-powered-by',
     'x-shopify-stage',
     'x-sorting-hat-podid',
     'x-vercel-id',
 ]);
+const PASSIVE_HEADER_NAMES = new Set(PASSIVE_TECHNOLOGY_HEADER_NAMES);
 function normalizedResponseHeaders(value) {
     const input = value && typeof value === 'object' && !Array.isArray(value)
         ? value
@@ -148,7 +150,10 @@ const TECHNOLOGY_SIGNATURES = [
     },
     {
         id: 'craft-cms', name: 'Craft CMS', category: 'content management',
-        evidence: [generatorEvidence(/^craft cms(?:\s|$)/i, 'Generator metadata identifies Craft CMS.')],
+        evidence: [
+            generatorEvidence(/^craft cms(?:\s|$)/i, 'Generator metadata identifies Craft CMS.'),
+            responseHeaderEvidence('x-powered-by', /(?:^|,\s*)craft cms(?:\s|$|\/)/i, 'The passive X-Powered-By response header identifies Craft CMS.'),
+        ],
     },
     {
         id: 'typo3', name: 'TYPO3 CMS', category: 'content management',
@@ -198,17 +203,23 @@ const TECHNOLOGY_SIGNATURES = [
     },
     {
         id: 'opencart', name: 'OpenCart', category: 'commerce',
-        evidence: [generatorEvidence(/^opencart(?:\s|$)/i, 'Generator metadata identifies OpenCart.')],
+        evidence: [
+            generatorEvidence(/^opencart(?:\s|$)/i, 'Generator metadata identifies OpenCart.'),
+            htmlEvidence(['index.php?route=common/home', 'image/catalog/opencart-logo.png'], 'Static markup contains OpenCart routing or default asset conventions.'),
+        ],
     },
     {
         id: 'prestashop', name: 'PrestaShop', category: 'commerce',
-        evidence: [generatorEvidence(/^prestashop(?:\s|$)/i, 'Generator metadata identifies PrestaShop.')],
+        evidence: [
+            generatorEvidence(/^prestashop(?:\s|$)/i, 'Generator metadata identifies PrestaShop.'),
+            htmlEvidence(['/modules/ps_'], 'Static resource paths use PrestaShop module conventions.'),
+        ],
     },
     {
         id: 'wix', name: 'Wix', category: 'site builder',
         requiresNonResourceEvidence: true,
         evidence: [
-            generatorEvidence(/^wix(?:\s|$)/i, 'Generator metadata identifies Wix.'),
+            generatorEvidence(/^wix(?:\.com)?(?:\s|$)/i, 'Generator metadata identifies Wix.'),
             htmlEvidence(['data-mesh-id='], 'Static markup contains a Wix-specific document attribute.', 'medium'),
             resourceEvidence(['static.parastorage.com', 'wixstatic.com'], 'A retained resource origin uses Wix delivery infrastructure.'),
         ],
@@ -316,7 +327,10 @@ const TECHNOLOGY_SIGNATURES = [
     },
     {
         id: 'netlify', name: 'Netlify', category: 'delivery platform',
-        evidence: [serverEvidence(/^netlify(?:\s|$|\/)/i, 'The selected response server header identifies Netlify.')],
+        evidence: [
+            serverEvidence(/^netlify(?:\s|$|\/)/i, 'The selected response server header identifies Netlify.'),
+            responseHeaderEvidence('x-nf-request-id', null, 'A Netlify request identifier response header was observed.', 'medium'),
+        ],
     },
     {
         id: 'vercel', name: 'Vercel', category: 'delivery platform',
@@ -327,7 +341,7 @@ const TECHNOLOGY_SIGNATURES = [
     },
     {
         id: 'fastly', name: 'Fastly', category: 'delivery platform',
-        evidence: [responseHeaderEvidence('x-fastly-request-id', null, 'A Fastly request identifier response header was observed.', 'medium')],
+        evidence: [responseHeaderEvidence('x-served-by', /(?:^|,\s*)cache-[a-z0-9-]+-[a-z]{3}(?:\s*,|$)/i, 'The passive X-Served-By response header contains a Fastly cache-node identifier.', 'medium')],
     },
     {
         id: 'nginx', name: 'nginx', category: 'web server',
@@ -434,4 +448,4 @@ function analyzeWebsiteTechnology(input = {}) {
         browserLibraryProfile,
     };
 }
-export { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, MAX_TECHNOLOGY_HTML_CHARS, MAX_TECHNOLOGY_TAGS, TECHNOLOGY_PROFILE_VERSION, TECHNOLOGY_SIGNATURE_CATALOGUE, analyzeWebsiteTechnology, };
+export { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, MAX_TECHNOLOGY_HTML_CHARS, MAX_TECHNOLOGY_TAGS, PASSIVE_TECHNOLOGY_HEADER_NAMES, TECHNOLOGY_PROFILE_VERSION, TECHNOLOGY_SIGNATURE_CATALOGUE, analyzeWebsiteTechnology, };