@slicedearth/whoisleuth-cli 1.32.0

package/frontend/src/lib/analysis/page-baseline.js

@@ -0,0 +1,272 @@
+// Browser-local official-site baseline model for Brand Profiles. This module
+// deliberately accepts only bounded, normalized page-identity fields. Raw
+// HTML, URLs, headers, redirect inventories, and parser diagnostics never
+// cross this storage boundary.
+import { normalizeDomain } from "./case-model.js";
+import { isInformativeFaviconHash } from "./utils.js";
+export const PAGE_BASELINE_VERSION = 1;
+export const PAGE_IDENTITY_VERSION = 3;
+export const PAGE_FINGERPRINT_VERSION = 1;
+export const MAX_BASELINE_TITLE_LENGTH = 200;
+export const MAX_BASELINE_RESOURCE_HOSTS = 30;
+export const MAX_BASELINE_IDENTIFIERS = 30;
+const SHA256_RE = /^[a-f0-9]{64}$/i;
+const SIMHASH_RE = /^[a-f0-9]{16}$/i;
+const CONTROL_RE = /[\x00-\x1f\x7f]/;
+const MAX_TIMESTAMP_LENGTH = 64;
+const MAX_HTML_TOKENS = 4096;
+const MAX_TEXT_TOKENS = 8192;
+const MAX_FORMS = 50;
+const MAX_FORM_CONTROLS = 500;
+function record(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : null;
+}
+function timestamp(value) {
+    if (typeof value !== 'string' || value.length > MAX_TIMESTAMP_LENGTH || CONTROL_RE.test(value))
+        return null;
+    const parsed = new Date(value);
+    return Number.isNaN(parsed.getTime()) ? null : parsed.toISOString();
+}
+function count(value, maximum) {
+    return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 && value <= maximum
+        ? value
+        : null;
+}
+function sha256(value) {
+    return typeof value === 'string' && SHA256_RE.test(value) ? value.toLowerCase() : null;
+}
+function boundedTitle(value) {
+    if (typeof value !== 'string' || value.length > MAX_BASELINE_TITLE_LENGTH || CONTROL_RE.test(value))
+        return null;
+    return value.trim().replace(/\s+/g, ' ') || null;
+}
+function shaComponent(raw, countKey, maximum) {
+    const value = record(raw);
+    if (!value || value.algorithm !== 'sha256')
+        return null;
+    const digest = sha256(value.value);
+    const observedCount = count(value[countKey], maximum);
+    if (!digest || observedCount === null)
+        return null;
+    return {
+        algorithm: 'sha256',
+        value: digest,
+        [countKey]: observedCount,
+        truncated: value.truncated === true,
+    };
+}
+function visibleTextComponent(raw) {
+    const value = record(raw);
+    if (!value || value.algorithm !== 'simhash64-v1' || typeof value.value !== 'string' || !SIMHASH_RE.test(value.value))
+        return null;
+    const tokenCount = count(value.tokenCount, MAX_TEXT_TOKENS);
+    const featureCount = count(value.featureCount, MAX_TEXT_TOKENS);
+    if (tokenCount === null || featureCount === null)
+        return null;
+    return {
+        algorithm: 'simhash64-v1',
+        value: value.value.toLowerCase(),
+        tokenCount,
+        featureCount,
+        truncated: value.truncated === true,
+    };
+}
+function domComponent(raw) {
+    const value = record(raw);
+    if (!value || value.algorithm !== 'sha256' || value.parser !== 'static-tag-sequence-v1')
+        return null;
+    const digest = sha256(value.value);
+    const nodeCount = count(value.nodeCount, MAX_HTML_TOKENS);
+    if (!digest || nodeCount === null)
+        return null;
+    const similarity = value.similarity == null ? null : visibleTextComponent(value.similarity);
+    return {
+        algorithm: 'sha256',
+        value: digest,
+        nodeCount,
+        parser: 'static-tag-sequence-v1',
+        truncated: value.truncated === true,
+        ...(value.similarity == null ? {} : { similarity }),
+    };
+}
+function formComponent(raw) {
+    const value = record(raw);
+    if (!value || value.algorithm !== 'sha256')
+        return null;
+    const digest = sha256(value.value);
+    const formCount = count(value.formCount, MAX_FORMS);
+    const controlCount = count(value.controlCount, MAX_FORM_CONTROLS);
+    if (!digest || formCount === null || controlCount === null)
+        return null;
+    return {
+        algorithm: 'sha256',
+        value: digest,
+        formCount,
+        controlCount,
+        truncated: value.truncated === true,
+    };
+}
+function resourceHostComponent(raw) {
+    const value = record(raw);
+    if (!value || value.algorithm !== 'set-sha256' || !Array.isArray(value.values))
+        return null;
+    const values = new Set();
+    for (const candidate of value.values.slice(0, MAX_BASELINE_RESOURCE_HOSTS * 4)) {
+        const host = normalizeDomain(candidate);
+        if (host)
+            values.add(host);
+        if (values.size >= MAX_BASELINE_RESOURCE_HOSTS)
+            break;
+    }
+    const sorted = [...values].sort();
+    const digest = value.value === null ? null : sha256(value.value);
+    if ((sorted.length > 0 && !digest) || (sorted.length === 0 && value.value !== null))
+        return null;
+    return {
+        algorithm: 'set-sha256',
+        value: digest,
+        values: sorted,
+        truncated: value.truncated === true || value.values.length > MAX_BASELINE_RESOURCE_HOSTS,
+    };
+}
+function identifierComponent(raw) {
+    const value = record(raw);
+    if (!value || value.algorithm !== 'set-sha256' || !Array.isArray(value.values))
+        return null;
+    const values = [];
+    const seen = new Set();
+    for (const candidate of value.values.slice(0, MAX_BASELINE_IDENTIFIERS * 4)) {
+        const item = record(candidate);
+        if (!item || typeof item.type !== 'string' || typeof item.value !== 'string')
+            continue;
+        if (!/^[a-z-]{1,40}$/.test(item.type) || !/^[A-Z0-9-]{1,64}$/.test(item.value))
+            continue;
+        const key = `${item.type}:${item.value}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        values.push({ type: item.type, value: item.value });
+        if (values.length >= MAX_BASELINE_IDENTIFIERS)
+            break;
+    }
+    values.sort((left, right) => left.type.localeCompare(right.type) || left.value.localeCompare(right.value));
+    const digest = value.value === null ? null : sha256(value.value);
+    if ((values.length > 0 && !digest) || (values.length === 0 && value.value !== null))
+        return null;
+    return {
+        algorithm: 'set-sha256',
+        value: digest,
+        values,
+        truncated: value.truncated === true || value.values.length > MAX_BASELINE_IDENTIFIERS,
+    };
+}
+function faviconSha(value) {
+    return sha256(value);
+}
+function faviconPHash(value) {
+    return typeof value === 'string' && isInformativeFaviconHash(value) ? value.toLowerCase() : null;
+}
+/**
+ * Strictly normalizes a persisted baseline and drops every unknown field.
+ * Unsupported/future schema versions and incomplete core fingerprint shapes
+ * fail closed so old code never misinterprets newer evidence.
+ * @param {unknown} raw
+ */
+export function normalizePageBaseline(raw) {
+    const value = record(raw);
+    if (!value || value.baselineVersion !== PAGE_BASELINE_VERSION)
+        return null;
+    const domain = normalizeDomain(value.domain);
+    const lookupDomain = normalizeDomain(value.lookupDomain);
+    const observedAt = timestamp(value.observedAt);
+    const pageIdentityVersion = count(value.pageIdentityVersion, PAGE_IDENTITY_VERSION);
+    const fingerprintVersion = count(value.fingerprintVersion, PAGE_FINGERPRINT_VERSION);
+    const normalizedHtml = shaComponent(value.normalizedHtml, 'tokenCount', MAX_HTML_TOKENS);
+    const domStructure = domComponent(value.domStructure);
+    if (!domain || !lookupDomain || !observedAt || pageIdentityVersion !== PAGE_IDENTITY_VERSION
+        || fingerprintVersion !== PAGE_FINGERPRINT_VERSION || !normalizedHtml || !domStructure)
+        return null;
+    const visibleText = value.visibleText == null ? null : visibleTextComponent(value.visibleText);
+    const formStructure = value.formStructure == null ? null : formComponent(value.formStructure);
+    const resourceHosts = resourceHostComponent(value.resourceHosts);
+    const trackingIdentifiers = identifierComponent(value.trackingIdentifiers);
+    if (!resourceHosts || !trackingIdentifiers)
+        return null;
+    const rawDomStructure = record(value.domStructure);
+    const optionalComponentInvalid = (value.visibleText != null && !visibleText)
+        || (value.formStructure != null && !formStructure)
+        || (rawDomStructure?.similarity != null && !domStructure.similarity);
+    const truncated = value.truncated === true || optionalComponentInvalid || normalizedHtml.truncated || domStructure.truncated
+        || visibleText?.truncated === true || formStructure?.truncated === true
+        || resourceHosts.truncated || trackingIdentifiers.truncated;
+    return {
+        baselineVersion: PAGE_BASELINE_VERSION,
+        domain,
+        lookupDomain,
+        observedAt,
+        pageIdentityVersion,
+        fingerprintVersion,
+        pageTitle: boundedTitle(value.pageTitle),
+        canonicalHost: value.canonicalHost == null ? null : normalizeDomain(value.canonicalHost) || null,
+        faviconHash: faviconSha(value.faviconHash),
+        faviconPHash: faviconPHash(value.faviconPHash),
+        normalizedHtml,
+        visibleText,
+        domStructure,
+        formStructure,
+        resourceHosts,
+        trackingIdentifiers,
+        complete: value.complete === true && !truncated,
+        truncated,
+    };
+}
+function canonicalHost(value) {
+    const item = record(value);
+    if (!item || typeof item.url !== 'string' || item.url.length > 2048 || CONTROL_RE.test(item.url))
+        return null;
+    try {
+        const url = new URL(item.url);
+        return ['http:', 'https:'].includes(url.protocol) && !url.username && !url.password
+            ? normalizeDomain(url.hostname) || null
+            : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Builds a baseline from one deep availability response. Returns null when
+ * the response does not contain the current core page-fingerprint contract.
+ * @param {unknown} rawDomain
+ * @param {unknown} rawAvailability
+ */
+export function createPageBaseline(rawDomain, rawAvailability) {
+    const domain = normalizeDomain(rawDomain);
+    const availability = record(rawAvailability);
+    const identity = record(availability?.pageIdentity);
+    const fingerprints = record(identity?.fingerprints);
+    if (!domain || !availability || !identity || !fingerprints || identity.source !== 'html')
+        return null;
+    return normalizePageBaseline({
+        baselineVersion: PAGE_BASELINE_VERSION,
+        domain,
+        lookupDomain: normalizeDomain(availability.domain) || domain,
+        observedAt: identity.observedAt,
+        pageIdentityVersion: identity.identityVersion,
+        fingerprintVersion: fingerprints.fingerprintVersion,
+        pageTitle: availability.pageTitle,
+        canonicalHost: canonicalHost(identity.canonical),
+        faviconHash: availability.faviconHash,
+        faviconPHash: availability.faviconPHash,
+        normalizedHtml: fingerprints.normalizedHtml,
+        visibleText: fingerprints.visibleText,
+        domStructure: fingerprints.domStructure,
+        formStructure: fingerprints.formStructure,
+        resourceHosts: fingerprints.resourceHosts,
+        trackingIdentifiers: fingerprints.identifiers,
+        complete: identity.complete === true && fingerprints.complete === true,
+        truncated: identity.truncated === true || fingerprints.truncated === true,
+    });
+}

package/frontend/src/lib/analysis/page-similarity.js

@@ -0,0 +1,179 @@
+// Explainable, browser-local comparison of two bounded page baselines. This
+// module deliberately reports each component independently: it does not
+// produce an aggregate similarity score, risk factor, or maliciousness
+// verdict. Inputs pass through the strict baseline normalizer before use.
+import { normalizePageBaseline } from "./page-baseline.js";
+import { hammingDistanceHex } from "./utils.js";
+export const PAGE_COMPARISON_VERSION = 3;
+function component(id, label, method, status, outcome, detail, partial) {
+    return { id, label, method, status, outcome, detail, partial, sharedValues: [] };
+}
+/**
+ * Compares two mandatory digest components. A match or difference applies to
+ * the bounded captured material only; the partial flag keeps a capped capture
+ * from being presented as a conclusion about the complete page.
+ * @param {string} id
+ * @param {string} label
+ * @param {string} detailLabel
+ */
+function digestComponent(id, label, detailLabel, reference, observed) {
+    const same = reference.value === observed.value;
+    const partial = reference.truncated || observed.truncated;
+    return component(id, label, 'Exact SHA-256 equality', same ? 'same' : 'different', `${same ? 'Same' : 'Different'} captured digest${partial ? ' · partial evidence' : ''}`, `${detailLabel} hashes ${same ? 'are equal' : 'differ'} for the bounded material captured on each page.`, partial);
+}
+function visibleTextComponent(reference, observed) {
+    if (!reference && !observed) {
+        return component('visible_text', 'Visible text', '64-bit SimHash distance', 'not_observed', 'Not observed', 'Neither capture produced a visible-text fingerprint.', false);
+    }
+    if (!reference || !observed) {
+        return component('visible_text', 'Visible text', '64-bit SimHash distance', 'unavailable', 'Not comparable', 'Only one capture produced a visible-text fingerprint.', Boolean(reference?.truncated || observed?.truncated));
+    }
+    const distance = hammingDistanceHex(reference.value, observed.value);
+    if (distance === null) {
+        return component('visible_text', 'Visible text', '64-bit SimHash distance', 'unavailable', 'Not comparable', 'The visible-text fingerprints use an unsupported or malformed representation.', true);
+    }
+    const agreementPercent = Math.round(((64 - distance) / 64) * 100);
+    return {
+        ...component('visible_text', 'Visible text', '64-bit SimHash distance', distance === 0 ? 'same' : 'different', `${agreementPercent}% bit agreement${reference.truncated || observed.truncated ? ' · partial evidence' : ''}`, `The fingerprints differ by ${distance} of 64 bits. Bit agreement is a fuzzy comparison aid, not a percentage of copied text.`, reference.truncated || observed.truncated),
+        hammingDistance: distance,
+        agreementPercent,
+    };
+}
+function domStructureComponent(reference, observed) {
+    const partial = reference.truncated || observed.truncated
+        || reference.similarity?.truncated === true || observed.similarity?.truncated === true;
+    if (reference.value === observed.value) {
+        return component('dom_structure', 'DOM structure', 'Exact SHA-256 equality', 'same', `Same captured structure${partial ? ' · partial evidence' : ''}`, 'The exact static tag-sequence digests are equal for the bounded material captured on each page.', partial);
+    }
+    if (!reference.similarity || !observed.similarity) {
+        return component('dom_structure', 'DOM structure', 'Exact SHA-256 equality', 'different', `Different captured structure${partial ? ' · partial evidence' : ''}`, 'The exact static tag-sequence digests differ. At least one older capture lacks the optional standards-token similarity fingerprint.', partial);
+    }
+    const distance = hammingDistanceHex(reference.similarity.value, observed.similarity.value);
+    if (distance === null) {
+        return component('dom_structure', 'DOM structure', 'Exact SHA-256, then 64-bit SimHash distance', 'unavailable', 'Not comparable', 'The DOM-structure similarity fingerprints use an unsupported or malformed representation.', true);
+    }
+    const agreementPercent = Math.round(((64 - distance) / 64) * 100);
+    const near = distance <= 8;
+    return {
+        ...component('dom_structure', 'DOM structure', 'Exact SHA-256, then 64-bit SimHash distance', near ? 'overlap' : 'different', `${near ? 'Similar' : 'Different'} captured structure · ${agreementPercent}% bit agreement${partial ? ' · partial evidence' : ''}`, `The standards-token fingerprints differ by ${distance} of 64 bits. The bounded eight-bit proximity threshold is a review lead and does not establish copying, ownership, intent, safety, or maliciousness.`, partial),
+        hammingDistance: distance,
+        agreementPercent,
+    };
+}
+function formComponent(reference, observed) {
+    if (!reference && !observed) {
+        return component('form_structure', 'Form structure', 'Exact SHA-256 equality', 'not_observed', 'No forms fingerprinted', 'Neither capture contained a form structure to fingerprint.', false);
+    }
+    if (!reference || !observed) {
+        return component('form_structure', 'Form structure', 'Exact SHA-256 equality', 'unavailable', 'Not comparable', 'Only one capture produced a form-structure fingerprint.', Boolean(reference?.truncated || observed?.truncated));
+    }
+    return digestComponent('form_structure', 'Form structure', 'Form-structure', reference, observed);
+}
+function faviconComponent(reference, observed) {
+    if (!reference.faviconHash && !observed.faviconHash && !reference.faviconPHash && !observed.faviconPHash) {
+        return component('favicon', 'Favicon', 'Exact SHA-256 or 64-bit dHash distance', 'not_observed', 'Not observed', 'Neither capture produced a favicon fingerprint.', false);
+    }
+    if (reference.faviconHash && observed.faviconHash && reference.faviconHash === observed.faviconHash) {
+        return component('favicon', 'Favicon', 'Exact SHA-256 equality', 'same', 'Same favicon digest', 'The bounded favicon byte digests are equal.', false);
+    }
+    if (!reference.faviconPHash || !observed.faviconPHash) {
+        return component('favicon', 'Favicon', 'Exact SHA-256 or 64-bit dHash distance', 'unavailable', 'Not comparable', 'The exact favicon digests differ or are incomplete, and both captures did not retain an informative perceptual fingerprint.', false);
+    }
+    const distance = hammingDistanceHex(reference.faviconPHash, observed.faviconPHash);
+    if (distance === null) {
+        return component('favicon', 'Favicon', '64-bit dHash distance', 'unavailable', 'Not comparable', 'The favicon perceptual fingerprints use an unsupported or malformed representation.', true);
+    }
+    const near = distance <= 6;
+    return {
+        ...component('favicon', 'Favicon', '64-bit dHash distance', distance === 0 ? 'same' : near ? 'overlap' : 'different', distance === 0 ? 'Same perceptual fingerprint' : near ? 'Perceptually similar' : 'Perceptually different', `The favicon perceptual fingerprints differ by ${distance} of 64 bits. Similarity is a review lead and does not establish copying, ownership, intent, safety, or maliciousness.`, false),
+        hammingDistance: distance,
+        agreementPercent: Math.round(((64 - distance) / 64) * 100),
+    };
+}
+function intersect(left, right) {
+    const rightSet = new Set(right);
+    return left.filter((value) => rightSet.has(value));
+}
+/**
+ * @param {string} id
+ * @param {string} label
+ * @param {string} noun
+ */
+function setComponent(id, label, noun, reference, observed, key) {
+    const referenceValues = reference.values.map(key);
+    const observedValues = observed.values.map(key);
+    const sharedValues = intersect(referenceValues, observedValues);
+    const partial = reference.truncated || observed.truncated;
+    const quantity = (count) => `${count} ${noun}${count === 1 ? '' : 's'}`;
+    if (!referenceValues.length && !observedValues.length) {
+        return {
+            ...component(id, label, 'Bounded set overlap', 'not_observed', `No ${noun}s observed`, `Neither capture retained any ${noun}s.`, partial),
+            referenceCount: 0,
+            observedCount: 0,
+            sharedCount: 0,
+            sharedValues: [],
+        };
+    }
+    const same = referenceValues.length === observedValues.length && sharedValues.length === referenceValues.length;
+    const status = same ? 'same' : sharedValues.length ? 'overlap' : 'different';
+    const outcome = same
+        ? `Same ${noun} set`
+        : sharedValues.length
+            ? `${quantity(sharedValues.length)} shared`
+            : `No shared ${noun}s`;
+    return {
+        ...component(id, label, 'Bounded set overlap', status, `${outcome}${partial ? ' · partial evidence' : ''}`, `${quantity(sharedValues.length)} shared across ${quantity(referenceValues.length)} in the baseline and ${quantity(observedValues.length)} in this capture.`, partial),
+        referenceCount: referenceValues.length,
+        observedCount: observedValues.length,
+        sharedCount: sharedValues.length,
+        sharedValues,
+    };
+}
+/**
+ * Compares an official-site baseline with a current bounded page capture.
+ * Returns null unless both inputs satisfy the current strict baseline schema.
+ * @param {unknown} rawReference
+ * @param {unknown} rawObserved
+ */
+export function comparePageBaselines(rawReference, rawObserved) {
+    const reference = normalizePageBaseline(rawReference);
+    const observed = normalizePageBaseline(rawObserved);
+    if (!reference || !observed)
+        return null;
+    const components = [
+        digestComponent('normalized_html', 'Normalized HTML', 'Normalized-HTML', reference.normalizedHtml, observed.normalizedHtml),
+        visibleTextComponent(reference.visibleText, observed.visibleText),
+        domStructureComponent(reference.domStructure, observed.domStructure),
+        formComponent(reference.formStructure, observed.formStructure),
+        faviconComponent(reference, observed),
+        setComponent('resource_hosts', 'External resource hosts', 'host', reference.resourceHosts, observed.resourceHosts, (value) => value),
+        setComponent('tracking_identifiers', 'Tracking identifiers', 'identifier', reference.trackingIdentifiers, observed.trackingIdentifiers, (value) => `${value.type}:${value.value}`),
+    ];
+    const counts = { same: 0, overlap: 0, different: 0, notObserved: 0, unavailable: 0 };
+    for (const item of components) {
+        if (item.status === 'not_observed')
+            counts.notObserved += 1;
+        else if (item.status === 'unavailable')
+            counts.unavailable += 1;
+        else
+            counts[item.status] += 1;
+    }
+    return {
+        comparisonVersion: PAGE_COMPARISON_VERSION,
+        reference: {
+            domain: reference.domain,
+            observedAt: reference.observedAt,
+            complete: reference.complete,
+            truncated: reference.truncated,
+        },
+        observed: {
+            domain: observed.domain,
+            observedAt: observed.observedAt,
+            complete: observed.complete,
+            truncated: observed.truncated,
+        },
+        partial: reference.truncated || observed.truncated || components.some((item) => item.partial),
+        components,
+        counts,
+    };
+}