@slicedearth/whoisleuth-cli 1.32.0

package/frontend/src/lib/analysis/watchlist-store.js

@@ -0,0 +1,127 @@
+// Pure browser-local watchlist collection model. Watchlist history owns the
+// evidence shape and diff semantics; this module owns collection names, schema
+// migration, import merging, and exact serialized-byte accounting.
+import { MAX_WATCHLIST_DOMAINS, normalizeWatchlistEntry } from "./watchlist-history.js";
+export const WATCHLIST_SCHEMA_VERSION = 2;
+export const WATCHLIST_SCHEMA = 'whoisleuth.watchlists';
+export const MAX_WATCHLISTS = 100;
+export const MAX_WATCHLIST_INPUTS = MAX_WATCHLISTS * 4;
+export const MAX_WATCHLIST_NAME_LENGTH = 100;
+// Cases and other analyst stores share the same origin quota. A two-megabyte
+// ceiling accommodates practical Bulk results while failing before the browser
+// turns storage pressure into an opaque quota exception.
+export const MAX_WATCHLIST_STORE_BYTES = 2 * 1024 * 1024;
+const BLOCKED_NAMES = new Set(['__proto__', 'prototype', 'constructor']);
+const CONTROL_RE = /[\x00-\x1f\x7f]/;
+function plainRecord(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : null;
+}
+function isEnvelope(value) {
+    return Boolean(value && value.schema === WATCHLIST_SCHEMA && plainRecord(value.watchlists));
+}
+export function normalizeWatchlistName(value) {
+    if (typeof value !== 'string' || CONTROL_RE.test(value))
+        return '';
+    const name = value.trim();
+    if (name.length > MAX_WATCHLIST_NAME_LENGTH)
+        return '';
+    return name && !BLOCKED_NAMES.has(name.toLowerCase()) ? name : '';
+}
+function watchlistMap(raw) {
+    const value = plainRecord(raw);
+    if (!value)
+        return null;
+    return isEnvelope(value) ? plainRecord(value.watchlists) : value;
+}
+export function watchlistStoreVersion(raw) {
+    const value = plainRecord(raw);
+    if (!value || !isEnvelope(value))
+        return value ? 1 : null;
+    return typeof value.version === 'number' && Number.isFinite(value.version) && value.version > 0 ? value.version : null;
+}
+function defineEntry(target, name, entry) {
+    Object.defineProperty(target, name, { value: entry, writable: true, enumerable: true, configurable: true });
+}
+export function normalizeWatchlistStore(raw) {
+    const source = watchlistMap(raw);
+    const watchlists = {};
+    if (!source)
+        return { schema: WATCHLIST_SCHEMA, version: WATCHLIST_SCHEMA_VERSION, watchlists };
+    for (const [rawName, rawEntry] of Object.entries(source).slice(0, MAX_WATCHLIST_INPUTS)) {
+        const name = normalizeWatchlistName(rawName);
+        const entry = plainRecord(rawEntry);
+        if (!name || !entry || !Array.isArray(entry.results) || entry.results.length > MAX_WATCHLIST_DOMAINS)
+            continue;
+        defineEntry(watchlists, name, normalizeWatchlistEntry(entry));
+        if (Object.keys(watchlists).length >= MAX_WATCHLISTS)
+            break;
+    }
+    return { schema: WATCHLIST_SCHEMA, version: WATCHLIST_SCHEMA_VERSION, watchlists };
+}
+function byteLength(value) {
+    return new TextEncoder().encode(value).byteLength;
+}
+export function assertWatchlistStoreBudget(watchlists) {
+    const store = normalizeWatchlistStore(watchlists);
+    if (byteLength(JSON.stringify(store)) > MAX_WATCHLIST_STORE_BYTES) {
+        throw new Error('Watchlist storage is full. Export and remove a watchlist before saving more.');
+    }
+    return store;
+}
+export function serializeWatchlistStore(watchlists) {
+    return JSON.stringify(assertWatchlistStoreBudget(watchlists));
+}
+function validateImportShape(raw) {
+    const value = plainRecord(raw);
+    if (!value || value.schema !== WATCHLIST_SCHEMA) {
+        throw new Error('This JSON file is not a WHOISleuth watchlist export.');
+    }
+    if (!plainRecord(value.watchlists)) {
+        throw new Error('Expected a current WHOISleuth watchlist export.');
+    }
+}
+export function mergeWatchlistStores(localRaw, importedRaw) {
+    validateImportShape(importedRaw);
+    const importedVersion = watchlistStoreVersion(importedRaw);
+    if (importedVersion !== null && importedVersion > WATCHLIST_SCHEMA_VERSION) {
+        throw new Error(`This watchlist file uses newer schema ${importedVersion}. Update the app before importing it.`);
+    }
+    if (importedVersion !== WATCHLIST_SCHEMA_VERSION) {
+        throw new Error(`Expected a WHOISleuth watchlist export using schema ${WATCHLIST_SCHEMA_VERSION}.`);
+    }
+    const local = normalizeWatchlistStore(localRaw).watchlists;
+    const source = watchlistMap(importedRaw) || {};
+    const entries = Object.entries(source);
+    let added = 0;
+    let updated = 0;
+    let skipped = Math.max(0, entries.length - MAX_WATCHLIST_INPUTS);
+    for (const [rawName, rawEntry] of entries.slice(0, MAX_WATCHLIST_INPUTS)) {
+        const name = normalizeWatchlistName(rawName);
+        const entry = plainRecord(rawEntry);
+        if (!name || !entry || !Array.isArray(entry.results) || entry.results.length > MAX_WATCHLIST_DOMAINS) {
+            skipped++;
+            continue;
+        }
+        if (Object.prototype.hasOwnProperty.call(local, name))
+            updated++;
+        else if (Object.keys(local).length >= MAX_WATCHLISTS) {
+            skipped++;
+            continue;
+        }
+        else
+            added++;
+        defineEntry(local, name, normalizeWatchlistEntry(entry));
+    }
+    return { watchlists: local, added, updated, skipped };
+}
+export function buildWatchlistExport(watchlists, nowIso = new Date().toISOString()) {
+    const parsed = typeof nowIso === 'string' ? Date.parse(nowIso) : Number.NaN;
+    return {
+        schema: WATCHLIST_SCHEMA,
+        version: WATCHLIST_SCHEMA_VERSION,
+        exportedAt: Number.isFinite(parsed) ? new Date(parsed).toISOString() : new Date().toISOString(),
+        watchlists: normalizeWatchlistStore(watchlists).watchlists,
+    };
+}

package/frontend/src/lib/analysis/website-snapshot-model.js

@@ -0,0 +1,302 @@
+import { normalizeDomain } from "./case-model.js";
+export const WEBSITE_SNAPSHOT_SCHEMA = 'whoisleuth.website-profile-snapshots';
+export const WEBSITE_SNAPSHOT_SCHEMA_VERSION = 3;
+export const MAX_WEBSITE_SNAPSHOTS = 60;
+export const MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN = 12;
+export const MAX_WEBSITE_SNAPSHOT_STORE_BYTES = 512 * 1024;
+export const MAX_WEBSITE_SNAPSHOT_IMPORT_BYTES = 768 * 1024;
+const CONTROL_RE = /[\u0000-\u001f\u007f]/u;
+const DIGEST_RE = /^[a-f0-9]{16,128}$/iu;
+const SHA256_RE = /^[a-f0-9]{64}$/iu;
+const SERIAL_RE = /^[a-f0-9]{1,128}$/iu;
+function record(value) {
+    return value && typeof value === 'object' && !Array.isArray(value) ? value : null;
+}
+function text(value, maximum) {
+    return typeof value === 'string' && value.length <= maximum && !CONTROL_RE.test(value)
+        ? value.trim()
+        : '';
+}
+function timestamp(value) {
+    const candidate = text(value, 64);
+    const parsed = Date.parse(candidate);
+    return Number.isFinite(parsed) ? new Date(parsed).toISOString() : '';
+}
+function digest(value) {
+    const candidate = text(value, 128).toLowerCase();
+    return DIGEST_RE.test(candidate) ? candidate : null;
+}
+function values(raw, limit, normalize) {
+    if (!Array.isArray(raw))
+        return [];
+    const output = [];
+    const seen = new Set();
+    for (const value of raw.slice(0, limit * 4)) {
+        const normalized = normalize(value);
+        if (!normalized)
+            continue;
+        const key = JSON.stringify(normalized);
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        output.push(normalized);
+        if (output.length >= limit)
+            break;
+    }
+    return output;
+}
+function technology(value) {
+    const item = record(value);
+    const id = text(item?.id, 80);
+    const name = text(item?.name, 120);
+    if (!id || !name)
+        return null;
+    return { id, name, category: text(item?.category, 80) || 'technology', confidence: text(item?.confidence, 40) || 'unknown' };
+}
+function posture(value) {
+    const item = record(value);
+    const id = text(item?.id, 80);
+    const state = text(item?.state, 40);
+    return id && state ? { id, state } : null;
+}
+function source(value) {
+    const item = record(value);
+    const sourceName = text(item?.source, 40);
+    const state = text(item?.state, 40);
+    return sourceName && state ? { source: sourceName, state } : null;
+}
+function nullableBoolean(value) {
+    return typeof value === 'boolean' ? value : null;
+}
+function nullableText(value, maximum) {
+    return text(value, maximum) || null;
+}
+function certificate(value) {
+    const item = record(value);
+    const fingerprintSha256 = text(item?.fingerprintSha256, 64).toLowerCase();
+    if (!SHA256_RE.test(fingerprintSha256))
+        return null;
+    const spkiCandidate = text(item?.spkiFingerprintSha256, 64).toLowerCase();
+    const serialCandidate = text(item?.serialNumber, 128).toLowerCase();
+    return {
+        observationVersion: 1,
+        source: 'tls',
+        collectionDepth: 'deep',
+        fingerprintSha256,
+        spkiFingerprintSha256: SHA256_RE.test(spkiCandidate) ? spkiCandidate : null,
+        issuer: nullableText(item?.issuer, 180),
+        subject: nullableText(item?.subject, 180),
+        serialNumber: SERIAL_RE.test(serialCandidate) ? serialCandidate : null,
+        validFrom: timestamp(item?.validFrom) || null,
+        validTo: timestamp(item?.validTo) || null,
+        authorized: nullableBoolean(item?.authorized),
+        hostnameMatches: nullableBoolean(item?.hostnameMatches),
+        validity: nullableText(item?.validity, 40),
+        complete: item?.complete === true,
+        truncated: item?.truncated === true,
+    };
+}
+function identity(value) {
+    const item = record(value);
+    return {
+        normalizedHtml: digest(item?.normalizedHtml),
+        visibleText: digest(item?.visibleText),
+        domStructure: digest(item?.domStructure),
+        formStructure: digest(item?.formStructure),
+        resourceHosts: digest(item?.resourceHosts),
+        trackingIdentifiers: digest(item?.trackingIdentifiers),
+        faviconHash: digest(item?.faviconHash),
+    };
+}
+function origin(value) {
+    const candidate = text(value, 500);
+    if (!candidate)
+        return '';
+    try {
+        const parsed = new URL(candidate);
+        return ['http:', 'https:'].includes(parsed.protocol)
+            && !parsed.username
+            && !parsed.password
+            && parsed.pathname === '/'
+            && !parsed.search
+            && !parsed.hash
+            ? parsed.origin.toLowerCase()
+            : '';
+    }
+    catch {
+        return '';
+    }
+}
+function identityValues(value) {
+    const item = record(value);
+    const resourceHosts = values(item?.resourceHosts, 30, (candidate) => normalizeDomain(candidate));
+    const trackingIdentifiers = values(item?.trackingIdentifiers, 30, (candidate) => {
+        const tracker = record(candidate);
+        const type = text(tracker?.type, 40).toLowerCase();
+        const trackerValue = text(tracker?.value, 64).toUpperCase();
+        return /^[a-z-]{1,40}$/u.test(type) && /^[A-Z0-9-]{1,64}$/u.test(trackerValue)
+            ? { type, value: trackerValue }
+            : null;
+    });
+    const formActionOrigins = values(item?.formActionOrigins, 20, origin);
+    return { resourceHosts, trackingIdentifiers, formActionOrigins };
+}
+export function normalizeWebsiteProfileSnapshot(raw) {
+    const value = record(raw);
+    const domain = normalizeDomain(value?.domain);
+    const observedAt = timestamp(value?.observedAt);
+    const savedAt = timestamp(value?.savedAt);
+    const id = text(value?.id, 128);
+    if (!domain || !observedAt || !savedAt || !id)
+        return null;
+    return {
+        id,
+        domain,
+        observedAt,
+        savedAt,
+        complete: value?.complete === true,
+        truncated: value?.truncated === true,
+        technologies: values(value?.technologies, 40, technology),
+        posture: values(value?.posture, 40, posture),
+        identity: identity(value?.identity),
+        identityValues: identityValues(value?.identityValues),
+        sources: values(value?.sources, 16, source),
+        certificate: certificate(value?.certificate),
+    };
+}
+export function normalizeWebsiteSnapshotStore(raw) {
+    const value = record(raw);
+    if (value?.schema === WEBSITE_SNAPSHOT_SCHEMA
+        && Number(value?.version) > WEBSITE_SNAPSHOT_SCHEMA_VERSION) {
+        throw new Error('This website-snapshot collection uses a newer schema and cannot be read safely.');
+    }
+    const sourceValues = Array.isArray(raw) ? raw : Array.isArray(value?.snapshots) ? value.snapshots : [];
+    const snapshots = values(sourceValues, MAX_WEBSITE_SNAPSHOTS * 2, normalizeWebsiteProfileSnapshot);
+    const perDomain = new Map();
+    const retained = [];
+    for (const snapshot of snapshots.sort((left, right) => right.savedAt.localeCompare(left.savedAt))) {
+        const count = perDomain.get(snapshot.domain) ?? 0;
+        if (count >= MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN || retained.length >= MAX_WEBSITE_SNAPSHOTS)
+            continue;
+        perDomain.set(snapshot.domain, count + 1);
+        retained.push(snapshot);
+    }
+    return { schema: WEBSITE_SNAPSHOT_SCHEMA, version: WEBSITE_SNAPSHOT_SCHEMA_VERSION, snapshots: retained };
+}
+export function websiteSnapshotStoreVersion(raw) {
+    const value = record(raw);
+    return Number.isSafeInteger(value?.version) ? Number(value?.version) : WEBSITE_SNAPSHOT_SCHEMA_VERSION;
+}
+export function serializeWebsiteSnapshotStore(raw) {
+    const store = normalizeWebsiteSnapshotStore(raw);
+    const serialized = JSON.stringify(store);
+    if (new TextEncoder().encode(serialized).byteLength > MAX_WEBSITE_SNAPSHOT_STORE_BYTES) {
+        throw new Error('Website snapshots exceed the 512 KiB browser-local limit.');
+    }
+    return serialized;
+}
+export function saveWebsiteSnapshot(localRaw, candidateRaw) {
+    const candidate = normalizeWebsiteProfileSnapshot(candidateRaw);
+    if (!candidate)
+        throw new Error('The website snapshot is incomplete or invalid.');
+    const current = normalizeWebsiteSnapshotStore(localRaw).snapshots;
+    const snapshots = current.filter((item) => item.id !== candidate.id);
+    snapshots.unshift(candidate);
+    return normalizeWebsiteSnapshotStore(snapshots).snapshots;
+}
+export function deleteWebsiteSnapshot(localRaw, id) {
+    return normalizeWebsiteSnapshotStore(localRaw).snapshots.filter((item) => item.id !== id);
+}
+export function buildWebsiteSnapshotExport(raw, generatedAt = new Date().toISOString()) {
+    const store = normalizeWebsiteSnapshotStore(raw);
+    return { ...store, generatedAt: timestamp(generatedAt) || new Date().toISOString() };
+}
+export function mergeWebsiteSnapshots(localRaw, incomingRaw) {
+    const local = normalizeWebsiteSnapshotStore(localRaw).snapshots;
+    const incoming = normalizeWebsiteSnapshotStore(incomingRaw).snapshots;
+    const byId = new Map(local.map((item) => [item.id, item]));
+    let added = 0;
+    let updated = 0;
+    for (const item of incoming) {
+        if (byId.has(item.id))
+            updated += 1;
+        else
+            added += 1;
+        byId.set(item.id, item);
+    }
+    const snapshots = normalizeWebsiteSnapshotStore([...byId.values()]).snapshots;
+    return { snapshots, added, updated, skipped: Math.max(0, incoming.length - added - updated) };
+}
+function compareMap(field, before, after) {
+    const changes = [];
+    for (const key of [...new Set([...before.keys(), ...after.keys()])].sort()) {
+        const left = before.get(key) ?? null;
+        const right = after.get(key) ?? null;
+        if (left === right)
+            continue;
+        changes.push({ field: `${field}.${key}`, state: left === null ? 'added' : right === null ? 'removed' : 'changed', before: left, after: right });
+    }
+    return changes;
+}
+export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
+    const before = normalizeWebsiteProfileSnapshot(beforeRaw);
+    const after = normalizeWebsiteProfileSnapshot(afterRaw);
+    if (!before || !after || before.domain !== after.domain) {
+        return { compatible: false, changes: [{ field: 'snapshot', state: 'incomparable', before: before?.domain ?? null, after: after?.domain ?? null }] };
+    }
+    const changes = [
+        ...compareMap('technology', new Map(before.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}`])), new Map(after.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}`]))),
+        ...compareMap('posture', new Map(before.posture.map((item) => [item.id, item.state])), new Map(after.posture.map((item) => [item.id, item.state]))),
+        ...compareMap('source', new Map(before.sources.map((item) => [item.source, item.state])), new Map(after.sources.map((item) => [item.source, item.state]))),
+    ];
+    for (const key of Object.keys(before.identity)) {
+        const left = before.identity[key];
+        const right = after.identity[key];
+        if (left === right)
+            continue;
+        changes.push({ field: `identity.${key}`, state: left === null || right === null ? 'unavailable' : 'changed', before: left, after: right });
+    }
+    changes.push(...compareMap('identityValues.resourceHosts', new Map(before.identityValues.resourceHosts.map((item) => [item, item])), new Map(after.identityValues.resourceHosts.map((item) => [item, item]))), ...compareMap('identityValues.trackingIdentifiers', new Map(before.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), new Map(after.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value]))), ...compareMap('identityValues.formActionOrigins', new Map(before.identityValues.formActionOrigins.map((item) => [item, item])), new Map(after.identityValues.formActionOrigins.map((item) => [item, item]))));
+    if (before.certificate || after.certificate) {
+        if (!before.certificate || !after.certificate) {
+            changes.push({
+                field: 'certificate.observation',
+                state: 'unavailable',
+                before: before.certificate?.fingerprintSha256 ?? null,
+                after: after.certificate?.fingerprintSha256 ?? null,
+            });
+        }
+        else {
+            const certificateFields = [
+                'fingerprintSha256',
+                'spkiFingerprintSha256',
+                'issuer',
+                'subject',
+                'serialNumber',
+                'validFrom',
+                'validTo',
+                'authorized',
+                'hostnameMatches',
+                'validity',
+                'complete',
+                'truncated',
+            ];
+            for (const key of certificateFields) {
+                const left = before.certificate[key];
+                const right = after.certificate[key];
+                if (left === right)
+                    continue;
+                changes.push({
+                    field: `certificate.${key}`,
+                    state: left === null || right === null ? 'unavailable' : 'changed',
+                    before: left === null ? null : String(left),
+                    after: right === null ? null : String(right),
+                });
+            }
+        }
+    }
+    if (before.complete !== after.complete || before.truncated !== after.truncated) {
+        changes.push({ field: 'completeness', state: 'incomparable', before: `${before.complete}/${before.truncated}`, after: `${after.complete}/${after.truncated}` });
+    }
+    return { compatible: true, changes };
+}