@slicedearth/whoisleuth-cli 1.32.0

package/frontend/src/lib/analysis/shortlist-model.js

@@ -0,0 +1,209 @@
+// Pure browser-local shortlist model. The browser wrapper owns persistence
+// and downloads; this module owns schema migration, record normalization,
+// import merging, and exact serialized-byte accounting.
+import { compactWatchlistResults } from "./watchlist-history.js";
+import { normalizeRiskModelVersion } from "./scoring.js";
+export const SHORTLIST_SCHEMA = 'whoisleuth.shortlist';
+export const SHORTLIST_SCHEMA_VERSION = 2;
+export const MAX_SHORTLIST_ENTRIES = 500;
+export const MAX_SHORTLIST_INPUTS = MAX_SHORTLIST_ENTRIES * 4;
+export const MAX_SHORTLIST_STORE_BYTES = 1024 * 1024;
+export const MAX_SHORTLIST_FACTORS = 20;
+const MAX_FACTOR_LABEL_LENGTH = 200;
+const MAX_TIMESTAMP_LENGTH = 64;
+const CONTROL_RE = /[\x00-\x1f\x7f]/;
+const EPOCH = new Date(0).toISOString();
+function plainRecord(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : null;
+}
+function isEnvelope(value) {
+    return Boolean(value && value.schema === SHORTLIST_SCHEMA && Array.isArray(value.entries));
+}
+function entryList(raw) {
+    if (Array.isArray(raw))
+        return raw;
+    const value = plainRecord(raw);
+    return isEnvelope(value) ? value.entries : null;
+}
+export function shortlistStoreVersion(raw) {
+    if (Array.isArray(raw))
+        return 1;
+    const value = plainRecord(raw);
+    if (!value || !isEnvelope(value))
+        return null;
+    return typeof value.version === 'number' && Number.isFinite(value.version) && value.version > 0
+        ? value.version
+        : null;
+}
+function score(value) {
+    return typeof value === 'number' && Number.isFinite(value)
+        ? Math.max(0, Math.min(100, Math.round(value)))
+        : null;
+}
+function timestamp(value, fallback = EPOCH) {
+    if (typeof value !== 'string' || value.length > MAX_TIMESTAMP_LENGTH || CONTROL_RE.test(value))
+        return fallback;
+    const parsed = Date.parse(value);
+    return Number.isFinite(parsed) ? new Date(parsed).toISOString() : fallback;
+}
+function factors(value) {
+    if (!Array.isArray(value))
+        return [];
+    const normalized = [];
+    for (const item of value.slice(0, MAX_SHORTLIST_FACTORS * 4)) {
+        const factor = plainRecord(item);
+        if (!factor || typeof factor.label !== 'string' || CONTROL_RE.test(factor.label))
+            continue;
+        const label = factor.label.slice(0, MAX_FACTOR_LABEL_LENGTH * 4).replace(/\s+/g, ' ').trim().slice(0, MAX_FACTOR_LABEL_LENGTH).trim();
+        const points = typeof factor.points === 'number' && Number.isFinite(factor.points)
+            ? Math.max(-100, Math.min(100, Math.round(factor.points)))
+            : null;
+        if (!label || points === null)
+            continue;
+        normalized.push({ label, points });
+        if (normalized.length >= MAX_SHORTLIST_FACTORS)
+            break;
+    }
+    return normalized;
+}
+/** Normalize one shortlist record while retaining only known compact fields. */
+export function normalizeShortlistRecord(raw, options = {}) {
+    const value = plainRecord(raw);
+    if (!value)
+        return null;
+    const compact = compactWatchlistResults([value])[0];
+    if (!compact)
+        return null;
+    const riskScore = score(value.riskScore);
+    return {
+        ...compact,
+        availability: typeof compact.availability === 'string' ? compact.availability : 'unknown',
+        riskModelVersion: riskScore === null ? null : normalizeRiskModelVersion(value.riskModelVersion),
+        riskScore,
+        riskFactors: factors(value.riskFactors),
+        opportunityScore: score(value.opportunityScore),
+        savedAt: timestamp(value.savedAt, timestamp(options.fallbackTimestamp, EPOCH)),
+    };
+}
+/** Normalize an internal record collection or current stored envelope. */
+export function normalizeShortlistStore(raw) {
+    const source = entryList(raw);
+    const byDomain = new Map();
+    if (source) {
+        for (const item of source.slice(0, MAX_SHORTLIST_INPUTS)) {
+            const record = normalizeShortlistRecord(item);
+            if (!record)
+                continue;
+            byDomain.set(record.domain, record);
+            if (byDomain.size >= MAX_SHORTLIST_ENTRIES)
+                break;
+        }
+    }
+    return { schema: SHORTLIST_SCHEMA, version: SHORTLIST_SCHEMA_VERSION, entries: [...byDomain.values()] };
+}
+function byteLength(value) {
+    return new TextEncoder().encode(value).byteLength;
+}
+export function assertShortlistStoreBudget(records) {
+    const store = normalizeShortlistStore(Array.isArray(records) ? records : entryList(records));
+    if (byteLength(JSON.stringify(store)) > MAX_SHORTLIST_STORE_BYTES) {
+        throw new Error('Shortlist storage is full. Export and remove entries before saving more.');
+    }
+    return store;
+}
+export function serializeShortlistStore(records) {
+    return JSON.stringify(assertShortlistStoreBudget(records));
+}
+/** Add, refresh, or remove an explicit bounded selection in one transaction. */
+export function setShortlistSelection(localRaw, selectedRaw, selected, nowIso = new Date().toISOString()) {
+    const local = normalizeShortlistStore(localRaw).entries;
+    const byDomain = new Map(local.map((record) => [record.domain, record]));
+    const candidates = Array.isArray(selectedRaw) ? selectedRaw.slice(0, MAX_SHORTLIST_INPUTS) : [];
+    let added = 0;
+    let updated = 0;
+    let removed = 0;
+    let skipped = Math.max(0, (Array.isArray(selectedRaw) ? selectedRaw.length : 0) - MAX_SHORTLIST_INPUTS);
+    const seen = new Set();
+    for (const candidate of candidates) {
+        const record = normalizeShortlistRecord(candidate, { fallbackTimestamp: nowIso });
+        if (!record || seen.has(record.domain)) {
+            skipped += 1;
+            continue;
+        }
+        seen.add(record.domain);
+        if (selected) {
+            if (!byDomain.has(record.domain) && byDomain.size >= MAX_SHORTLIST_ENTRIES) {
+                skipped += 1;
+                continue;
+            }
+            if (byDomain.has(record.domain))
+                updated += 1;
+            else
+                added += 1;
+            byDomain.set(record.domain, record);
+        }
+        else if (byDomain.delete(record.domain)) {
+            removed += 1;
+        }
+    }
+    return {
+        entries: assertShortlistStoreBudget([...byDomain.values()]).entries,
+        added,
+        updated,
+        removed,
+        skipped,
+    };
+}
+function validateImport(raw) {
+    const value = plainRecord(raw);
+    if (!value || value.schema !== SHORTLIST_SCHEMA || !Array.isArray(value.entries)) {
+        throw new Error('Expected a current WHOISleuth shortlist export.');
+    }
+}
+export function mergeShortlistStores(localRaw, importedRaw) {
+    validateImport(importedRaw);
+    const importedVersion = shortlistStoreVersion(importedRaw);
+    if (importedVersion !== null && importedVersion > SHORTLIST_SCHEMA_VERSION) {
+        throw new Error(`This shortlist file uses newer schema ${importedVersion}. Update the app before importing it.`);
+    }
+    if (importedVersion !== SHORTLIST_SCHEMA_VERSION) {
+        throw new Error(`Expected a WHOISleuth shortlist export using schema ${SHORTLIST_SCHEMA_VERSION}.`);
+    }
+    const local = normalizeShortlistStore(localRaw).entries;
+    const byDomain = new Map(local.map((record) => [record.domain, record]));
+    const input = entryList(importedRaw) || [];
+    const imported = new Map();
+    let skipped = Math.max(0, input.length - MAX_SHORTLIST_INPUTS);
+    for (const item of input.slice(0, MAX_SHORTLIST_INPUTS)) {
+        const record = normalizeShortlistRecord(item);
+        if (!record) {
+            skipped++;
+            continue;
+        }
+        if (imported.has(record.domain))
+            skipped++;
+        imported.set(record.domain, record);
+    }
+    let added = 0;
+    let updated = 0;
+    for (const record of imported.values()) {
+        if (byDomain.has(record.domain))
+            updated++;
+        else if (byDomain.size >= MAX_SHORTLIST_ENTRIES) {
+            skipped++;
+            continue;
+        }
+        else
+            added++;
+        byDomain.set(record.domain, record);
+    }
+    return { entries: [...byDomain.values()], added, updated, skipped };
+}
+export function buildShortlistExport(records, nowIso = new Date().toISOString()) {
+    return {
+        ...normalizeShortlistStore(records),
+        exportedAt: timestamp(nowIso, new Date().toISOString()),
+    };
+}

package/frontend/src/lib/analysis/utils.js

@@ -0,0 +1,265 @@
+// Browser-safe analysis helpers shared by the Svelte tools.
+import { serializeCsvCell, serializeCsvRows } from "../../../../lib/csv.mjs";
+// Deliberately conservative (no +tags, no comments, no quoted local parts) -
+// this only gates whether a WHOIS/RDAP-sourced string is safe to drop into a
+// mailto: URI as the recipient, not a general email validator. mailto:
+// treats a comma as an additional-recipient separator (RFC 6068), so a
+// registrant/abuse-contact field containing e.g. "victim@x.com,cc@evil.com"
+// would silently add a second recipient to the outreach/abuse draft the user
+// opens - rejecting anything outside a single plain address closes that off.
+const SIMPLE_EMAIL_RE = /^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/;
+const MAX_ENTITY_DISPLAY_LENGTH = 300;
+export function isValidEmailAddress(value) {
+    return typeof value === 'string' && SIMPLE_EMAIL_RE.test(value.trim());
+}
+// RDAP and compact availability results represent a registrar as an entity,
+// while some WHOIS paths retain the historical string field. Normalize both
+// forms before they reach UI summaries or browser-local evidence; blindly
+// calling String() on an entity would persist "[object Object]" and hide real
+// registrar changes. The backend already bounds these values, but this client
+// boundary revalidates the API response before retaining it.
+function plainRecord(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : null;
+}
+export function entityDisplayName(value) {
+    let candidate = value;
+    const record = plainRecord(value);
+    if (record) {
+        candidate = record.name || record.org || record.handle;
+    }
+    if (typeof candidate !== 'string' && typeof candidate !== 'number')
+        return null;
+    const normalized = String(candidate)
+        .replace(/[\u0000-\u001f\u007f]/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .slice(0, MAX_ENTITY_DISPLAY_LENGTH);
+    return normalized || null;
+}
+// Hamming distance (0-64) between two 16-hex perceptual dHash strings (see
+// lib/perceptual-hash.mts), or null if either isn't a well-formed hash. Smaller
+// = more visually similar. Shared by the brand-profile near-match check and
+// bulk favicon clustering; mirrors the backend's hammingDistanceHex. The two
+// implementations target separate Node and browser execution boundaries.
+const HEX_HASH_RE = /^[0-9a-f]{16}$/;
+export function hammingDistanceHex(a, b) {
+    if (typeof a !== 'string' || typeof b !== 'string')
+        return null;
+    if (!HEX_HASH_RE.test(a) || !HEX_HASH_RE.test(b))
+        return null;
+    let distance = 0;
+    for (let i = 0; i < 16; i += 1) {
+        let diff = parseInt(a.charAt(i), 16) ^ parseInt(b.charAt(i), 16);
+        while (diff) {
+            distance += diff & 1;
+            diff >>= 1;
+        }
+    }
+    return distance;
+}
+// A dHash carries usable structure only when its set-bit count is away from
+// the degenerate extremes - a near-all-zero (or near-all-one) hash comes from
+// a solid/monotonic icon and collides with every other such icon (see
+// lib/perceptual-hash.mts's MIN_INFORMATIVE_BITS, mirrored here). Generation
+// already rejects these, but this is re-checked at every comparison so a
+// degenerate hash saved into a Brand Profile before that guard existed can't
+// keep producing false near-matches.
+const MIN_INFORMATIVE_HASH_BITS = 10;
+export function isInformativeFaviconHash(hex) {
+    if (typeof hex !== 'string' || !HEX_HASH_RE.test(hex))
+        return false;
+    let bits = 0;
+    for (let i = 0; i < 16; i += 1) {
+        let n = parseInt(hex.charAt(i), 16);
+        while (n) {
+            bits += n & 1;
+            n >>= 1;
+        }
+    }
+    return bits >= MIN_INFORMATIVE_HASH_BITS && bits <= 64 - MIN_INFORMATIVE_HASH_BITS;
+}
+function faviconRecord(value) {
+    const record = plainRecord(value);
+    if (!record || typeof record.domain !== 'string')
+        return null;
+    const faviconHash = typeof record.faviconHash === 'string' && record.faviconHash
+        ? record.faviconHash
+        : null;
+    const faviconPHash = typeof record.faviconPHash === 'string' && HEX_HASH_RE.test(record.faviconPHash)
+        ? record.faviconPHash
+        : null;
+    return faviconHash || faviconPHash
+        ? { domain: record.domain, faviconHash, faviconPHash }
+        : null;
+}
+export function groupBySimilarFavicon(records, maxDistance) {
+    const items = Array.isArray(records)
+        ? records.map(faviconRecord).filter((record) => record !== null)
+        : [];
+    const parent = items.map((_, i) => i);
+    const find = (x) => {
+        let root = x;
+        while ((parent[root] ?? root) !== root)
+            root = parent[root] ?? root;
+        while ((parent[x] ?? x) !== root) {
+            const next = parent[x] ?? x;
+            parent[x] = root;
+            x = next;
+        }
+        return root;
+    };
+    const union = (a, b) => { const ra = find(a); const rb = find(b); if (ra !== rb)
+        parent[ra] = rb; };
+    // Exact-hash buckets first (cheap, and the only signal for undecodable icons).
+    const firstByHash = new Map();
+    items.forEach((r, i) => {
+        if (!r.faviconHash)
+            return;
+        const existing = firstByHash.get(r.faviconHash);
+        if (existing !== undefined)
+            union(i, existing);
+        else
+            firstByHash.set(r.faviconHash, i);
+    });
+    // Perceptual near-matches among records with an *informative* phash -
+    // degenerate hashes (solid/monotonic icons) are skipped so they don't all
+    // cluster together; they can still group via an exact-hash match above.
+    const withPhash = [];
+    items.forEach((r, i) => {
+        if (isInformativeFaviconHash(r.faviconPHash))
+            withPhash.push({ i, phash: r.faviconPHash });
+    });
+    for (let a = 0; a < withPhash.length; a += 1) {
+        for (let b = a + 1; b < withPhash.length; b += 1) {
+            const left = withPhash[a];
+            const right = withPhash[b];
+            if (!left || !right)
+                continue;
+            const distance = hammingDistanceHex(left.phash, right.phash);
+            if (distance !== null && distance <= maxDistance)
+                union(left.i, right.i);
+        }
+    }
+    const groups = new Map();
+    items.forEach((r, i) => {
+        const root = find(i);
+        if (!groups.has(root))
+            groups.set(root, []);
+        groups.get(root)?.push(r.domain);
+    });
+    return [...groups.values()].filter((domains) => domains.length >= 2);
+}
+export function toCsvValue(v) {
+    return serializeCsvCell(v);
+}
+export function rowsToCsv(rows) {
+    return serializeCsvRows(rows);
+}
+function splitDelimitedLine(line, delimiter = ',') {
+    const result = [];
+    let cur = '';
+    let inQuotes = false;
+    for (let i = 0; i < line.length; i += 1) {
+        const ch = line[i];
+        if (ch === '"') {
+            if (inQuotes && line[i + 1] === '"') {
+                cur += '"';
+                i += 1;
+            }
+            else {
+                inQuotes = !inQuotes;
+            }
+        }
+        else if (ch === delimiter && !inQuotes) {
+            result.push(cur);
+            cur = '';
+        }
+        else {
+            cur += ch;
+        }
+    }
+    result.push(cur);
+    return result.map((s) => s.trim());
+}
+const DOMAIN_HEADER_NAMES = ['domain', 'domain_name', 'domain name', 'hostname', 'name'];
+function detectDelimiter(line) {
+    const candidates = [',', ';', '\t'];
+    let best = null;
+    let bestCount = 0;
+    for (const delimiter of candidates) {
+        const count = splitDelimitedLine(line, delimiter).length - 1;
+        if (count > bestCount) {
+            best = delimiter;
+            bestCount = count;
+        }
+    }
+    return best;
+}
+// Used only to distinguish a pasted delimiter-separated query list from a
+// headerless multi-column CSV. Server-side classifyQuery remains the
+// authoritative validator when the scan runs.
+function looksLikeLookupToken(value) {
+    const token = String(value || '').trim();
+    if (!token || /\s/.test(token))
+        return false;
+    if (/^AS\d+$/i.test(token) || /^\d{1,3}(?:\.\d{1,3}){3}$/.test(token) || token.includes(':'))
+        return true;
+    try {
+        const url = new URL(/^[a-z][a-z\d+.-]*:\/\//i.test(token) ? token : `https://${token}`);
+        return url.hostname.includes('.');
+    }
+    catch {
+        return false;
+    }
+}
+export function parseDomainInput(text) {
+    const normalized = String(text || '').replace(/^\uFEFF/, '');
+    const lines = normalized.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+    if (lines.length === 0)
+        return { entries: [], duplicates: 0, usedHeader: false };
+    const firstLine = lines[0] ?? '';
+    const delimiter = detectDelimiter(firstLine);
+    const firstCells = delimiter ? splitDelimitedLine(firstLine, delimiter) : [firstLine];
+    const header = firstCells.map((cell) => cell.toLowerCase());
+    const domainColIdx = header.findIndex((cell) => DOMAIN_HEADER_NAMES.includes(cell));
+    const usedHeader = domainColIdx !== -1;
+    let candidates;
+    if (usedHeader) {
+        candidates = lines.slice(1).map((line) => {
+            const rowDelimiter = delimiter || detectDelimiter(line);
+            const cells = rowDelimiter ? splitDelimitedLine(line, rowDelimiter) : [line];
+            return cells[domainColIdx] || '';
+        });
+    }
+    else {
+        const rows = lines.map((line) => {
+            const rowDelimiter = detectDelimiter(line);
+            return rowDelimiter ? splitDelimitedLine(line, rowDelimiter) : [line];
+        });
+        const flattened = rows.flat().map((cell) => cell.trim()).filter(Boolean);
+        // If every cell looks like a query, this is a pasted comma/semicolon list
+        // (possibly wrapped over several lines). Otherwise retain column zero so
+        // a headerless "domain,notes" CSV does not scan its notes as domains.
+        candidates = lines.length === 1 || (flattened.length > 0 && flattened.every(looksLikeLookupToken))
+            ? flattened
+            : rows.map((cells) => cells[0] || '');
+    }
+    const entries = [];
+    const seen = new Set();
+    let duplicates = 0;
+    for (const candidate of candidates) {
+        const value = String(candidate || '').trim();
+        if (!value)
+            continue;
+        const key = value.toLowerCase();
+        if (seen.has(key)) {
+            duplicates += 1;
+            continue;
+        }
+        seen.add(key);
+        entries.push(value);
+    }
+    return { entries, duplicates, usedHeader };
+}