@slicedearth/whoisleuth-cli 1.34.0 → 1.39.1

package/frontend/src/lib/analysis/lookup-presentation.js

@@ -0,0 +1,74 @@
+export const LOOKUP_PRESENTATION_STORAGE_KEY = 'whoisleuth:lookup-presentation:v1';
+export const LOOKUP_EVIDENCE_DENSITIES = Object.freeze([
+    Object.freeze({ id: 'summary', label: 'Summary', detail: 'Overview and source-state ledger, with detailed sections reduced to headings.' }),
+    Object.freeze({ id: 'standard', label: 'Standard', detail: 'Evidence sections remain available while the raw response stays collapsed until you choose to inspect it.' }),
+    Object.freeze({ id: 'full', label: 'Full', detail: 'Every settled evidence section, including the bounded raw response view.' }),
+]);
+export const LOOKUP_TASK_VIEWS = Object.freeze([
+    Object.freeze({ id: 'general', label: 'General investigation' }),
+    Object.freeze({ id: 'acquisition', label: 'Acquisition review' }),
+    Object.freeze({ id: 'brand', label: 'Brand review' }),
+    Object.freeze({ id: 'incident', label: 'Incident response' }),
+    Object.freeze({ id: 'owned', label: 'Owned-domain posture' }),
+]);
+const DENSITIES = new Set(LOOKUP_EVIDENCE_DENSITIES.map((option) => option.id));
+const TASKS = new Set(LOOKUP_TASK_VIEWS.map((option) => option.id));
+export function normalizeLookupEvidenceDensity(value) {
+    return typeof value === 'string' && DENSITIES.has(value)
+        ? value
+        : 'standard';
+}
+export function normalizeLookupTaskView(value) {
+    return typeof value === 'string' && TASKS.has(value)
+        ? value
+        : 'general';
+}
+export function readLookupPresentation(storage) {
+    try {
+        const stored = JSON.parse(storage.getItem(LOOKUP_PRESENTATION_STORAGE_KEY) || 'null');
+        if (stored?.version === 1) {
+            return {
+                density: normalizeLookupEvidenceDensity(stored.density),
+                task: normalizeLookupTaskView(stored.task),
+            };
+        }
+    }
+    catch {
+        // Invalid or unavailable browser storage falls back to the stable defaults.
+    }
+    return { density: 'standard', task: 'general' };
+}
+export function writeLookupPresentation(storage, state) {
+    try {
+        storage.setItem(LOOKUP_PRESENTATION_STORAGE_KEY, JSON.stringify({
+            version: 1,
+            density: normalizeLookupEvidenceDensity(state.density),
+            task: normalizeLookupTaskView(state.task),
+        }));
+    }
+    catch {
+        // The caller's in-memory selection remains valid when storage is unavailable.
+    }
+}
+const PRIORITY = Object.freeze({
+    general: Object.freeze(['overview', 'web-evidence', 'registry', 'external-intelligence', 'case-response', 'raw-data']),
+    acquisition: Object.freeze(['overview', 'registry', 'web-evidence', 'case-response', 'external-intelligence', 'raw-data']),
+    brand: Object.freeze(['overview', 'web-evidence', 'external-intelligence', 'registry', 'case-response', 'raw-data']),
+    incident: Object.freeze(['overview', 'external-intelligence', 'web-evidence', 'registry', 'case-response', 'raw-data']),
+    owned: Object.freeze(['overview', 'registry', 'web-evidence', 'case-response', 'external-intelligence', 'raw-data']),
+});
+const INITIAL_PANELS = {
+    general: [],
+    acquisition: ['dns', 'network-context', 'tls'],
+    brand: ['http', 'page-identity', 'credential-surface', 'structured-identity', 'technology'],
+    incident: ['http', 'tls', 'credential-surface', 'security-posture'],
+    owned: ['dns', 'network-context', 'tls', 'security-posture'],
+};
+export function lookupTaskInitiallyExpands(task, panel) {
+    return INITIAL_PANELS[normalizeLookupTaskView(task)].includes(panel);
+}
+export function prioritizeLookupSectionLinks(links, task) {
+    const order = PRIORITY[normalizeLookupTaskView(task)];
+    const rank = new Map(order.map((id, index) => [`#${id}`, index]));
+    return [...links].sort((left, right) => ((rank.get(left.href) ?? order.length) - (rank.get(right.href) ?? order.length)));
+}

package/frontend/src/lib/analysis/lookup-response.js

@@ -0,0 +1,2 @@
+// Environment-neutral Lookup response contract shared with the backend.
+export * from "../../../../lib/lookup-response-contract.mjs";

package/frontend/src/lib/analysis/lookup-source-refresh.js

@@ -0,0 +1,304 @@
+export const LOOKUP_SOURCE_REFRESH_VERSION = 1;
+export const LOOKUP_SOURCE_STALE_AFTER_DAYS = 7;
+export const LOOKUP_FRESHNESS_POLICY_VERSION = 1;
+export const LOOKUP_SOURCE_REFRESH_TIMEOUT_MS = 40_000;
+export const MAX_LOOKUP_SOURCE_REFRESH_KEYS = 512;
+export const MAX_LOOKUP_SOURCE_REFRESH_BYTES = 2 * 1024 * 1024;
+export const MAX_LOOKUP_SOURCE_REFRESH_HISTORY = 12;
+const DOMAIN_EVIDENCE_IDS = new Set([
+    'availability',
+    'client-behavior',
+    'dns',
+    'http',
+    'page-identity',
+    'page-role',
+    'security-posture',
+    'technology',
+    'tls',
+]);
+const NETWORK_EVIDENCE_IDS = new Set(['availability', 'dns', 'reverse-dns', 'network-context']);
+const WEB_EVIDENCE_IDS = new Set(['http', 'tls', 'page-identity', 'page-role', 'client-behavior', 'security-posture', 'technology']);
+const TASK_FRESHNESS_THRESHOLDS = Object.freeze({
+    general: Object.freeze({ registration: 30, network: 7, web: 3 }),
+    acquisition: Object.freeze({ registration: 7, network: 3, web: 3 }),
+    brand: Object.freeze({ registration: 30, network: 3, web: 1 }),
+    incident: Object.freeze({ registration: 14, network: 1, web: 1 }),
+    owned: Object.freeze({ registration: 30, network: 7, web: 3 }),
+});
+function freshnessDays(value, fallback) {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? Math.max(1, Math.min(365, Math.round(parsed))) : fallback;
+}
+export function buildLookupFreshnessPolicy(task, input) {
+    const defaults = TASK_FRESHNESS_THRESHOLDS[task];
+    const custom = input?.id === 'analyst-custom';
+    return {
+        version: LOOKUP_FRESHNESS_POLICY_VERSION,
+        id: custom ? 'analyst-custom' : 'task-default',
+        task,
+        thresholdsDays: custom ? {
+            registration: freshnessDays(input?.thresholdsDays?.registration, defaults.registration),
+            network: freshnessDays(input?.thresholdsDays?.network, defaults.network),
+            web: freshnessDays(input?.thresholdsDays?.web, defaults.web),
+        } : { ...defaults },
+    };
+}
+function observedAgeDays(observedAt, now) {
+    if (typeof observedAt !== 'string' || typeof now !== 'string')
+        return null;
+    const observedMs = Date.parse(observedAt);
+    const nowMs = Date.parse(now);
+    if (!Number.isFinite(observedMs) || !Number.isFinite(nowMs))
+        return null;
+    return Math.max(0, Math.floor((nowMs - observedMs) / 86_400_000));
+}
+function limited(entries, ids) {
+    return entries.some((entry) => ids.has(entry.id) && entry.manualReviewSuggested);
+}
+function availableIds(entries, ids) {
+    return entries.filter((entry) => ids.has(entry.id)).map((entry) => entry.id).slice(0, 12);
+}
+export function buildLookupSourceRefreshPlan(ledger, observedAt, now = new Date().toISOString(), options = {}) {
+    const ageDays = observedAgeDays(observedAt, now);
+    const freshnessPolicy = buildLookupFreshnessPolicy(options.task ?? 'general', options.freshnessPolicy);
+    const entries = ledger.entries.slice(0, 24);
+    const plans = [];
+    const groups = [
+        {
+            id: 'rdap',
+            label: 'Registry RDAP',
+            endpoint: '/api/rdap',
+            ids: new Set(['rdap']),
+            disclosure: 'Starts one bounded registry RDAP operation for this target.',
+            threshold: 'registration',
+        },
+        {
+            id: 'whois',
+            label: 'WHOIS',
+            endpoint: '/api/whois',
+            ids: new Set(['whois']),
+            disclosure: 'Starts one bounded referral-aware WHOIS operation for this target.',
+            threshold: 'registration',
+        },
+        {
+            id: 'availability',
+            label: 'Domain evidence',
+            endpoint: '/api/availability',
+            ids: DOMAIN_EVIDENCE_IDS,
+            disclosure: 'Repeats the bounded domain-evidence branch, including eligible DNS, HTTP, page, and TLS work for the selected depth.',
+            threshold: 'network',
+        },
+    ];
+    for (const group of groups) {
+        const evidenceIds = availableIds(entries, group.ids);
+        if (!evidenceIds.length)
+            continue;
+        const isLimited = limited(entries, group.ids);
+        const thresholds = evidenceIds.map((id) => WEB_EVIDENCE_IDS.has(id)
+            ? freshnessPolicy.thresholdsDays.web
+            : NETWORK_EVIDENCE_IDS.has(id)
+                ? freshnessPolicy.thresholdsDays.network
+                : freshnessPolicy.thresholdsDays[group.threshold]);
+        const staleAfterDays = Math.min(...thresholds);
+        const evidenceAges = evidenceIds
+            .map((id) => observedAgeDays(options.observedAtByEvidence?.[id] ?? observedAt, now))
+            .filter((value) => value !== null);
+        const groupAgeDays = evidenceAges.length ? Math.max(...evidenceAges) : ageDays;
+        const groupStale = groupAgeDays !== null && groupAgeDays >= staleAfterDays;
+        if (!isLimited && !groupStale)
+            continue;
+        plans.push({
+            id: group.id,
+            label: group.label,
+            endpoint: group.endpoint,
+            evidenceIds,
+            reason: isLimited ? 'limited' : 'stale',
+            requestDisclosure: group.disclosure,
+            supersedesObservedAt: typeof observedAt === 'string' && Number.isFinite(Date.parse(observedAt))
+                ? new Date(observedAt).toISOString()
+                : null,
+            ageDays: groupAgeDays,
+            staleAfterDays,
+        });
+    }
+    const stale = plans.some((item) => item.reason === 'stale');
+    return {
+        version: LOOKUP_SOURCE_REFRESH_VERSION,
+        stale,
+        ageDays,
+        freshnessPolicy,
+        items: plans,
+        limitations: [
+            'A source refresh is displayed separately and never merged into the original unified Lookup envelope.',
+            'Run a complete Lookup before saving, comparing, or exporting replacement evidence collected at one review time.',
+            'A retry can remain partial or unavailable and never proves that missing evidence is absent.',
+            'Freshness thresholds organise review only. They do not make an older observation false or a newer observation complete.',
+        ],
+    };
+}
+function record(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : {};
+}
+async function boundedJson(response) {
+    const declaredLength = Number(response.headers.get('content-length'));
+    if (Number.isFinite(declaredLength) && declaredLength > MAX_LOOKUP_SOURCE_REFRESH_BYTES) {
+        throw new Error('oversized');
+    }
+    if (!response.body)
+        return {};
+    const reader = response.body.getReader();
+    const chunks = [];
+    let total = 0;
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            if (!value)
+                continue;
+            total += value.byteLength;
+            if (total > MAX_LOOKUP_SOURCE_REFRESH_BYTES)
+                throw new Error('oversized');
+            chunks.push(value);
+        }
+    }
+    finally {
+        if (total > MAX_LOOKUP_SOURCE_REFRESH_BYTES)
+            await reader.cancel().catch(() => { });
+        reader.releaseLock();
+    }
+    const bytes = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    try {
+        return JSON.parse(new TextDecoder().decode(bytes));
+    }
+    catch {
+        return {};
+    }
+}
+function text(value, maximum = 180) {
+    return String(value ?? '')
+        .replace(/[\u0000-\u001f\u007f]+/gu, ' ')
+        .replace(/\s+/gu, ' ')
+        .trim()
+        .slice(0, maximum);
+}
+function summarizeSource(plan, body, observedAt, depth, supersedesObservedAt) {
+    const base = {
+        version: LOOKUP_SOURCE_REFRESH_VERSION,
+        id: plan.id,
+        observedAt,
+        reason: plan.reason,
+        evidenceIds: plan.evidenceIds.slice(0, 12),
+        supersedesObservedAt,
+    };
+    if (plan.id === 'rdap') {
+        const upstreamStatus = Number(body.upstreamStatus);
+        const parsed = record(body.parsed);
+        const complete = upstreamStatus === 200 && Object.keys(parsed).length > 0;
+        return {
+            ...base,
+            state: complete ? 'complete' : 'limited',
+            detail: complete
+                ? `Registry RDAP returned a validated ${upstreamStatus} response.`
+                : `Registry RDAP returned ${Number.isFinite(upstreamStatus) ? `HTTP ${upstreamStatus}` : 'no complete structured record'}.`,
+            observedAt,
+        };
+    }
+    if (plan.id === 'whois') {
+        const chain = Array.isArray(body.chain) ? body.chain : [];
+        const chainStatus = text(record(body.parsed).chainStatus, 40);
+        const complete = chain.length > 0 && chainStatus === 'complete';
+        return {
+            ...base,
+            state: complete ? 'complete' : 'limited',
+            detail: complete
+                ? `WHOIS returned a complete ${chain.length}-hop referral chain.`
+                : `WHOIS returned ${chain.length} hop${chain.length === 1 ? '' : 's'} with ${chainStatus || 'unknown'} chain status.`,
+            observedAt,
+        };
+    }
+    const state = text(body.state, 40) || 'unknown';
+    const complete = body.deepScanComplete === true
+        || (depth === 'fast' && state !== 'unknown');
+    const sourceStates = ['dns', 'http', 'tls']
+        .map((key) => text(record(body[key]).status, 40))
+        .filter(Boolean);
+    return {
+        ...base,
+        state: complete ? 'complete' : 'limited',
+        detail: `Domain evidence returned ${state}${sourceStates.length ? `; DNS, HTTP, and TLS states: ${sourceStates.join(', ')}` : ''}.`,
+        observedAt,
+    };
+}
+export async function requestLookupSourceRefresh(plan, query, depth, options = {}) {
+    const normalizedQuery = text(query, 4096);
+    if (!normalizedQuery)
+        return { ok: false, message: 'A valid Lookup target is required.' };
+    const timeoutMs = Number.isFinite(options.timeoutMs)
+        ? Math.max(1, Math.min(LOOKUP_SOURCE_REFRESH_TIMEOUT_MS, Math.round(Number(options.timeoutMs))))
+        : LOOKUP_SOURCE_REFRESH_TIMEOUT_MS;
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort('timeout'), timeoutMs);
+    try {
+        const url = `${plan.endpoint}?q=${encodeURIComponent(normalizedQuery)}${plan.id === 'availability' && depth === 'fast' ? '&fast=true' : ''}`;
+        const response = await (options.fetchImpl ?? fetch)(url, {
+            credentials: 'same-origin',
+            signal: controller.signal,
+        });
+        const body = record(await boundedJson(response));
+        if (!response.ok) {
+            return {
+                ok: false,
+                message: text(body.error, 240) || `Source refresh failed with HTTP ${response.status}.`,
+            };
+        }
+        if (Object.keys(body).length > MAX_LOOKUP_SOURCE_REFRESH_KEYS) {
+            return { ok: false, message: 'Source refresh returned an oversized record.' };
+        }
+        return {
+            ok: true,
+            value: summarizeSource(plan, body, options.now?.() ?? new Date().toISOString(), depth, options.supersedesObservedAt === undefined
+                ? plan.supersedesObservedAt
+                : options.supersedesObservedAt),
+        };
+    }
+    catch (cause) {
+        if (cause instanceof Error && cause.message === 'oversized') {
+            return { ok: false, message: 'Source refresh returned an oversized response.' };
+        }
+        return {
+            ok: false,
+            message: controller.signal.aborted ? 'Source refresh timed out.' : 'Source refresh could not be completed.',
+        };
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+export function mergeLookupSourceRefreshLedger(current, result) {
+    const candidates = [
+        ...(current?.version === LOOKUP_SOURCE_REFRESH_VERSION ? current.entries : []),
+        result,
+    ];
+    const byObservation = new Map();
+    for (const entry of candidates.slice(-MAX_LOOKUP_SOURCE_REFRESH_HISTORY * 4)) {
+        byObservation.set(`${entry.id}:${entry.observedAt}`, entry);
+    }
+    const ordered = [...byObservation.values()]
+        .sort((left, right) => Date.parse(left.observedAt) - Date.parse(right.observedAt));
+    const truncated = Boolean(current?.truncated)
+        || ordered.length > MAX_LOOKUP_SOURCE_REFRESH_HISTORY
+        || candidates.length > MAX_LOOKUP_SOURCE_REFRESH_HISTORY;
+    return {
+        version: LOOKUP_SOURCE_REFRESH_VERSION,
+        entries: ordered.slice(-MAX_LOOKUP_SOURCE_REFRESH_HISTORY),
+        truncated,
+    };
+}