@slicedearth/whoisleuth-cli 1.36.1 → 1.43.0

package/frontend/src/lib/analysis/external-findings-import.js

@@ -0,0 +1,320 @@
+import { normalizeDomain, openOrCreateCase, updateCase, } from "./case-model.js";
+export const EXTERNAL_FINDINGS_SCHEMA = 'whoisleuth.external-findings';
+export const EXTERNAL_FINDINGS_VERSION = 3;
+export const MAX_EXTERNAL_FINDINGS_IMPORT_BYTES = 384 * 1024;
+export const MAX_EXTERNAL_FINDINGS = 100;
+export const MAX_EXTERNAL_FINDINGS_PER_DOMAIN = 20;
+export const MAX_EXTERNAL_FINDING_DOMAINS = 25;
+export const EXTERNAL_FINDING_CATEGORIES = [
+    'certificate',
+    'dns',
+    'http',
+    'malware',
+    'other',
+    'page',
+    'registration',
+    'reputation',
+];
+const CATEGORIES = new Set(EXTERNAL_FINDING_CATEGORIES);
+const COMPLETENESS = new Set(['complete', 'inconclusive', 'partial', 'unknown']);
+const CONTROL_RE = /[\u0000-\u001f\u007f]/u;
+const ROOT_KEYS = new Set(['schema', 'schemaVersion', 'source', 'findings']);
+const SOURCE_KEYS = new Set(['name', 'reference', 'collectedAt']);
+const FINDING_KEYS = new Set([
+    'domain',
+    'category',
+    'evidenceClass',
+    'summary',
+    'observedAt',
+    'completeness',
+    'limitations',
+    'reference',
+    'structuredObservation',
+]);
+const STRUCTURED_OBSERVATION_KEYS = new Set([
+    'sourceSchema',
+    'sourceVersion',
+    'field',
+    'value',
+    'issuer',
+    'notAfter',
+]);
+const STRUCTURED_OBSERVATION_SCHEMAS = new Set([
+    'whoisleuth.domain-observation-rows',
+    'whoisleuth.dns-observation-rows',
+    'whoisleuth.certificate-observation-rows',
+]);
+function record(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : null;
+}
+function hasOnlyKeys(value, allowed) {
+    return Object.keys(value).every((key) => allowed.has(key));
+}
+function requiredText(value, maximum, label) {
+    if (typeof value !== 'string'
+        || !value.trim()
+        || value.length > maximum
+        || CONTROL_RE.test(value)) {
+        throw new Error(`${label} must be non-empty, bounded text without control characters.`);
+    }
+    return value.trim();
+}
+function optionalText(value, maximum, label) {
+    if (value === undefined || value === null || value === '')
+        return null;
+    return requiredText(value, maximum, label);
+}
+function iso(value, label, optional = false) {
+    if (optional && (value === undefined || value === null || value === ''))
+        return null;
+    const text = requiredText(value, 64, label);
+    const parsed = Date.parse(text);
+    if (!Number.isFinite(parsed))
+        throw new Error(`${label} must be a valid date and time.`);
+    return new Date(parsed).toISOString();
+}
+function limitations(value, index) {
+    if (value === undefined)
+        return [];
+    if (!Array.isArray(value) || value.length > 8) {
+        throw new Error(`Finding ${index + 1} limitations must be an array with no more than 8 entries.`);
+    }
+    const unique = new Set();
+    for (const item of value)
+        unique.add(requiredText(item, 240, `Finding ${index + 1} limitation`));
+    return [...unique];
+}
+function structuredObservation(value, index) {
+    if (value === undefined || value === null)
+        return null;
+    const item = record(value);
+    if (!item || !hasOnlyKeys(item, STRUCTURED_OBSERVATION_KEYS)) {
+        throw new Error(`Finding ${index + 1} structured observation has an invalid shape or additional fields.`);
+    }
+    if (typeof item.sourceSchema !== 'string' || !STRUCTURED_OBSERVATION_SCHEMAS.has(item.sourceSchema)) {
+        throw new Error(`Finding ${index + 1} structured observation source schema is unsupported.`);
+    }
+    if (item.sourceVersion !== 1) {
+        throw new Error(`Finding ${index + 1} structured observation source version is unsupported.`);
+    }
+    const field = requiredText(item.field, 40, `Finding ${index + 1} structured observation field`);
+    const observationValue = requiredText(item.value, 500, `Finding ${index + 1} structured observation value`);
+    const issuer = optionalText(item.issuer, 160, `Finding ${index + 1} certificate issuer`);
+    const notAfter = iso(item.notAfter, `Finding ${index + 1} certificate expiry`, true);
+    const certificateSchema = item.sourceSchema === 'whoisleuth.certificate-observation-rows';
+    if (!certificateSchema && (issuer !== null || notAfter !== null)) {
+        throw new Error(`Finding ${index + 1} non-certificate observation cannot declare certificate metadata.`);
+    }
+    return {
+        sourceSchema: item.sourceSchema,
+        sourceVersion: 1,
+        field,
+        value: observationValue,
+        issuer,
+        notAfter,
+    };
+}
+function findingKey(finding, sourceName) {
+    return [
+        finding.domain,
+        finding.category,
+        finding.evidenceClass,
+        finding.summary,
+        finding.observedAt,
+        finding.completeness,
+        sourceName,
+    ].join('\u0000');
+}
+export function parseExternalFindingsDocument(value) {
+    const root = record(value);
+    if (!root || !hasOnlyKeys(root, ROOT_KEYS)) {
+        throw new Error('External findings must use the documented object shape without additional top-level fields.');
+    }
+    if (root.schema !== EXTERNAL_FINDINGS_SCHEMA
+        || ![1, 2, EXTERNAL_FINDINGS_VERSION].includes(Number(root.schemaVersion))) {
+        throw new Error(`External findings must use ${EXTERNAL_FINDINGS_SCHEMA} schema version 1, 2, or ${EXTERNAL_FINDINGS_VERSION}.`);
+    }
+    const sourceValue = record(root.source);
+    if (!sourceValue || !hasOnlyKeys(sourceValue, SOURCE_KEYS)) {
+        throw new Error('External findings require a bounded source object without additional fields.');
+    }
+    const source = {
+        name: requiredText(sourceValue.name, 80, 'Source name'),
+        reference: optionalText(sourceValue.reference, 500, 'Source reference'),
+        collectedAt: iso(sourceValue.collectedAt, 'Source collection time', true),
+    };
+    if (!Array.isArray(root.findings) || !root.findings.length || root.findings.length > MAX_EXTERNAL_FINDINGS) {
+        throw new Error(`External findings must contain between 1 and ${MAX_EXTERNAL_FINDINGS} findings.`);
+    }
+    const normalized = [];
+    const seen = new Set();
+    const domainCounts = new Map();
+    for (const [index, raw] of root.findings.entries()) {
+        const item = record(raw);
+        if (!item || !hasOnlyKeys(item, FINDING_KEYS)) {
+            throw new Error(`Finding ${index + 1} has an invalid shape or additional fields.`);
+        }
+        const domainInput = requiredText(item.domain, 253, `Finding ${index + 1} domain`);
+        const domain = normalizeDomain(domainInput);
+        if (!domain)
+            throw new Error(`Finding ${index + 1} domain is invalid.`);
+        if (typeof item.category !== 'string' || !CATEGORIES.has(item.category)) {
+            throw new Error(`Finding ${index + 1} category is unsupported.`);
+        }
+        if (typeof item.completeness !== 'string' || !COMPLETENESS.has(item.completeness)) {
+            throw new Error(`Finding ${index + 1} completeness is unsupported.`);
+        }
+        const evidenceClass = item.evidenceClass === undefined && root.schemaVersion === 1
+            ? 'provider_report'
+            : item.evidenceClass;
+        if (evidenceClass !== 'deployment_observation' && evidenceClass !== 'provider_report') {
+            throw new Error(`Finding ${index + 1} evidence class is unsupported.`);
+        }
+        const finding = {
+            domain,
+            category: item.category,
+            evidenceClass,
+            summary: requiredText(item.summary, 900, `Finding ${index + 1} summary`),
+            observedAt: iso(item.observedAt, `Finding ${index + 1} observation time`),
+            completeness: item.completeness,
+            limitations: limitations(item.limitations, index),
+            reference: optionalText(item.reference, 500, `Finding ${index + 1} reference`),
+            structuredObservation: structuredObservation(item.structuredObservation, index),
+        };
+        const key = findingKey(finding, source.name);
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        const count = (domainCounts.get(domain) ?? 0) + 1;
+        if (count > MAX_EXTERNAL_FINDINGS_PER_DOMAIN) {
+            throw new Error(`External findings exceed the ${MAX_EXTERNAL_FINDINGS_PER_DOMAIN}-finding per-domain limit.`);
+        }
+        domainCounts.set(domain, count);
+        if (domainCounts.size > MAX_EXTERNAL_FINDING_DOMAINS) {
+            throw new Error(`External findings exceed the ${MAX_EXTERNAL_FINDING_DOMAINS}-domain limit.`);
+        }
+        normalized.push(finding);
+    }
+    if (!normalized.length)
+        throw new Error('External findings did not contain a unique valid finding.');
+    return {
+        schema: EXTERNAL_FINDINGS_SCHEMA,
+        schemaVersion: EXTERNAL_FINDINGS_VERSION,
+        source,
+        findings: normalized,
+    };
+}
+function pinValue(finding) {
+    return finding.reference
+        ? `${finding.summary} Reference: ${finding.reference}`
+        : finding.summary;
+}
+function importedSourceLabel(finding, sourceName) {
+    return finding.evidenceClass === 'deployment_observation'
+        ? `Deployment observation: ${sourceName}`
+        : `Provider report: ${sourceName}`;
+}
+function existingPinKey(recordValue, finding, sourceName) {
+    const expectedLabel = `External ${finding.category} finding`;
+    const expectedValue = finding.structuredObservation?.value ?? pinValue(finding);
+    const expectedSource = importedSourceLabel(finding, sourceName);
+    const observation = finding.structuredObservation;
+    return recordValue.evidencePins.some((pin) => (pin.label === expectedLabel
+        && pin.value === expectedValue
+        && pin.source === expectedSource
+        && pin.observedAt === finding.observedAt
+        && pin.completeness === finding.completeness
+        && (!observation || (pin.field === observation.field
+            && pin.sourceSchema?.collection === 'external_observations'
+            && pin.sourceSchema.schema === observation.sourceSchema
+            && pin.sourceSchema.version === observation.sourceVersion)))) ? findingKey(finding, sourceName) : '';
+}
+function structuredPinFields(finding) {
+    const observation = finding.structuredObservation;
+    if (!observation)
+        return {};
+    return {
+        field: observation.field,
+        category: finding.category,
+        sourceSchema: {
+            collection: 'external_observations',
+            schema: observation.sourceSchema,
+            version: observation.sourceVersion,
+        },
+    };
+}
+export function mergeExternalFindingsIntoCases(current, document, now = new Date().toISOString()) {
+    let cases = [...current];
+    const createdDomains = new Set();
+    const updatedDomains = new Set();
+    let findingsAdded = 0;
+    let duplicatesSkipped = 0;
+    for (const finding of document.findings) {
+        const existing = cases.find((candidate) => candidate.domain === finding.domain) ?? null;
+        const opened = openOrCreateCase(cases, { domain: finding.domain, source: 'import' }, now);
+        cases = opened.cases;
+        const target = cases.find((candidate) => candidate.id === opened.record.id) ?? opened.record;
+        if (existingPinKey(target, finding, document.source.name)) {
+            duplicatesSkipped += 1;
+            continue;
+        }
+        const sourceLimitations = [
+            finding.evidenceClass === 'deployment_observation'
+                ? `Imported as an observation made by ${document.source.name}; this browser session did not collect or independently verify it.`
+                : `Reported by ${document.source.name}; WHOISleuth did not collect or independently verify this provider finding.`,
+            ...(document.source.reference ? [`Source reference: ${document.source.reference}`] : []),
+            ...finding.limitations,
+        ];
+        const existingPinIds = new Set(target.evidencePins.map((pin) => pin.id));
+        const updated = updateCase(cases, target.id, {
+            evidencePin: {
+                ...structuredPinFields(finding),
+                label: `External ${finding.category} finding`,
+                value: finding.structuredObservation?.value ?? pinValue(finding),
+                source: importedSourceLabel(finding, document.source.name),
+                observedAt: finding.observedAt,
+                completeness: finding.completeness,
+                limitations: sourceLimitations,
+            },
+        }, now);
+        cases = updated.cases;
+        const addedPin = updated.record.evidencePins.find((pin) => !existingPinIds.has(pin.id)) ?? null;
+        if (addedPin) {
+            cases = updateCase(cases, updated.record.id, {
+                sighting: {
+                    state: finding.evidenceClass === 'deployment_observation'
+                        ? 'observed_by_deployment'
+                        : 'reported_by_provider',
+                    category: finding.category === 'dns'
+                        ? 'delegation'
+                        : finding.category === 'certificate'
+                            ? 'certificate'
+                            : finding.category === 'registration'
+                                ? 'registration'
+                                : finding.category === 'http' || finding.category === 'page'
+                                    ? 'website'
+                                    : 'other',
+                    source: document.source.name,
+                    observedAt: finding.observedAt,
+                    completeness: finding.completeness,
+                    evidencePinId: addedPin.id,
+                    limitations: sourceLimitations,
+                },
+            }, now).cases;
+        }
+        findingsAdded += 1;
+        if (existing)
+            updatedDomains.add(finding.domain);
+        else
+            createdDomains.add(finding.domain);
+    }
+    return {
+        cases,
+        casesCreated: createdDomains.size,
+        casesUpdated: updatedDomains.size,
+        findingsAdded,
+        duplicatesSkipped,
+    };
+}

package/frontend/src/lib/analysis/investigation-capsule.js

@@ -0,0 +1,105 @@
+import { canonicalArtifactJson, sha256ArtifactDigest } from "./artifact-integrity.js";
+export const INVESTIGATION_CAPSULE_SCHEMA = 'whoisleuth.investigation-capsule';
+export const INVESTIGATION_CAPSULE_VERSION = 1;
+function safeVersion(value) {
+    return typeof value === 'string' && /^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u.test(value)
+        ? value
+        : '0.0.0';
+}
+function safeSchema(value) {
+    return typeof value === 'string' && /^[a-z0-9.-]{1,120}$/u.test(value) ? value : 'unknown';
+}
+function safeSchemaVersion(value) {
+    return Number.isSafeInteger(value) && Number(value) > 0 && Number(value) <= 10_000 ? Number(value) : 0;
+}
+function iso(value) {
+    if (typeof value !== 'string' || !Number.isFinite(Date.parse(value)))
+        return new Date().toISOString();
+    return new Date(value).toISOString();
+}
+function analystProjection(record) {
+    if (!record)
+        return null;
+    return {
+        caseId: record.id,
+        status: record.status,
+        disposition: record.disposition,
+        decisions: record.decisions.map(({ id, summary, rationale, evidencePinIds, createdAt }) => ({
+            id,
+            summary,
+            rationale,
+            evidencePinIds: [...evidencePinIds],
+            createdAt,
+        })),
+        assertions: record.assertions.map(({ id, kind, statement, rationale, evidencePinIds, state, createdAt, updatedAt }) => ({
+            id,
+            kind,
+            statement,
+            rationale,
+            evidencePinIds: [...evidencePinIds],
+            state,
+            createdAt,
+            updatedAt,
+        })),
+    };
+}
+export async function buildInvestigationCapsule(input) {
+    const analystRecords = input.includeAnalystRecords ? analystProjection(input.caseRecord) : null;
+    const [evidenceDigest, briefDigest, graphDigest, analystRecordsDigest] = await Promise.all([
+        sha256ArtifactDigest(input.lookupEvidence),
+        sha256ArtifactDigest(input.brief),
+        sha256ArtifactDigest(input.graph),
+        analystRecords ? sha256ArtifactDigest(analystRecords) : Promise.resolve(null),
+    ]);
+    const evidenceSchema = safeSchema(input.lookupEvidence.schema);
+    const evidenceVersion = safeSchemaVersion(input.lookupEvidence.schemaVersion);
+    return {
+        schema: INVESTIGATION_CAPSULE_SCHEMA,
+        schemaVersion: INVESTIGATION_CAPSULE_VERSION,
+        generatedAt: iso(input.generatedAt),
+        application: { name: 'WHOISleuth', version: safeVersion(input.applicationVersion) },
+        target: { value: input.brief.target, type: input.brief.targetType },
+        sourceContracts: [
+            { id: 'lookup-evidence', schema: evidenceSchema, version: evidenceVersion, digest: evidenceDigest, embedded: false },
+            { id: 'investigation-brief', schema: input.brief.schema, version: input.brief.schemaVersion, digest: briefDigest, embedded: true },
+            { id: 'asset-graph', schema: 'whoisleuth.lookup-asset-graph', version: input.graph.version, digest: graphDigest, embedded: true },
+            ...(analystRecords && analystRecordsDigest
+                ? [{ id: 'analyst-records', schema: 'whoisleuth.case-analyst-records', version: 1, digest: analystRecordsDigest, embedded: true }]
+                : []),
+        ],
+        investigationBrief: input.brief,
+        graphSnapshot: input.graph,
+        analystRecords,
+        integrity: { algorithm: 'SHA-256', briefDigest, graphDigest, analystRecordsDigest },
+        limitations: [
+            'The Lookup evidence file is linked by digest but is not embedded; keep or share that exact file separately when verification is required.',
+            'Digest verification can detect changed content but does not establish who created the capsule. No digital signature is applied.',
+            'The graph and brief are bounded projections of collected and derived evidence, not attribution, ownership, safety, availability, or maliciousness conclusions.',
+            analystRecords
+                ? 'Analyst decisions and assertions were deliberately included; review them for sensitive or personal information before sharing.'
+                : 'Analyst decisions, assertions, notes, contacts, actions, and raw source payloads are excluded.',
+        ],
+    };
+}
+export async function verifyInvestigationCapsule(capsule) {
+    const [briefDigest, graphDigest, analystRecordsDigest] = await Promise.all([
+        sha256ArtifactDigest(capsule.investigationBrief),
+        sha256ArtifactDigest(capsule.graphSnapshot),
+        capsule.analystRecords ? sha256ArtifactDigest(capsule.analystRecords) : Promise.resolve(null),
+    ]);
+    const brief = briefDigest === capsule.integrity.briefDigest;
+    const graph = graphDigest === capsule.integrity.graphDigest;
+    const analystRecords = capsule.analystRecords
+        ? analystRecordsDigest === capsule.integrity.analystRecordsDigest
+        : capsule.integrity.analystRecordsDigest === null
+            ? null
+            : false;
+    return { valid: brief && graph && analystRecords !== false, brief, graph, analystRecords };
+}
+export function investigationCapsuleFilename(capsule) {
+    const target = capsule.target.value.toLowerCase().replace(/[^a-z0-9.-]+/gu, '-').replace(/^-|-$/gu, '').slice(0, 80) || 'lookup';
+    return `whoisleuth-investigation-capsule-${target}-${capsule.generatedAt.slice(0, 10)}.json`;
+}
+export function serializeInvestigationCapsule(capsule) {
+    return `${canonicalArtifactJson(capsule)}\n`;
+}