@slicedearth/whoisleuth-cli 2.3.0 → 2.5.1

package/packages/investigation/investigation-package.mjs

@@ -0,0 +1,200 @@
+import { zipSync } from 'fflate';
+import { MAX_INVESTIGATION_PACKAGE_ENTRIES, MAX_INVESTIGATION_PACKAGE_BYTES } from "../contracts/investigation-package-limits.mjs";
+export { MAX_INVESTIGATION_PACKAGE_ENTRIES, MAX_INVESTIGATION_PACKAGE_BYTES } from "../contracts/investigation-package-limits.mjs";
+import { extractBoundedZipEntries } from "../interchange/bounded-zip-extraction.mjs";
+import { canonicalArtifactJsonV2, sha256ArtifactBytes, sha256ArtifactDigestV2 } from "../evidence/artifact-integrity.mjs";
+import { INVESTIGATION_CAPSULE_SCHEMA, investigationCapsuleSourceIdentity } from "./investigation-capsule.mjs";
+import { matchCaptureArtifacts, readWebCaptureManifest } from "../interchange/web-capture-import.mjs";
+import { MAX_WEB_CAPTURE_MANIFEST_BYTES, WEB_CAPTURE_MANIFEST_SCHEMA } from "../contracts/web-capture.mjs";
+import { MAX_INVESTIGATION_MANIFEST_ARTIFACTS, MAX_INVESTIGATION_MANIFEST_ARTIFACT_BYTES, MAX_INVESTIGATION_MANIFEST_DOCUMENT_BYTES, MAX_INVESTIGATION_MANIFEST_TOTAL_BYTES, investigationJsonMetadata, prepareInvestigationManifest, readInvestigationManifest, } from "./investigation-manifest.mjs";
+export const INVESTIGATION_PACKAGE_MANIFEST_PATH = 'manifest.json';
+const MANIFEST_PATH = INVESTIGATION_PACKAGE_MANIFEST_PATH;
+const MAX_PACKAGE_ENTRIES = MAX_INVESTIGATION_PACKAGE_ENTRIES;
+export function investigationPackageEntryPath(id) {
+    const match = /^artifact-([1-9]\d{0,2})$/u.exec(id);
+    if (!match || Number(match[1]) > MAX_INVESTIGATION_MANIFEST_ARTIFACTS)
+        throw new TypeError('Invalid investigation package entry identity.');
+    return `artifacts/${id}`;
+}
+export function investigationPackagePath(name) {
+    if (typeof name !== 'string')
+        throw new TypeError('Investigation package entry paths must be strings.');
+    if (name === MANIFEST_PATH)
+        return name;
+    if (!name.startsWith('artifacts/') || investigationPackageEntryPath(name.slice('artifacts/'.length)) !== name)
+        throw new TypeError('Investigation package contains an unexpected entry path.');
+    return name;
+}
+export async function prepareInvestigationPackageEntries(input, generatedAt, applicationVersion) {
+    const prepared = await prepareInvestigationManifest(input, generatedAt, applicationVersion);
+    const files = new Map();
+    files.set(MANIFEST_PATH, new TextEncoder().encode(canonicalArtifactJsonV2(prepared.manifest)));
+    for (const [index, item] of prepared.manifest.artifacts.entries())
+        files.set(investigationPackageEntryPath(item.id), prepared.contents[index]);
+    return Object.freeze({ manifest: prepared.manifest, files });
+}
+/** Capture a directory selection before hashing; no caller-owned bytes survive. */
+function capturePackageEntries(input) {
+    if (!(input instanceof Map) || input.size < 2 || input.size > MAX_PACKAGE_ENTRIES)
+        throw new TypeError('Investigation package has an invalid entry count.');
+    const files = new Map();
+    let total = 0;
+    for (const [name, bytes] of input) {
+        investigationPackagePath(name);
+        const maximum = name === MANIFEST_PATH ? MAX_INVESTIGATION_MANIFEST_DOCUMENT_BYTES : MAX_INVESTIGATION_MANIFEST_ARTIFACT_BYTES;
+        if (!(bytes instanceof Uint8Array) || bytes.byteLength < 1 || bytes.byteLength > maximum)
+            throw new TypeError('Investigation package entry exceeds its byte limit.');
+        if (name !== MANIFEST_PATH) {
+            if (bytes.byteLength > MAX_INVESTIGATION_MANIFEST_TOTAL_BYTES - total)
+                throw new TypeError('Investigation package exceeds its combined byte limit.');
+            total += bytes.byteLength;
+        }
+        files.set(name, new Uint8Array(bytes));
+    }
+    if (!files.has(MANIFEST_PATH))
+        throw new TypeError('Investigation package is missing its manifest.');
+    return files;
+}
+/** A stable container representation, independent of directory listing order. */
+export function encodeInvestigationPackageEntries(input) {
+    const captured = capturePackageEntries(input);
+    const files = Object.create(null);
+    for (const name of [...captured.keys()].sort())
+        files[name] = captured.get(name);
+    // Stored entries avoid re-compressing images and large JSON while retaining
+    // exact source bytes. The reader also admits independently bounded deflate.
+    const bytes = zipSync(files, { level: 0, mtime: new Date(1980, 0, 1) });
+    if (bytes.byteLength > MAX_INVESTIGATION_PACKAGE_BYTES)
+        throw new TypeError('Investigation package exceeds its byte limit.');
+    return bytes;
+}
+export async function buildInvestigationPackage(input, generatedAt, applicationVersion) {
+    const prepared = await prepareInvestigationPackageEntries(input, generatedAt, applicationVersion);
+    return Object.freeze({ manifest: prepared.manifest, bytes: encodeInvestigationPackageEntries(prepared.files) });
+}
+export async function inspectInvestigationPackage(input) {
+    if (!(input instanceof Uint8Array) || input.byteLength < 22 || input.byteLength > MAX_INVESTIGATION_PACKAGE_BYTES)
+        throw new TypeError(`Investigation package input must fit within ${MAX_INVESTIGATION_PACKAGE_BYTES} bytes.`);
+    // The bounded reader verifies local/central identity and every selected CRC.
+    // All entries are selected: an unverified trailing file cannot hide in a pack.
+    const extracted = extractBoundedZipEntries(input, {
+        maximumEntries: MAX_PACKAGE_ENTRIES,
+        maximumSelectedBytes: MAX_INVESTIGATION_MANIFEST_TOTAL_BYTES + MAX_INVESTIGATION_MANIFEST_DOCUMENT_BYTES,
+        selectedBytesExceededMessage: 'Investigation package exceeds its inflated byte limit.',
+        metadataMismatchMessage: 'Investigation package ZIP metadata or CRC is invalid.',
+        keyForName: investigationPackagePath,
+        inspect(info) {
+            const key = investigationPackagePath(info.name);
+            const maximumBytes = key === MANIFEST_PATH ? MAX_INVESTIGATION_MANIFEST_DOCUMENT_BYTES : MAX_INVESTIGATION_MANIFEST_ARTIFACT_BYTES;
+            if (info.originalSize < 1 || info.originalSize > maximumBytes)
+                throw new TypeError('Investigation package entry exceeds its byte limit.');
+            return { key, selected: true, maximumBytes, exceededMessage: 'Investigation package entry exceeds its inflated byte limit.' };
+        },
+    });
+    return inspectOwnedPackageEntries(extracted.files);
+}
+export async function inspectInvestigationPackageEntries(input) {
+    return inspectOwnedPackageEntries(capturePackageEntries(input));
+}
+async function inspectOwnedPackageEntries(files) {
+    const manifestBytes = files.get(MANIFEST_PATH);
+    if (!manifestBytes)
+        throw new TypeError('Investigation package is missing its manifest.');
+    const manifest = await readInvestigationManifest(new TextDecoder('utf-8', { fatal: true }).decode(manifestBytes));
+    if (files.size !== manifest.artifacts.length + 1)
+        throw new TypeError('Investigation package has missing or unlisted files.');
+    const entries = [];
+    const contents = new Map();
+    const captureDeclarations = new Map();
+    const jsonSources = new Map();
+    for (const entry of manifest.artifacts) {
+        const content = files.get(investigationPackageEntryPath(entry.id));
+        if (!content)
+            throw new TypeError('Investigation package is missing a declared file.');
+        const isJson = !('mediaType' in entry) || entry.mediaType === 'application/json';
+        const byteLength = content.byteLength === entry.byteLength;
+        const rawDigest = await sha256ArtifactBytes(content) === entry.contentDigestSha256;
+        let canonicalDigest = isJson ? false : null;
+        let schema = isJson ? false : null;
+        let version = isJson ? false : null;
+        let issue = null;
+        let interpretation = isJson ? 'not_checked' : 'opaque';
+        if (isJson) {
+            try {
+                const metadata = investigationJsonMetadata(new TextDecoder('utf-8', { fatal: true }).decode(content));
+                canonicalDigest = await sha256ArtifactDigestV2(metadata.value) === entry.canonicalDigestSha256;
+                schema = metadata.schema === entry.schema;
+                version = metadata.version === entry.version;
+                if (!metadata.object)
+                    interpretation = 'unsupported_json_value';
+                if (metadata.object && byteLength && rawDigest && canonicalDigest && schema && version) {
+                    if (metadata.schema === WEB_CAPTURE_MANIFEST_SCHEMA) {
+                        try {
+                            if (content.byteLength > MAX_WEB_CAPTURE_MANIFEST_BYTES)
+                                throw new TypeError('Capture manifest is oversized.');
+                            captureDeclarations.set(entry.id, readWebCaptureManifest(metadata.object).artifacts);
+                        }
+                        catch {
+                            captureDeclarations.set(entry.id, null);
+                        }
+                    }
+                    const query = metadata.object.query;
+                    const queryObject = query && typeof query === 'object' && !Array.isArray(query) ? query : null;
+                    jsonSources.set(entry.id, {
+                        schema: metadata.schema, version: metadata.version, digest: entry.canonicalDigestSha256,
+                        query: queryObject ? { submitted: queryObject.submitted, type: queryObject.type } : null,
+                        target: metadata.schema === INVESTIGATION_CAPSULE_SCHEMA ? metadata.object.target : null,
+                        references: metadata.schema === INVESTIGATION_CAPSULE_SCHEMA ? metadata.object.sourceContracts : null,
+                    });
+                }
+            }
+            catch {
+                issue = 'The declared JSON file is not valid bounded UTF-8 JSON.';
+            }
+        }
+        const verified = byteLength && rawDigest && canonicalDigest !== false && schema !== false && version !== false;
+        entries.push(Object.freeze({ entry, state: verified ? 'identity_verified' : 'rejected',
+            checks: Object.freeze({ byteLength, rawDigest, canonicalDigest, schema, version }),
+            interpretation,
+            issue: issue ?? (verified ? null : 'File content does not match its manifest identity.'),
+        }));
+        // Rejected bytes remain in the selected archive but cannot be used by an
+        // importing caller as if they had passed identity verification.
+        if (verified)
+            contents.set(entry.id, content);
+    }
+    const links = [];
+    const captureManifests = [...captureDeclarations].map(([entryId, declarations]) => {
+        if (!declarations)
+            return { entryId, state: 'rejected', artifacts: [] };
+        const artifacts = matchCaptureArtifacts(declarations, entries
+            .filter(candidate => candidate.entry.id !== entryId && candidate.state === 'identity_verified')
+            .map(candidate => ({ id: candidate.entry.id, bytes: candidate.entry.byteLength, sha256: candidate.entry.contentDigestSha256 })));
+        return { entryId, state: artifacts.every(artifact => artifact.state === 'matched') ? 'matched' : 'missing_attachments', artifacts };
+    });
+    for (const [id, capsule] of jsonSources) {
+        if (capsule.schema !== INVESTIGATION_CAPSULE_SCHEMA)
+            continue;
+        const references = Array.isArray(capsule.references) && capsule.references.length <= 4
+            ? capsule.references.filter((item) => item && typeof item === 'object' && item.id === 'lookup-evidence') : [];
+        if (references.length !== 1) {
+            links.push({ capsuleEntryId: id, sourceEntryId: null, state: 'mismatch' });
+            continue;
+        }
+        const reference = references[0];
+        const candidates = [...jsonSources].filter(([otherId, source]) => otherId !== id && source.schema === reference.schema && source.digest === reference.digest);
+        if (candidates.length !== 1) {
+            const state = candidates.length > 1 ? 'ambiguous' : [...jsonSources.values()].some((source) => source.schema === reference.schema) ? 'mismatch' : 'missing';
+            links.push({ capsuleEntryId: id, sourceEntryId: null, state });
+            continue;
+        }
+        const [sourceEntryId, source] = candidates[0];
+        const identity = investigationCapsuleSourceIdentity(capsule.target, reference, source);
+        links.push({ capsuleEntryId: id, sourceEntryId, state: identity.linked ? 'linked' : 'mismatch' });
+    }
+    return Object.freeze({ manifest, entries: Object.freeze(entries), contents, links: Object.freeze(links), captureManifests: Object.freeze(captureManifests),
+        identityVerified: entries.every((entry) => entry.state === 'identity_verified'),
+        storageEffect: 'none',
+        signatureTrust: 'not_checked', timestampAssurance: 'not_checked', factualAccuracy: 'not_established',
+    });
+}

package/packages/investigation/link-intake.mjs

@@ -0,0 +1,179 @@
+import { canonicalRegistrableDomain } from "../analysis/registrable-domain.mjs";
+import { parseCredentialFreeHttpUrl } from "../evidence/lookup-target.mjs";
+import { MAX_INTAKE_LINKS, MAX_INTAKE_URL_LENGTH, MAX_EMBEDDED_LINK_DEPTH, MAX_AUTH_SCOPES } from "../contracts/message-intake.mjs";
+const EXCLUDED_INPUTS = {
+    malformed: 'Malformed or overlong links were not reviewed',
+    credentials: 'Links containing credentials were not reviewed',
+    protocol: 'Non-HTTP(S) links were not reviewed',
+    port: 'Links using non-default ports were not reviewed',
+    host: 'IP-address or non-registrable-host links were not reviewed',
+    qrNetwork: 'QR network settings were not interpreted',
+    qrContact: 'QR contact details were not interpreted',
+    qrProtocol: 'QR non-HTTP(S) payloads were not interpreted',
+    qrHost: 'QR hostnames without an HTTP(S) scheme were not offered for collection',
+    qrText: 'QR text without an HTTP(S) link was not interpreted',
+};
+const AUTH_KEYS = ['client_id', 'scope', 'redirect_uri', 'response_type', 'prompt'];
+const EMBEDDED_KEYS = new Set(['url', 'u', 'target', 'redirect', 'redirect_uri', 'redirect_url', 'return', 'returnurl', 'continue', 'next', 'dest', 'destination']);
+const safeIdentifier = (value) => value && /^[a-zA-Z0-9._~:-]{1,160}$/u.test(value) ? value : null;
+const tokens = (value) => (value ?? '').split(/\s+/u).filter(Boolean);
+export function reviewAuthorisationLink(url) {
+    const params = url.searchParams;
+    const authorisation = params.has('client_id') && (params.has('scope') || params.has('response_type') || params.has('redirect_uri'));
+    const device = /\/(?:device|devicecode|devicelogin)(?:\/|$)/iu.test(url.pathname);
+    if (!authorisation && !device)
+        return null;
+    const duplicateParameters = AUTH_KEYS.filter(key => params.getAll(key).length > 1);
+    const single = (key) => duplicateParameters.includes(key) ? null : params.get(key);
+    const scopes = tokens(single('scope'));
+    const retainedScopes = scopes.filter(scope => /^[a-zA-Z0-9._:/-]{1,160}$/u.test(scope)).slice(0, MAX_AUTH_SCOPES);
+    const redirect = parseCredentialFreeHttpUrl(single('redirect_uri'), MAX_INTAKE_URL_LENGTH);
+    const responseTypes = tokens(single('response_type')).filter(value => ['code', 'token', 'id_token', 'none'].includes(value));
+    const prompt = tokens(single('prompt')).filter(value => ['none', 'login', 'consent', 'select_account', 'create'].includes(value));
+    return {
+        kind: authorisation ? 'authorisation_parameters' : 'device_code_reference',
+        clientId: safeIdentifier(single('client_id')), scopes: retainedScopes,
+        redirectOrigin: redirect?.origin ?? null, responseTypes, prompt, duplicateParameters,
+        omittedParameters: scopes.length !== retainedScopes.length || (params.has('client_id') && !safeIdentifier(single('client_id')))
+            || (params.has('redirect_uri') && !redirect) || tokens(single('response_type')).length !== responseTypes.length
+            || tokens(single('prompt')).length !== prompt.length,
+    };
+}
+export function refangIntakeUrl(raw) {
+    // Only the scheme and authority are defanged. Path/query/fragment bytes may be
+    // significant to an exact supplied target, even when they resemble defanging.
+    return raw.replace(/^(https?|hxxps?)(?::|\[:\])\/\/([^/?#]*)/iu, (_match, scheme, authority) => `${scheme.toLowerCase().replace('hxxp', 'http')}://${authority.replace(/\[\.\]|\(\.\)/gu, '.')}`);
+}
+const HTTP_PREFIX = /^(?:https?|hxxps?)(?::|\[:\])\/\//iu;
+function proseUrl(value) {
+    // Prose delimiters are not part of a URL; balanced path brackets are. Direct
+    // supplied links bypass this text-scanning cleanup entirely.
+    let result = value.replace(/[.,;!?]+$/u, '');
+    let round = 0, square = 0;
+    for (const char of result) {
+        if (char === '(')
+            round++;
+        else if (char === ')')
+            round--;
+        else if (char === '[')
+            square++;
+        else if (char === ']')
+            square--;
+    }
+    for (;;) {
+        const end = result.at(-1);
+        if (end === ')' && round < 0)
+            round++;
+        else if (end === ']' && square < 0)
+            square++;
+        else
+            return result;
+        result = result.slice(0, -1).replace(/[.,;!?]+$/u, '');
+    }
+}
+function displayedHost(value) {
+    const bounded = value.trim();
+    if (bounded.length > MAX_INTAKE_URL_LENGTH || /\s/u.test(bounded))
+        return null;
+    const parsed = parseCredentialFreeHttpUrl(refangIntakeUrl(HTTP_PREFIX.test(bounded) ? bounded : `https://${bounded}`), MAX_INTAKE_URL_LENGTH);
+    return parsed && canonicalRegistrableDomain(parsed.hostname) ? parsed.hostname : null;
+}
+/** No resolution or request occurs here; embedded targets remain supplied text. */
+export function createLinkIntake() {
+    const links = [], targets = [];
+    const seen = new Set();
+    const exclusions = new Map();
+    let rejected = 0, bounded = false;
+    function exclude(reason, isLink = true) {
+        if (isLink)
+            rejected++;
+        exclusions.set(reason, (exclusions.get(reason) ?? 0) + 1);
+    }
+    function add(raw, source, displayed = '', parentId = null, depth = 0, location) {
+        if (links.length >= MAX_INTAKE_LINKS) {
+            bounded = true;
+            return;
+        }
+        const value = refangIntakeUrl(raw.trim());
+        const url = parseCredentialFreeHttpUrl(value, MAX_INTAKE_URL_LENGTH);
+        if (!url) {
+            let reason = 'malformed';
+            if (value.length <= MAX_INTAKE_URL_LENGTH && !/[\x00-\x20\x7f]/u.test(value)) {
+                try {
+                    const parsed = new URL(value);
+                    if (!['http:', 'https:'].includes(parsed.protocol))
+                        reason = 'protocol';
+                    else if (parsed.username || parsed.password)
+                        reason = 'credentials';
+                }
+                catch { /* Retain only the fixed malformed-input category. */ }
+            }
+            exclude(reason);
+            return;
+        }
+        if (url.port) {
+            exclude('port');
+            return;
+        }
+        if (!canonicalRegistrableDomain(url.hostname)) {
+            exclude('host');
+            return;
+        }
+        const shown = displayedHost(displayed);
+        const key = JSON.stringify([url.href, source, parentId, shown, location ?? null]);
+        if (seen.has(key))
+            return;
+        seen.add(key);
+        const id = `link-${links.length + 1}`;
+        links.push({ id, origin: url.origin, hostname: url.hostname, registrationDomain: canonicalRegistrableDomain(url.hostname),
+            source, ...(location ? { location } : {}), parentId, displayedHostname: shown, displayedDestination: shown ? shown === url.hostname ? 'same_host' : 'different_host' : 'not_a_hostname',
+            hasPrivateLocation: url.pathname !== '/' || Boolean(url.search || url.hash), authorisation: reviewAuthorisationLink(url) });
+        targets.push({ id, exactUrl: url.href });
+        for (const [name, value] of url.searchParams) {
+            if (!EMBEDDED_KEYS.has(name.toLowerCase()) || !HTTP_PREFIX.test(value))
+                continue;
+            if (depth >= MAX_EMBEDDED_LINK_DEPTH) {
+                bounded = true;
+                continue;
+            }
+            add(value, 'embedded_parameter', '', id, depth + 1, location);
+        }
+    }
+    function addText(text, source = 'text', location) {
+        // The caller admits bytes before this scan; iteration stops at the work bound.
+        for (const match of text.matchAll(/(?:https?|hxxps?)(?::|\[:\])\/\/[^\s<>"'`]+/giu)) {
+            add(match[0].length > MAX_INTAKE_URL_LENGTH ? match[0] : proseUrl(match[0]), source, '', null, 0, location);
+            if (links.length >= MAX_INTAKE_LINKS) {
+                bounded = true;
+                break;
+            }
+        }
+    }
+    function addQr(text, source = 'qr', location) {
+        const value = refangIntakeUrl(text.trim());
+        const scheme = /^([a-z][a-z0-9+.-]*):/iu.exec(value)?.[1]?.toLowerCase();
+        if (/^WIFI:/iu.test(value)) {
+            exclude('qrNetwork', false);
+            return;
+        }
+        if (/^(?:BEGIN:VCARD|MECARD:)/iu.test(value)) {
+            exclude('qrContact', false);
+            return;
+        }
+        if (scheme && !['http', 'https'].includes(scheme)) {
+            exclude('qrProtocol', false);
+            return;
+        }
+        if (!/(?:https?|hxxps?):\/\//iu.test(value)) {
+            exclude(displayedHost(value) ? 'qrHost' : 'qrText', false);
+            return;
+        }
+        addText(value, source, location);
+    }
+    return { add, addText, addQr, result: () => ({ links: [...links], targets: [...targets], rejected, bounded,
+            limitations: Object.keys(EXCLUDED_INPUTS).flatMap(reason => {
+                const count = exclusions.get(reason);
+                return count ? [`${EXCLUDED_INPUTS[reason]} (${count}).`] : [];
+            }),
+        }) };
+}

package/packages/investigation/lookup-asset-graph-inputs.mjs

@@ -0,0 +1,38 @@
+/** Counts describe the projection input, not the collector's upstream universe. */
+export function createGraphInputReader() {
+    const coverage = new Map();
+    function values(id, value, maximum, normalise, identity) {
+        const input = Array.isArray(value) ? value : [];
+        const inspected = Math.min(input.length, maximum * 4);
+        const result = [];
+        const seen = new Set();
+        let invalid = 0;
+        let duplicates = 0;
+        let omitted = 0;
+        for (let index = 0; index < inspected; index += 1) {
+            const item = normalise(input[index]);
+            if (item === null) {
+                invalid += 1;
+                continue;
+            }
+            const key = identity?.(item);
+            if (key !== undefined && seen.has(key)) {
+                duplicates += 1;
+                continue;
+            }
+            if (key !== undefined)
+                seen.add(key);
+            if (result.length === maximum) {
+                omitted += 1;
+                continue;
+            }
+            result.push(item);
+        }
+        coverage.set(id, { id, supplied: input.length, inspected, admitted: result.length, invalid, duplicates, omitted });
+        return result;
+    }
+    return { values, coverage };
+}
+export function graphInputIsIncomplete(row) {
+    return row.supplied > row.inspected || row.invalid > 0 || row.omitted > 0;
+}