@slicedearth/whoisleuth-cli 2.5.1 → 2.6.0

package/packages/investigation/investigation-manifest.mjs

@@ -2,12 +2,13 @@ import { boundedJsonLimitsForBytes, parseBoundedJson, parseBoundedJsonObject } f
 import { normalizeBoundedSemanticVersion } from "../analysis/semantic-version.mjs";
 import { canonicalArtifactJsonV2, sha256ArtifactBytes, sha256ArtifactDigestV2, SORTED_JSON_V2 } from "../evidence/artifact-integrity.mjs";
 import { array, digest, enumeration, exact, fail, integer, iso, strings, text, validateIntegrity } from "../evidence/artifact-structure.mjs";
+import { readImageDerivationDeclaration } from "../evidence/image-regions.mjs";
 import { SELECTED_FILE_MEDIA_TYPES as INVESTIGATION_FILE_MEDIA_TYPES } from "../contracts/selected-file-limits.mjs";
 import { MAX_INVESTIGATION_MANIFEST_ARTIFACTS, MAX_INVESTIGATION_MANIFEST_TOTAL_BYTES, MAX_INVESTIGATION_MANIFEST_ARTIFACT_BYTES } from "../contracts/investigation-package-limits.mjs";
 export { MAX_INVESTIGATION_MANIFEST_ARTIFACTS, MAX_INVESTIGATION_MANIFEST_TOTAL_BYTES, MAX_INVESTIGATION_MANIFEST_ARTIFACT_BYTES, INVESTIGATION_FILE_MEDIA_TYPES };
 export const INVESTIGATION_MANIFEST_SCHEMA = 'whoisleuth.investigation-manifest';
-export const INVESTIGATION_MANIFEST_VERSION = 3;
-export const SUPPORTED_INVESTIGATION_MANIFEST_VERSIONS = [2, INVESTIGATION_MANIFEST_VERSION];
+export const INVESTIGATION_MANIFEST_VERSION = 4;
+export const SUPPORTED_INVESTIGATION_MANIFEST_VERSIONS = [2, 3, INVESTIGATION_MANIFEST_VERSION];
 // Independently bounds the path-free metadata, not the selected file content.
 import { MAX_INVESTIGATION_MANIFEST_DOCUMENT_BYTES } from "../contracts/investigation-package-limits.mjs";
 export { MAX_INVESTIGATION_MANIFEST_DOCUMENT_BYTES } from "../contracts/investigation-package-limits.mjs";
@@ -78,7 +79,10 @@ export async function prepareInvestigationManifest(input, generatedAt, applicati
             throw new TypeError('Manifest artefacts exceed the combined limit.');
         totalBytes += bytes.byteLength;
         const mediaType = enumeration(artifact.mediaType ?? (typeof artifact.content === 'string' ? 'application/json' : 'application/octet-stream'), INVESTIGATION_FILE_MEDIA_TYPES, 'Artefact media type');
-        return { bytes, mediaType, source: sourceDeclaration(artifact.source) };
+        const imageDerivation = artifact.imageDerivation == null ? null : readImageDerivationDeclaration(artifact.imageDerivation);
+        if (imageDerivation && mediaType !== 'image/png')
+            throw new TypeError('Image derivation requires a PNG media declaration.');
+        return { bytes, mediaType, source: sourceDeclaration(artifact.source), imageDerivation };
     });
     const artifacts = [];
     for (const [index, file] of selected.entries()) {
@@ -90,7 +94,7 @@ export async function prepareInvestigationManifest(input, generatedAt, applicati
             byteLength: file.bytes.byteLength,
             contentDigestSha256: await sha256ArtifactBytes(file.bytes),
             canonicalDigestSha256: metadata ? await sha256ArtifactDigestV2(metadata.value) : null,
-            mediaType: file.mediaType, source: file.source,
+            mediaType: file.mediaType, source: file.source, imageDerivation: file.imageDerivation,
         }));
     }
     const unsigned = Object.freeze({
@@ -109,6 +113,7 @@ export async function prepareInvestigationManifest(input, generatedAt, applicati
             'Source identities, observation times and media types are declarations, not independently authenticated evidence. Missing source times remain unknown.',
             'Packaging events use the local clock. They do not establish earlier custody, a trusted timestamp, a signature, source accuracy or current validity.',
             'Digests establish content identity. JSON parsing and declared schema metadata do not establish that a source format is supported or its claims are true.',
+            'Image derivation declares an immediate parent fingerprint and editing operations, not proof of transformation or complete redaction. Parent bytes need not be included; missing declarations leave editing history unknown.',
         ]),
     });
     const manifest = Object.freeze({ ...unsigned, integrity: Object.freeze({
@@ -124,7 +129,8 @@ export async function buildInvestigationManifest(input, generatedAt, application
 }
 export function validateInvestigationManifest(value) {
     const version = integer(value.version, 'Investigation manifest version', 2, INVESTIGATION_MANIFEST_VERSION);
-    const current = version === INVESTIGATION_MANIFEST_VERSION;
+    const current = version >= 3;
+    const withImageDerivation = version >= 4;
     const root = exact(value, ['schema', 'version', 'generatedAt', 'application', 'workflow', 'configuration', 'artifacts', 'steps', 'summary', 'limitations', 'integrity', ...(current ? ['audience'] : [])], 'Investigation manifest');
     if (root.schema !== INVESTIGATION_MANIFEST_SCHEMA)
         fail('Investigation manifest schema');
@@ -148,7 +154,7 @@ export function validateInvestigationManifest(value) {
         fail('Investigation manifest step count');
     let totalBytes = 0;
     for (const [index, candidate] of artifacts.entries()) {
-        const item = exact(candidate, ['sequence', 'id', 'schema', 'version', 'byteLength', 'contentDigestSha256', 'canonicalDigestSha256', ...(current ? ['mediaType', 'source'] : [])], `Investigation manifest artifact ${index + 1}`);
+        const item = exact(candidate, ['sequence', 'id', 'schema', 'version', 'byteLength', 'contentDigestSha256', 'canonicalDigestSha256', ...(current ? ['mediaType', 'source'] : []), ...(withImageDerivation ? ['imageDerivation'] : [])], `Investigation manifest artifact ${index + 1}`);
         if (integer(item.sequence, 'Investigation manifest artifact sequence', 1, artifacts.length) !== index + 1 || item.id !== `artifact-${index + 1}`)
             fail('Investigation manifest artifact order');
         if (item.schema !== null)
@@ -162,6 +168,11 @@ export function validateInvestigationManifest(value) {
             digest(item.canonicalDigestSha256, 'Investigation manifest canonical digest');
         else if (item.canonicalDigestSha256 !== null || item.schema !== null || item.version !== null)
             fail('Opaque artefact JSON metadata');
+        if (withImageDerivation && item.imageDerivation !== null) {
+            readImageDerivationDeclaration(item.imageDerivation);
+            if (mediaType !== 'image/png')
+                fail('Image derivation media type');
+        }
         if (current) {
             const source = exact(item.source, ['identity', 'observedAt'], 'Artefact source');
             if (source.identity !== null)
@@ -189,12 +200,27 @@ export async function readInvestigationManifest(raw) {
         throw new TypeError('Investigation manifest failed its integrity check.');
     return value;
 }
+/** Matches selected manifest entries, not unselected attachment declarations. */
+export function investigationImageParentIncluded(manifest, artifact) {
+    if (!('imageDerivation' in artifact) || !artifact.imageDerivation)
+        return null;
+    const parent = artifact.imageDerivation.source;
+    return manifest.artifacts.some(entry => entry.id !== artifact.id
+        && entry.contentDigestSha256 === parent.digestSha256 && entry.byteLength === parent.byteLength);
+}
 export function formatInvestigationManifest(manifest) {
     const lines = ['Investigation manifest', `Workflow       ${manifest.workflow}`, `Tool version   ${manifest.application.version}`,
         `Artifacts      ${manifest.summary.artifactCount}`, `Total bytes    ${manifest.summary.totalBytes}`,
         `Configuration  ${manifest.configuration.digestSha256 ?? 'not supplied'}`, `Integrity      ${manifest.integrity.digestSha256}`, '', 'Ordered artefacts:'];
-    for (const artifact of manifest.artifacts)
+    for (const artifact of manifest.artifacts) {
         lines.push(`  ${artifact.sequence}. ${artifact.schema ?? (artifact.canonicalDigestSha256 ? 'unversioned JSON' : 'opaque file')}${artifact.version ? ` v${artifact.version}` : ''} · ${artifact.contentDigestSha256}`);
+        if ('imageDerivation' in artifact && artifact.imageDerivation) {
+            const declaration = artifact.imageDerivation;
+            lines.push(`     Declared image derivation: ${declaration.method}; ${declaration.operations.join(', ')}`, `     Parent: ${declaration.source.digestSha256} · ${declaration.source.byteLength} bytes`, `     Parent entry: ${investigationImageParentIncluded(manifest, artifact) ? 'declared in this manifest' : 'not declared in this manifest'}`, '     Declared edits are not proof of transformation or complete redaction.');
+        }
+        else if ('mediaType' in artifact && artifact.mediaType.startsWith('image/'))
+            lines.push('     Editing history: not declared');
+    }
     lines.push('', 'No source file paths or artefact contents are retained in this manifest.', '');
     return lines.join('\n');
 }

package/packages/investigation/link-intake.mjs

@@ -71,12 +71,39 @@ function proseUrl(value) {
         result = result.slice(0, -1).replace(/[.,;!?]+$/u, '');
     }
 }
-function displayedHost(value) {
+function admittedDestination(url, registrationDomain, normalisation = []) {
+    return { state: 'parsed', hostname: url.hostname, origin: url.origin, registrationDomain,
+        hasPrivateLocation: url.pathname !== '/' || Boolean(url.search || url.hash), normalisation };
+}
+export function projectIntakeDestination(value, allowBareHostname = false) {
+    const empty = (state) => ({ state, hostname: null, origin: null, registrationDomain: null, hasPrivateLocation: false, normalisation: [] });
+    if (value.length > MAX_INTAKE_URL_LENGTH)
+        return empty('unsupported');
     const bounded = value.trim();
-    if (bounded.length > MAX_INTAKE_URL_LENGTH || /\s/u.test(bounded))
-        return null;
-    const parsed = parseCredentialFreeHttpUrl(refangIntakeUrl(HTTP_PREFIX.test(bounded) ? bounded : `https://${bounded}`), MAX_INTAKE_URL_LENGTH);
-    return parsed && canonicalRegistrableDomain(parsed.hostname) ? parsed.hostname : null;
+    if (!bounded)
+        return empty('missing');
+    if (/[\s\\\p{Cf}]/u.test(bounded))
+        return empty('unsupported');
+    const explicitScheme = HTTP_PREFIX.test(bounded);
+    if (!explicitScheme && !allowBareHostname)
+        return empty('unsupported');
+    const refanged = refangIntakeUrl(explicitScheme ? bounded : `https://${bounded}`);
+    const parsed = parseCredentialFreeHttpUrl(refanged, MAX_INTAKE_URL_LENGTH);
+    const registrationDomain = parsed && canonicalRegistrableDomain(parsed.hostname);
+    if (!parsed || !registrationDomain || parsed.port)
+        return empty('unsupported');
+    return admittedDestination(parsed, registrationDomain, [
+        ...(!explicitScheme ? ['assumed_https_for_hostname_parsing'] : []),
+        ...(refanged !== (explicitScheme ? bounded : `https://${bounded}`) ? ['refanged_scheme_or_authority'] : []),
+        ...(/[^\x00-\x7f]/u.test(bounded.split(/[/?#]/u).slice(0, explicitScheme ? 3 : 1).join('/')) ? ['unicode_hostname_to_ascii'] : []),
+    ]);
+}
+export function compareIntakeDestinations(displayed, destination) {
+    return displayed.state !== 'parsed' || destination.state !== 'parsed' ? 'insufficient_evidence'
+        : displayed.hostname === destination.hostname ? 'same_host' : 'different_host';
+}
+function displayedHost(value) {
+    return projectIntakeDestination(value, true).hostname;
 }
 /** No resolution or request occurs here; embedded targets remain supplied text. */
 export function createLinkIntake() {
@@ -115,7 +142,8 @@ export function createLinkIntake() {
             exclude('port');
             return;
         }
-        if (!canonicalRegistrableDomain(url.hostname)) {
+        const registrationDomain = canonicalRegistrableDomain(url.hostname);
+        if (!registrationDomain) {
             exclude('host');
             return;
         }
@@ -125,8 +153,10 @@ export function createLinkIntake() {
             return;
         seen.add(key);
         const id = `link-${links.length + 1}`;
-        links.push({ id, origin: url.origin, hostname: url.hostname, registrationDomain: canonicalRegistrableDomain(url.hostname),
-            source, ...(location ? { location } : {}), parentId, displayedHostname: shown, displayedDestination: shown ? shown === url.hostname ? 'same_host' : 'different_host' : 'not_a_hostname',
+        // Compare the already admitted canonical URL, not its differently bounded raw spelling.
+        const compared = compareIntakeDestinations(projectIntakeDestination(displayed, true), admittedDestination(url, registrationDomain));
+        links.push({ id, origin: url.origin, hostname: url.hostname, registrationDomain,
+            source, ...(location ? { location } : {}), parentId, displayedHostname: shown, displayedDestination: compared === 'insufficient_evidence' ? 'not_a_hostname' : compared,
             hasPrivateLocation: url.pathname !== '/' || Boolean(url.search || url.hash), authorisation: reviewAuthorisationLink(url) });
         targets.push({ id, exactUrl: url.href });
         for (const [name, value] of url.searchParams) {

package/packages/investigation/message-intake.mjs

@@ -2,6 +2,8 @@ import PostalMime, {} from 'postal-mime';
 import { parse, defaultTreeAdapter } from 'parse5';
 import { sha256ArtifactBytes } from "../evidence/artifact-integrity.mjs";
 import { createLinkIntake } from "./link-intake.mjs";
+import { createIndicatorIntake, MAX_INTAKE_INDICATOR_TEXT } from "./intake-indicators.mjs";
+import { reviewPhoneCandidates } from "./intake-phones.mjs";
 import { AUTHENTICATION_METHODS, addressDomains, authenticationServiceDomains, dkimSigningDomains } from "./mail-header-identity.mjs";
 import { createIntakeReport } from "./intake-report.mjs";
 export { assertMessageBytes } from "./intake-report.mjs";
@@ -9,24 +11,27 @@ import { requestedActionHints } from "./requested-action-hints.mjs";
 import { reviewMailAuthentication, aggregateMailAuthentication } from "./mail-authentication-review.mjs";
 import { MAX_AUTHENTICATION_HEADERS } from "../contracts/mail-authentication.mjs";
 import { MAX_MESSAGE_INTAKE_BYTES, MAX_MESSAGE_PARTS, MAX_MESSAGE_DEPTH, MAX_MESSAGE_HTML_NODES } from "../contracts/message-intake.mjs";
+const HTML_TEXT_BLOCK = /^(?:address|article|aside|blockquote|body|dd|div|dl|dt|fieldset|figcaption|figure|footer|form|h[1-6]|header|hr|li|main|nav|ol|p|pre|section|table|tr|ul)$/u;
 /** All parsing is inert. Only HTTP(S) link targets are offered for deliberate collection. */
 export async function reviewMessageInput(bytes, kind, reviewedAt, qrText) {
     const base = await createIntakeReport(bytes, kind, reviewedAt);
     if (!['text', 'email', 'calendar', 'qr'].includes(kind))
         throw new TypeError('Use the selected-file intake entry point for this format.');
-    const links = createLinkIntake(), identities = [], authenticationClaims = [];
+    const links = createLinkIntake(), indicators = createIndicatorIntake(), identities = [], authenticationClaims = [];
     const actionHints = new Set(), bounds = new Set();
     const authenticationHeaders = [];
     const messageParts = [];
     let omittedAuthenticationHeaders = 0;
     let partialAuthentication = false;
     let reviewedParts = 0, unreviewedAttachments = 0, decodedBytes = 0;
-    const text = (value, source) => {
+    const text = (value, source, partId, includeIndicators = true) => {
         for (const hint of requestedActionHints(value))
             actionHints.add(hint);
         links.addText(value, source);
+        if (includeIndicators)
+            indicators.addText(value, source, { partId, page: null });
     };
-    function html(value) {
+    function html(value, partId) {
         // parse5 never renders, loads resources, or runs supplied script.
         let allocated = 0;
         const htmlBound = new Error('HTML work bound');
@@ -47,13 +52,31 @@ export async function reviewMessageInput(bytes, kind, reviewedAt, qrText) {
             return;
         }
         const queue = [tree];
+        const indicatorText = [];
+        let indicatorCharacters = 0, indicatorOverflow = false;
+        const appendIndicatorText = (value) => {
+            if (indicatorOverflow)
+                return;
+            if (value.length > MAX_INTAKE_INDICATOR_TEXT - indicatorCharacters) {
+                indicatorOverflow = true;
+                indicatorText.length = 0;
+                indicators.markPartial();
+                return;
+            }
+            indicatorCharacters += value.length;
+            indicatorText.push(value);
+        };
         let nodes = 0;
         while (queue.length) {
+            const node = queue.pop();
+            if (node === null) {
+                appendIndicatorText('\n');
+                continue;
+            }
             if (++nodes > MAX_MESSAGE_HTML_NODES) {
                 bounds.add('HTML nodes');
                 break;
             }
-            const node = queue.pop();
             if ('tagName' in node) {
                 const attrs = new Map(node.attrs.map(attr => [attr.name, attr.value]));
                 if (node.tagName === 'a' && attrs.has('href')) {
@@ -74,15 +97,27 @@ export async function reviewMessageInput(bytes, kind, reviewedAt, qrText) {
                     links.add(attrs.get('src'), 'html_frame');
                 if (node.tagName === 'script' || node.tagName === 'style')
                     continue;
+                if (HTML_TEXT_BLOCK.test(node.tagName)) {
+                    appendIndicatorText('\n');
+                    queue.push(null);
+                }
+                else if (node.tagName === 'br' || node.tagName === 'td' || node.tagName === 'th')
+                    appendIndicatorText(' ');
+            }
+            if ('value' in node) {
+                text(node.value, 'text', partId, false);
+                // Inline markup must not separate a credential label from its value.
+                // Block boundaries keep unrelated visible paragraphs independently usable.
+                appendIndicatorText(node.value.replace(/\s+/gu, ' '));
             }
-            if ('value' in node)
-                text(node.value, 'text');
             if ('childNodes' in node)
                 for (let index = node.childNodes.length - 1; index >= 0; index--)
                     queue.push(node.childNodes[index]);
         }
+        if (!indicatorOverflow)
+            indicators.addText(indicatorText.join(''), 'text', { partId, page: null });
     }
-    function calendar(value) {
+    function calendar(value, partId) {
         const unfolded = value.replace(/\r?\n[ \t]/gu, '');
         // No attendee names, addresses, descriptions or event identifiers enter the report.
         for (const line of unfolded.split(/\r?\n/u)) {
@@ -91,7 +126,7 @@ export async function reviewMessageInput(bytes, kind, reviewedAt, qrText) {
                 continue;
             const key = line.slice(0, colon).split(';', 1)[0]?.toUpperCase();
             if (['URL', 'DESCRIPTION', 'LOCATION', 'X-ALT-DESC'].includes(key ?? ''))
-                text(line.slice(colon + 1).replace(/\\n/giu, '\n').replace(/\\([,;\\])/gu, '$1'), 'calendar');
+                text(line.slice(colon + 1).replace(/\\n/giu, '\n').replace(/\\([,;\\])/gu, '$1'), 'calendar', partId);
         }
     }
     function identity(part, role, domains) {
@@ -142,9 +177,9 @@ export async function reviewMessageInput(bytes, kind, reviewedAt, qrText) {
         messageParts.push({ part, parentPart, digestSha256: parentPart === null ? base.source.digestSha256 : await sha256ArtifactBytes(input), byteLength: input.byteLength });
         headers(mail, part);
         if (mail.text)
-            text(mail.text, 'text');
+            text(mail.text, 'text', `message-${part}`);
         if (mail.html)
-            html(mail.html);
+            html(mail.html, `message-${part}`);
         for (const attachment of mail.attachments) {
             if (reviewedParts >= MAX_MESSAGE_PARTS) {
                 bounds.add('Message parts');
@@ -155,8 +190,9 @@ export async function reviewMessageInput(bytes, kind, reviewedAt, qrText) {
             if (attachment.mimeType === 'message/rfc822')
                 await message(body, depth + 1, part);
             else if (attachment.mimeType === 'text/calendar') {
-                reviewedParts++;
-                calendar(new TextDecoder().decode(body));
+                const calendarPart = ++reviewedParts;
+                messageParts.push({ part: calendarPart, parentPart: part, digestSha256: await sha256ArtifactBytes(body), byteLength: body.byteLength });
+                calendar(new TextDecoder().decode(body), `message-${calendarPart}`);
             }
             else {
                 unreviewedAttachments++;
@@ -179,17 +215,21 @@ export async function reviewMessageInput(bytes, kind, reviewedAt, qrText) {
         reviewedParts = 1;
         const value = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
         if (kind === 'calendar')
-            calendar(value);
+            calendar(value, 'input');
         else
-            text(value, 'text');
+            text(value, 'text', 'input');
     }
     const result = links.result();
     for (const limitation of result.limitations)
         bounds.add(limitation);
     if (result.bounded)
         bounds.add('Link extraction');
+    const indicatorResult = indicators.result();
+    if (indicatorResult.indicatorCoverage.state === 'partial')
+        bounds.add('Indicator extraction');
+    const phoneReview = kind === 'text' ? reviewPhoneCandidates(new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes), base.source.digestSha256) : undefined;
     return { report: { ...base,
             coverage: { state: bounds.size || unreviewedAttachments || partialAuthentication ? 'partial' : 'reviewed', reviewedParts, unreviewedAttachments, rejectedLinks: result.rejected, boundsReached: [...bounds] },
             identities, authenticationClaims, authenticationReview: { headers: authenticationHeaders, omittedHeaders: omittedAuthenticationHeaders }, messageParts,
-            links: result.links, actionHints: [...actionHints] }, targets: result.targets };
+            ...indicatorResult, links: result.links, actionHints: [...actionHints] }, targets: result.targets, ...(phoneReview ? { phoneReview } : {}) };
 }