@slicedearth/whoisleuth-cli 2.1.0 → 2.3.0

package/packages/workspace/website-snapshot-model.mjs

@@ -296,14 +296,89 @@ export function mergeWebsiteSnapshots(localRaw, incomingRaw) {
     const snapshots = normalizeWebsiteSnapshotStore([...byId.values()]).snapshots;
     return { snapshots, added, updated, skipped: Math.max(0, incoming.length - added - updated) };
 }
-function compareMap(field, before, after) {
+const MAX_RECONCILED_WEBSITE_SOURCES = 16;
+const UNAVAILABLE_WEBSITE_SOURCE_STATES = new Set([
+    'blocked', 'error', 'not_found', 'rate_limited', 'unavailable', 'unsupported',
+]);
+export function canonicalWebsiteSourceState(value) {
+    return value.trim().toLowerCase().replace(/[\s-]+/gu, '_');
+}
+function websiteSourceStateSeverity(value) {
+    if (UNAVAILABLE_WEBSITE_SOURCE_STATES.has(value))
+        return 2;
+    return ['complete', 'success'].includes(value) ? 0 : 1;
+}
+export function reconciledWebsiteSources(snapshot) {
+    const selected = new Map();
+    for (const sourceEntry of snapshot.sources.slice(0, MAX_RECONCILED_WEBSITE_SOURCES)) {
+        const sourceId = sourceEntry.source.trim().toLowerCase();
+        if (!sourceId)
+            continue;
+        const state = canonicalWebsiteSourceState(sourceEntry.state);
+        const severity = websiteSourceStateSeverity(state);
+        const current = selected.get(sourceId);
+        if (!current || severity > current.severity || (severity === current.severity && state.localeCompare(current.state) < 0)) {
+            selected.set(sourceId, { state, severity });
+        }
+    }
+    return [...selected.entries()]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([sourceName, value]) => ({ source: sourceName, state: value.state }));
+}
+export function websiteWithReconciledSources(snapshot) {
+    return { ...snapshot, sources: reconciledWebsiteSources(snapshot) };
+}
+export function reconciledWebsiteSourceState(snapshot, sourceId) {
+    const canonicalId = sourceId.trim().toLowerCase();
+    return reconciledWebsiteSources(snapshot)
+        .find((sourceEntry) => sourceEntry.source === canonicalId)?.state ?? null;
+}
+function completeWebsiteSourceState(value) {
+    const state = canonicalWebsiteSourceState(value);
+    return state === 'complete' || state === 'success';
+}
+export function websiteSnapshotFieldSource(field) {
+    if (field.startsWith('certificate.'))
+        return 'tls';
+    if (field.startsWith('source.'))
+        return field.slice('source.'.length);
+    if (field.startsWith('dependency.')) {
+        const recordType = field.slice('dependency.'.length).split(':')[0];
+        return recordType === 'HTTP' ? 'http' : 'dns';
+    }
+    if (['technology', 'posture', 'identity'].some((family) => field.startsWith(family)))
+        return 'http';
+    return '';
+}
+export function websiteSnapshotFieldComplete(snapshot, field) {
+    const sourceName = websiteSnapshotFieldSource(field);
+    const retainedSource = sourceName ? reconciledWebsiteSourceState(snapshot, sourceName) : null;
+    const retainedSourceComplete = !retainedSource || completeWebsiteSourceState(retainedSource);
+    if (field.startsWith('certificate.')) {
+        return Boolean(snapshot.certificate?.complete && !snapshot.certificate.truncated && retainedSourceComplete);
+    }
+    return snapshot.complete && !snapshot.truncated && retainedSourceComplete;
+}
+export function websiteSnapshotComparisonEvidenceComplete(snapshot) {
+    return websiteSnapshotFieldComplete(snapshot, 'technology')
+        && websiteSnapshotFieldComplete(snapshot, 'posture')
+        && websiteSnapshotFieldComplete(snapshot, 'identity')
+        && websiteSnapshotFieldComplete(snapshot, 'dependency.CNAME')
+        && (!snapshot.certificate || websiteSnapshotFieldComplete(snapshot, 'certificate.observation'));
+}
+function compareMap(field, before, after, options = {}) {
     const changes = [];
     for (const key of [...new Set([...before.keys(), ...after.keys()])].sort()) {
         const left = before.get(key) ?? null;
         const right = after.get(key) ?? null;
         if (left === right)
             continue;
-        changes.push({ field: `${field}.${key}`, state: left === null ? 'added' : right === null ? 'removed' : 'changed', before: left, after: right });
+        const state = left === null
+            ? options.beforeComplete?.(key) === false ? 'incomparable' : 'added'
+            : right === null
+                ? options.afterComplete?.(key) === false ? 'incomparable' : 'removed'
+                : 'changed';
+        changes.push({ field: `${field}.${key}`, state, before: left, after: right });
     }
     return changes;
 }
@@ -313,6 +388,7 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
     if (!before || !after || before.domain !== after.domain) {
         return {
             compatible: false,
+            complete: false,
             changes: [{ field: 'snapshot', state: 'incomparable', before: before?.domain ?? null, after: after?.domain ?? null }],
             dependencyTransitions: [],
         };
@@ -328,20 +404,29 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
         ? 'legacy_unknown'
         : postureVersionBefore === postureVersionAfter ? 'comparable' : 'detector_changed';
     const changes = [
-        ...(technologyComparability === 'comparable' ? compareMap('technology', new Map(before.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}|${item.roles.join(',')}`])), new Map(after.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}|${item.roles.join(',')}`]))) : [{
+        ...(technologyComparability === 'comparable' ? compareMap('technology', new Map(before.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}|${item.roles.join(',')}`])), new Map(after.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}|${item.roles.join(',')}`])), {
+            beforeComplete: () => websiteSnapshotFieldComplete(before, 'technology'),
+            afterComplete: () => websiteSnapshotFieldComplete(after, 'technology'),
+        }) : [{
                 field: 'technology.profileVersion',
                 state: technologyComparability === 'detector_changed' ? 'changed' : 'incomparable',
                 before: technologyVersionBefore === null ? null : String(technologyVersionBefore),
                 after: technologyVersionAfter === null ? null : String(technologyVersionAfter),
             }]),
-        ...(postureComparability === 'comparable' ? compareMap('posture', new Map(before.posture.map((item) => [item.id, item.state])), new Map(after.posture.map((item) => [item.id, item.state]))) : [{
+        ...(postureComparability === 'comparable' ? compareMap('posture', new Map(before.posture.map((item) => [item.id, item.state])), new Map(after.posture.map((item) => [item.id, item.state])), {
+            beforeComplete: () => websiteSnapshotFieldComplete(before, 'posture'),
+            afterComplete: () => websiteSnapshotFieldComplete(after, 'posture'),
+        }) : [{
                 field: 'posture.profileVersion',
                 state: postureComparability === 'detector_changed' ? 'changed' : 'incomparable',
                 before: postureVersionBefore === null ? null : String(postureVersionBefore),
                 after: postureVersionAfter === null ? null : String(postureVersionAfter),
             }]),
         ...compareMap('source', new Map(before.sources.map((item) => [item.source, item.state])), new Map(after.sources.map((item) => [item.source, item.state]))),
-        ...compareMap('dependency', new Map(before.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`])), new Map(after.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`]))),
+        ...compareMap('dependency', new Map(before.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`])), new Map(after.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`])), {
+            beforeComplete: (key) => websiteSnapshotFieldComplete(before, `dependency.${key}`),
+            afterComplete: (key) => websiteSnapshotFieldComplete(after, `dependency.${key}`),
+        }),
     ];
     const dependencyTransitions = [];
     const beforeDependencies = new Map(before.dependencies.map((item) => [`${item.recordType}:${item.target}`, item]));
@@ -365,10 +450,10 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
                 detail: 'An earlier active dependency is unresolved in the later retained observation. Resolver or collection failure remains possible.',
             });
         }
-        else if (!left && right) {
+        else if (!left && right && websiteSnapshotFieldComplete(before, `dependency.${key}`)) {
             dependencyTransitions.push({ target: right.target, recordType: right.recordType, state: 'added', detail: 'This dependency first appears in the later retained snapshot.' });
         }
-        else if (left && !right && after.complete) {
+        else if (left && !right && websiteSnapshotFieldComplete(after, `dependency.${key}`)) {
             dependencyTransitions.push({ target: left.target, recordType: left.recordType, state: 'removed', detail: 'This dependency is not represented in the later complete snapshot. It may have been intentionally removed or replaced.' });
         }
         if (dependencyTransitions.length >= 20)
@@ -381,7 +466,16 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
             continue;
         changes.push({ field: `identity.${key}`, state: left === null || right === null ? 'unavailable' : 'changed', before: left, after: right });
     }
-    changes.push(...compareMap('identityValues.resourceHosts', new Map(before.identityValues.resourceHosts.map((item) => [item, item])), new Map(after.identityValues.resourceHosts.map((item) => [item, item]))), ...compareMap('identityValues.trackingIdentifiers', new Map(before.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), new Map(after.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value]))), ...compareMap('identityValues.formActionOrigins', new Map(before.identityValues.formActionOrigins.map((item) => [item, item])), new Map(after.identityValues.formActionOrigins.map((item) => [item, item]))));
+    changes.push(...compareMap('identityValues.resourceHosts', new Map(before.identityValues.resourceHosts.map((item) => [item, item])), new Map(after.identityValues.resourceHosts.map((item) => [item, item])), {
+        beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.resourceHosts'),
+        afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.resourceHosts'),
+    }), ...compareMap('identityValues.trackingIdentifiers', new Map(before.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), new Map(after.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), {
+        beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.trackingIdentifiers'),
+        afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.trackingIdentifiers'),
+    }), ...compareMap('identityValues.formActionOrigins', new Map(before.identityValues.formActionOrigins.map((item) => [item, item])), new Map(after.identityValues.formActionOrigins.map((item) => [item, item])), {
+        beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.formActionOrigins'),
+        afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.formActionOrigins'),
+    }));
     if (before.certificate || after.certificate) {
         if (!before.certificate || !after.certificate) {
             changes.push({
@@ -425,6 +519,10 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
     }
     return {
         compatible: true,
+        complete: websiteSnapshotComparisonEvidenceComplete(before)
+            && websiteSnapshotComparisonEvidenceComplete(after)
+            && technologyComparability === 'comparable'
+            && postureComparability === 'comparable',
         profileComparability: {
             technology: technologyComparability,
             securityPosture: postureComparability,

package/packages/workspace/workspace-archive.mjs

@@ -528,6 +528,7 @@ export async function previewWorkspaceArchive(raw, localInput, options = {}) {
                 skipped: result.skipped ?? 0,
                 pruned: result.pruned ?? 0,
                 brandProfileReferencesOmitted: result.brandProfileReferencesOmitted ?? 0,
+                authoredHistoryOmitted: result.authoredHistoryOmitted ?? 0,
                 selected,
                 normalizedSettings: result.settings || null,
             });
@@ -567,6 +568,7 @@ export async function previewWorkspaceArchive(raw, localInput, options = {}) {
                 skipped: dependentSection.recordCount,
                 pruned: 0,
                 brandProfileReferencesOmitted: 0,
+                authoredHistoryOmitted: 0,
                 selected: false,
                 normalizedSettings: dependentId === 'settings' ? null : dependentSection.normalizedSettings ?? null,
             };

package/cli/evidence-command-arguments.mjs

@@ -1,156 +0,0 @@
-import { CliUsageError } from "./errors.mjs";
-export function parseInspectArchiveArguments(argv) {
-    let source = null;
-    let passphraseSource = null;
-    let search = null;
-    let reveal = false;
-    let requireMatch = false;
-    let expectedContentDigest = null;
-    let output = 'terminal';
-    let quiet = false;
-    let color = true;
-    for (let index = 0; index < argv.length; index++) {
-        const argument = argv[index];
-        if (argument === undefined)
-            break;
-        if (argument === '--json') {
-            if (output !== 'terminal')
-                throw new CliUsageError('--json may be supplied only once.');
-            output = 'json';
-        }
-        else if (argument === '--passphrase-file') {
-            if (passphraseSource !== null)
-                throw new CliUsageError('--passphrase-file may be supplied only once.');
-            const value = argv[++index];
-            if (!value || value.startsWith('-'))
-                throw new CliUsageError('--passphrase-file requires one bounded UTF-8 file.');
-            passphraseSource = value;
-        }
-        else if (argument === '--search') {
-            if (search !== null)
-                throw new CliUsageError('--search may be supplied only once.');
-            const value = argv[++index];
-            if (!value || value.startsWith('-'))
-                throw new CliUsageError('--search requires one exact bounded value.');
-            search = value;
-        }
-        else if (argument === '--reveal') {
-            if (reveal)
-                throw new CliUsageError('--reveal may be supplied only once.');
-            reveal = true;
-        }
-        else if (argument === '--require-match') {
-            if (requireMatch)
-                throw new CliUsageError('--require-match may be supplied only once.');
-            requireMatch = true;
-        }
-        else if (argument === '--expect-content-digest') {
-            if (expectedContentDigest !== null)
-                throw new CliUsageError('--expect-content-digest may be supplied only once.');
-            const value = argv[++index];
-            if (!value || !/^sha256:[a-f0-9]{64}$/u.test(value)) {
-                throw new CliUsageError('--expect-content-digest requires sha256 followed by 64 lowercase hexadecimal characters.');
-            }
-            expectedContentDigest = value;
-        }
-        else if (argument === '--quiet')
-            quiet = true;
-        else if (argument === '--no-color')
-            color = false;
-        else if (argument.startsWith('-'))
-            throw new CliUsageError(`Unknown option "${argument}".`);
-        else if (source === null)
-            source = argument;
-        else
-            throw new CliUsageError('inspect-archive accepts one optional JSON file. Otherwise pipe one archive on stdin.');
-    }
-    if (reveal && search === null)
-        throw new CliUsageError('--reveal requires --search.');
-    if (requireMatch && search === null)
-        throw new CliUsageError('--require-match requires --search.');
-    if (quiet && output !== 'terminal')
-        throw new CliUsageError('--quiet cannot be combined with machine-readable output.');
-    return {
-        action: 'inspect-archive',
-        source,
-        passphraseSource,
-        search,
-        reveal,
-        requireMatch,
-        expectedContentDigest,
-        output,
-        quiet,
-        color,
-    };
-}
-export function parseSignArtifactArguments(argv) {
-    let source = null;
-    let privateKeySource = null;
-    for (let index = 0; index < argv.length; index++) {
-        const argument = argv[index];
-        if (argument === undefined)
-            break;
-        if (argument === '--private-key-file') {
-            if (privateKeySource !== null)
-                throw new CliUsageError('--private-key-file may be supplied only once.');
-            const value = argv[++index];
-            if (!value || value.startsWith('-'))
-                throw new CliUsageError('--private-key-file requires one bounded PEM file.');
-            privateKeySource = value;
-        }
-        else if (argument.startsWith('-'))
-            throw new CliUsageError(`Unknown option "${argument}".`);
-        else if (source === null)
-            source = argument;
-        else
-            throw new CliUsageError('sign-artifact accepts one optional JSON file. Otherwise pipe one artefact on stdin.');
-    }
-    if (!privateKeySource)
-        throw new CliUsageError('sign-artifact requires --private-key-file.');
-    return { action: 'sign-artifact', source, privateKeySource };
-}
-export function parseVerifySignatureArguments(argv) {
-    let source = null;
-    let publicKeySource = null;
-    let output = 'terminal';
-    let quiet = false;
-    let color = true;
-    for (let index = 0; index < argv.length; index++) {
-        const argument = argv[index];
-        if (argument === undefined)
-            break;
-        if (argument === '--json') {
-            if (output !== 'terminal')
-                throw new CliUsageError('--json may be supplied only once.');
-            output = 'json';
-        }
-        else if (argument === '--public-key-file') {
-            if (publicKeySource !== null)
-                throw new CliUsageError('--public-key-file may be supplied only once.');
-            const value = argv[++index];
-            if (!value || value.startsWith('-'))
-                throw new CliUsageError('--public-key-file requires one bounded PEM file.');
-            publicKeySource = value;
-        }
-        else if (argument === '--quiet')
-            quiet = true;
-        else if (argument === '--no-color')
-            color = false;
-        else if (argument.startsWith('-'))
-            throw new CliUsageError(`Unknown option "${argument}".`);
-        else if (source === null)
-            source = argument;
-        else
-            throw new CliUsageError('verify-signature accepts one optional JSON file. Otherwise pipe one package on stdin.');
-    }
-    if (quiet && output !== 'terminal')
-        throw new CliUsageError('--quiet cannot be combined with machine-readable output.');
-    return {
-        action: 'verify-signature',
-        source,
-        publicKeySource,
-        output,
-        quiet,
-        color,
-    };
-}