@slicedearth/whoisleuth-cli 2.3.0 → 2.5.1

package/packages/workspace/watchlist-history.mjs

@@ -2,10 +2,12 @@
 // the watchlist entry for rescanning, while history stores compact changes
 // against a last-known baseline so browser-local storage does not grow by one complete
 // result set on every check.
-import { explainRiskScore, normalizeRiskModelVersion } from "../../lib/risk-scoring.mjs";
+import { explainRiskScore, normalizeRiskModelVersion } from "../analysis/risk-scoring.mjs";
 import { HTTP_SECURITY_HEADER_TOKENS, normalizeHttpSummary } from "../cases/http-summary.mjs";
-import { normalizeDomain } from "../cases/case-model.mjs";
+import { normalizeDomain } from "../evidence/domain-name.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
+import { registryDateIso } from "../evidence/registry-dates.mjs";
+import { mergeWebCollectionQuality, normalizeWebCollectionQuality, webCollectionAllowsComparison } from "../evidence/collection-quality.mjs";
 import { MAX_WATCHLIST_CHANGES_PER_EVENT, MAX_WATCHLIST_DOMAINS, MAX_WATCHLIST_HISTORY_DOMAIN_OPTIONS, MAX_WATCHLIST_HISTORY_EVENTS, MAX_WATCHLIST_INPUT_RECORDS, MAX_WATCHLIST_MUTATION_TYPES, MAX_WATCHLIST_NAMESERVERS, } from "../contracts/workspace-portability.mjs";
 export { MAX_WATCHLIST_CHANGES_PER_EVENT, MAX_WATCHLIST_DOMAINS, MAX_WATCHLIST_HISTORY_DOMAIN_OPTIONS, MAX_WATCHLIST_HISTORY_EVENTS, MAX_WATCHLIST_INPUT_RECORDS, MAX_WATCHLIST_MUTATION_TYPES, MAX_WATCHLIST_NAMESERVERS, } from "../contracts/workspace-portability.mjs";
 const MAX_TEXT_LENGTH = 300;
@@ -144,9 +146,6 @@ function boundedText(value, maximum = MAX_TEXT_LENGTH, allowNull = true) {
     const normalized = value.slice(0, maximum * 4).replace(/\s+/g, ' ').trim().slice(0, maximum).trim();
     return normalized || (allowNull ? null : '');
 }
-function isoTimestamp(value, fallback = new Date(0).toISOString()) {
-    return normalizeExplicitIsoTimestamp(value) ?? fallback;
-}
 function boundedInteger(value, maximum, fallback = 0) {
     return Number.isSafeInteger(value) && Number(value) >= 0 && Number(value) <= maximum
         ? Number(value)
@@ -155,8 +154,7 @@ function boundedInteger(value, maximum, fallback = 0) {
 function normalizeDateValue(value) {
     if (typeof value !== 'string' || value.length > MAX_TIMESTAMP_LENGTH || CONTROL_RE.test(value))
         return null;
-    const parsed = Date.parse(value);
-    return Number.isFinite(parsed) ? new Date(parsed).toISOString().slice(0, 10) : null;
+    return registryDateIso(value)?.slice(0, 10) ?? null;
 }
 function inferredScanDepth(record) {
     if (record.scanDepth === 'deep' || record.scanDepth === 'fast')
@@ -205,11 +203,14 @@ function compactRecord(value) {
     if (!domain)
         return null;
     const scanDepth = inferredScanDepth(record);
+    const webCollectionQuality = normalizeWebCollectionQuality(record.webCollectionQuality, scanDepth);
     const httpSummary = normalizeHttpSummary(record) || {};
     const providedRiskScore = typeof record.riskScore === 'number' && Number.isFinite(record.riskScore)
         ? Math.max(0, Math.min(100, Math.round(record.riskScore)))
         : null;
-    const computedRisk = scanDepth === 'deep' && providedRiskScore === null ? explainRiskScore(record) : null;
+    const computedRisk = scanDepth === 'deep' && providedRiskScore === null
+        && (!webCollectionQuality || (record.riskScore === undefined
+            && webCollectionAllowsComparison('riskScore', webCollectionQuality, scanDepth))) ? explainRiskScore(record) : null;
     const riskScore = scanDepth === 'deep' ? (providedRiskScore ?? computedRisk?.score ?? null) : null;
     const riskModelVersion = riskScore === null
         ? null
@@ -219,6 +220,7 @@ function compactRecord(value) {
     return {
         domain,
         scanDepth,
+        ...(webCollectionQuality ? { webCollectionQuality } : {}),
         availability: hasString(AVAILABILITY_VALUES, record.availability) ? record.availability : null,
         registrarName: boundedText(record.registrarName),
         nameservers: normalizeNameservers(record.nameservers),
@@ -260,6 +262,8 @@ export function compactWatchlistResults(results) {
 function isComparable(field, value, record) {
     if (DEEP_FIELDS.has(field) && record.scanDepth !== 'deep')
         return false;
+    if (!webCollectionAllowsComparison(field, record.webCollectionQuality, record.scanDepth))
+        return false;
     if (field === 'availability')
         return hasString(CONCLUSIVE_AVAILABILITY, value);
     if (field === 'nameservers')
@@ -358,7 +362,7 @@ export function mergeWatchlistBaseline(baseline, current) {
     return current.map((next) => {
         const previous = previousByDomain.get(next.domain);
         const updated = {
-            ...(previous || {}),
+            ...(previous || next),
             domain: next.domain,
             scanDepth: previous?.scanDepth === 'deep' || next.scanDepth === 'deep' ? 'deep' : 'fast',
         };
@@ -370,9 +374,12 @@ export function mergeWatchlistBaseline(baseline, current) {
         // mean "unversioned" to the comparator, but retaining the key makes the
         // stored/exported contract unambiguous and prevents consumers from having
         // to infer why a score was deliberately excluded from comparison.
-        if (next.scanDepth === 'deep' && typeof next.riskScore === 'number') {
+        if (next.scanDepth === 'deep' && typeof next.riskScore === 'number' && isComparable('riskScore', next.riskScore, next)) {
             updated.riskModelVersion = normalizeRiskModelVersion(next.riskModelVersion);
         }
+        const quality = mergeWebCollectionQuality(previous?.webCollectionQuality, next.webCollectionQuality);
+        if (quality)
+            updated.webCollectionQuality = quality;
         return updated;
     });
 }
@@ -452,12 +459,13 @@ function normalizeStoredFieldValue(field, value) {
 }
 function initialHistoryEvent(entry, baseline) {
     return {
-        checkedAt: entry.updatedAt || new Date(0).toISOString(),
+        checkedAt: entry.updatedAt,
         mode: 'saved',
         resultCount: Array.isArray(entry.results) ? entry.results.length : baseline.length,
         conclusiveCount: baseline.filter((record) => hasString(CONCLUSIVE_AVAILABILITY, record.availability)).length,
         changeCount: 0,
         omittedChanges: 0,
+        webComparisonLimitedCount: limitedWebComparisonCount(baseline),
         changes: [],
     };
 }
@@ -480,7 +488,7 @@ export function normalizeWatchlistEntry(entry) {
                 : [];
             const rawMode = event.mode;
             return {
-                checkedAt: isoTimestamp(event.checkedAt, new Date(0).toISOString()),
+                checkedAt: normalizeExplicitIsoTimestamp(event.checkedAt),
                 mode: typeof rawMode === 'string' && ['fast', 'deep', 'saved'].includes(rawMode)
                     ? rawMode
                     : 'saved',
@@ -488,12 +496,14 @@ export function normalizeWatchlistEntry(entry) {
                 conclusiveCount: boundedInteger(event.conclusiveCount, MAX_WATCHLIST_DOMAINS),
                 changeCount: boundedInteger(event.changeCount, MAX_CHANGE_COUNT, changes.length),
                 omittedChanges: boundedInteger(event.omittedChanges, MAX_CHANGE_COUNT),
+                ...(event.webComparisonLimitedCount !== undefined
+                    ? { webComparisonLimitedCount: boundedInteger(event.webComparisonLimitedCount, MAX_WATCHLIST_DOMAINS) } : {}),
                 changes,
             };
         }).slice(-MAX_WATCHLIST_HISTORY_EVENTS)
         : [];
     const normalized = {
-        updatedAt: isoTimestamp(input.updatedAt, new Date(0).toISOString()),
+        updatedAt: normalizeExplicitIsoTimestamp(input.updatedAt),
         results,
         baseline,
         history,
@@ -502,13 +512,18 @@ export function normalizeWatchlistEntry(entry) {
         normalized.history.push(initialHistoryEvent(normalized, baseline));
     return normalized;
 }
+function limitedWebComparisonCount(current, baseline = []) {
+    const previous = new Map(baseline.map(record => [record.domain, record]));
+    return current.filter(record => record.scanDepth === 'deep' && (!webCollectionAllowsComparison('riskScore', record.webCollectionQuality, record.scanDepth)
+        || (previous.has(record.domain) && !webCollectionAllowsComparison('riskScore', previous.get(record.domain).webCollectionQuality, previous.get(record.domain).scanDepth)))).length;
+}
 /**
  * @param {object | null} existingEntry
  * @param {object[]} results
  * @param {{ checkedAt?: string, mode?: string, ignoredDomains?: Set<string> }} [options]
  */
 export function appendWatchlistScan(existingEntry, results, options = {}) {
-    const checkedAt = isoTimestamp(options.checkedAt, new Date().toISOString());
+    const checkedAt = options.checkedAt === undefined ? new Date().toISOString() : normalizeExplicitIsoTimestamp(options.checkedAt);
     const mode = typeof options.mode === 'string' && ['fast', 'deep', 'saved'].includes(options.mode)
         ? options.mode
         : 'saved';
@@ -525,6 +540,7 @@ export function appendWatchlistScan(existingEntry, results, options = {}) {
         conclusiveCount: current.filter((record) => hasString(CONCLUSIVE_AVAILABILITY, record.availability)).length,
         changeCount: changes.length,
         omittedChanges: Math.max(0, changes.length - storedChanges.length),
+        webComparisonLimitedCount: limitedWebComparisonCount(current, previous?.baseline),
         changes: storedChanges,
     };
     const history = [...(previous?.history || []), event].slice(-MAX_WATCHLIST_HISTORY_EVENTS);

package/packages/workspace/watchlist-store.mjs

@@ -4,7 +4,7 @@
 import { MAX_WATCHLIST_DOMAINS, normalizeWatchlistEntry } from "./watchlist-history.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
 import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
-import { MAX_WATCHLIST_INPUTS, MAX_WATCHLIST_NAME_LENGTH, MAX_WATCHLISTS, MAX_WATCHLIST_STORE_BYTES, WATCHLIST_SCHEMA, WATCHLIST_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
+import { MAX_WATCHLIST_INPUTS, MAX_WATCHLIST_NAME_LENGTH, MAX_WATCHLISTS, MAX_WATCHLIST_STORE_BYTES, WATCHLIST_SCHEMA, WATCHLIST_SCHEMA_VERSION, WATCHLIST_BROWSER_SUPPORTED_VERSIONS, WATCHLIST_EXPORT_SUPPORTED_VERSIONS, } from "../contracts/workspace-portability.mjs";
 export { MAX_WATCHLIST_INPUTS, MAX_WATCHLIST_NAME_LENGTH, MAX_WATCHLISTS, MAX_WATCHLIST_STORE_BYTES, WATCHLIST_SCHEMA, WATCHLIST_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
 const BLOCKED_NAMES = new Set(['__proto__', 'prototype', 'constructor']);
 const CONTROL_RE = /[\x00-\x1f\x7f]/;
@@ -39,12 +39,21 @@ export function watchlistStoreVersion(raw) {
 function defineEntry(target, name, entry) {
     Object.defineProperty(target, name, { value: entry, writable: true, enumerable: true, configurable: true });
 }
+function assertHistoricalQuality(entry, version) {
+    if (version !== 2)
+        return;
+    for (const values of [entry.results, entry.baseline]) {
+        if (Array.isArray(values) && values.some(value => plainRecord(value)?.webCollectionQuality !== undefined)) {
+            throw new TypeError('Web collection quality requires Watchlist schema 3 or later; historical evidence was not reinterpreted.');
+        }
+    }
+}
 export function normalizeWatchlistStore(raw) {
     assertWorkspaceInputGraph(raw, 'Watchlist store');
     const root = plainRecord(raw);
     if (root?.schema === WATCHLIST_SCHEMA)
         assertWorkspaceDeclaredVersion(raw, 'Watchlist store');
-    if (root?.schema === WATCHLIST_SCHEMA && watchlistStoreVersion(root) !== WATCHLIST_SCHEMA_VERSION) {
+    if (root?.schema === WATCHLIST_SCHEMA && !WATCHLIST_BROWSER_SUPPORTED_VERSIONS.includes(Number(watchlistStoreVersion(root)))) {
         throw new Error(`Watchlist schema ${String(root.version)} is unsupported; no data was changed.`);
     }
     const source = watchlistMap(raw);
@@ -56,6 +65,7 @@ export function normalizeWatchlistStore(raw) {
         const entry = plainRecord(rawEntry);
         if (!name || !entry || !Array.isArray(entry.results) || entry.results.length > MAX_WATCHLIST_DOMAINS)
             continue;
+        assertHistoricalQuality(entry, root?.schema === WATCHLIST_SCHEMA ? watchlistStoreVersion(root) : null);
         defineEntry(watchlists, name, normalizeWatchlistEntry(entry));
         if (Object.keys(watchlists).length >= MAX_WATCHLISTS)
             break;
@@ -93,8 +103,8 @@ export function mergeWatchlistStores(localRaw, importedRaw) {
     if (importedVersion !== null && importedVersion > WATCHLIST_SCHEMA_VERSION) {
         throw new Error(`This watchlist file uses newer schema ${importedVersion}. Update the app before importing it.`);
     }
-    if (importedVersion !== WATCHLIST_SCHEMA_VERSION) {
-        throw new Error(`Expected a WHOISleuth watchlist export using schema ${WATCHLIST_SCHEMA_VERSION}.`);
+    if (!WATCHLIST_EXPORT_SUPPORTED_VERSIONS.includes(Number(importedVersion))) {
+        throw new Error('Expected a supported WHOISleuth watchlist export.');
     }
     const local = normalizeWatchlistStore(localRaw).watchlists;
     const source = watchlistMap(importedRaw) || {};
@@ -109,15 +119,23 @@ export function mergeWatchlistStores(localRaw, importedRaw) {
             skipped++;
             continue;
         }
-        if (Object.prototype.hasOwnProperty.call(local, name))
+        assertHistoricalQuality(entry, importedVersion);
+        const normalized = normalizeWatchlistEntry(entry);
+        if (Object.prototype.hasOwnProperty.call(local, name)) {
+            const localTime = local[name].updatedAt;
+            if (!normalized.updatedAt || !localTime || normalized.updatedAt <= localTime) {
+                skipped++;
+                continue;
+            }
             updated++;
+        }
         else if (Object.keys(local).length >= MAX_WATCHLISTS) {
             skipped++;
             continue;
         }
         else
             added++;
-        defineEntry(local, name, normalizeWatchlistEntry(entry));
+        defineEntry(local, name, normalized);
     }
     return { watchlists: local, added, updated, skipped };
 }

package/packages/workspace/website-snapshot-model.mjs

@@ -1,7 +1,8 @@
-import { normalizeDomain } from "../cases/case-model.mjs";
+import { normalizeDomain } from "../evidence/domain-name.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
+import { validWebObservationMode } from "../evidence/lookup-target.mjs";
 import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
-import { MAX_WEBSITE_SNAPSHOTS, MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN, MAX_WEBSITE_SNAPSHOT_STORE_BYTES, SUPPORTED_WEBSITE_SNAPSHOT_SCHEMA_VERSIONS, WEBSITE_SNAPSHOT_SCHEMA, WEBSITE_SNAPSHOT_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
+import { MAX_WEBSITE_SNAPSHOTS, MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN, MAX_WEBSITE_SNAPSHOT_STORE_BYTES, PAGE_FINGERPRINT_PARSERS, SUPPORTED_WEBSITE_SNAPSHOT_SCHEMA_VERSIONS, WEBSITE_SNAPSHOT_SCHEMA, WEBSITE_SNAPSHOT_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
 export { MAX_WEBSITE_SNAPSHOTS, MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN, MAX_WEBSITE_SNAPSHOT_IMPORT_BYTES, MAX_WEBSITE_SNAPSHOT_STORE_BYTES, SUPPORTED_WEBSITE_SNAPSHOT_SCHEMA_VERSIONS, WEBSITE_SNAPSHOT_SCHEMA, WEBSITE_SNAPSHOT_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
 const CONTROL_RE = /[\u0000-\u001f\u007f]/u;
 const DIGEST_RE = /^[a-f0-9]{16,128}$/iu;
@@ -10,6 +11,10 @@ const SERIAL_RE = /^[a-f0-9]{1,128}$/iu;
 const SNAPSHOT_TECHNOLOGY_ROLES = new Set([
     'observed_edge', 'application_platform', 'framework_runtime', 'embedded_dependency',
 ]);
+export const WEBSITE_SNAPSHOT_COLLECTION_LIMITS = Object.freeze({
+    technologies: 40, posture: 40, sources: 16, dependencies: 20,
+    resourceHosts: 30, trackingIdentifiers: 30, formActionOrigins: 20,
+});
 function record(value) {
     return ordinaryWorkspaceRecord(value, 'Website-snapshot input');
 }
@@ -67,18 +72,26 @@ function profileVersion(value) {
         ? Number(value)
         : null;
 }
-function profileProvenance(value, legacy = false) {
+function profileProvenance(value, sourceVersion) {
+    const legacy = sourceVersion === 4;
     if (legacy) {
         return {
             technology: { version: null, state: 'legacy_unknown' },
             securityPosture: { version: null, state: 'legacy_unknown' },
+            pageFingerprint: { version: 1, state: 'known' },
         };
     }
     const item = record(value);
     const technologyItem = record(item?.technology);
     const securityPostureItem = record(item?.securityPosture);
+    const pageFingerprintItem = record(item?.pageFingerprint);
     const technologyVersion = profileVersion(technologyItem?.version);
     const securityPostureVersion = profileVersion(securityPostureItem?.version);
+    // Exact published snapshot writers 4 and 5 used fingerprint algorithm 1.
+    // Unversioned records do not receive invented historical provenance.
+    const candidateFingerprintVersion = sourceVersion === 5 ? 1 : profileVersion(pageFingerprintItem?.version);
+    const pageFingerprintVersion = candidateFingerprintVersion !== null && Object.hasOwn(PAGE_FINGERPRINT_PARSERS, candidateFingerprintVersion)
+        ? candidateFingerprintVersion : null;
     return {
         technology: {
             version: technologyVersion,
@@ -88,6 +101,7 @@ function profileProvenance(value, legacy = false) {
             version: securityPostureVersion,
             state: securityPostureVersion === null ? 'legacy_unknown' : 'known',
         },
+        pageFingerprint: { version: pageFingerprintVersion, state: pageFingerprintVersion === null ? 'legacy_unknown' : 'known' },
     };
 }
 function posture(value) {
@@ -183,8 +197,8 @@ function origin(value) {
 }
 function identityValues(value) {
     const item = record(value);
-    const resourceHosts = values(item?.resourceHosts, 30, (candidate) => normalizeDomain(candidate));
-    const trackingIdentifiers = values(item?.trackingIdentifiers, 30, (candidate) => {
+    const resourceHosts = values(item?.resourceHosts, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.resourceHosts, (candidate) => normalizeDomain(candidate));
+    const trackingIdentifiers = values(item?.trackingIdentifiers, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.trackingIdentifiers, (candidate) => {
         const tracker = record(candidate);
         const type = text(tracker?.type, 40).toLowerCase();
         const trackerValue = text(tracker?.value, 64).toUpperCase();
@@ -192,7 +206,7 @@ function identityValues(value) {
             ? { type, value: trackerValue }
             : null;
     });
-    const formActionOrigins = values(item?.formActionOrigins, 20, origin);
+    const formActionOrigins = values(item?.formActionOrigins, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.formActionOrigins, origin);
     return { resourceHosts, trackingIdentifiers, formActionOrigins };
 }
 export function normalizeWebsiteProfileSnapshot(raw, sourceVersion) {
@@ -203,20 +217,24 @@ export function normalizeWebsiteProfileSnapshot(raw, sourceVersion) {
     const id = text(value?.id, 128);
     if (!domain || !observedAt || !savedAt || !id)
         return null;
+    const acceptsWebScope = sourceVersion === undefined || sourceVersion >= 6;
+    if (acceptsWebScope && !validWebObservationMode(value?.webObservationMode))
+        return null;
     return {
         id,
         domain,
+        ...(acceptsWebScope && value?.webObservationMode === 'selected_url' ? { webObservationMode: value.webObservationMode } : {}),
         observedAt,
         savedAt,
         complete: value?.complete === true,
         truncated: value?.truncated === true,
-        profileProvenance: profileProvenance(value?.profileProvenance, sourceVersion === 4),
-        technologies: values(value?.technologies, 40, technology),
-        posture: values(value?.posture, 40, posture),
+        profileProvenance: profileProvenance(value?.profileProvenance, sourceVersion),
+        technologies: values(value?.technologies, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.technologies, technology),
+        posture: values(value?.posture, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.posture, posture),
         identity: identity(value?.identity),
         identityValues: identityValues(value?.identityValues),
-        sources: values(value?.sources, 16, source),
-        dependencies: values(value?.dependencies, 20, dependency),
+        sources: values(value?.sources, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.sources, source),
+        dependencies: values(value?.dependencies, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.dependencies, dependency),
         certificate: certificate(value?.certificate),
     };
 }
@@ -284,17 +302,42 @@ export function mergeWebsiteSnapshots(localRaw, incomingRaw) {
     const local = normalizeWebsiteSnapshotStore(localRaw).snapshots;
     const incoming = normalizeWebsiteSnapshotStore(incomingRaw).snapshots;
     const byId = new Map(local.map((item) => [item.id, item]));
+    const perDomain = new Map();
+    for (const item of local)
+        perDomain.set(item.domain, (perDomain.get(item.domain) ?? 0) + 1);
+    const byteLength = (value) => new TextEncoder().encode(JSON.stringify(value)).byteLength;
+    let bytes = byteLength({ schema: WEBSITE_SNAPSHOT_SCHEMA, version: WEBSITE_SNAPSHOT_SCHEMA_VERSION, snapshots: local });
+    if (bytes > MAX_WEBSITE_SNAPSHOT_STORE_BYTES)
+        throw new Error('Existing website snapshots exceed the storage limit; no imported changes were applied.');
     let added = 0;
     let updated = 0;
+    const supplied = record(incomingRaw)?.snapshots;
+    let skipped = Array.isArray(supplied) ? supplied.length - incoming.length : 0;
+    let capacitySkipped = 0;
     for (const item of incoming) {
-        if (byId.has(item.id))
+        const existing = byId.get(item.id);
+        const targetCount = (perDomain.get(item.domain) ?? 0) - Number(existing?.domain === item.domain);
+        const delta = byteLength(item) - (existing ? byteLength(existing) : 0) + (!existing && byId.size ? 1 : 0);
+        if ((!existing && byId.size >= MAX_WEBSITE_SNAPSHOTS)
+            || targetCount >= MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN
+            || bytes + delta > MAX_WEBSITE_SNAPSHOT_STORE_BYTES) {
+            skipped += 1;
+            capacitySkipped += 1;
+            continue;
+        }
+        if (existing) {
             updated += 1;
+            perDomain.set(existing.domain, (perDomain.get(existing.domain) ?? 1) - 1);
+        }
         else
             added += 1;
         byId.set(item.id, item);
+        perDomain.set(item.domain, (perDomain.get(item.domain) ?? 0) + 1);
+        bytes += delta;
     }
     const snapshots = normalizeWebsiteSnapshotStore([...byId.values()]).snapshots;
-    return { snapshots, added, updated, skipped: Math.max(0, incoming.length - added - updated) };
+    return { snapshots, added, updated, skipped,
+        reason: capacitySkipped ? `${capacitySkipped} imported website snapshots did not fit the remaining per-domain or total storage capacity. Existing local snapshots are preserved.` : '' };
 }
 const MAX_RECONCILED_WEBSITE_SOURCES = 16;
 const UNAVAILABLE_WEBSITE_SOURCE_STATES = new Set([
@@ -366,6 +409,13 @@ export function websiteSnapshotComparisonEvidenceComplete(snapshot) {
         && websiteSnapshotFieldComplete(snapshot, 'dependency.CNAME')
         && (!snapshot.certificate || websiteSnapshotFieldComplete(snapshot, 'certificate.observation'));
 }
+export function websiteSnapshotProfileComparability(before, after, profile) {
+    const left = before.profileProvenance[profile].version;
+    const right = after.profileProvenance[profile].version;
+    if (left === null || right === null)
+        return 'legacy_unknown';
+    return left === right ? 'comparable' : 'detector_changed';
+}
 function compareMap(field, before, after, options = {}) {
     const changes = [];
     for (const key of [...new Set([...before.keys(), ...after.keys()])].sort()) {
@@ -394,15 +444,13 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
         };
     }
     const technologyVersionBefore = before.profileProvenance.technology.version;
+    const selectedPage = before.webObservationMode === 'selected_url' || after.webObservationMode === 'selected_url';
     const technologyVersionAfter = after.profileProvenance.technology.version;
     const postureVersionBefore = before.profileProvenance.securityPosture.version;
     const postureVersionAfter = after.profileProvenance.securityPosture.version;
-    const technologyComparability = technologyVersionBefore === null || technologyVersionAfter === null
-        ? 'legacy_unknown'
-        : technologyVersionBefore === technologyVersionAfter ? 'comparable' : 'detector_changed';
-    const postureComparability = postureVersionBefore === null || postureVersionAfter === null
-        ? 'legacy_unknown'
-        : postureVersionBefore === postureVersionAfter ? 'comparable' : 'detector_changed';
+    const technologyComparability = websiteSnapshotProfileComparability(before, after, 'technology');
+    const postureComparability = websiteSnapshotProfileComparability(before, after, 'securityPosture');
+    const fingerprintComparability = websiteSnapshotProfileComparability(before, after, 'pageFingerprint');
     const changes = [
         ...(technologyComparability === 'comparable' ? compareMap('technology', new Map(before.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}|${item.roles.join(',')}`])), new Map(after.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}|${item.roles.join(',')}`])), {
             beforeComplete: () => websiteSnapshotFieldComplete(before, 'technology'),
@@ -462,20 +510,30 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
     for (const key of Object.keys(before.identity)) {
         const left = before.identity[key];
         const right = after.identity[key];
+        if (key !== 'faviconHash' && fingerprintComparability !== 'comparable')
+            continue;
         if (left === right)
             continue;
         changes.push({ field: `identity.${key}`, state: left === null || right === null ? 'unavailable' : 'changed', before: left, after: right });
     }
-    changes.push(...compareMap('identityValues.resourceHosts', new Map(before.identityValues.resourceHosts.map((item) => [item, item])), new Map(after.identityValues.resourceHosts.map((item) => [item, item])), {
-        beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.resourceHosts'),
-        afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.resourceHosts'),
-    }), ...compareMap('identityValues.trackingIdentifiers', new Map(before.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), new Map(after.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), {
-        beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.trackingIdentifiers'),
-        afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.trackingIdentifiers'),
-    }), ...compareMap('identityValues.formActionOrigins', new Map(before.identityValues.formActionOrigins.map((item) => [item, item])), new Map(after.identityValues.formActionOrigins.map((item) => [item, item])), {
-        beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.formActionOrigins'),
-        afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.formActionOrigins'),
-    }));
+    if (fingerprintComparability !== 'comparable') {
+        changes.push({
+            field: 'identity.profileVersion', state: 'incomparable',
+            before: before.profileProvenance.pageFingerprint.version?.toString() ?? null,
+            after: after.profileProvenance.pageFingerprint.version?.toString() ?? null,
+        });
+    }
+    else
+        changes.push(...compareMap('identityValues.resourceHosts', new Map(before.identityValues.resourceHosts.map((item) => [item, item])), new Map(after.identityValues.resourceHosts.map((item) => [item, item])), {
+            beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.resourceHosts'),
+            afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.resourceHosts'),
+        }), ...compareMap('identityValues.trackingIdentifiers', new Map(before.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), new Map(after.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), {
+            beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.trackingIdentifiers'),
+            afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.trackingIdentifiers'),
+        }), ...compareMap('identityValues.formActionOrigins', new Map(before.identityValues.formActionOrigins.map((item) => [item, item])), new Map(after.identityValues.formActionOrigins.map((item) => [item, item])), {
+            beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.formActionOrigins'),
+            afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.formActionOrigins'),
+        }));
     if (before.certificate || after.certificate) {
         if (!before.certificate || !after.certificate) {
             changes.push({
@@ -519,15 +577,22 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
     }
     return {
         compatible: true,
-        complete: websiteSnapshotComparisonEvidenceComplete(before)
+        complete: !selectedPage && websiteSnapshotComparisonEvidenceComplete(before)
             && websiteSnapshotComparisonEvidenceComplete(after)
             && technologyComparability === 'comparable'
-            && postureComparability === 'comparable',
+            && postureComparability === 'comparable'
+            && fingerprintComparability === 'comparable',
         profileComparability: {
             technology: technologyComparability,
             securityPosture: postureComparability,
+            pageFingerprint: fingerprintComparability,
         },
-        changes,
-        dependencyTransitions,
+        changes: selectedPage ? [
+            { field: 'web.collectionTarget', state: 'incomparable', before: before.webObservationMode ?? 'homepage', after: after.webObservationMode ?? 'homepage' },
+            ...changes.map((change) => change.field.startsWith('certificate.') || change.field.startsWith('source.') || change.field === 'completeness'
+                || /^dependency\.(?:CNAME|HTTPS|NS|MX)(?::|$)/u.test(change.field)
+                ? change : { ...change, state: 'incomparable' }),
+        ] : changes,
+        dependencyTransitions: selectedPage ? dependencyTransitions.filter((transition) => transition.recordType !== 'HTTP') : dependencyTransitions,
     };
 }

package/packages/workspace/workspace-archive-crypto.mjs

@@ -2,13 +2,13 @@
 // This protects a downloaded file while it is locked. It does not encrypt the
 // active IndexedDB workspace or defend an unlocked page from same-origin code.
 import { WORKSPACE_ARCHIVE_SCHEMA, WORKSPACE_ARCHIVE_VERSION, SUPPORTED_WORKSPACE_ARCHIVE_VERSIONS, isSupportedWorkspaceArchiveVersion, readWorkspaceArchive, } from "./workspace-archive.mjs";
-import { parseBoundedJson } from "../../lib/bounded-json.mjs";
+import { boundedJsonLimitsForBytes, parseBoundedJson } from "../analysis/bounded-json.mjs";
 import { ENCRYPTED_WORKSPACE_ARCHIVE_SCHEMA, ENCRYPTED_WORKSPACE_ARCHIVE_VERSION, MAX_ENCRYPTED_WORKSPACE_ARCHIVE_BYTES, MAX_WORKSPACE_ARCHIVE_BYTES, MAX_WORKSPACE_ARCHIVE_PASSPHRASE_BYTES, MIN_WORKSPACE_ARCHIVE_PASSPHRASE_CHARACTERS, WORKSPACE_ARCHIVE_PBKDF2_ITERATIONS, } from "../contracts/case-portability.mjs";
 import { assertWorkspaceInputGraph } from "./hostile-input.mjs";
+import { AES_KEY_BITS, arrayBuffer, assertPassphrase, cryptoProvider, deriveArchiveKey } from "../evidence/passphrase-encryption.mjs";
 export { ENCRYPTED_WORKSPACE_ARCHIVE_SCHEMA, ENCRYPTED_WORKSPACE_ARCHIVE_VERSION, MAX_ENCRYPTED_WORKSPACE_ARCHIVE_BYTES, MAX_WORKSPACE_ARCHIVE_PASSPHRASE_BYTES, MIN_WORKSPACE_ARCHIVE_PASSPHRASE_CHARACTERS, WORKSPACE_ARCHIVE_PBKDF2_ITERATIONS, };
 const SALT_BYTES = 16;
 const IV_BYTES = 12;
-const AES_KEY_BITS = 256;
 const AES_GCM_TAG_BITS = 128;
 const MAX_CIPHERTEXT_BASE64URL_CHARACTERS = Math.ceil((MAX_WORKSPACE_ARCHIVE_BYTES + AES_GCM_TAG_BITS / 8) * 4 / 3);
 const BASE64URL_RE = /^[A-Za-z0-9_-]+$/;
@@ -25,25 +25,6 @@ function hasExactKeys(value, expected) {
     const keys = Object.keys(value);
     return keys.length === expected.length && keys.every((key) => expected.includes(key));
 }
-function arrayBuffer(bytes) {
-    return bytes.slice().buffer;
-}
-function assertPassphrase(passphrase) {
-    if (typeof passphrase !== 'string') {
-        throw new Error(`Use a backup passphrase with at least ${MIN_WORKSPACE_ARCHIVE_PASSPHRASE_CHARACTERS} characters.`);
-    }
-    if (passphrase.length > MAX_WORKSPACE_ARCHIVE_PASSPHRASE_BYTES) {
-        throw new Error('Backup passphrases are limited to 1024 UTF-8 bytes.');
-    }
-    if (Array.from(passphrase).length < MIN_WORKSPACE_ARCHIVE_PASSPHRASE_CHARACTERS) {
-        throw new Error(`Use a backup passphrase with at least ${MIN_WORKSPACE_ARCHIVE_PASSPHRASE_CHARACTERS} characters.`);
-    }
-    const bytes = encoder.encode(passphrase);
-    if (bytes.byteLength > MAX_WORKSPACE_ARCHIVE_PASSPHRASE_BYTES) {
-        throw new Error('Backup passphrases are limited to 1024 UTF-8 bytes.');
-    }
-    return bytes;
-}
 function base64url(bytes) {
     let encoded = '';
     for (let index = 0; index < bytes.length; index += 3) {
@@ -112,7 +93,7 @@ function validTimestamp(value) {
     return Number.isFinite(timestamp) && new Date(timestamp).toISOString() === value;
 }
 function validateEnvelope(raw) {
-    assertWorkspaceInputGraph(raw, 'Encrypted workspace archive');
+    assertWorkspaceInputGraph(raw, 'Encrypted workspace archive', { maximumBytes: MAX_ENCRYPTED_WORKSPACE_ARCHIVE_BYTES });
     const value = record(raw);
     if (!value || value.schema !== ENCRYPTED_WORKSPACE_ARCHIVE_SCHEMA) {
         throw new Error('This file is not an encrypted WHOISleuth workspace archive.');
@@ -197,26 +178,6 @@ function validateEnvelope(raw) {
         ciphertext,
     };
 }
-function cryptoProvider(provider = globalThis.crypto) {
-    if (!provider
-        || typeof provider.getRandomValues !== 'function'
-        || typeof provider.subtle?.importKey !== 'function'
-        || typeof provider.subtle?.deriveKey !== 'function'
-        || typeof provider.subtle?.encrypt !== 'function'
-        || typeof provider.subtle?.decrypt !== 'function') {
-        throw new Error('Encrypted workspace archives are unavailable in this browser.');
-    }
-    return provider;
-}
-async function deriveArchiveKey(provider, passphraseBytes, salt, usages) {
-    const material = await provider.subtle.importKey('raw', arrayBuffer(passphraseBytes), 'PBKDF2', false, ['deriveKey']);
-    return provider.subtle.deriveKey({
-        name: 'PBKDF2',
-        hash: 'SHA-256',
-        salt: arrayBuffer(salt),
-        iterations: WORKSPACE_ARCHIVE_PBKDF2_ITERATIONS,
-    }, material, { name: 'AES-GCM', length: AES_KEY_BITS }, false, usages);
-}
 export function isEncryptedWorkspaceArchive(raw) {
     return record(raw)?.schema === ENCRYPTED_WORKSPACE_ARCHIVE_SCHEMA;
 }
@@ -227,15 +188,19 @@ export function inspectEncryptedWorkspaceArchive(raw) {
 }
 /** Encrypt one already-built workspace archive using browser-native Web Crypto. */
 export async function encryptWorkspaceArchive(archive, passphrase, provider) {
-    await readWorkspaceArchive(archive);
-    const archiveVersion = record(archive)?.version;
-    if (!isSupportedWorkspaceArchiveVersion(archiveVersion)) {
-        throw new Error('The workspace archive declares an unsupported content contract.');
-    }
+    assertWorkspaceInputGraph(archive, 'Workspace archive', { maximumBytes: MAX_WORKSPACE_ARCHIVE_BYTES });
     const plaintext = JSON.stringify(archive);
     const plaintextBytes = encoder.encode(plaintext);
     if (plaintextBytes.byteLength > MAX_WORKSPACE_ARCHIVE_BYTES) {
-        throw new Error('Workspace archives are limited to 10 MiB. Export smaller collections separately before trying again.');
+        throw new Error(`Workspace archives are limited to ${MAX_WORKSPACE_ARCHIVE_BYTES / 1024 / 1024} MiB. Export smaller collections separately before trying again.`);
+    }
+    // Validation and encryption consume the same snapshot, including when the
+    // caller changes the original object while checksum verification is pending.
+    const snapshot = JSON.parse(plaintext);
+    await readWorkspaceArchive(snapshot);
+    const archiveVersion = record(snapshot)?.version;
+    if (!isSupportedWorkspaceArchiveVersion(archiveVersion)) {
+        throw new Error('The workspace archive declares an unsupported content contract.');
     }
     const crypto = cryptoProvider(provider);
     const passphraseBytes = assertPassphrase(passphrase);
@@ -278,30 +243,45 @@ export async function encryptWorkspaceArchive(archive, passphrase, provider) {
 }
 /** Decrypt and parse one encrypted archive. The ordinary archive reader remains authoritative. */
 export async function decryptWorkspaceArchive(raw, passphrase, provider) {
+    return (await decryptWorkspaceArchiveWithMetadata(raw, passphrase, provider)).archive;
+}
+/** Authenticate one envelope and return its inner JSON with verified byte metadata. */
+export async function decryptWorkspaceArchiveWithMetadata(raw, passphrase, provider) {
     const { envelope, salt, iv, ciphertext } = validateEnvelope(raw);
     const crypto = cryptoProvider(provider);
     const passphraseBytes = assertPassphrase(passphrase);
     const { ciphertext: _ciphertext, ...metadata } = envelope;
     try {
-        const key = await deriveArchiveKey(crypto, passphraseBytes, salt, ['decrypt']);
-        const plaintext = await crypto.subtle.decrypt({
-            name: 'AES-GCM',
-            iv: arrayBuffer(iv),
-            additionalData: arrayBuffer(encoder.encode(canonicalMetadata(metadata))),
-            tagLength: AES_GCM_TAG_BITS,
-        }, key, arrayBuffer(ciphertext));
+        let plaintext;
+        try {
+            const key = await deriveArchiveKey(crypto, passphraseBytes, salt, ['decrypt']);
+            plaintext = await crypto.subtle.decrypt({
+                name: 'AES-GCM',
+                iv: arrayBuffer(iv),
+                additionalData: arrayBuffer(encoder.encode(canonicalMetadata(metadata))),
+                tagLength: AES_GCM_TAG_BITS,
+            }, key, arrayBuffer(ciphertext));
+        }
+        catch {
+            // Authentication failure cannot distinguish a wrong key from tampering.
+            throw new Error('The backup passphrase is incorrect or the encrypted file is corrupted.');
+        }
         if (plaintext.byteLength > MAX_WORKSPACE_ARCHIVE_BYTES) {
             throw new Error('The decrypted workspace archive exceeds its byte limit.');
         }
-        return parseBoundedJson(decoder.decode(plaintext), {
+        let json;
+        try {
+            json = decoder.decode(plaintext);
+        }
+        catch {
+            throw new Error('The backup decrypted, but its contents are not valid UTF-8.');
+        }
+        const archive = parseBoundedJson(json, {
             label: 'Decrypted workspace archive',
             maximumBytes: MAX_WORKSPACE_ARCHIVE_BYTES,
+            limits: boundedJsonLimitsForBytes(MAX_WORKSPACE_ARCHIVE_BYTES),
         });
-    }
-    catch (cause) {
-        if (cause instanceof Error && cause.message === 'The decrypted workspace archive exceeds its byte limit.')
-            throw cause;
-        throw new Error('The backup passphrase is incorrect or the encrypted file is corrupted.');
+        return { archive, ciphertextBytes: ciphertext.byteLength };
     }
     finally {
         passphraseBytes.fill(0);