@slicedearth/whoisleuth-cli 2.4.0 → 2.6.0

package/packages/workspace/watchlist-store.mjs

@@ -1,13 +1,103 @@
 // Pure browser-local watchlist collection model. Watchlist history owns the
 // evidence shape and diff semantics; this module owns collection names, schema
 // migration, import merging, and exact serialized-byte accounting.
-import { MAX_WATCHLIST_DOMAINS, normalizeWatchlistEntry } from "./watchlist-history.mjs";
+import { MAX_WATCHLIST_DOMAINS, normalizeWatchlistEntry, compactWatchlistResults, appendWatchlistScan, mergeWatchlistBaseline, watchlistActiveDomains, assertWatchlistEditable } from "./watchlist-history.mjs";
+import { mergeWatchDomainMetadata, normalizeWatchDomainMetadata } from "./brand-candidate-workflow.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
 import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
-import { MAX_WATCHLIST_INPUTS, MAX_WATCHLIST_NAME_LENGTH, MAX_WATCHLISTS, MAX_WATCHLIST_STORE_BYTES, WATCHLIST_SCHEMA, WATCHLIST_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
+import { boundedJsonLimitsForBytes, parseBoundedJson } from "../analysis/bounded-json.mjs";
+import { MAX_WATCHLIST_INPUTS, MAX_WATCHLIST_NAME_LENGTH, MAX_WATCHLISTS, MAX_WATCHLIST_STORE_BYTES, MAX_WATCHLIST_PORTABLE_BYTES, WATCHLIST_RECOVERY_METADATA_BYTES, serialiseWorkspacePortableJson, WATCHLIST_SCHEMA, WATCHLIST_SCHEMA_VERSION, WATCHLIST_BROWSER_SUPPORTED_VERSIONS, WATCHLIST_EXPORT_SUPPORTED_VERSIONS, } from "../contracts/workspace-portability.mjs";
 export { MAX_WATCHLIST_INPUTS, MAX_WATCHLIST_NAME_LENGTH, MAX_WATCHLISTS, MAX_WATCHLIST_STORE_BYTES, WATCHLIST_SCHEMA, WATCHLIST_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
 const BLOCKED_NAMES = new Set(['__proto__', 'prototype', 'constructor']);
 const CONTROL_RE = /[\x00-\x1f\x7f]/;
+/** Restore evidence while retaining every local active member and its context. */
+export function planHostedWatchlistRestore(current, name, input) {
+    const normalizedName = normalizeWatchlistName(name);
+    if (!normalizedName)
+        throw new Error('Hosted watchlist name is invalid.');
+    const target = resolveWatchlistMutationTarget(normalizeWatchlistStore(current).watchlists, normalizedName);
+    const snapshot = normalizeWatchlistEntry(input);
+    assertWatchlistEditable(snapshot);
+    const before = new Set(target.previous ? watchlistActiveDomains(target.previous) : []);
+    const metadata = normalizeWatchDomainMetadata(target.previous?.domainMetadata, snapshot.results.map(record => record.domain));
+    const after = new Set(metadata.map(record => record.domain));
+    // Check bytes as well as membership before requesting consent.
+    assertWatchlistStoreBudget({ ...current, [target.name]: { ...snapshot, domainMetadata: metadata } });
+    return freezePreview(structuredClone({ name: target.name, previous: target.previous, snapshot,
+        retained: [...before].sort(), added: [...after].filter(domain => !before.has(domain)).sort(),
+        removed: [], capacity: MAX_WATCHLIST_DOMAINS }));
+}
+export function applyReviewedHostedWatchlistRestore(current, reviewed) {
+    const fresh = planHostedWatchlistRestore(current, reviewed.name, reviewed.snapshot);
+    if (JSON.stringify(fresh) !== JSON.stringify(reviewed))
+        throw new Error('The watchlist changed after review. Preview its membership again; nothing was overwritten.');
+    return assertWatchlistStoreBudget({ ...current, [fresh.name]: { ...fresh.snapshot,
+            domainMetadata: normalizeWatchDomainMetadata(fresh.previous?.domainMetadata, fresh.snapshot.results.map(record => record.domain)) } }).watchlists;
+}
+export function resolveWatchlistMutationTarget(current, requestedName) {
+    const existingName = Object.keys(current).find(candidate => candidate.toLowerCase() === requestedName.toLowerCase());
+    const previous = existingName ? current[existingName] ?? null : null;
+    assertWatchlistEditable(previous);
+    if (!previous && Object.keys(current).length >= MAX_WATCHLISTS)
+        throw new Error('Watchlist storage is full. Export and remove a watchlist before saving more.');
+    return { name: existingName || requestedName, previous };
+}
+function freezePreview(value) {
+    if (value && typeof value === 'object') {
+        for (const child of Object.values(value))
+            freezePreview(child);
+        Object.freeze(value);
+    }
+    return value;
+}
+/** Review current membership and the exact compact submitted evidence together. */
+export function planWatchlistUpdate(current, name, input, mode, operation) {
+    const normalizedName = normalizeWatchlistName(name);
+    if (!normalizedName)
+        throw new Error('Watchlist names must be 1–100 characters and use a safe name.');
+    if (!Array.isArray(input) || input.length > MAX_WATCHLIST_DOMAINS)
+        throw new Error(`Watchlists are limited to ${MAX_WATCHLIST_DOMAINS} domains.`);
+    if (!['fast', 'deep'].includes(mode) || !['merge', 'replace'].includes(operation))
+        throw new Error('Choose a supported Monitor update operation.');
+    const target = resolveWatchlistMutationTarget(normalizeWatchlistStore(current).watchlists, normalizedName);
+    const submitted = compactWatchlistResults(input);
+    if (!submitted.length || submitted.length !== input.length)
+        throw new Error('The selected Monitor results contain invalid or repeated domains. Nothing was saved.');
+    const before = new Set(target.previous ? watchlistActiveDomains(target.previous) : []);
+    const after = new Set([...(operation === 'merge' ? before : []), ...submitted.map(record => record.domain)]);
+    if (after.size > MAX_WATCHLIST_DOMAINS)
+        throw new Error(`The merged watchlist exceeds its ${MAX_WATCHLIST_DOMAINS}-domain limit. Existing members were preserved.`);
+    return freezePreview(structuredClone({ name: target.name, operation, mode, previous: target.previous, input: submitted,
+        retained: [...before].filter(domain => after.has(domain)).sort(),
+        added: [...after].filter(domain => !before.has(domain)).sort(),
+        removed: [...before].filter(domain => !after.has(domain)).sort() }));
+}
+/** Reconcile consent inside the transaction, then retain only bounded history. */
+export function applyReviewedWatchlistUpdate(current, reviewed, checkedAt = new Date().toISOString()) {
+    const fresh = planWatchlistUpdate(current, reviewed.name, reviewed.input, reviewed.mode, reviewed.operation);
+    if (JSON.stringify(fresh) !== JSON.stringify(reviewed))
+        throw new Error('The watchlist changed after review. Preview its membership again; nothing was overwritten.');
+    // An explicit replacement removes active membership, not retained history.
+    const selected = new Set(fresh.input.map(record => record.domain));
+    const previous = fresh.operation === 'replace' && fresh.previous
+        ? { ...fresh.previous, results: fresh.previous.results.filter(record => selected.has(record.domain)),
+            domainMetadata: fresh.previous.domainMetadata.filter(record => selected.has(record.domain)) }
+        : fresh.previous;
+    const appended = appendWatchlistScan(previous, fresh.input, { mode: fresh.mode, checkedAt });
+    let entry = appended.entry;
+    if (fresh.operation === 'merge' && fresh.previous) {
+        const records = new Map(fresh.previous.results.map(record => [record.domain, record]));
+        for (const record of fresh.input)
+            records.set(record.domain, record);
+        const results = [...records.values()];
+        entry = normalizeWatchlistEntry({ ...entry, results,
+            baseline: mergeWatchlistBaseline(fresh.previous.baseline, results),
+            domainMetadata: fresh.previous.domainMetadata });
+    }
+    const all = { ...current };
+    Object.defineProperty(all, fresh.name, { value: entry, enumerable: true, writable: true, configurable: true });
+    return { watchlists: assertWatchlistStoreBudget(all).watchlists, changes: appended.changes };
+}
 function plainRecord(value) {
     return ordinaryWorkspaceRecord(value, 'Watchlist input');
 }
@@ -39,12 +129,34 @@ export function watchlistStoreVersion(raw) {
 function defineEntry(target, name, entry) {
     Object.defineProperty(target, name, { value: entry, writable: true, enumerable: true, configurable: true });
 }
+function assertHistoricalQuality(entry, version) {
+    if (version !== null && version < 6 && Object.hasOwn(entry, 'membershipRecovery'))
+        throw new TypeError('Watchlist membership recovery requires schema 6.');
+    if (version !== null && version < 5 && Object.hasOwn(entry, 'domainMetadata')) {
+        throw new TypeError('Domain watch metadata requires Watchlist schema 5; historical evidence was not reinterpreted.');
+    }
+    if (version !== null && version < 5) {
+        const values = [entry.results, entry.baseline, ...((Array.isArray(entry.history) ? entry.history : []).map(event => plainRecord(event)?.changes))];
+        if (values.some(rows => Array.isArray(rows) && rows.some(value => {
+            const row = plainRecord(value);
+            return row && (Object.hasOwn(row, 'hasExternalPasswordForm') || row.field === 'hasExternalPasswordForm');
+        })))
+            throw new TypeError('Password-form attribution requires Watchlist schema 5; historical evidence was not reinterpreted.');
+    }
+    if (version !== 2)
+        return;
+    for (const values of [entry.results, entry.baseline]) {
+        if (Array.isArray(values) && values.some(value => plainRecord(value)?.webCollectionQuality !== undefined)) {
+            throw new TypeError('Web collection quality requires Watchlist schema 3 or later; historical evidence was not reinterpreted.');
+        }
+    }
+}
 export function normalizeWatchlistStore(raw) {
     assertWorkspaceInputGraph(raw, 'Watchlist store');
     const root = plainRecord(raw);
     if (root?.schema === WATCHLIST_SCHEMA)
         assertWorkspaceDeclaredVersion(raw, 'Watchlist store');
-    if (root?.schema === WATCHLIST_SCHEMA && watchlistStoreVersion(root) !== WATCHLIST_SCHEMA_VERSION) {
+    if (root?.schema === WATCHLIST_SCHEMA && !WATCHLIST_BROWSER_SUPPORTED_VERSIONS.includes(Number(watchlistStoreVersion(root)))) {
         throw new Error(`Watchlist schema ${String(root.version)} is unsupported; no data was changed.`);
     }
     const source = watchlistMap(raw);
@@ -56,7 +168,8 @@ export function normalizeWatchlistStore(raw) {
         const entry = plainRecord(rawEntry);
         if (!name || !entry || !Array.isArray(entry.results) || entry.results.length > MAX_WATCHLIST_DOMAINS)
             continue;
-        defineEntry(watchlists, name, normalizeWatchlistEntry(entry));
+        assertHistoricalQuality(entry, root?.schema === WATCHLIST_SCHEMA ? watchlistStoreVersion(root) : null);
+        defineEntry(watchlists, name, normalizeWatchlistEntry(entry, { recoverLegacyMembership: root?.schema === WATCHLIST_SCHEMA && root.version === 5 }));
         if (Object.keys(watchlists).length >= MAX_WATCHLISTS)
             break;
     }
@@ -67,7 +180,8 @@ function byteLength(value) {
 }
 export function assertWatchlistStoreBudget(watchlists) {
     const store = normalizeWatchlistStore(watchlists);
-    if (byteLength(JSON.stringify(store)) > MAX_WATCHLIST_STORE_BYTES) {
+    const recoveryBytes = Object.values(store.watchlists).filter(entry => entry.membershipRecovery).length * WATCHLIST_RECOVERY_METADATA_BYTES;
+    if (byteLength(JSON.stringify(store)) - recoveryBytes > MAX_WATCHLIST_STORE_BYTES) {
         throw new Error('Watchlist storage is full. Export and remove a watchlist before saving more.');
     }
     return store;
@@ -93,8 +207,8 @@ export function mergeWatchlistStores(localRaw, importedRaw) {
     if (importedVersion !== null && importedVersion > WATCHLIST_SCHEMA_VERSION) {
         throw new Error(`This watchlist file uses newer schema ${importedVersion}. Update the app before importing it.`);
     }
-    if (importedVersion !== WATCHLIST_SCHEMA_VERSION) {
-        throw new Error(`Expected a WHOISleuth watchlist export using schema ${WATCHLIST_SCHEMA_VERSION}.`);
+    if (!WATCHLIST_EXPORT_SUPPORTED_VERSIONS.includes(Number(importedVersion))) {
+        throw new Error('Expected a supported WHOISleuth watchlist export.');
     }
     const local = normalizeWatchlistStore(localRaw).watchlists;
     const source = watchlistMap(importedRaw) || {};
@@ -109,12 +223,29 @@ export function mergeWatchlistStores(localRaw, importedRaw) {
             skipped++;
             continue;
         }
-        const normalized = normalizeWatchlistEntry(entry);
+        assertHistoricalQuality(entry, importedVersion);
+        const normalized = normalizeWatchlistEntry(entry, { recoverLegacyMembership: importedVersion === 5 });
         if (Object.prototype.hasOwnProperty.call(local, name)) {
-            if (normalized.updatedAt <= local[name].updatedAt) {
-                skipped++;
+            if (normalized.membershipRecovery || local[name].membershipRecovery) {
+                if (JSON.stringify(normalized) === JSON.stringify(local[name])) {
+                    skipped++;
+                    continue;
+                }
+                throw new Error('A watchlist awaiting membership recovery cannot be merged or replaced. Import it under a separate name to preserve both records.');
+            }
+            const localTime = local[name].updatedAt;
+            const previous = local[name];
+            const domainMetadata = mergeWatchDomainMetadata(previous.domainMetadata, Object.hasOwn(entry, 'domainMetadata') ? normalized.domainMetadata : []);
+            if (!normalized.updatedAt || !localTime || normalized.updatedAt <= localTime) {
+                if (JSON.stringify(domainMetadata) !== JSON.stringify(previous.domainMetadata)) {
+                    defineEntry(local, name, { ...previous, domainMetadata });
+                    updated++;
+                }
+                else
+                    skipped++;
                 continue;
             }
+            normalized.domainMetadata = domainMetadata;
             updated++;
         }
         else if (Object.keys(local).length >= MAX_WATCHLISTS) {
@@ -135,6 +266,16 @@ export function buildWatchlistExport(watchlists, nowIso = new Date().toISOString
         schema: WATCHLIST_SCHEMA,
         version: WATCHLIST_SCHEMA_VERSION,
         exportedAt,
-        watchlists: normalizeWatchlistStore(watchlists).watchlists,
+        watchlists: assertWatchlistStoreBudget(watchlists).watchlists,
     };
 }
+/** Preserve readable formatting when it fits; maximum stores need compact JSON. */
+export function serializeWatchlistExport(watchlists, nowIso) {
+    const value = buildWatchlistExport(watchlists, nowIso);
+    const formatted = serialiseWorkspacePortableJson(value);
+    return byteLength(formatted) <= MAX_WATCHLIST_PORTABLE_BYTES ? formatted : JSON.stringify(value);
+}
+export function parseWatchlistExport(json) {
+    return parseBoundedJson(json, { label: 'Watchlist import', maximumBytes: MAX_WATCHLIST_PORTABLE_BYTES,
+        limits: boundedJsonLimitsForBytes(MAX_WATCHLIST_PORTABLE_BYTES) });
+}

package/packages/workspace/website-snapshot-model.mjs

@@ -1,4 +1,4 @@
-import { normalizeDomain } from "../cases/case-model.mjs";
+import { normalizeDomain } from "../evidence/domain-name.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
 import { validWebObservationMode } from "../evidence/lookup-target.mjs";
 import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
@@ -302,17 +302,42 @@ export function mergeWebsiteSnapshots(localRaw, incomingRaw) {
     const local = normalizeWebsiteSnapshotStore(localRaw).snapshots;
     const incoming = normalizeWebsiteSnapshotStore(incomingRaw).snapshots;
     const byId = new Map(local.map((item) => [item.id, item]));
+    const perDomain = new Map();
+    for (const item of local)
+        perDomain.set(item.domain, (perDomain.get(item.domain) ?? 0) + 1);
+    const byteLength = (value) => new TextEncoder().encode(JSON.stringify(value)).byteLength;
+    let bytes = byteLength({ schema: WEBSITE_SNAPSHOT_SCHEMA, version: WEBSITE_SNAPSHOT_SCHEMA_VERSION, snapshots: local });
+    if (bytes > MAX_WEBSITE_SNAPSHOT_STORE_BYTES)
+        throw new Error('Existing website snapshots exceed the storage limit; no imported changes were applied.');
     let added = 0;
     let updated = 0;
+    const supplied = record(incomingRaw)?.snapshots;
+    let skipped = Array.isArray(supplied) ? supplied.length - incoming.length : 0;
+    let capacitySkipped = 0;
     for (const item of incoming) {
-        if (byId.has(item.id))
+        const existing = byId.get(item.id);
+        const targetCount = (perDomain.get(item.domain) ?? 0) - Number(existing?.domain === item.domain);
+        const delta = byteLength(item) - (existing ? byteLength(existing) : 0) + (!existing && byId.size ? 1 : 0);
+        if ((!existing && byId.size >= MAX_WEBSITE_SNAPSHOTS)
+            || targetCount >= MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN
+            || bytes + delta > MAX_WEBSITE_SNAPSHOT_STORE_BYTES) {
+            skipped += 1;
+            capacitySkipped += 1;
+            continue;
+        }
+        if (existing) {
             updated += 1;
+            perDomain.set(existing.domain, (perDomain.get(existing.domain) ?? 1) - 1);
+        }
         else
             added += 1;
         byId.set(item.id, item);
+        perDomain.set(item.domain, (perDomain.get(item.domain) ?? 0) + 1);
+        bytes += delta;
     }
     const snapshots = normalizeWebsiteSnapshotStore([...byId.values()]).snapshots;
-    return { snapshots, added, updated, skipped: Math.max(0, incoming.length - added - updated) };
+    return { snapshots, added, updated, skipped,
+        reason: capacitySkipped ? `${capacitySkipped} imported website snapshots did not fit the remaining per-domain or total storage capacity. Existing local snapshots are preserved.` : '' };
 }
 const MAX_RECONCILED_WEBSITE_SOURCES = 16;
 const UNAVAILABLE_WEBSITE_SOURCE_STATES = new Set([

package/packages/workspace/workspace-archive-crypto.mjs

@@ -2,7 +2,7 @@
 // This protects a downloaded file while it is locked. It does not encrypt the
 // active IndexedDB workspace or defend an unlocked page from same-origin code.
 import { WORKSPACE_ARCHIVE_SCHEMA, WORKSPACE_ARCHIVE_VERSION, SUPPORTED_WORKSPACE_ARCHIVE_VERSIONS, isSupportedWorkspaceArchiveVersion, readWorkspaceArchive, } from "./workspace-archive.mjs";
-import { boundedJsonLimitsForBytes, parseBoundedJson } from "../../lib/bounded-json.mjs";
+import { boundedJsonLimitsForBytes, parseBoundedJson } from "../analysis/bounded-json.mjs";
 import { ENCRYPTED_WORKSPACE_ARCHIVE_SCHEMA, ENCRYPTED_WORKSPACE_ARCHIVE_VERSION, MAX_ENCRYPTED_WORKSPACE_ARCHIVE_BYTES, MAX_WORKSPACE_ARCHIVE_BYTES, MAX_WORKSPACE_ARCHIVE_PASSPHRASE_BYTES, MIN_WORKSPACE_ARCHIVE_PASSPHRASE_CHARACTERS, WORKSPACE_ARCHIVE_PBKDF2_ITERATIONS, } from "../contracts/case-portability.mjs";
 import { assertWorkspaceInputGraph } from "./hostile-input.mjs";
 import { AES_KEY_BITS, arrayBuffer, assertPassphrase, cryptoProvider, deriveArchiveKey } from "../evidence/passphrase-encryption.mjs";

package/packages/workspace/workspace-archive.mjs

@@ -1,7 +1,8 @@
 // Pure, bounded workspace archive composition. Existing typed store models remain the
 // authority for normalization and merge semantics; this module only packages
 // their portable contracts, verifies integrity, and previews conflicts.
-import { buildCaseExport, enforceStoreBudget, mergeCases, } from "../cases/case-model.mjs";
+import { buildCaseExport, enforceStoreBudget } from "../cases/case-storage-model.mjs";
+import { mergeCases } from "../cases/case-migration-model.mjs";
 import { assertBrandProfileStoreBudget, buildBrandProfileExport, mergeBrandProfiles, normalizeBrandProfileId, } from "./brand-profile-model.mjs";
 import { assertCampaignStoreBudget, buildCampaignExport, mergeCampaigns, } from "./campaign-model.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
@@ -584,15 +585,14 @@ function previewVerifiedWorkspaceArchive(archive, localInput, options) {
             results.push({
                 ...section,
                 status: 'ready',
-                reason: definition.id === 'settings' && selected && (result.skipped ?? 0) > 0
-                    ? result.reason ?? 'The imported workspace preference was skipped. Existing browser-local context is preserved.'
-                    : section.reason,
+                reason: selected && result.reason ? result.reason : section.reason,
                 added: result.added ?? 0,
                 updated: result.updated ?? 0,
                 skipped: result.skipped ?? 0,
                 pruned: result.pruned ?? 0,
                 brandProfileReferencesOmitted: result.brandProfileReferencesOmitted ?? 0,
                 authoredHistoryOmitted: result.authoredHistoryOmitted ?? 0,
+                evidenceHistoryOmitted: result.evidenceHistoryOmitted ?? 0,
                 selected,
                 normalizedSettings: result.settings || null,
             });
@@ -633,6 +633,7 @@ function previewVerifiedWorkspaceArchive(archive, localInput, options) {
                 pruned: 0,
                 brandProfileReferencesOmitted: 0,
                 authoredHistoryOmitted: 0,
+                evidenceHistoryOmitted: 0,
                 selected: false,
                 normalizedSettings: dependentId === 'settings' ? null : dependentSection.normalizedSettings ?? null,
             };