@slicedearth/whoisleuth-cli 2.4.0 → 2.6.0

package/packages/workspace/investigation-template-model.mjs

@@ -1,8 +1,27 @@
-import { INVESTIGATION_RECIPES, normalizeInvestigationGuideTemplateSnapshot, } from "./investigation-guide.mjs";
+import { INVESTIGATION_RECIPES, normalizeInvestigationGuideTemplateSnapshot, SAFE_TEMPLATE_ID_RE, } from "./investigation-guide.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
 import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
+import { exact, HEX_DIGEST_RE, text } from "../evidence/artifact-structure.mjs";
+import { SORTED_JSON_V2 } from "../evidence/artifact-integrity.mjs";
 import { INVESTIGATION_TEMPLATE_SCHEMA, INVESTIGATION_TEMPLATE_SUPPORTED_VERSIONS, INVESTIGATION_TEMPLATE_VERSION, MAX_INVESTIGATION_TEMPLATES, MAX_INVESTIGATION_TEMPLATE_STORE_BYTES, } from "../contracts/workspace-portability.mjs";
 export { INVESTIGATION_TEMPLATE_SCHEMA, INVESTIGATION_TEMPLATE_SUPPORTED_VERSIONS, INVESTIGATION_TEMPLATE_VERSION, MAX_INVESTIGATION_TEMPLATES, MAX_INVESTIGATION_TEMPLATE_IMPORT_BYTES, MAX_INVESTIGATION_TEMPLATE_STORE_BYTES, } from "../contracts/workspace-portability.mjs";
+export function readTemplateLessonRevision(raw) {
+    if (raw === undefined)
+        return undefined;
+    assertWorkspaceInputGraph(raw, 'Template revision provenance');
+    const value = exact(raw, ['parentTemplateId', 'parentContentSha256', 'lessonContentSha256', 'canonicalization', 'applicability', 'rationale'], 'Template revision provenance');
+    if (typeof value.parentTemplateId !== 'string' || !SAFE_TEMPLATE_ID_RE.test(value.parentTemplateId)
+        || value.canonicalization !== SORTED_JSON_V2
+        || typeof value.parentContentSha256 !== 'string' || !HEX_DIGEST_RE.test(value.parentContentSha256)
+        || typeof value.lessonContentSha256 !== 'string' || !HEX_DIGEST_RE.test(value.lessonContentSha256))
+        throw new TypeError('Template revision provenance is invalid.');
+    const applicability = text(value.applicability, 'Template applicability', 400).trim();
+    const rationale = text(value.rationale, 'Template revision reason', 400).trim();
+    if (!applicability || !rationale)
+        throw new TypeError('Template applicability and revision reason must not be empty.');
+    return { parentTemplateId: value.parentTemplateId, parentContentSha256: value.parentContentSha256, lessonContentSha256: value.lessonContentSha256,
+        canonicalization: SORTED_JSON_V2, applicability, rationale };
+}
 function record(value) {
     return ordinaryWorkspaceRecord(value, 'Investigation-template input');
 }
@@ -17,11 +36,13 @@ function template(raw) {
     const snapshot = normalizeInvestigationGuideTemplateSnapshot(value);
     const createdAt = timestamp(value?.createdAt);
     const updatedAt = timestamp(value?.updatedAt);
+    const lessonRevision = readTemplateLessonRevision(value?.lessonRevision);
     return snapshot && createdAt && updatedAt
-        ? { ...snapshot, createdAt, updatedAt }
+        ? { ...snapshot, createdAt, updatedAt, ...(lessonRevision ? { lessonRevision } : {}) }
         : null;
 }
 export function normalizeInvestigationTemplate(raw) {
+    assertWorkspaceInputGraph(raw, 'Investigation template');
     return template(raw);
 }
 export function normalizeInvestigationTemplateStore(raw) {
@@ -33,6 +54,9 @@ export function normalizeInvestigationTemplateStore(raw) {
         throw new Error(`Investigation-template schema ${String(value.version)} is unsupported; no data was changed.`);
     }
     const source = Array.isArray(raw) ? raw : Array.isArray(value?.templates) ? value.templates : [];
+    if (value?.version === 2 && source.some(candidate => record(candidate)?.lessonRevision !== undefined)) {
+        throw new Error('Template revision provenance requires schema 3; no data was changed.');
+    }
     const templates = [];
     const seen = new Set();
     for (const candidate of source.slice(0, MAX_INVESTIGATION_TEMPLATES * 2)) {

package/packages/workspace/page-baseline.mjs

@@ -2,8 +2,8 @@
 // deliberately accepts only bounded, normalized page-identity fields. Raw
 // HTML, URLs, headers, redirect inventories, and parser diagnostics never
 // cross this storage boundary.
-import { normalizeDomain } from "../cases/case-model.mjs";
-import { isInformativePerceptualHash as isInformativeFaviconHash } from "../../lib/perceptual-hash-comparison.mjs";
+import { normalizeDomain } from "../evidence/domain-name.mjs";
+import { isInformativePerceptualHash as isInformativeFaviconHash } from "../analysis/perceptual-hash-comparison.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
 import { PAGE_FINGERPRINT_TOKEN_LIMITS } from "../contracts/page-fingerprints.mjs";
 import { MAX_BASELINE_IDENTIFIERS, MAX_BASELINE_RESOURCE_HOSTS, MAX_BASELINE_TITLE_LENGTH, PAGE_BASELINE_VERSION, PAGE_FINGERPRINT_PARSERS, PAGE_IDENTITY_VERSION, } from "../contracts/workspace-portability.mjs";

package/packages/workspace/relationship-observation-model.mjs

@@ -1,7 +1,7 @@
-import { normalizeDomain } from "../cases/case-model.mjs";
+import { normalizeDomain } from "../evidence/domain-name.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
-import { RELATIONSHIP_EVIDENCE_VERSION } from "../contracts/offline-comparison.mjs";
-import { RELATIONSHIP_TYPES, qualifyRelationshipSources } from "../comparison/relationship-provenance.mjs";
+import { RELATIONSHIP_EVIDENCE_VERSION, RELATIONSHIP_SOURCE_PROVENANCE_VERSION, RELATIONSHIP_PUBLIC_ADDRESS_VERSION } from "../contracts/offline-comparison.mjs";
+import { RELATIONSHIP_TYPES, qualifyRelationshipSources, isNonPublicAddressRelationship, NON_PUBLIC_RELATIONSHIP_LIMITATION, FAVICON_RELATIONSHIP_DESCRIPTION } from "../comparison/relationship-provenance.mjs";
 import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
 import { MAX_RELATIONSHIP_OBSERVATIONS, MAX_RELATIONSHIP_OBSERVATION_DOMAINS, MAX_RELATIONSHIP_OBSERVATION_INPUTS, MAX_RELATIONSHIP_OBSERVATION_LIMITATIONS, MAX_RELATIONSHIP_OBSERVATION_STORE_BYTES, MAX_RELATIONSHIP_OBSERVATION_VALUE_LENGTH, RELATIONSHIP_OBSERVATION_SCHEMA, RELATIONSHIP_OBSERVATION_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
 export { MAX_RELATIONSHIP_OBSERVATIONS, MAX_RELATIONSHIP_OBSERVATION_DOMAINS, MAX_RELATIONSHIP_OBSERVATION_INPUTS, MAX_RELATIONSHIP_OBSERVATION_LIMITATIONS, MAX_RELATIONSHIP_OBSERVATION_STORE_BYTES, MAX_RELATIONSHIP_OBSERVATION_VALUE_LENGTH, RELATIONSHIP_OBSERVATION_SCHEMA, RELATIONSHIP_OBSERVATION_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
@@ -34,9 +34,9 @@ const TYPE_METADATA = Object.freeze({
         description: 'These pages exposed the same recognised public tracking identifier in bounded static HTML.',
     }),
     favicon: Object.freeze({
-        label: 'Similar favicon',
+        label: 'Connected favicon matches',
         method: 'Exact SHA-256 or perceptual dHash distance ≤ 6',
-        description: 'These domains used an identical or perceptually similar favicon in the retained scan.',
+        description: FAVICON_RELATIONSHIP_DESCRIPTION,
     }),
     official_asset: Object.freeze({
         label: 'Official asset host match',
@@ -234,16 +234,17 @@ export function normalizeRelationshipObservation(raw, sourceStoreVersion = RELAT
         throw new TypeError('This retained relationship uses a newer source-evidence version; no evidence was interpreted.');
     if (sourceStoreVersion === 1 && sourceVersion > 2)
         throw new TypeError('Relationship schema 1 cannot contain newer source evidence; no evidence was interpreted.');
-    const qualified = qualifyRelationshipSources(sourceVersion === RELATIONSHIP_EVIDENCE_VERSION ? value.sourceEvidence : [], domains, { truncated: value.truncated === true, type });
+    const qualified = qualifyRelationshipSources(sourceVersion >= RELATIONSHIP_SOURCE_PROVENANCE_VERSION ? value.sourceEvidence : [], domains, { truncated: value.truncated === true, type });
+    const nonPublicAddress = isNonPublicAddressRelationship(type, normalizedValue);
     return {
         id,
         type,
-        label: metadata.label,
+        label: nonPublicAddress ? 'Shared non-public DNS answer' : metadata.label,
         method: metadata.method,
         normalizedValue,
         displayValue: type === 'favicon' ? '' : normalizedValue,
         domains,
-        description: metadata.description,
+        description: nonPublicAddress ? NON_PUBLIC_RELATIONSHIP_LIMITATION : metadata.description,
         classification: 'derived',
         source: 'bulk_relationship_analysis',
         sourceVersion,
@@ -255,7 +256,7 @@ export function normalizeRelationshipObservation(raw, sourceStoreVersion = RELAT
         complete: value.complete === true && qualified.complete,
         truncated: qualified.truncated
             || (Array.isArray(value.domains) && value.domains.length > MAX_RELATIONSHIP_OBSERVATION_DOMAINS),
-        limitations: observationLimitations(value.limitations),
+        limitations: observationLimitations([...(nonPublicAddress ? [NON_PUBLIC_RELATIONSHIP_LIMITATION] : []), ...(Array.isArray(value.limitations) ? value.limitations : [])]),
     };
 }
 export function normalizeRelationshipObservationStore(raw) {
@@ -296,7 +297,10 @@ export function createRelationshipObservation(raw, options = {}) {
     const sourceVersion = Object.hasOwn(options, 'sourceVersion') ? positiveInteger(options.sourceVersion) : RELATIONSHIP_EVIDENCE_VERSION;
     if (!sourceVersion || sourceVersion > RELATIONSHIP_EVIDENCE_VERSION)
         throw new TypeError('This relationship needs a supported source-evidence version.');
-    const qualified = qualifyRelationshipSources(sourceVersion === RELATIONSHIP_EVIDENCE_VERSION ? raw.sourceEvidence : [], domains, { truncated: options.truncated === true, type });
+    const nonPublicAddress = isNonPublicAddressRelationship(type, normalizedValue);
+    if (nonPublicAddress && sourceVersion >= RELATIONSHIP_PUBLIC_ADDRESS_VERSION)
+        throw new TypeError('Non-public DNS answers cannot be retained as new public-infrastructure relationships.');
+    const qualified = qualifyRelationshipSources(sourceVersion >= RELATIONSHIP_SOURCE_PROVENANCE_VERSION ? raw.sourceEvidence : [], domains, { truncated: options.truncated === true, type });
     // Older callers can supply their original scan timestamp. It does not
     // qualify the missing source metadata, and an explicit current projection
     // never substitutes that timestamp for unknown contributing-source times.
@@ -308,12 +312,12 @@ export function createRelationshipObservation(raw, options = {}) {
     return {
         id: observationId(canonical),
         type,
-        label: metadata.label,
+        label: nonPublicAddress ? 'Shared non-public DNS answer' : metadata.label,
         method: metadata.method,
         normalizedValue,
         displayValue: type === 'favicon' ? '' : normalizedValue,
         domains,
-        description: metadata.description,
+        description: nonPublicAddress ? NON_PUBLIC_RELATIONSHIP_LIMITATION : metadata.description,
         classification: 'derived',
         source: 'bulk_relationship_analysis',
         sourceVersion,
@@ -323,7 +327,7 @@ export function createRelationshipObservation(raw, options = {}) {
         complete: options.complete !== false && qualified.complete,
         truncated: qualified.truncated
             || (Array.isArray(raw.domains) && raw.domains.length > MAX_RELATIONSHIP_OBSERVATION_DOMAINS),
-        limitations: observationLimitations(options.limitations),
+        limitations: observationLimitations([...(nonPublicAddress ? [NON_PUBLIC_RELATIONSHIP_LIMITATION] : []), ...(Array.isArray(options.limitations) ? options.limitations : [])]),
     };
 }
 function byteLength(value) {

package/packages/workspace/shortlist-model.mjs

@@ -2,8 +2,8 @@
 // and downloads; this module owns schema migration, record normalization,
 // import merging, and exact serialized-byte accounting.
 import { compactWatchlistResults } from "./watchlist-history.mjs";
-import { normalizeOpportunityModelVersion } from "../../lib/opportunity-scoring.mjs";
-import { normalizeRiskModelVersion } from "../../lib/risk-scoring.mjs";
+import { normalizeOpportunityModelVersion } from "../analysis/opportunity-scoring.mjs";
+import { normalizeRiskModelVersion } from "../analysis/risk-scoring.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
 import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
 import { MAX_SHORTLIST_ENTRIES, MAX_SHORTLIST_FACTORS, MAX_SHORTLIST_INPUTS, MAX_SHORTLIST_STORE_BYTES, SHORTLIST_SCHEMA, SHORTLIST_SCHEMA_VERSION, SUPPORTED_SHORTLIST_SCHEMA_VERSIONS, } from "../contracts/workspace-portability.mjs";
@@ -69,9 +69,11 @@ export function normalizeShortlistRecord(raw, options = {}) {
     const compact = compactWatchlistResults([value])[0];
     if (!compact)
         return null;
+    // Shortlist's published schema does not retain the newer local watch evidence.
+    const { hasExternalPasswordForm: _localWatchEvidence, ...shortlistCompact } = compact;
     const riskScore = score(value.riskScore);
     return {
-        ...compact,
+        ...shortlistCompact,
         availability: typeof compact.availability === 'string' ? compact.availability : 'unknown',
         riskModelVersion: riskScore === null ? null : normalizeRiskModelVersion(value.riskModelVersion),
         riskScore,
@@ -120,12 +122,13 @@ export function serializeShortlistStore(records) {
     return JSON.stringify(assertShortlistStoreBudget(records));
 }
 /** Add, refresh, or remove an explicit bounded selection in one transaction. */
-export function setShortlistSelection(localRaw, selectedRaw, selected, nowIso = new Date().toISOString()) {
+export function setShortlistSelection(localRaw, selectedRaw, selected, nowIso = new Date().toISOString(), existing = 'refresh') {
     const local = normalizeShortlistStore(localRaw).entries;
     const byDomain = new Map(local.map((record) => [record.domain, record]));
     const candidates = Array.isArray(selectedRaw) ? selectedRaw.slice(0, MAX_SHORTLIST_INPUTS) : [];
     let added = 0;
     let updated = 0;
+    let retained = 0;
     let removed = 0;
     let skipped = Math.max(0, (Array.isArray(selectedRaw) ? selectedRaw.length : 0) - MAX_SHORTLIST_INPUTS);
     const seen = new Set();
@@ -137,6 +140,10 @@ export function setShortlistSelection(localRaw, selectedRaw, selected, nowIso =
         }
         seen.add(record.domain);
         if (selected) {
+            if (existing === 'retain' && byDomain.has(record.domain)) {
+                retained += 1;
+                continue;
+            }
             if (!byDomain.has(record.domain) && byDomain.size >= MAX_SHORTLIST_ENTRIES) {
                 skipped += 1;
                 continue;
@@ -155,6 +162,7 @@ export function setShortlistSelection(localRaw, selectedRaw, selected, nowIso =
         entries: assertShortlistStoreBudget([...byDomain.values()]).entries,
         added,
         updated,
+        retained,
         removed,
         skipped,
     };
@@ -188,14 +196,23 @@ export function mergeShortlistStores(localRaw, importedRaw) {
             skipped++;
             continue;
         }
-        if (imported.has(record.domain))
+        const prior = imported.get(record.domain);
+        if (prior)
             skipped++;
-        imported.set(record.domain, record);
+        if (!prior || Date.parse(record.savedAt) > Date.parse(prior.savedAt))
+            imported.set(record.domain, record);
     }
     let added = 0;
     let updated = 0;
     for (const record of imported.values()) {
-        if (byDomain.has(record.domain))
+        const existing = byDomain.get(record.domain);
+        // A backup must not roll back an observation. Equal-time conflicts retain
+        // the local record rather than choosing whichever file was imported last.
+        if (existing && Date.parse(record.savedAt) <= Date.parse(existing.savedAt)) {
+            skipped++;
+            continue;
+        }
+        if (existing)
             updated++;
         else if (byDomain.size >= MAX_SHORTLIST_ENTRIES) {
             skipped++;

package/packages/workspace/watchlist-history.mjs

@@ -2,12 +2,14 @@
 // the watchlist entry for rescanning, while history stores compact changes
 // against a last-known baseline so browser-local storage does not grow by one complete
 // result set on every check.
-import { explainRiskScore, normalizeRiskModelVersion } from "../../lib/risk-scoring.mjs";
+import { explainRiskScore, normalizeRiskModelVersion } from "../analysis/risk-scoring.mjs";
 import { HTTP_SECURITY_HEADER_TOKENS, normalizeHttpSummary } from "../cases/http-summary.mjs";
-import { normalizeDomain } from "../cases/case-model.mjs";
+import { normalizeDomain } from "../evidence/domain-name.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
 import { registryDateIso } from "../evidence/registry-dates.mjs";
-import { MAX_WATCHLIST_CHANGES_PER_EVENT, MAX_WATCHLIST_DOMAINS, MAX_WATCHLIST_HISTORY_DOMAIN_OPTIONS, MAX_WATCHLIST_HISTORY_EVENTS, MAX_WATCHLIST_INPUT_RECORDS, MAX_WATCHLIST_MUTATION_TYPES, MAX_WATCHLIST_NAMESERVERS, } from "../contracts/workspace-portability.mjs";
+import { normalizeWatchDomainMetadata } from "./brand-candidate-workflow.mjs";
+import { mergeWebCollectionQuality, normalizeWebCollectionQuality, webCollectionAllowsComparison } from "../evidence/collection-quality.mjs";
+import { MAX_WATCHLIST_CHANGES_PER_EVENT, MAX_WATCHLIST_DOMAINS, MAX_WATCHLIST_HISTORY_DOMAIN_OPTIONS, MAX_WATCHLIST_HISTORY_EVENTS, MAX_WATCHLIST_INPUT_RECORDS, MAX_WATCHLIST_MUTATION_TYPES, MAX_WATCHLIST_NAMESERVERS, WATCHLIST_RECOVERY_METADATA, } from "../contracts/workspace-portability.mjs";
 export { MAX_WATCHLIST_CHANGES_PER_EVENT, MAX_WATCHLIST_DOMAINS, MAX_WATCHLIST_HISTORY_DOMAIN_OPTIONS, MAX_WATCHLIST_HISTORY_EVENTS, MAX_WATCHLIST_INPUT_RECORDS, MAX_WATCHLIST_MUTATION_TYPES, MAX_WATCHLIST_NAMESERVERS, } from "../contracts/workspace-portability.mjs";
 const MAX_TEXT_LENGTH = 300;
 const MAX_TITLE_LENGTH = 200;
@@ -36,6 +38,7 @@ const DEEP_FIELDS = new Set([
     'faviconMatch',
     'faviconNearMatch',
     'hasPasswordField',
+    'hasExternalPasswordForm',
     'phishingLanguageMatch',
     'reusesOfficialAssets',
     'riskModelVersion',
@@ -67,6 +70,7 @@ const FIELD_LABELS = {
     faviconMatch: 'Official favicon match',
     faviconNearMatch: 'Official favicon near-match',
     hasPasswordField: 'Password form',
+    hasExternalPasswordForm: 'External password-form destination',
     phishingLanguageMatch: 'Phishing language',
     reusesOfficialAssets: 'Official asset reuse',
     riskScore: 'Risk score',
@@ -93,6 +97,7 @@ const HISTORY_CATEGORY_FIELDS = {
         'faviconMatch',
         'faviconNearMatch',
         'hasPasswordField',
+        'hasExternalPasswordForm',
         'phishingLanguageMatch',
         'reusesOfficialAssets',
     ]),
@@ -145,9 +150,6 @@ function boundedText(value, maximum = MAX_TEXT_LENGTH, allowNull = true) {
     const normalized = value.slice(0, maximum * 4).replace(/\s+/g, ' ').trim().slice(0, maximum).trim();
     return normalized || (allowNull ? null : '');
 }
-function isoTimestamp(value, fallback = new Date(0).toISOString()) {
-    return normalizeExplicitIsoTimestamp(value) ?? fallback;
-}
 function boundedInteger(value, maximum, fallback = 0) {
     return Number.isSafeInteger(value) && Number(value) >= 0 && Number(value) <= maximum
         ? Number(value)
@@ -205,11 +207,14 @@ function compactRecord(value) {
     if (!domain)
         return null;
     const scanDepth = inferredScanDepth(record);
+    const webCollectionQuality = normalizeWebCollectionQuality(record.webCollectionQuality, scanDepth);
     const httpSummary = normalizeHttpSummary(record) || {};
     const providedRiskScore = typeof record.riskScore === 'number' && Number.isFinite(record.riskScore)
         ? Math.max(0, Math.min(100, Math.round(record.riskScore)))
         : null;
-    const computedRisk = scanDepth === 'deep' && providedRiskScore === null ? explainRiskScore(record) : null;
+    const computedRisk = scanDepth === 'deep' && providedRiskScore === null
+        && (!webCollectionQuality || (record.riskScore === undefined
+            && webCollectionAllowsComparison('riskScore', webCollectionQuality, scanDepth))) ? explainRiskScore(record) : null;
     const riskScore = scanDepth === 'deep' ? (providedRiskScore ?? computedRisk?.score ?? null) : null;
     const riskModelVersion = riskScore === null
         ? null
@@ -219,6 +224,7 @@ function compactRecord(value) {
     return {
         domain,
         scanDepth,
+        ...(webCollectionQuality ? { webCollectionQuality } : {}),
         availability: hasString(AVAILABILITY_VALUES, record.availability) ? record.availability : null,
         registrarName: boundedText(record.registrarName),
         nameservers: normalizeNameservers(record.nameservers),
@@ -235,6 +241,7 @@ function compactRecord(value) {
         faviconMatch: typeof record.faviconMatch === 'boolean' ? record.faviconMatch : null,
         faviconNearMatch: typeof record.faviconNearMatch === 'boolean' ? record.faviconNearMatch : null,
         hasPasswordField: typeof record.hasPasswordField === 'boolean' ? record.hasPasswordField : null,
+        hasExternalPasswordForm: typeof record.hasExternalPasswordForm === 'boolean' ? record.hasExternalPasswordForm : null,
         phishingLanguageMatch: boundedText(record.phishingLanguageMatch, MAX_TITLE_LENGTH),
         reusesOfficialAssets: typeof record.reusesOfficialAssets === 'boolean' ? record.reusesOfficialAssets : null,
         riskModelVersion,
@@ -260,6 +267,8 @@ export function compactWatchlistResults(results) {
 function isComparable(field, value, record) {
     if (DEEP_FIELDS.has(field) && record.scanDepth !== 'deep')
         return false;
+    if (!webCollectionAllowsComparison(field, record.webCollectionQuality, record.scanDepth))
+        return false;
     if (field === 'availability')
         return hasString(CONCLUSIVE_AVAILABILITY, value);
     if (field === 'nameservers')
@@ -297,7 +306,7 @@ function classifyChange(field, before, after) {
         }
         return { kind: 'availability_changed', tone: 'warn' };
     }
-    if (['faviconMatch', 'faviconNearMatch', 'hasPasswordField', 'reusesOfficialAssets'].includes(field) && before === false && after === true) {
+    if (['faviconMatch', 'faviconNearMatch', 'hasPasswordField', 'hasExternalPasswordForm', 'reusesOfficialAssets'].includes(field) && before === false && after === true) {
         return { kind: 'risk_signal_added', tone: 'danger' };
     }
     if (field === 'phishingLanguageMatch' && !before && after)
@@ -358,7 +367,7 @@ export function mergeWatchlistBaseline(baseline, current) {
     return current.map((next) => {
         const previous = previousByDomain.get(next.domain);
         const updated = {
-            ...(previous || {}),
+            ...(previous || next),
             domain: next.domain,
             scanDepth: previous?.scanDepth === 'deep' || next.scanDepth === 'deep' ? 'deep' : 'fast',
         };
@@ -370,9 +379,12 @@ export function mergeWatchlistBaseline(baseline, current) {
         // mean "unversioned" to the comparator, but retaining the key makes the
         // stored/exported contract unambiguous and prevents consumers from having
         // to infer why a score was deliberately excluded from comparison.
-        if (next.scanDepth === 'deep' && typeof next.riskScore === 'number') {
+        if (next.scanDepth === 'deep' && typeof next.riskScore === 'number' && isComparable('riskScore', next.riskScore, next)) {
             updated.riskModelVersion = normalizeRiskModelVersion(next.riskModelVersion);
         }
+        const quality = mergeWebCollectionQuality(previous?.webCollectionQuality, next.webCollectionQuality);
+        if (quality)
+            updated.webCollectionQuality = quality;
         return updated;
     });
 }
@@ -452,17 +464,28 @@ function normalizeStoredFieldValue(field, value) {
 }
 function initialHistoryEvent(entry, baseline) {
     return {
-        checkedAt: entry.updatedAt || new Date(0).toISOString(),
+        checkedAt: entry.updatedAt,
         mode: 'saved',
         resultCount: Array.isArray(entry.results) ? entry.results.length : baseline.length,
         conclusiveCount: baseline.filter((record) => hasString(CONCLUSIVE_AVAILABILITY, record.availability)).length,
         changeCount: 0,
         omittedChanges: 0,
+        webComparisonLimitedCount: limitedWebComparisonCount(baseline),
         changes: [],
     };
 }
+/** Active membership excludes historical baselines and includes unobserved candidates. */
+export function watchlistActiveDomains(entry) {
+    if (entry.membershipRecovery)
+        return [];
+    return normalizeWatchDomainMetadata(entry.domainMetadata, entry.results.map(record => record.domain)).map(record => record.domain);
+}
+export function assertWatchlistEditable(entry) {
+    if (entry?.membershipRecovery)
+        throw new Error('This older watchlist is paused for membership recovery. Export its preserved records and create a separate watchlist with at most 2,000 selected domains.');
+}
 /** @param {object} entry */
-export function normalizeWatchlistEntry(entry) {
+export function normalizeWatchlistEntry(entry, options = {}) {
     const input = plainRecord(entry) || {};
     const rawResults = Array.isArray(input.results) ? input.results : [];
     const results = compactWatchlistResults(rawResults);
@@ -480,7 +503,7 @@ export function normalizeWatchlistEntry(entry) {
                 : [];
             const rawMode = event.mode;
             return {
-                checkedAt: isoTimestamp(event.checkedAt, new Date(0).toISOString()),
+                checkedAt: normalizeExplicitIsoTimestamp(event.checkedAt),
                 mode: typeof rawMode === 'string' && ['fast', 'deep', 'saved'].includes(rawMode)
                     ? rawMode
                     : 'saved',
@@ -488,31 +511,47 @@ export function normalizeWatchlistEntry(entry) {
                 conclusiveCount: boundedInteger(event.conclusiveCount, MAX_WATCHLIST_DOMAINS),
                 changeCount: boundedInteger(event.changeCount, MAX_CHANGE_COUNT, changes.length),
                 omittedChanges: boundedInteger(event.omittedChanges, MAX_CHANGE_COUNT),
+                ...(event.webComparisonLimitedCount !== undefined
+                    ? { webComparisonLimitedCount: boundedInteger(event.webComparisonLimitedCount, MAX_WATCHLIST_DOMAINS) } : {}),
                 changes,
             };
         }).slice(-MAX_WATCHLIST_HISTORY_EVENTS)
         : [];
+    const metadata = normalizeWatchDomainMetadata(input.domainMetadata);
+    const membership = new Set([...metadata.map(record => record.domain), ...results.map(record => record.domain)]);
+    const recovery = membership.size > MAX_WATCHLIST_DOMAINS
+        && (options.recoverLegacyMembership || input.membershipRecovery === 'legacy_overflow');
+    if (input.membershipRecovery !== undefined && (!recovery || input.membershipRecovery !== 'legacy_overflow'))
+        throw new TypeError('Watchlist membership recovery state is invalid.');
     const normalized = {
-        updatedAt: isoTimestamp(input.updatedAt, new Date(0).toISOString()),
+        updatedAt: normalizeExplicitIsoTimestamp(input.updatedAt),
         results,
         baseline,
         history,
+        domainMetadata: recovery ? metadata : normalizeWatchDomainMetadata(metadata, results.map(record => record.domain)),
+        ...(recovery ? WATCHLIST_RECOVERY_METADATA : {}),
     };
-    if (normalized.history.length === 0)
+    if (normalized.history.length === 0 && (results.length || baseline.length))
         normalized.history.push(initialHistoryEvent(normalized, baseline));
     return normalized;
 }
+function limitedWebComparisonCount(current, baseline = []) {
+    const previous = new Map(baseline.map(record => [record.domain, record]));
+    return current.filter(record => record.scanDepth === 'deep' && (!webCollectionAllowsComparison('riskScore', record.webCollectionQuality, record.scanDepth)
+        || (previous.has(record.domain) && !webCollectionAllowsComparison('riskScore', previous.get(record.domain).webCollectionQuality, previous.get(record.domain).scanDepth)))).length;
+}
 /**
  * @param {object | null} existingEntry
  * @param {object[]} results
  * @param {{ checkedAt?: string, mode?: string, ignoredDomains?: Set<string> }} [options]
  */
 export function appendWatchlistScan(existingEntry, results, options = {}) {
-    const checkedAt = isoTimestamp(options.checkedAt, new Date().toISOString());
+    const checkedAt = options.checkedAt === undefined ? new Date().toISOString() : normalizeExplicitIsoTimestamp(options.checkedAt);
     const mode = typeof options.mode === 'string' && ['fast', 'deep', 'saved'].includes(options.mode)
         ? options.mode
         : 'saved';
     const previous = existingEntry ? normalizeWatchlistEntry(existingEntry) : null;
+    assertWatchlistEditable(previous);
     const current = compactWatchlistResults(results);
     const changes = previous
         ? diffWatchlistBaseline(previous.baseline, current, options.ignoredDomains || new Set())
@@ -525,6 +564,7 @@ export function appendWatchlistScan(existingEntry, results, options = {}) {
         conclusiveCount: current.filter((record) => hasString(CONCLUSIVE_AVAILABILITY, record.availability)).length,
         changeCount: changes.length,
         omittedChanges: Math.max(0, changes.length - storedChanges.length),
+        webComparisonLimitedCount: limitedWebComparisonCount(current, previous?.baseline),
         changes: storedChanges,
     };
     const history = [...(previous?.history || []), event].slice(-MAX_WATCHLIST_HISTORY_EVENTS);
@@ -534,6 +574,7 @@ export function appendWatchlistScan(existingEntry, results, options = {}) {
             results: current,
             baseline: mergeWatchlistBaseline(previous?.baseline || [], current),
             history,
+            domainMetadata: normalizeWatchDomainMetadata(previous?.domainMetadata, current.map(record => record.domain)),
         },
         changes,
     };
@@ -578,6 +619,8 @@ export function watchlistHistoryDomains(entry) {
         if (record?.domain)
             current.add(record.domain);
     }
+    for (const metadata of normalized.domainMetadata)
+        current.add(metadata.domain);
     for (const event of normalized.history) {
         for (const change of event.changes) {
             if (change.domain && !current.has(change.domain))