@slicedearth/whoisleuth-cli 2.2.0 → 2.4.0

package/packages/workspace/investigation-template-model.mjs

@@ -85,6 +85,9 @@ export function saveInvestigationTemplate(localRaw, candidateRaw) {
     if (!candidate)
         throw new Error('The investigation template is incomplete or invalid.');
     const local = normalizeInvestigationTemplateStore(localRaw).templates;
+    if (!local.some((item) => item.id === candidate.id) && local.length >= MAX_INVESTIGATION_TEMPLATES) {
+        throw new Error(`Investigation-template storage is full (${MAX_INVESTIGATION_TEMPLATES} templates). Export and remove a template before saving another; no templates were changed.`);
+    }
     return normalizeInvestigationTemplateStore([
         candidate,
         ...local.filter((item) => item.id !== candidate.id),
@@ -126,11 +129,23 @@ export function mergeInvestigationTemplates(localRaw, incomingRaw) {
     const byId = new Map(local.map((item) => [item.id, item]));
     let added = 0;
     let updated = 0;
+    let skipped = 0;
     for (const item of incoming) {
-        if (byId.has(item.id))
+        const existing = byId.get(item.id);
+        if (existing) {
+            if (item.updatedAt <= existing.updatedAt) {
+                skipped += 1;
+                continue;
+            }
             updated += 1;
-        else
+        }
+        else {
+            if (byId.size >= MAX_INVESTIGATION_TEMPLATES) {
+                skipped += 1;
+                continue;
+            }
             added += 1;
+        }
         byId.set(item.id, item);
     }
     const templates = normalizeInvestigationTemplateStore([...byId.values()]).templates;
@@ -138,7 +153,7 @@ export function mergeInvestigationTemplates(localRaw, incomingRaw) {
         templates,
         added,
         updated,
-        skipped: Math.max(0, incoming.length - added - updated),
-        pruned: Math.max(0, byId.size - templates.length),
+        skipped,
+        pruned: 0,
     };
 }

package/packages/workspace/page-baseline.mjs

@@ -5,12 +5,12 @@
 import { normalizeDomain } from "../cases/case-model.mjs";
 import { isInformativePerceptualHash as isInformativeFaviconHash } from "../../lib/perceptual-hash-comparison.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
-import { MAX_BASELINE_IDENTIFIERS, MAX_BASELINE_RESOURCE_HOSTS, MAX_BASELINE_TITLE_LENGTH, PAGE_BASELINE_VERSION, PAGE_FINGERPRINT_VERSION, PAGE_IDENTITY_VERSION, } from "../contracts/workspace-portability.mjs";
-export { MAX_BASELINE_IDENTIFIERS, MAX_BASELINE_RESOURCE_HOSTS, MAX_BASELINE_TITLE_LENGTH, PAGE_BASELINE_VERSION, PAGE_FINGERPRINT_VERSION, PAGE_IDENTITY_VERSION, } from "../contracts/workspace-portability.mjs";
+import { PAGE_FINGERPRINT_TOKEN_LIMITS } from "../contracts/page-fingerprints.mjs";
+import { MAX_BASELINE_IDENTIFIERS, MAX_BASELINE_RESOURCE_HOSTS, MAX_BASELINE_TITLE_LENGTH, PAGE_BASELINE_VERSION, PAGE_FINGERPRINT_PARSERS, PAGE_IDENTITY_VERSION, } from "../contracts/workspace-portability.mjs";
+export { MAX_BASELINE_IDENTIFIERS, MAX_BASELINE_RESOURCE_HOSTS, MAX_BASELINE_TITLE_LENGTH, PAGE_BASELINE_VERSION, PAGE_FINGERPRINT_VERSION, PAGE_FINGERPRINT_PARSERS, PAGE_IDENTITY_VERSION, } from "../contracts/workspace-portability.mjs";
 const SHA256_RE = /^[a-f0-9]{64}$/i;
 const SIMHASH_RE = /^[a-f0-9]{16}$/i;
 const CONTROL_RE = /[\x00-\x1f\x7f]/;
-const MAX_HTML_TOKENS = 4096;
 const MAX_TEXT_TOKENS = 8192;
 const MAX_FORMS = 50;
 const MAX_FORM_CONTROLS = 500;
@@ -50,12 +50,12 @@ function shaComponent(raw, countKey, maximum) {
         truncated: value.truncated === true,
     };
 }
-function visibleTextComponent(raw) {
+function visibleTextComponent(raw, maximum = MAX_TEXT_TOKENS) {
     const value = record(raw);
     if (!value || value.algorithm !== 'simhash64-v1' || typeof value.value !== 'string' || !SIMHASH_RE.test(value.value))
         return null;
-    const tokenCount = count(value.tokenCount, MAX_TEXT_TOKENS);
-    const featureCount = count(value.featureCount, MAX_TEXT_TOKENS);
+    const tokenCount = count(value.tokenCount, maximum);
+    const featureCount = count(value.featureCount, maximum);
     if (tokenCount === null || featureCount === null)
         return null;
     return {
@@ -66,20 +66,21 @@ function visibleTextComponent(raw) {
         truncated: value.truncated === true,
     };
 }
-function domComponent(raw) {
+function domComponent(raw, fingerprintVersion) {
     const value = record(raw);
-    if (!value || value.algorithm !== 'sha256' || value.parser !== 'static-tag-sequence-v1')
+    if (!value || value.algorithm !== 'sha256' || value.parser !== PAGE_FINGERPRINT_PARSERS[fingerprintVersion])
         return null;
     const digest = sha256(value.value);
-    const nodeCount = count(value.nodeCount, MAX_HTML_TOKENS);
+    const maximum = PAGE_FINGERPRINT_TOKEN_LIMITS[fingerprintVersion];
+    const nodeCount = count(value.nodeCount, maximum);
     if (!digest || nodeCount === null)
         return null;
-    const similarity = value.similarity == null ? null : visibleTextComponent(value.similarity);
+    const similarity = value.similarity == null ? null : visibleTextComponent(value.similarity, maximum);
     return {
         algorithm: 'sha256',
         value: digest,
         nodeCount,
-        parser: 'static-tag-sequence-v1',
+        parser: PAGE_FINGERPRINT_PARSERS[fingerprintVersion],
         truncated: value.truncated === true,
         ...(value.similarity == null ? {} : { similarity }),
     };
@@ -175,11 +176,13 @@ export function normalizePageBaseline(raw) {
     const lookupDomain = normalizeDomain(value.lookupDomain);
     const observedAt = timestamp(value.observedAt);
     const pageIdentityVersion = count(value.pageIdentityVersion, PAGE_IDENTITY_VERSION);
-    const fingerprintVersion = count(value.fingerprintVersion, PAGE_FINGERPRINT_VERSION);
-    const normalizedHtml = shaComponent(value.normalizedHtml, 'tokenCount', MAX_HTML_TOKENS);
-    const domStructure = domComponent(value.domStructure);
+    if (typeof value.fingerprintVersion !== 'number' || !Object.hasOwn(PAGE_FINGERPRINT_PARSERS, value.fingerprintVersion))
+        return null;
+    const fingerprintVersion = value.fingerprintVersion;
+    const normalizedHtml = shaComponent(value.normalizedHtml, 'tokenCount', PAGE_FINGERPRINT_TOKEN_LIMITS[fingerprintVersion]);
+    const domStructure = domComponent(value.domStructure, fingerprintVersion);
     if (!domain || !lookupDomain || !observedAt || pageIdentityVersion !== PAGE_IDENTITY_VERSION
-        || fingerprintVersion !== PAGE_FINGERPRINT_VERSION || !normalizedHtml || !domStructure)
+        || !normalizedHtml || !domStructure)
         return null;
     const visibleText = value.visibleText == null ? null : visibleTextComponent(value.visibleText);
     const formStructure = value.formStructure == null ? null : formComponent(value.formStructure);

package/packages/workspace/relationship-observation-model.mjs

@@ -1,16 +1,11 @@
 import { normalizeDomain } from "../cases/case-model.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
+import { RELATIONSHIP_EVIDENCE_VERSION } from "../contracts/offline-comparison.mjs";
+import { RELATIONSHIP_TYPES, qualifyRelationshipSources } from "../comparison/relationship-provenance.mjs";
 import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
 import { MAX_RELATIONSHIP_OBSERVATIONS, MAX_RELATIONSHIP_OBSERVATION_DOMAINS, MAX_RELATIONSHIP_OBSERVATION_INPUTS, MAX_RELATIONSHIP_OBSERVATION_LIMITATIONS, MAX_RELATIONSHIP_OBSERVATION_STORE_BYTES, MAX_RELATIONSHIP_OBSERVATION_VALUE_LENGTH, RELATIONSHIP_OBSERVATION_SCHEMA, RELATIONSHIP_OBSERVATION_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
 export { MAX_RELATIONSHIP_OBSERVATIONS, MAX_RELATIONSHIP_OBSERVATION_DOMAINS, MAX_RELATIONSHIP_OBSERVATION_INPUTS, MAX_RELATIONSHIP_OBSERVATION_LIMITATIONS, MAX_RELATIONSHIP_OBSERVATION_STORE_BYTES, MAX_RELATIONSHIP_OBSERVATION_VALUE_LENGTH, RELATIONSHIP_OBSERVATION_SCHEMA, RELATIONSHIP_OBSERVATION_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
-export const RELATIONSHIP_OBSERVATION_TYPES = Object.freeze([
-    'nameserver_set',
-    'ip_address',
-    'certificate',
-    'tracking_identifier',
-    'favicon',
-    'official_asset',
-]);
+export const RELATIONSHIP_OBSERVATION_TYPES = RELATIONSHIP_TYPES;
 const CONTROL_RE = /[\u0000-\u001f\u007f]/u;
 const SAFE_ID_RE = /^relationship-[a-z0-9]{1,16}-[a-z0-9]{1,16}$/u;
 const SHA256_RE = /^[a-f0-9]{64}$/iu;
@@ -205,7 +200,7 @@ export function relationshipObservationId(raw) {
 }
 export function relationshipObservationStoreVersion(raw) {
     const value = record(raw);
-    return positiveInteger(value?.version);
+    return typeof value?.version === 'number' && Number.isSafeInteger(value.version) && value.version > 0 ? value.version : null;
 }
 function observationList(raw) {
     if (Array.isArray(raw))
@@ -213,7 +208,7 @@ function observationList(raw) {
     const value = record(raw);
     return Array.isArray(value?.observations) ? value.observations : [];
 }
-export function normalizeRelationshipObservation(raw) {
+export function normalizeRelationshipObservation(raw, sourceStoreVersion = RELATIONSHIP_OBSERVATION_SCHEMA_VERSION) {
     const value = record(raw);
     if (!value || typeof value.type !== 'string' || !TYPES.has(value.type))
         return null;
@@ -230,8 +225,16 @@ export function normalizeRelationshipObservation(raw) {
     const id = observationId(canonical);
     const observedAt = timestamp(value.observedAt);
     const retainedAt = timestamp(value.retainedAt);
-    if (!observedAt || !retainedAt)
+    if (!retainedAt || (value.observedAt !== null && !observedAt))
         return null;
+    const sourceVersion = positiveInteger(value.sourceVersion) || 1;
+    if (value.sourceVersion !== undefined && !positiveInteger(value.sourceVersion))
+        throw new TypeError('This retained relationship uses an unsupported source-evidence version; no evidence was interpreted.');
+    if (sourceVersion > RELATIONSHIP_EVIDENCE_VERSION)
+        throw new TypeError('This retained relationship uses a newer source-evidence version; no evidence was interpreted.');
+    if (sourceStoreVersion === 1 && sourceVersion > 2)
+        throw new TypeError('Relationship schema 1 cannot contain newer source evidence; no evidence was interpreted.');
+    const qualified = qualifyRelationshipSources(sourceVersion === RELATIONSHIP_EVIDENCE_VERSION ? value.sourceEvidence : [], domains, { truncated: value.truncated === true, type });
     return {
         id,
         type,
@@ -243,11 +246,14 @@ export function normalizeRelationshipObservation(raw) {
         description: metadata.description,
         classification: 'derived',
         source: 'bulk_relationship_analysis',
-        sourceVersion: positiveInteger(value.sourceVersion) || 1,
-        observedAt,
+        sourceVersion,
+        // Legacy scan timestamps remain available, but cannot qualify missing
+        // contributing-source timestamps or turn the old complete flag into proof.
+        observedAt: qualified.observedAt ?? (observedAt || null),
+        sourceEvidence: qualified.sourceEvidence,
         retainedAt,
-        complete: value.complete === true,
-        truncated: value.truncated === true
+        complete: value.complete === true && qualified.complete,
+        truncated: qualified.truncated
             || (Array.isArray(value.domains) && value.domains.length > MAX_RELATIONSHIP_OBSERVATION_DOMAINS),
         limitations: observationLimitations(value.limitations),
     };
@@ -255,9 +261,13 @@ export function normalizeRelationshipObservation(raw) {
 export function normalizeRelationshipObservationStore(raw) {
     assertWorkspaceInputGraph(raw, 'Relationship-observation store');
     assertWorkspaceDeclaredVersion(raw, 'Relationship-observation store');
+    const sourceStoreVersion = relationshipObservationStoreVersion(raw) ?? RELATIONSHIP_OBSERVATION_SCHEMA_VERSION;
+    if (sourceStoreVersion > RELATIONSHIP_OBSERVATION_SCHEMA_VERSION) {
+        throw new TypeError(`This relationship-observation store uses newer schema ${sourceStoreVersion}; no evidence was interpreted.`);
+    }
     const byId = new Map();
     for (const candidate of observationList(raw).slice(0, MAX_RELATIONSHIP_OBSERVATION_INPUTS)) {
-        const observation = normalizeRelationshipObservation(candidate);
+        const observation = normalizeRelationshipObservation(candidate, sourceStoreVersion);
         if (!observation)
             continue;
         const existing = byId.get(observation.id);
@@ -283,7 +293,15 @@ export function createRelationshipObservation(raw, options = {}) {
         throw new Error('That relationship does not contain a supported bounded value and at least one valid domain.');
     }
     const now = new Date().toISOString();
-    const observedAt = timestamp(options.observedAt) || now;
+    const sourceVersion = Object.hasOwn(options, 'sourceVersion') ? positiveInteger(options.sourceVersion) : RELATIONSHIP_EVIDENCE_VERSION;
+    if (!sourceVersion || sourceVersion > RELATIONSHIP_EVIDENCE_VERSION)
+        throw new TypeError('This relationship needs a supported source-evidence version.');
+    const qualified = qualifyRelationshipSources(sourceVersion === RELATIONSHIP_EVIDENCE_VERSION ? raw.sourceEvidence : [], domains, { truncated: options.truncated === true, type });
+    // Older callers can supply their original scan timestamp. It does not
+    // qualify the missing source metadata, and an explicit current projection
+    // never substitutes that timestamp for unknown contributing-source times.
+    const observedAt = qualified.observedAt
+        ?? (!Object.hasOwn(raw, 'sourceEvidence') ? timestamp(options.observedAt) || null : null);
     const retainedAt = timestamp(options.retainedAt) || now;
     const canonical = canonicalIdentity(type, normalizedValue, domains);
     const metadata = TYPE_METADATA[type];
@@ -298,11 +316,12 @@ export function createRelationshipObservation(raw, options = {}) {
         description: metadata.description,
         classification: 'derived',
         source: 'bulk_relationship_analysis',
-        sourceVersion: positiveInteger(options.sourceVersion) || 1,
+        sourceVersion,
         observedAt,
+        sourceEvidence: qualified.sourceEvidence,
         retainedAt,
-        complete: options.complete === true,
-        truncated: options.truncated === true
+        complete: options.complete !== false && qualified.complete,
+        truncated: qualified.truncated
             || (Array.isArray(raw.domains) && raw.domains.length > MAX_RELATIONSHIP_OBSERVATION_DOMAINS),
         limitations: observationLimitations(options.limitations),
     };
@@ -356,12 +375,11 @@ export function deleteRelationshipObservation(rawStore, id) {
 export function mergeRelationshipObservations(localRaw, importedRaw) {
     assertWorkspaceInputGraph(localRaw, 'Local relationship-observation store');
     assertWorkspaceInputGraph(importedRaw, 'Imported relationship-observation document');
-    assertWorkspacePortableVersion(importedRaw, RELATIONSHIP_OBSERVATION_SCHEMA_VERSION, 'Imported relationship-observation document');
+    const importedVersion = assertWorkspacePortableVersion(importedRaw, RELATIONSHIP_OBSERVATION_SCHEMA_VERSION, 'Imported relationship-observation document');
     const importedRecord = record(importedRaw);
     if (importedRecord?.schema !== undefined && importedRecord.schema !== RELATIONSHIP_OBSERVATION_SCHEMA) {
         throw new Error('This JSON section is not a WHOISleuth relationship-observation export.');
     }
-    const importedVersion = relationshipObservationStoreVersion(importedRaw);
     if (importedVersion !== null && importedVersion > RELATIONSHIP_OBSERVATION_SCHEMA_VERSION) {
         throw new Error(`This relationship-observation section uses newer schema ${importedVersion}. Update the app before importing it.`);
     }
@@ -372,7 +390,7 @@ export function mergeRelationshipObservations(localRaw, importedRaw) {
     let updated = 0;
     let skipped = Math.max(0, importedInput.length - MAX_RELATIONSHIP_OBSERVATION_INPUTS);
     for (const candidate of importedInput.slice(0, MAX_RELATIONSHIP_OBSERVATION_INPUTS)) {
-        const observation = normalizeRelationshipObservation(candidate);
+        const observation = normalizeRelationshipObservation(candidate, importedVersion ?? RELATIONSHIP_OBSERVATION_SCHEMA_VERSION);
         if (!observation) {
             skipped += 1;
             continue;

package/packages/workspace/watchlist-history.mjs

@@ -6,6 +6,7 @@ import { explainRiskScore, normalizeRiskModelVersion } from "../../lib/risk-scor
 import { HTTP_SECURITY_HEADER_TOKENS, normalizeHttpSummary } from "../cases/http-summary.mjs";
 import { normalizeDomain } from "../cases/case-model.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
+import { registryDateIso } from "../evidence/registry-dates.mjs";
 import { MAX_WATCHLIST_CHANGES_PER_EVENT, MAX_WATCHLIST_DOMAINS, MAX_WATCHLIST_HISTORY_DOMAIN_OPTIONS, MAX_WATCHLIST_HISTORY_EVENTS, MAX_WATCHLIST_INPUT_RECORDS, MAX_WATCHLIST_MUTATION_TYPES, MAX_WATCHLIST_NAMESERVERS, } from "../contracts/workspace-portability.mjs";
 export { MAX_WATCHLIST_CHANGES_PER_EVENT, MAX_WATCHLIST_DOMAINS, MAX_WATCHLIST_HISTORY_DOMAIN_OPTIONS, MAX_WATCHLIST_HISTORY_EVENTS, MAX_WATCHLIST_INPUT_RECORDS, MAX_WATCHLIST_MUTATION_TYPES, MAX_WATCHLIST_NAMESERVERS, } from "../contracts/workspace-portability.mjs";
 const MAX_TEXT_LENGTH = 300;
@@ -155,8 +156,7 @@ function boundedInteger(value, maximum, fallback = 0) {
 function normalizeDateValue(value) {
     if (typeof value !== 'string' || value.length > MAX_TIMESTAMP_LENGTH || CONTROL_RE.test(value))
         return null;
-    const parsed = Date.parse(value);
-    return Number.isFinite(parsed) ? new Date(parsed).toISOString().slice(0, 10) : null;
+    return registryDateIso(value)?.slice(0, 10) ?? null;
 }
 function inferredScanDepth(record) {
     if (record.scanDepth === 'deep' || record.scanDepth === 'fast')

package/packages/workspace/watchlist-store.mjs

@@ -109,15 +109,21 @@ export function mergeWatchlistStores(localRaw, importedRaw) {
             skipped++;
             continue;
         }
-        if (Object.prototype.hasOwnProperty.call(local, name))
+        const normalized = normalizeWatchlistEntry(entry);
+        if (Object.prototype.hasOwnProperty.call(local, name)) {
+            if (normalized.updatedAt <= local[name].updatedAt) {
+                skipped++;
+                continue;
+            }
             updated++;
+        }
         else if (Object.keys(local).length >= MAX_WATCHLISTS) {
             skipped++;
             continue;
         }
         else
             added++;
-        defineEntry(local, name, normalizeWatchlistEntry(entry));
+        defineEntry(local, name, normalized);
     }
     return { watchlists: local, added, updated, skipped };
 }

package/packages/workspace/website-snapshot-model.mjs

@@ -1,7 +1,8 @@
 import { normalizeDomain } from "../cases/case-model.mjs";
 import { normalizeExplicitIsoTimestamp } from "../evidence/observation.mjs";
+import { validWebObservationMode } from "../evidence/lookup-target.mjs";
 import { assertWorkspaceDeclaredVersion, assertWorkspaceInputGraph, assertWorkspacePortableVersion, ordinaryWorkspaceRecord } from "./hostile-input.mjs";
-import { MAX_WEBSITE_SNAPSHOTS, MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN, MAX_WEBSITE_SNAPSHOT_STORE_BYTES, SUPPORTED_WEBSITE_SNAPSHOT_SCHEMA_VERSIONS, WEBSITE_SNAPSHOT_SCHEMA, WEBSITE_SNAPSHOT_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
+import { MAX_WEBSITE_SNAPSHOTS, MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN, MAX_WEBSITE_SNAPSHOT_STORE_BYTES, PAGE_FINGERPRINT_PARSERS, SUPPORTED_WEBSITE_SNAPSHOT_SCHEMA_VERSIONS, WEBSITE_SNAPSHOT_SCHEMA, WEBSITE_SNAPSHOT_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
 export { MAX_WEBSITE_SNAPSHOTS, MAX_WEBSITE_SNAPSHOTS_PER_DOMAIN, MAX_WEBSITE_SNAPSHOT_IMPORT_BYTES, MAX_WEBSITE_SNAPSHOT_STORE_BYTES, SUPPORTED_WEBSITE_SNAPSHOT_SCHEMA_VERSIONS, WEBSITE_SNAPSHOT_SCHEMA, WEBSITE_SNAPSHOT_SCHEMA_VERSION, } from "../contracts/workspace-portability.mjs";
 const CONTROL_RE = /[\u0000-\u001f\u007f]/u;
 const DIGEST_RE = /^[a-f0-9]{16,128}$/iu;
@@ -10,6 +11,10 @@ const SERIAL_RE = /^[a-f0-9]{1,128}$/iu;
 const SNAPSHOT_TECHNOLOGY_ROLES = new Set([
     'observed_edge', 'application_platform', 'framework_runtime', 'embedded_dependency',
 ]);
+export const WEBSITE_SNAPSHOT_COLLECTION_LIMITS = Object.freeze({
+    technologies: 40, posture: 40, sources: 16, dependencies: 20,
+    resourceHosts: 30, trackingIdentifiers: 30, formActionOrigins: 20,
+});
 function record(value) {
     return ordinaryWorkspaceRecord(value, 'Website-snapshot input');
 }
@@ -67,18 +72,26 @@ function profileVersion(value) {
         ? Number(value)
         : null;
 }
-function profileProvenance(value, legacy = false) {
+function profileProvenance(value, sourceVersion) {
+    const legacy = sourceVersion === 4;
     if (legacy) {
         return {
             technology: { version: null, state: 'legacy_unknown' },
             securityPosture: { version: null, state: 'legacy_unknown' },
+            pageFingerprint: { version: 1, state: 'known' },
         };
     }
     const item = record(value);
     const technologyItem = record(item?.technology);
     const securityPostureItem = record(item?.securityPosture);
+    const pageFingerprintItem = record(item?.pageFingerprint);
     const technologyVersion = profileVersion(technologyItem?.version);
     const securityPostureVersion = profileVersion(securityPostureItem?.version);
+    // Exact published snapshot writers 4 and 5 used fingerprint algorithm 1.
+    // Unversioned records do not receive invented historical provenance.
+    const candidateFingerprintVersion = sourceVersion === 5 ? 1 : profileVersion(pageFingerprintItem?.version);
+    const pageFingerprintVersion = candidateFingerprintVersion !== null && Object.hasOwn(PAGE_FINGERPRINT_PARSERS, candidateFingerprintVersion)
+        ? candidateFingerprintVersion : null;
     return {
         technology: {
             version: technologyVersion,
@@ -88,6 +101,7 @@ function profileProvenance(value, legacy = false) {
             version: securityPostureVersion,
             state: securityPostureVersion === null ? 'legacy_unknown' : 'known',
         },
+        pageFingerprint: { version: pageFingerprintVersion, state: pageFingerprintVersion === null ? 'legacy_unknown' : 'known' },
     };
 }
 function posture(value) {
@@ -183,8 +197,8 @@ function origin(value) {
 }
 function identityValues(value) {
     const item = record(value);
-    const resourceHosts = values(item?.resourceHosts, 30, (candidate) => normalizeDomain(candidate));
-    const trackingIdentifiers = values(item?.trackingIdentifiers, 30, (candidate) => {
+    const resourceHosts = values(item?.resourceHosts, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.resourceHosts, (candidate) => normalizeDomain(candidate));
+    const trackingIdentifiers = values(item?.trackingIdentifiers, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.trackingIdentifiers, (candidate) => {
         const tracker = record(candidate);
         const type = text(tracker?.type, 40).toLowerCase();
         const trackerValue = text(tracker?.value, 64).toUpperCase();
@@ -192,7 +206,7 @@ function identityValues(value) {
             ? { type, value: trackerValue }
             : null;
     });
-    const formActionOrigins = values(item?.formActionOrigins, 20, origin);
+    const formActionOrigins = values(item?.formActionOrigins, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.formActionOrigins, origin);
     return { resourceHosts, trackingIdentifiers, formActionOrigins };
 }
 export function normalizeWebsiteProfileSnapshot(raw, sourceVersion) {
@@ -203,20 +217,24 @@ export function normalizeWebsiteProfileSnapshot(raw, sourceVersion) {
     const id = text(value?.id, 128);
     if (!domain || !observedAt || !savedAt || !id)
         return null;
+    const acceptsWebScope = sourceVersion === undefined || sourceVersion >= 6;
+    if (acceptsWebScope && !validWebObservationMode(value?.webObservationMode))
+        return null;
     return {
         id,
         domain,
+        ...(acceptsWebScope && value?.webObservationMode === 'selected_url' ? { webObservationMode: value.webObservationMode } : {}),
         observedAt,
         savedAt,
         complete: value?.complete === true,
         truncated: value?.truncated === true,
-        profileProvenance: profileProvenance(value?.profileProvenance, sourceVersion === 4),
-        technologies: values(value?.technologies, 40, technology),
-        posture: values(value?.posture, 40, posture),
+        profileProvenance: profileProvenance(value?.profileProvenance, sourceVersion),
+        technologies: values(value?.technologies, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.technologies, technology),
+        posture: values(value?.posture, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.posture, posture),
         identity: identity(value?.identity),
         identityValues: identityValues(value?.identityValues),
-        sources: values(value?.sources, 16, source),
-        dependencies: values(value?.dependencies, 20, dependency),
+        sources: values(value?.sources, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.sources, source),
+        dependencies: values(value?.dependencies, WEBSITE_SNAPSHOT_COLLECTION_LIMITS.dependencies, dependency),
         certificate: certificate(value?.certificate),
     };
 }
@@ -296,14 +314,96 @@ export function mergeWebsiteSnapshots(localRaw, incomingRaw) {
     const snapshots = normalizeWebsiteSnapshotStore([...byId.values()]).snapshots;
     return { snapshots, added, updated, skipped: Math.max(0, incoming.length - added - updated) };
 }
-function compareMap(field, before, after) {
+const MAX_RECONCILED_WEBSITE_SOURCES = 16;
+const UNAVAILABLE_WEBSITE_SOURCE_STATES = new Set([
+    'blocked', 'error', 'not_found', 'rate_limited', 'unavailable', 'unsupported',
+]);
+export function canonicalWebsiteSourceState(value) {
+    return value.trim().toLowerCase().replace(/[\s-]+/gu, '_');
+}
+function websiteSourceStateSeverity(value) {
+    if (UNAVAILABLE_WEBSITE_SOURCE_STATES.has(value))
+        return 2;
+    return ['complete', 'success'].includes(value) ? 0 : 1;
+}
+export function reconciledWebsiteSources(snapshot) {
+    const selected = new Map();
+    for (const sourceEntry of snapshot.sources.slice(0, MAX_RECONCILED_WEBSITE_SOURCES)) {
+        const sourceId = sourceEntry.source.trim().toLowerCase();
+        if (!sourceId)
+            continue;
+        const state = canonicalWebsiteSourceState(sourceEntry.state);
+        const severity = websiteSourceStateSeverity(state);
+        const current = selected.get(sourceId);
+        if (!current || severity > current.severity || (severity === current.severity && state.localeCompare(current.state) < 0)) {
+            selected.set(sourceId, { state, severity });
+        }
+    }
+    return [...selected.entries()]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([sourceName, value]) => ({ source: sourceName, state: value.state }));
+}
+export function websiteWithReconciledSources(snapshot) {
+    return { ...snapshot, sources: reconciledWebsiteSources(snapshot) };
+}
+export function reconciledWebsiteSourceState(snapshot, sourceId) {
+    const canonicalId = sourceId.trim().toLowerCase();
+    return reconciledWebsiteSources(snapshot)
+        .find((sourceEntry) => sourceEntry.source === canonicalId)?.state ?? null;
+}
+function completeWebsiteSourceState(value) {
+    const state = canonicalWebsiteSourceState(value);
+    return state === 'complete' || state === 'success';
+}
+export function websiteSnapshotFieldSource(field) {
+    if (field.startsWith('certificate.'))
+        return 'tls';
+    if (field.startsWith('source.'))
+        return field.slice('source.'.length);
+    if (field.startsWith('dependency.')) {
+        const recordType = field.slice('dependency.'.length).split(':')[0];
+        return recordType === 'HTTP' ? 'http' : 'dns';
+    }
+    if (['technology', 'posture', 'identity'].some((family) => field.startsWith(family)))
+        return 'http';
+    return '';
+}
+export function websiteSnapshotFieldComplete(snapshot, field) {
+    const sourceName = websiteSnapshotFieldSource(field);
+    const retainedSource = sourceName ? reconciledWebsiteSourceState(snapshot, sourceName) : null;
+    const retainedSourceComplete = !retainedSource || completeWebsiteSourceState(retainedSource);
+    if (field.startsWith('certificate.')) {
+        return Boolean(snapshot.certificate?.complete && !snapshot.certificate.truncated && retainedSourceComplete);
+    }
+    return snapshot.complete && !snapshot.truncated && retainedSourceComplete;
+}
+export function websiteSnapshotComparisonEvidenceComplete(snapshot) {
+    return websiteSnapshotFieldComplete(snapshot, 'technology')
+        && websiteSnapshotFieldComplete(snapshot, 'posture')
+        && websiteSnapshotFieldComplete(snapshot, 'identity')
+        && websiteSnapshotFieldComplete(snapshot, 'dependency.CNAME')
+        && (!snapshot.certificate || websiteSnapshotFieldComplete(snapshot, 'certificate.observation'));
+}
+export function websiteSnapshotProfileComparability(before, after, profile) {
+    const left = before.profileProvenance[profile].version;
+    const right = after.profileProvenance[profile].version;
+    if (left === null || right === null)
+        return 'legacy_unknown';
+    return left === right ? 'comparable' : 'detector_changed';
+}
+function compareMap(field, before, after, options = {}) {
     const changes = [];
     for (const key of [...new Set([...before.keys(), ...after.keys()])].sort()) {
         const left = before.get(key) ?? null;
         const right = after.get(key) ?? null;
         if (left === right)
             continue;
-        changes.push({ field: `${field}.${key}`, state: left === null ? 'added' : right === null ? 'removed' : 'changed', before: left, after: right });
+        const state = left === null
+            ? options.beforeComplete?.(key) === false ? 'incomparable' : 'added'
+            : right === null
+                ? options.afterComplete?.(key) === false ? 'incomparable' : 'removed'
+                : 'changed';
+        changes.push({ field: `${field}.${key}`, state, before: left, after: right });
     }
     return changes;
 }
@@ -313,35 +413,43 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
     if (!before || !after || before.domain !== after.domain) {
         return {
             compatible: false,
+            complete: false,
             changes: [{ field: 'snapshot', state: 'incomparable', before: before?.domain ?? null, after: after?.domain ?? null }],
             dependencyTransitions: [],
         };
     }
     const technologyVersionBefore = before.profileProvenance.technology.version;
+    const selectedPage = before.webObservationMode === 'selected_url' || after.webObservationMode === 'selected_url';
     const technologyVersionAfter = after.profileProvenance.technology.version;
     const postureVersionBefore = before.profileProvenance.securityPosture.version;
     const postureVersionAfter = after.profileProvenance.securityPosture.version;
-    const technologyComparability = technologyVersionBefore === null || technologyVersionAfter === null
-        ? 'legacy_unknown'
-        : technologyVersionBefore === technologyVersionAfter ? 'comparable' : 'detector_changed';
-    const postureComparability = postureVersionBefore === null || postureVersionAfter === null
-        ? 'legacy_unknown'
-        : postureVersionBefore === postureVersionAfter ? 'comparable' : 'detector_changed';
+    const technologyComparability = websiteSnapshotProfileComparability(before, after, 'technology');
+    const postureComparability = websiteSnapshotProfileComparability(before, after, 'securityPosture');
+    const fingerprintComparability = websiteSnapshotProfileComparability(before, after, 'pageFingerprint');
     const changes = [
-        ...(technologyComparability === 'comparable' ? compareMap('technology', new Map(before.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}|${item.roles.join(',')}`])), new Map(after.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}|${item.roles.join(',')}`]))) : [{
+        ...(technologyComparability === 'comparable' ? compareMap('technology', new Map(before.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}|${item.roles.join(',')}`])), new Map(after.technologies.map((item) => [item.id, `${item.name}|${item.category}|${item.confidence}|${item.roles.join(',')}`])), {
+            beforeComplete: () => websiteSnapshotFieldComplete(before, 'technology'),
+            afterComplete: () => websiteSnapshotFieldComplete(after, 'technology'),
+        }) : [{
                 field: 'technology.profileVersion',
                 state: technologyComparability === 'detector_changed' ? 'changed' : 'incomparable',
                 before: technologyVersionBefore === null ? null : String(technologyVersionBefore),
                 after: technologyVersionAfter === null ? null : String(technologyVersionAfter),
             }]),
-        ...(postureComparability === 'comparable' ? compareMap('posture', new Map(before.posture.map((item) => [item.id, item.state])), new Map(after.posture.map((item) => [item.id, item.state]))) : [{
+        ...(postureComparability === 'comparable' ? compareMap('posture', new Map(before.posture.map((item) => [item.id, item.state])), new Map(after.posture.map((item) => [item.id, item.state])), {
+            beforeComplete: () => websiteSnapshotFieldComplete(before, 'posture'),
+            afterComplete: () => websiteSnapshotFieldComplete(after, 'posture'),
+        }) : [{
                 field: 'posture.profileVersion',
                 state: postureComparability === 'detector_changed' ? 'changed' : 'incomparable',
                 before: postureVersionBefore === null ? null : String(postureVersionBefore),
                 after: postureVersionAfter === null ? null : String(postureVersionAfter),
             }]),
         ...compareMap('source', new Map(before.sources.map((item) => [item.source, item.state])), new Map(after.sources.map((item) => [item.source, item.state]))),
-        ...compareMap('dependency', new Map(before.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`])), new Map(after.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`]))),
+        ...compareMap('dependency', new Map(before.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`])), new Map(after.dependencies.map((item) => [`${item.recordType}:${item.target}`, `${item.state}|${item.qualification}`])), {
+            beforeComplete: (key) => websiteSnapshotFieldComplete(before, `dependency.${key}`),
+            afterComplete: (key) => websiteSnapshotFieldComplete(after, `dependency.${key}`),
+        }),
     ];
     const dependencyTransitions = [];
     const beforeDependencies = new Map(before.dependencies.map((item) => [`${item.recordType}:${item.target}`, item]));
@@ -365,10 +473,10 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
                 detail: 'An earlier active dependency is unresolved in the later retained observation. Resolver or collection failure remains possible.',
             });
         }
-        else if (!left && right) {
+        else if (!left && right && websiteSnapshotFieldComplete(before, `dependency.${key}`)) {
             dependencyTransitions.push({ target: right.target, recordType: right.recordType, state: 'added', detail: 'This dependency first appears in the later retained snapshot.' });
         }
-        else if (left && !right && after.complete) {
+        else if (left && !right && websiteSnapshotFieldComplete(after, `dependency.${key}`)) {
             dependencyTransitions.push({ target: left.target, recordType: left.recordType, state: 'removed', detail: 'This dependency is not represented in the later complete snapshot. It may have been intentionally removed or replaced.' });
         }
         if (dependencyTransitions.length >= 20)
@@ -377,11 +485,30 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
     for (const key of Object.keys(before.identity)) {
         const left = before.identity[key];
         const right = after.identity[key];
+        if (key !== 'faviconHash' && fingerprintComparability !== 'comparable')
+            continue;
         if (left === right)
             continue;
         changes.push({ field: `identity.${key}`, state: left === null || right === null ? 'unavailable' : 'changed', before: left, after: right });
     }
-    changes.push(...compareMap('identityValues.resourceHosts', new Map(before.identityValues.resourceHosts.map((item) => [item, item])), new Map(after.identityValues.resourceHosts.map((item) => [item, item]))), ...compareMap('identityValues.trackingIdentifiers', new Map(before.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), new Map(after.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value]))), ...compareMap('identityValues.formActionOrigins', new Map(before.identityValues.formActionOrigins.map((item) => [item, item])), new Map(after.identityValues.formActionOrigins.map((item) => [item, item]))));
+    if (fingerprintComparability !== 'comparable') {
+        changes.push({
+            field: 'identity.profileVersion', state: 'incomparable',
+            before: before.profileProvenance.pageFingerprint.version?.toString() ?? null,
+            after: after.profileProvenance.pageFingerprint.version?.toString() ?? null,
+        });
+    }
+    else
+        changes.push(...compareMap('identityValues.resourceHosts', new Map(before.identityValues.resourceHosts.map((item) => [item, item])), new Map(after.identityValues.resourceHosts.map((item) => [item, item])), {
+            beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.resourceHosts'),
+            afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.resourceHosts'),
+        }), ...compareMap('identityValues.trackingIdentifiers', new Map(before.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), new Map(after.identityValues.trackingIdentifiers.map((item) => [`${item.type}:${item.value}`, item.value])), {
+            beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.trackingIdentifiers'),
+            afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.trackingIdentifiers'),
+        }), ...compareMap('identityValues.formActionOrigins', new Map(before.identityValues.formActionOrigins.map((item) => [item, item])), new Map(after.identityValues.formActionOrigins.map((item) => [item, item])), {
+            beforeComplete: () => websiteSnapshotFieldComplete(before, 'identityValues.formActionOrigins'),
+            afterComplete: () => websiteSnapshotFieldComplete(after, 'identityValues.formActionOrigins'),
+        }));
     if (before.certificate || after.certificate) {
         if (!before.certificate || !after.certificate) {
             changes.push({
@@ -425,11 +552,22 @@ export function compareWebsiteSnapshots(beforeRaw, afterRaw) {
     }
     return {
         compatible: true,
+        complete: !selectedPage && websiteSnapshotComparisonEvidenceComplete(before)
+            && websiteSnapshotComparisonEvidenceComplete(after)
+            && technologyComparability === 'comparable'
+            && postureComparability === 'comparable'
+            && fingerprintComparability === 'comparable',
         profileComparability: {
             technology: technologyComparability,
             securityPosture: postureComparability,
+            pageFingerprint: fingerprintComparability,
         },
-        changes,
-        dependencyTransitions,
+        changes: selectedPage ? [
+            { field: 'web.collectionTarget', state: 'incomparable', before: before.webObservationMode ?? 'homepage', after: after.webObservationMode ?? 'homepage' },
+            ...changes.map((change) => change.field.startsWith('certificate.') || change.field.startsWith('source.') || change.field === 'completeness'
+                || /^dependency\.(?:CNAME|HTTPS|NS|MX)(?::|$)/u.test(change.field)
+                ? change : { ...change, state: 'incomparable' }),
+        ] : changes,
+        dependencyTransitions: selectedPage ? dependencyTransitions.filter((transition) => transition.recordType !== 'HTTP') : dependencyTransitions,
     };
 }