@slicedearth/whoisleuth-cli 2.2.0 → 2.3.0

package/lib/static-html-analysis.mjs

@@ -73,7 +73,7 @@ const JAVASCRIPT_MEDIA_TYPES = new Set([
     'text/javascript1.1', 'text/javascript1.2', 'text/javascript1.3', 'text/javascript1.4',
     'text/javascript1.5', 'text/jscript', 'text/livescript', 'text/x-ecmascript', 'text/x-javascript',
 ]);
-function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LENGTH) {
+function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LENGTH, lowercase = true) {
     const attribute = attributes
         .slice(0, MAX_ATTRIBUTES_PER_TAG)
         .find((candidate) => candidate.name.toLowerCase() === name);
@@ -84,7 +84,7 @@ function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LEN
     }
     return {
         present: true,
-        value: attribute.value.trim().toLowerCase(),
+        value: lowercase ? attribute.value.trim().toLowerCase() : attribute.value.trim(),
         truncated: attributes.length > MAX_ATTRIBUTES_PER_TAG,
     };
 }
@@ -116,8 +116,8 @@ function formMethod(attributes) {
         truncated: method.truncated,
     };
 }
-function formAction(attributes, baseUrl) {
-    const action = attributeValue(attributes, 'action');
+function formAction(attributes, resolutionBase, documentOrigin) {
+    const action = attributeValue(attributes, 'action', MAX_FORM_ATTRIBUTE_LENGTH, false);
     if (!action.present && action.truncated) {
         return { relationship: 'unclassified', cleartext: false, truncated: true };
     }
@@ -127,16 +127,16 @@ function formAction(attributes, baseUrl) {
     if (action.value === null) {
         return { relationship: 'unclassified', cleartext: false, truncated: true };
     }
-    if (!baseUrl) {
+    if (!resolutionBase || !documentOrigin) {
         return { relationship: 'unclassified', cleartext: false, truncated: action.truncated };
     }
     try {
-        const parsed = new URL(action.value, baseUrl);
+        const parsed = new URL(action.value, resolutionBase);
         if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password || !parsed.hostname) {
             return { relationship: 'unclassified', cleartext: false, truncated: action.truncated };
         }
         return {
-            relationship: parsed.origin === baseUrl.origin ? 'sameOrigin' : 'external',
+            relationship: parsed.origin === documentOrigin ? 'sameOrigin' : 'external',
             cleartext: parsed.protocol === 'http:',
             truncated: action.truncated,
         };
@@ -238,7 +238,9 @@ function analyzeStaticHtml(value, options = {}) {
     const supplied = typeof value === 'string' ? value : '';
     const inputLimitReached = supplied.length > MAX_STATIC_HTML_CHARS;
     const html = supplied.slice(0, MAX_STATIC_HTML_CHARS);
-    const baseUrl = safeBaseUrl(options.baseUrl);
+    const documentUrl = safeBaseUrl(options.baseUrl);
+    let effectiveBaseUrl = documentUrl;
+    let baseHrefState = 'absent';
     const includeVisibleText = options.includeVisibleText === true;
     const markup = [];
     const visibleTextParts = [];
@@ -316,6 +318,17 @@ function analyzeStaticHtml(value, options = {}) {
     let scriptElementSeen = false;
     let nonVisibleTextDepth = 0;
     let visibleTextCharacters = 0;
+    let titleDepth = 0;
+    let titleCharacters = 0;
+    const titleParts = [];
+    const pendingFormActions = [];
+    function appendTitle(chars) {
+        if (titleDepth === 0 || !chars || titleCharacters >= 201)
+            return;
+        const retained = chars.slice(0, 201 - titleCharacters);
+        titleParts.push(retained);
+        titleCharacters += retained.length;
+    }
     function appendInlineScript(chars) {
         if (!activeInlineScript || !chars)
             return;
@@ -374,6 +387,29 @@ function analyzeStaticHtml(value, options = {}) {
                 implicitHeadTextDepth += 1;
             }
             const inPublicationHead = insideExplicitHead || (!headScopeClosed && !bodyContentStarted && HEAD_CONTEXT_TAGS.has(tagName));
+            if (tagName === 'base' && inPublicationHead && baseHrefState === 'absent') {
+                const href = attributeValue(token.attrs, 'href', MAX_FORM_ATTRIBUTE_LENGTH, false);
+                if (href.truncated && !href.present) {
+                    effectiveBaseUrl = documentUrl;
+                    baseHrefState = 'invalid';
+                }
+                else if (href.present) {
+                    try {
+                        const parsed = href.value !== null && documentUrl ? new URL(href.value, documentUrl) : null;
+                        if (!parsed || !['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password || !parsed.hostname) {
+                            throw new Error('Invalid document base URL.');
+                        }
+                        effectiveBaseUrl = parsed;
+                        baseHrefState = 'valid';
+                    }
+                    catch {
+                        effectiveBaseUrl = documentUrl;
+                        baseHrefState = 'invalid';
+                    }
+                }
+            }
+            if (tagName === 'title' && titleDepth === 0 && titleParts.length === 0)
+                titleDepth = 1;
             if (/^h[1-6]$/u.test(tagName)) {
                 const key = tagName;
                 publicationMetadata.headings[key] += 1;
@@ -566,11 +602,8 @@ function analyzeStaticHtml(value, options = {}) {
                     forms.formsObserved += 1;
                     const method = formMethod(token.attrs);
                     forms.methods[method.value] += 1;
-                    const action = formAction(token.attrs, baseUrl);
-                    forms.actions[action.relationship] += 1;
-                    if (action.cleartext)
-                        forms.actions.cleartext += 1;
-                    if (method.truncated || action.truncated)
+                    pendingFormActions.push(token.attrs);
+                    if (method.truncated || token.attrs.length > MAX_ATTRIBUTES_PER_TAG)
                         forms.truncated = true;
                 }
             }
@@ -612,6 +645,8 @@ function analyzeStaticHtml(value, options = {}) {
             const tagName = token.tagName.toLowerCase();
             if (tagName === 'script')
                 activeInlineScript = null;
+            if (tagName === 'title' && titleDepth > 0)
+                titleDepth = 0;
             if (tagName === 'head') {
                 insideExplicitHead = false;
                 headScopeClosed = true;
@@ -630,6 +665,7 @@ function analyzeStaticHtml(value, options = {}) {
                 headScopeClosed = true;
             }
             appendInlineScript(token.chars);
+            appendTitle(token.chars);
             appendVisibleText(token.chars);
         },
         onNullCharacter(token) {
@@ -638,10 +674,12 @@ function analyzeStaticHtml(value, options = {}) {
                 headScopeClosed = true;
             }
             appendInlineScript(token.chars);
+            appendTitle(token.chars);
             appendVisibleText(token.chars);
         },
         onWhitespaceCharacter(token) {
             appendInlineScript(token.chars);
+            appendTitle(token.chars);
             appendVisibleText(token.chars);
         },
         onComment() { },
@@ -650,6 +688,16 @@ function analyzeStaticHtml(value, options = {}) {
     };
     tokenizer = new Tokenizer({ sourceCodeLocationInfo: false }, handler);
     tokenizer.write(html, true);
+    for (const attributes of pendingFormActions) {
+        const action = formAction(attributes, effectiveBaseUrl, documentUrl?.origin ?? null);
+        forms.actions[action.relationship] += 1;
+        if (action.cleartext)
+            forms.actions.cleartext += 1;
+        if (action.truncated)
+            forms.truncated = true;
+    }
+    if (baseHrefState === 'invalid')
+        forms.truncated = true;
     publicationMetadata.truncated = publicationMetadata.truncated || inputLimitReached || publicationTagLimitReached;
     publicationMetadata.documentTruncated = inputLimitReached || publicationTagLimitReached;
     if (publicationMetadata.documentTruncated) {
@@ -671,6 +719,14 @@ function analyzeStaticHtml(value, options = {}) {
     publicationMetadata.renderBlockingCandidates.total = publicationMetadata.renderBlockingCandidates.script
         + publicationMetadata.renderBlockingCandidates.stylesheet;
     return {
+        title: (() => {
+            const value = titleParts.join('').replace(CONTROL_CHARACTER_RE_GLOBAL, ' ').replace(/\ufffd/gu, ' ').replace(/\s+/gu, ' ').trim();
+            if (!value)
+                return null;
+            return value.length > 200 ? `${value.slice(0, 200)}…` : value;
+        })(),
+        effectiveBaseUrl: effectiveBaseUrl?.toString() ?? null,
+        baseHrefState,
         markup: markup.join('\n'),
         visibleText: visibleTextParts.join('').replace(CONTROL_CHARACTER_RE_GLOBAL, ' ').replace(/\s+/gu, ' '),
         structureTokens,

package/lib/website-technology.mjs

@@ -9,6 +9,23 @@ import { createObservation } from "../packages/evidence/observation.mjs";
 import { MAX_STATIC_HTML_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, } from "./static-html-analysis.mjs";
 import { TECHNOLOGY_EVIDENCE_ROLE_ORDER, } from "./technology-evidence-role.mjs";
 import { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, TECHNOLOGY_PROFILE_VERSION, } from "./lookup-child-profile-contract.mjs";
+const TECHNOLOGY_CATEGORIES = Object.freeze([
+    'application runtime',
+    'content management',
+    'commerce',
+    'site builder',
+    'web framework',
+    'static site generator',
+    'web server',
+    'delivery platform',
+]);
+const TECHNOLOGY_EVIDENCE_SOURCES = Object.freeze([
+    'generator metadata',
+    'static HTML',
+    'resource origin',
+    'HTTP server header',
+    'passive response header',
+]);
 const MAX_TECHNOLOGY_HTML_CHARS = MAX_STATIC_HTML_CHARS;
 const MAX_TECHNOLOGY_TAG_LENGTH = MAX_TAG_LENGTH;
 const MAX_RESOURCE_ORIGINS = 30;
@@ -38,6 +55,29 @@ function normalizedResourceHosts(value) {
     }
     return hosts;
 }
+function applicationMarkup(markup, resourceHosts, effectiveBaseUrl, documentOrigin) {
+    let resolutionBase;
+    try {
+        resolutionBase = new URL(typeof effectiveBaseUrl === 'string' ? effectiveBaseUrl : 'https://document.invalid/');
+    }
+    catch {
+        resolutionBase = new URL('https://document.invalid/');
+    }
+    const comparedOrigin = typeof documentOrigin === 'string' && /^https?:\/\//iu.test(documentOrigin)
+        ? documentOrigin
+        : null;
+    return markup.replace(/\s(src|href|poster|data)="([^"]*)"/giu, (attribute, name, value) => {
+        try {
+            const parsed = new URL(value, resolutionBase);
+            const knownExternalHost = resourceHosts.has(parsed.hostname.toLowerCase());
+            const offOrigin = comparedOrigin ? parsed.origin !== comparedOrigin : knownExternalHost;
+            return knownExternalHost && offOrigin ? ` ${name.toLowerCase()}=""` : attribute;
+        }
+        catch {
+            return attribute;
+        }
+    });
+}
 const PASSIVE_TECHNOLOGY_HEADER_NAMES = Object.freeze([
     'cf-ray',
     'x-drupal-cache',
@@ -106,6 +146,17 @@ function htmlEvidence(markers, description, confidence = 'high') {
         matches: ({ html }) => markers.some((marker) => html.includes(marker)),
     };
 }
+function resourcePathEvidence(markers, description, confidence = 'high') {
+    return {
+        source: 'static HTML',
+        description,
+        confidence,
+        matches: ({ html }) => markers.some((marker) => html.includes(marker)),
+        roleFor: ({ applicationHtml }) => markers.some((marker) => applicationHtml.includes(marker))
+            ? undefined
+            : 'embedded_dependency',
+    };
+}
 function resourceEvidence(hosts, description) {
     return {
         source: 'resource origin',
@@ -142,7 +193,10 @@ function responseHeaderEvidence(name, pattern, description, confidence = 'high')
         },
     };
 }
-function evidenceRole(signature, evidence) {
+function evidenceRole(signature, evidence, context) {
+    const contextualRole = context ? evidence.roleFor?.(context) : undefined;
+    if (contextualRole)
+        return contextualRole;
     if (evidence.role)
         return evidence.role;
     if (evidence.source === 'resource origin')
@@ -159,7 +213,7 @@ const TECHNOLOGY_SIGNATURES = [
         id: 'wordpress', name: 'WordPress', category: 'content management',
         evidence: [
             generatorEvidence(/^wordpress(?:\s|$)/i, 'Generator metadata identifies WordPress.'),
-            htmlEvidence(['/wp-content/', '/wp-includes/'], 'Static resource paths use WordPress conventions.', 'medium'),
+            resourcePathEvidence(['/wp-content/', '/wp-includes/'], 'Static resource paths use WordPress conventions.', 'medium'),
         ],
     },
     {
@@ -222,8 +276,9 @@ const TECHNOLOGY_SIGNATURES = [
     {
         id: 'bigcommerce', name: 'BigCommerce', category: 'commerce',
         minimumEvidenceMatches: 2,
+        allowEmbeddedOnly: true,
         evidence: [
-            htmlEvidence(['cdn11.bigcommerce.com/s-', 'stencil-utils'], 'Static markup contains BigCommerce storefront asset markers.', 'medium'),
+            resourcePathEvidence(['cdn11.bigcommerce.com/s-', 'stencil-utils'], 'Static markup contains BigCommerce storefront asset markers.', 'medium'),
             resourcePatternEvidence(/^cdn\d+\.bigcommerce\.com$/i, 'A retained resource origin uses BigCommerce storefront delivery infrastructure.'),
         ],
     },
@@ -421,28 +476,41 @@ function analyzeWebsiteTechnology(input = {}) {
     }) : null;
     const context = {
         html: htmlAnalysis.markup,
+        applicationHtml: '',
         generator: boundedLowercase(input.generator, MAX_GENERATOR_INPUT),
         httpServer: boundedLowercase(input.httpServer, MAX_SERVER_INPUT),
         resourceHosts: normalizedResourceHosts(input.resourceOrigins),
         responseHeaders: normalizedResponseHeaders(input.responseHeaders),
     };
+    context.applicationHtml = applicationMarkup(context.html, context.resourceHosts, input.effectiveBaseUrl, input.documentOrigin);
     const findings = [];
     for (const signature of TECHNOLOGY_SIGNATURES) {
-        const matched = signature.evidence.filter((evidence) => evidence.matches(context));
-        if (matched.length < (signature.minimumEvidenceMatches || 1))
-            continue;
-        if (signature.requiresNonResourceEvidence && matched.every((evidence) => evidence.source === 'resource origin'))
+        const matched = signature.evidence
+            .filter((evidence) => evidence.matches(context))
+            .map((evidence) => ({ evidence, role: evidenceRole(signature, evidence, context) }));
+        const nonEmbeddedMatches = matched.filter((entry) => entry.role !== 'embedded_dependency');
+        const minimumEvidenceMatches = signature.minimumEvidenceMatches || 1;
+        const qualifiesForPlatform = matched.length >= minimumEvidenceMatches
+            && (minimumEvidenceMatches === 1 || nonEmbeddedMatches.length > 0)
+            && (!signature.requiresNonResourceEvidence || nonEmbeddedMatches.length > 0);
+        const contextualEmbeddedEvidence = matched.filter((entry) => (entry.role === 'embedded_dependency' && entry.evidence.roleFor !== undefined));
+        const selected = qualifiesForPlatform
+            ? matched
+            : signature.allowEmbeddedOnly
+                ? contextualEmbeddedEvidence.slice(0, 1)
+                : [];
+        if (selected.length === 0)
             continue;
-        const matchedEvidence = matched.slice(0, MAX_EVIDENCE_PER_TECHNOLOGY).map((evidence) => ({
+        const matchedEvidence = selected.slice(0, MAX_EVIDENCE_PER_TECHNOLOGY).map(({ evidence, role }) => ({
             source: evidence.source,
-            role: evidenceRole(signature, evidence),
+            role,
             description: evidence.description,
         }));
         findings.push({
             id: signature.id,
             name: signature.name,
             category: signature.category,
-            confidence: matched.some((evidence) => evidence.confidence === 'high') ? 'high' : 'medium',
+            confidence: selected.some(({ evidence }) => evidence.confidence === 'high') ? 'high' : 'medium',
             roles: TECHNOLOGY_EVIDENCE_ROLE_ORDER.filter((role) => matchedEvidence.some((evidence) => evidence.role === role)),
             evidence: matchedEvidence,
         });
@@ -493,4 +561,4 @@ function analyzeWebsiteTechnology(input = {}) {
         browserLibraryProfile,
     };
 }
-export { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, MAX_TECHNOLOGY_HTML_CHARS, MAX_TECHNOLOGY_TAGS, PASSIVE_TECHNOLOGY_HEADER_NAMES, TECHNOLOGY_PROFILE_VERSION, TECHNOLOGY_SIGNATURE_CATALOGUE, analyzeWebsiteTechnology, captureTechnologyResponseHeaders, };
+export { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, MAX_TECHNOLOGY_HTML_CHARS, MAX_TECHNOLOGY_TAGS, PASSIVE_TECHNOLOGY_HEADER_NAMES, TECHNOLOGY_CATEGORIES, TECHNOLOGY_EVIDENCE_SOURCES, TECHNOLOGY_PROFILE_VERSION, TECHNOLOGY_SIGNATURE_CATALOGUE, analyzeWebsiteTechnology, captureTechnologyResponseHeaders, };

package/lib/zone-intent-review.mjs

@@ -142,7 +142,11 @@ export function normaliseRdata(type, rawValue, origin, nameMode = 'absolute') {
         const family = type === 'A' ? 4 : 6;
         if (tokens.length !== 1 || isIP(tokens[0] ?? '') !== family)
             throw new TypeError(`${type} data must contain one valid address.`);
-        return { value: tokens[0].toLowerCase(), valueTreatment: 'normalised' };
+        const supplied = tokens[0];
+        const normalised = family === 4
+            ? supplied.split('.').map((part) => String(Number(part))).join('.')
+            : new URL(`http://[${supplied}]/`).hostname.slice(1, -1).toLowerCase();
+        return { value: normalised, valueTreatment: 'normalised' };
     }
     if (type === 'CNAME' || type === 'NS') {
         if (tokens.length !== 1)
@@ -152,7 +156,12 @@ export function normaliseRdata(type, rawValue, origin, nameMode = 'absolute') {
     if (type === 'MX') {
         if (tokens.length !== 2)
             throw new TypeError('MX data must contain preference and exchange.');
-        return { value: `${integerToken(tokens[0], 0, 65_535, 'MX preference')} ${host(tokens[1], 'MX exchange')}`, valueTreatment: 'normalised' };
+        const preference = integerToken(tokens[0], 0, 65_535, 'MX preference');
+        const exchangeToken = tokens[1];
+        if (exchangeToken === '.' && preference !== 0)
+            throw new TypeError('A Null MX exchange must use preference 0.');
+        const exchange = exchangeToken === '.' ? '.' : host(exchangeToken, 'MX exchange');
+        return { value: `${preference} ${exchange}`, valueTreatment: 'normalised' };
     }
     if (type === 'CAA') {
         if (tokens.length < 3)
@@ -380,7 +389,6 @@ function parseBindZone(zoneText, initialOrigin) {
                 if (tokens.length !== 2)
                     throw new TypeError('$ORIGIN requires one domain name.');
                 origin = domainName(tokens[1], origin, '$ORIGIN', 'master-file');
-                lastOwner = origin;
                 continue;
             }
             if (directive === '$TTL')
@@ -388,10 +396,12 @@ function parseBindZone(zoneText, initialOrigin) {
             if (directive?.startsWith('$'))
                 throw new TypeError(`${directive} is deliberately unsupported.`);
             let offset = 0;
-            const ownerToken = entry.ownerOmitted ? lastOwner : tokens[offset++];
-            if (!ownerToken)
+            const ownerToken = entry.ownerOmitted ? null : tokens[offset++];
+            if (!entry.ownerOmitted && !ownerToken)
                 throw new TypeError('Record owner is missing.');
-            const owner = ownerName(ownerToken, origin, 'record owner', 'master-file');
+            const owner = entry.ownerOmitted
+                ? lastOwner
+                : ownerName(ownerToken, origin, 'record owner', 'master-file');
             lastOwner = owner;
             let ttl = null;
             if (/^\d+$/u.test(tokens[offset] ?? ''))

package/package.json

@@ -33,7 +33,7 @@
   "bugs": {
     "url": "https://github.com/slicedearth/whoisleuth/issues"
   },
-  "version": "2.2.0",
+  "version": "2.3.0",
   "dependencies": {
     "@peculiar/x509": "2.0.0",
     "maxmind": "5.0.7",

package/packages/cases/case-migration-model.mjs

@@ -1,13 +1,10 @@
-import { CASE_DISPOSITIONS, CASE_IMPORT_VERSIONS, CASE_SCHEMA_VERSION, CASE_SOURCES, CASE_STATUSES, DEFAULT_DISPOSITION, DEFAULT_SOURCE, DEFAULT_STATUS, MAX_CASES, MAX_NOTES_PER_CASE, deterministicId, caseTimestampOrNull, normalizeCase, normalizeDomain, normalizeEvidenceHistory, normalizeNotes, normalizeReviewReasonCode, normalizeTags, objectRecord, safeId, } from "./case-record-model.mjs";
+import { CASE_IMPORT_VERSIONS, CASE_SCHEMA_VERSION, DEFAULT_DISPOSITION, DEFAULT_SOURCE, DEFAULT_STATUS, MAX_CASES, MAX_NOTES_PER_CASE, deterministicId, caseTimestampOrNull, normalizeCase, normalizeDomain, normalizeEvidenceHistory, normalizeNotes, normalizeReviewReasonCode, normalizeTags, objectRecord, safeId, isValidDisposition, isValidSource, isValidStatus, } from "./case-record-model.mjs";
 import { CASE_REPORT_SCHEMA, PUBLIC_CASE_SCHEMA_VERSION, caseReportVersionMatchesCase, CLI_CASE_PACK_CURRENT_REDACTION_KEYS, CLI_CASE_PACK_INTEGRITY_KEYS, CLI_CASE_PACK_LIMITATIONS, CLI_CASE_PACK_PACKET_KEYS, CLI_CASE_PACK_PUBLIC_REPORT_KEYS, CLI_CASE_PACK_REPORT_KEYS, CLI_CASE_PACK_ROOT_KEYS, CLI_CASE_PACK_SCHEMA, CLI_CASE_PACK_VERSION, } from "../contracts/case-portability.mjs";
 import { canonicalArtifactJsonV2 } from "../evidence/artifact-integrity.mjs";
 import { assertBoundedJsonStructure } from "../../lib/bounded-json.mjs";
 import { inspectCaseBrandProfileIds, unionCaseBrandProfileIds, } from "./case-brand-profile-references.mjs";
 import { caseInvestigationBranchReferences, mergeCaseInvestigationBranches, normalizeCaseInvestigationBranches, } from "./case-investigation-branch-model.mjs";
-import { buildCaseClosureLinkContext, mergeCaseActions, mergeCaseAssertions, mergeCaseClosureHistories, mergeCaseDecisions, mergeCaseEvidencePins, mergeCaseManualTrail, mergeCaseObservedEffectHistories, mergeCaseSightings, normalizeCaseActions, normalizeCaseAssertions, normalizeCaseClosureHistory, normalizeCaseDecisions, normalizeCaseEvidencePins, normalizeCaseManualTrail, normalizeCaseObservedEffectHistory, normalizeCaseSightings, } from "./case-response-model.mjs";
-const STATUS_VALUES = new Set(CASE_STATUSES.map((item) => item.value));
-const DISPOSITION_VALUES = new Set(CASE_DISPOSITIONS.map((item) => item.value));
-const SOURCE_VALUES = new Set(CASE_SOURCES.map((item) => item.value));
+import { buildCaseClosureLinkContext, mergeCaseActions, mergeCaseClosureHistories, mergeCaseObservedEffectHistories, MAX_CASE_ASSERTIONS, MAX_CASE_DECISIONS, MAX_CASE_EVIDENCE_PINS, MAX_CASE_MANUAL_TRAIL_EVENTS, MAX_CASE_SIGHTINGS, normalizeCaseActions, normalizeCaseAssertions, normalizeCaseClosureHistory, normalizeCaseDecisions, normalizeCaseEvidencePins, normalizeCaseManualTrail, normalizeCaseObservedEffectHistory, normalizeCaseSightings, } from "./case-response-model.mjs";
 export const MAX_CASE_INPUT_RECORDS = 2_000;
 function compareCodeUnits(left, right) {
     return left < right ? -1 : left > right ? 1 : 0;
@@ -51,7 +48,7 @@ export function normalizeCaseStore(raw) {
             throw new TypeError(`Unversioned Case stores are retired. Export or reset them explicitly before using Case schema ${CASE_SCHEMA_VERSION}; no data was changed.`);
         }
         if (!CASE_IMPORT_VERSIONS.includes(sourceVersion) && sourceVersion < CASE_SCHEMA_VERSION) {
-            throw new TypeError(`Case schema ${sourceVersion} is not part of the supported compatibility boundary. Exact published schemas 12 and 13 migrate to schema ${CASE_SCHEMA_VERSION}; no data was changed.`);
+            throw new TypeError(`Case schema ${sourceVersion} is not part of the supported compatibility boundary. Exact published schemas ${CASE_IMPORT_VERSIONS.filter((version) => version < CASE_SCHEMA_VERSION).join(', ')} migrate to schema ${CASE_SCHEMA_VERSION}; no data was changed.`);
         }
         if (sourceVersion > CASE_SCHEMA_VERSION) {
             throw new TypeError(`Case schema ${sourceVersion} is newer than the supported schema ${CASE_SCHEMA_VERSION}; no data was changed.`);
@@ -92,6 +89,17 @@ function assertCurrentCaseShape(raw) {
             if (Object.keys(record).length > 0 && !Object.hasOwn(record, 'routeObservedAt')) {
                 throw new TypeError(`Case schema ${CASE_SCHEMA_VERSION} response actions must declare their route observation time, including null; unreleased local Case checkpoints are not interpreted as the current format and no data was changed.`);
             }
+            if (Object.keys(record).length > 0 && !Object.hasOwn(record, 'routeReviewAfter')) {
+                throw new TypeError(`Case schema ${CASE_SCHEMA_VERSION} response actions must declare their route review deadline, including null; unreleased local Case checkpoints are not interpreted as the current format and no data was changed.`);
+            }
+        }
+        const decisions = itemRecord.decisions;
+        for (const decision of Array.isArray(decisions) ? decisions : []) {
+            const record = objectRecord(decision);
+            if (Object.keys(record).length > 0
+                && (!Object.hasOwn(record, 'confidence') || !Object.hasOwn(record, 'confidenceBasis'))) {
+                throw new TypeError(`Case schema ${CASE_SCHEMA_VERSION} analyst decisions must declare confidence and its basis; unreleased local Case checkpoints are not interpreted as the current format and no data was changed.`);
+            }
         }
     }
 }
@@ -140,9 +148,6 @@ function boundedCaseList(raw) {
  * @param {string} [nowIso]
  * @returns {CaseRecord}
  */
-function importScalar(value, valid) {
-    return typeof value === 'string' && valid.has(value) ? value : undefined;
-}
 /**
  * Validates one imported record into a patch. Unlike normalizeCase, absent or
  * invalid scalar fields stay `undefined` (never defaulted) and a missing/invalid
@@ -174,6 +179,8 @@ function extractImportPatch(raw, importedVersion) {
     const pinIds = new Set(evidencePins.map((item) => item.id));
     const actions = normalizeCaseActions(record.actions, normalizedFallback, { ...timestampOptions, validEvidencePinIds: pinIds });
     const assertions = normalizeCaseAssertions(record.assertions, normalizedFallback, pinIds, timestampOptions);
+    const decisions = normalizeCaseDecisions(record.decisions, normalizedFallback, pinIds, timestampOptions);
+    const manualTrail = normalizeCaseManualTrail(record.manualTrail, normalizedFallback, timestampOptions);
     const sightings = normalizeCaseSightings(record.sightings, normalizedFallback, pinIds, timestampOptions);
     const observedEffects = normalizeCaseObservedEffectHistory(importedVersion >= 13 ? record.observedEffects : undefined, normalizedFallback, pinIds, new Set(sightings.map((item) => item.id)), timestampOptions);
     const closures = normalizeCaseClosureHistory(importedVersion >= 13 ? record.closures : undefined, normalizedFallback, new Set(observedEffects.reviews.map((item) => item.id)), new Set(actions.map((item) => item.id)), timestampOptions, buildCaseClosureLinkContext(observedEffects, actions));
@@ -181,17 +188,26 @@ function extractImportPatch(raw, importedVersion) {
     const brandProfileReferences = importedVersion >= 12
         ? inspectCaseBrandProfileIds(record.brandProfileIds)
         : { ids: [], omitted: 0 };
+    const authoredHistoryOmitted = [
+        [record.evidencePins, evidencePins],
+        [record.decisions, decisions],
+        [record.assertions, assertions],
+        [record.manualTrail, manualTrail],
+        [record.sightings, sightings],
+    ].reduce((total, [candidates, retained]) => total
+        + Math.max(0, (Array.isArray(candidates) ? candidates.length : 0) - retained.length), 0);
     return {
         domain,
         rawId: typeof record.id === 'string' ? record.id : null,
-        status: importScalar(record.status, STATUS_VALUES),
-        disposition: importScalar(record.disposition, DISPOSITION_VALUES),
+        status: isValidStatus(record.status) ? record.status : undefined,
+        disposition: isValidDisposition(record.disposition) ? record.disposition : undefined,
         reviewReasonCode: Object.hasOwn(record, 'reviewReasonCode')
             ? normalizeReviewReasonCode(record.reviewReasonCode)
             : undefined,
         brandProfileIds: brandProfileReferences.ids,
         brandProfileReferencesOmitted: brandProfileReferences.omitted,
-        source: importScalar(record.source, SOURCE_VALUES),
+        authoredHistoryOmitted,
+        source: isValidSource(record.source) ? record.source : undefined,
         evidenceHistory: normalizeEvidenceHistory(rawEvidence, {
             source: 'import',
             fallback: importFallback,
@@ -199,10 +215,10 @@ function extractImportPatch(raw, importedVersion) {
             caseDomain: domain,
         }),
         evidencePins,
-        decisions: normalizeCaseDecisions(record.decisions, normalizedFallback, pinIds, timestampOptions),
+        decisions,
         actions,
         assertions,
-        manualTrail: normalizeCaseManualTrail(record.manualTrail, normalizedFallback, timestampOptions),
+        manualTrail,
         sightings,
         observedEffects,
         closures,
@@ -229,6 +245,22 @@ function unionNotes(a, b) {
     const notes = [...byId.values()].sort((left, right) => Date.parse(left.createdAt) - Date.parse(right.createdAt));
     return notes.slice(Math.max(0, notes.length - MAX_NOTES_PER_CASE));
 }
+function retainLocalAuthoredRecords(local, imported, maximum) {
+    const records = [...local];
+    const retainedIds = new Set(records.map((item) => item.id));
+    let omitted = 0;
+    for (const item of imported) {
+        if (retainedIds.has(item.id))
+            continue;
+        if (records.length >= maximum) {
+            omitted += 1;
+            continue;
+        }
+        records.push(item);
+        retainedIds.add(item.id);
+    }
+    return { records, omitted };
+}
 // Additive, deduplicated union of two evidence histories. Identical material
 // collapses (earliest firstCapturedAt, latest capturedAt), distinct snapshots
 // are retained subject to the per-case bound, and an older import can never
@@ -274,16 +306,29 @@ function caseFromPatch(patch, now) {
 function applyImportPatch(local, patch) {
     const importNewer = patch.updatedAt !== null && Date.parse(patch.updatedAt) > Date.parse(local.updatedAt);
     const fallback = patch.updatedAt || local.updatedAt;
-    const evidencePins = mergeCaseEvidencePins(local.evidencePins, patch.evidencePins, fallback);
+    const pinSelection = retainLocalAuthoredRecords(local.evidencePins, patch.evidencePins, MAX_CASE_EVIDENCE_PINS);
+    const evidencePins = normalizeCaseEvidencePins(pinSelection.records, fallback);
     const pinIds = new Set(evidencePins.map((item) => item.id));
     const actions = mergeCaseActions(local.actions, patch.actions, fallback, pinIds);
-    const assertions = mergeCaseAssertions(local.assertions, patch.assertions, fallback, pinIds);
+    const assertionSelection = retainLocalAuthoredRecords(local.assertions, patch.assertions, MAX_CASE_ASSERTIONS);
+    const assertions = normalizeCaseAssertions(assertionSelection.records, fallback, pinIds);
     const branchReferences = caseInvestigationBranchReferences({ evidencePins, actions, assertions });
-    const sightings = mergeCaseSightings(local.sightings, patch.sightings, fallback, pinIds);
+    const sightingSelection = retainLocalAuthoredRecords(local.sightings, patch.sightings, MAX_CASE_SIGHTINGS);
+    const sightings = normalizeCaseSightings(sightingSelection.records, fallback, pinIds);
     const sightingIds = new Set(sightings.map((item) => item.id));
     const observedEffects = mergeCaseObservedEffectHistories(local.observedEffects, patch.observedEffects, fallback, pinIds, sightingIds);
     const closures = mergeCaseClosureHistories(local.closures, patch.closures, fallback, new Set(observedEffects.reviews.map((item) => item.id)), new Set(actions.map((item) => item.id)), buildCaseClosureLinkContext(observedEffects, actions));
     const brandProfileReferences = unionCaseBrandProfileIds(local.brandProfileIds, patch.brandProfileIds);
+    const decisionSelection = retainLocalAuthoredRecords(local.decisions, patch.decisions, MAX_CASE_DECISIONS);
+    const decisions = normalizeCaseDecisions(decisionSelection.records, fallback, pinIds);
+    const trailSelection = retainLocalAuthoredRecords(local.manualTrail, patch.manualTrail, MAX_CASE_MANUAL_TRAIL_EVENTS);
+    const manualTrail = normalizeCaseManualTrail(trailSelection.records, fallback);
+    const authoredHistoryOmitted = patch.authoredHistoryOmitted
+        + pinSelection.omitted
+        + decisionSelection.omitted
+        + assertionSelection.omitted
+        + trailSelection.omitted
+        + sightingSelection.omitted;
     return { record: {
             ...local,
             status: patch.status !== undefined && importNewer ? patch.status : local.status,
@@ -293,10 +338,10 @@ function applyImportPatch(local, patch) {
             source: patch.source !== undefined && importNewer ? patch.source : local.source,
             evidenceHistory: mergeEvidenceHistories(local.evidenceHistory, patch.evidenceHistory, local.domain),
             evidencePins,
-            decisions: mergeCaseDecisions(local.decisions, patch.decisions, fallback, pinIds),
+            decisions,
             actions,
             assertions,
-            manualTrail: mergeCaseManualTrail(local.manualTrail, patch.manualTrail, fallback),
+            manualTrail,
             sightings,
             observedEffects,
             closures,
@@ -305,7 +350,7 @@ function applyImportPatch(local, patch) {
             notes: unionNotes(local.notes, patch.notes),
             createdAt: patch.createdAt && Date.parse(patch.createdAt) < Date.parse(local.createdAt) ? patch.createdAt : local.createdAt,
             updatedAt: importNewer ? (patch.updatedAt ?? local.updatedAt) : local.updatedAt,
-        }, brandProfileReferencesOmitted: brandProfileReferences.omitted };
+        }, brandProfileReferencesOmitted: brandProfileReferences.omitted, authoredHistoryOmitted };
 }
 function pickFreeId(preferred, domain, used) {
     const wanted = safeId(preferred);
@@ -418,7 +463,7 @@ function caseCollectionImportEnvelope(importedRaw) {
  * reinterpreted.
  * @param {CaseRecord[]} localCases
  * @param {unknown} importedRaw
- * @returns {{ cases: CaseRecord[], added: number, updated: number, skipped: number, brandProfileReferencesOmitted: number }}
+ * @returns {{ cases: CaseRecord[], added: number, updated: number, skipped: number, brandProfileReferencesOmitted: number, authoredHistoryOmitted: number }}
  */
 export function mergeCases(localCases, importedRaw) {
     assertBoundedJsonStructure(importedRaw, 'Case import', CASE_INPUT_JSON_LIMITS);
@@ -426,7 +471,7 @@ export function mergeCases(localCases, importedRaw) {
     const importedVersion = parseStoreVersion(importedEnvelope);
     if (importedVersion !== null && Number.isSafeInteger(importedVersion) && importedVersion < CASE_SCHEMA_VERSION
         && !CASE_IMPORT_VERSIONS.includes(importedVersion)) {
-        throw new Error(`Case schema ${importedVersion} is not part of the supported compatibility boundary. Exact published schemas 12 and 13 migrate to schema ${CASE_SCHEMA_VERSION}; no data was changed.`);
+        throw new Error(`Case schema ${importedVersion} is not part of the supported compatibility boundary. Exact published schemas ${CASE_IMPORT_VERSIONS.filter((version) => version < CASE_SCHEMA_VERSION).join(', ')} migrate to schema ${CASE_SCHEMA_VERSION}; no data was changed.`);
     }
     if (importedVersion !== null && Number.isSafeInteger(importedVersion) && importedVersion > CASE_SCHEMA_VERSION) {
         throw new Error(`Case schema ${importedVersion} is newer than the supported schema ${CASE_SCHEMA_VERSION}; no data was changed.`);
@@ -446,6 +491,7 @@ export function mergeCases(localCases, importedRaw) {
     const imported = boundedCaseList(importedEnvelope);
     let skipped = imported.omitted;
     let brandProfileReferencesOmitted = 0;
+    let authoredHistoryOmitted = 0;
     const fallback = new Date(0).toISOString();
     for (const item of imported.items) {
         const patch = extractImportPatch(item, supportedImportedVersion);
@@ -458,6 +504,7 @@ export function mergeCases(localCases, importedRaw) {
             const merged = applyImportPatch(existing, patch);
             byDomain.set(patch.domain, merged.record);
             brandProfileReferencesOmitted += patch.brandProfileReferencesOmitted + merged.brandProfileReferencesOmitted;
+            authoredHistoryOmitted += merged.authoredHistoryOmitted;
             updated += 1;
         }
         else if (byDomain.size < MAX_CASES) {
@@ -466,6 +513,7 @@ export function mergeCases(localCases, importedRaw) {
             usedIds.add(record.id);
             byDomain.set(patch.domain, record);
             brandProfileReferencesOmitted += patch.brandProfileReferencesOmitted;
+            authoredHistoryOmitted += patch.authoredHistoryOmitted;
             added += 1;
         }
         else {
@@ -478,5 +526,6 @@ export function mergeCases(localCases, importedRaw) {
         updated,
         skipped,
         brandProfileReferencesOmitted,
+        authoredHistoryOmitted,
     };
 }