@dataparade/cli 0.0.1

package/dist/src/ai-enrichment/providers/provider-contract.js

@@ -0,0 +1,703 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildProposalsValidationContext = buildProposalsValidationContext;
+exports.parseAgentFromPrompt = parseAgentFromPrompt;
+exports.aiDebugEnabled = aiDebugEnabled;
+exports.strictParseAndNormalizeProposals = strictParseAndNormalizeProposals;
+const openai_proposals_response_schema_1 = require("../openai-proposals-response.schema");
+const scan_paths_1 = require("../scan-paths");
+const scan_env_1 = require("../../config/scan-env");
+const AGENTS = new Set([
+    "tpAgent",
+    "propertyAgent",
+    "directionAgent",
+    "interactionAgent",
+]);
+const CANDIDATE_TYPES = new Set([
+    "third_party",
+    "node_property",
+    "flow_direction",
+    "missing_interaction",
+]);
+const FLOW_TYPES = new Set([
+    "api_call",
+    "database_query",
+    "message_queue",
+    "file_transfer",
+    "webhook",
+    "rpc",
+]);
+function pathsFromSlimComponentEntry(entry) {
+    const out = [];
+    const df = entry.detectedFrom;
+    if (Array.isArray(df)) {
+        for (const x of df) {
+            if (x && typeof x === "object") {
+                const fp = x.filePath;
+                if (typeof fp === "string" && fp.trim())
+                    out.push((0, scan_paths_1.normScanPath)(fp.trim()));
+            }
+        }
+    }
+    const sl = entry.sourceLocations;
+    if (Array.isArray(sl)) {
+        for (const x of sl) {
+            if (x && typeof x === "object") {
+                const fp = x.filePath;
+                if (typeof fp === "string" && fp.trim())
+                    out.push((0, scan_paths_1.normScanPath)(fp.trim()));
+            }
+        }
+    }
+    return out;
+}
+function buildProposalsValidationContext(userPrompt) {
+    let parsed;
+    try {
+        parsed = JSON.parse(userPrompt);
+    }
+    catch {
+        return null;
+    }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
+        return null;
+    const o = parsed;
+    const excerptPaths = new Set();
+    const excerptsByNormPath = new Map();
+    const rc = o.relevantFileContents;
+    if (rc && typeof rc === "object" && !Array.isArray(rc)) {
+        for (const key of Object.keys(rc)) {
+            const n = (0, scan_paths_1.normScanPath)(key);
+            excerptPaths.add(n);
+            const val = rc[key];
+            if (typeof val === "string")
+                excerptsByNormPath.set(n, val);
+        }
+    }
+    const allowedPathsByComponentId = new Map();
+    const cc = o.componentContext;
+    if (cc && typeof cc === "object" && !Array.isArray(cc)) {
+        for (const [compId, entry] of Object.entries(cc)) {
+            if (!entry || typeof entry !== "object" || Array.isArray(entry))
+                continue;
+            const set = new Set(pathsFromSlimComponentEntry(entry));
+            allowedPathsByComponentId.set(compId, set);
+        }
+    }
+    return { excerptPaths, excerptsByNormPath, allowedPathsByComponentId };
+}
+function posixBasename(p) {
+    const n = (0, scan_paths_1.normScanPath)(p);
+    const i = n.lastIndexOf("/");
+    return i >= 0 ? n.slice(i + 1) : n;
+}
+function findExcerptContent(normFp, excerptsByNormPath) {
+    if (excerptsByNormPath.has(normFp))
+        return excerptsByNormPath.get(normFp);
+    for (const [k, v] of excerptsByNormPath) {
+        if (normFp === k || normFp.endsWith(`/${k}`) || k.endsWith(`/${normFp}`)) {
+            return v;
+        }
+    }
+    const bn = posixBasename(normFp);
+    if (bn.length > 0) {
+        for (const [k, v] of excerptsByNormPath) {
+            if (posixBasename(k) === bn)
+                return v;
+        }
+    }
+    return undefined;
+}
+function pathsMatchAllowed(normFp, allowed) {
+    if (allowed.size === 0)
+        return true;
+    if (allowed.has(normFp))
+        return true;
+    for (const a of allowed) {
+        if (normFp === a)
+            return true;
+        if (normFp.endsWith(`/${a}`) || a.endsWith(`/${normFp}`))
+            return true;
+    }
+    const bn = posixBasename(normFp);
+    if (bn.length > 0) {
+        for (const a of allowed) {
+            if (posixBasename(a) === bn)
+                return true;
+        }
+    }
+    return false;
+}
+/**
+ * Ensure evidence points at payload files; when an excerpt exists, clamp line ranges to
+ * file length instead of dropping (models often overshoot endLine).
+ */
+function normalizeEvidenceRefForContext(ref, targetComponentId, ctx) {
+    if (!ctx)
+        return ref;
+    const normFp = (0, scan_paths_1.normScanPath)(ref.filePath);
+    const componentPaths = ctx.allowedPathsByComponentId.get(targetComponentId) ?? new Set();
+    const allowed = new Set([...ctx.excerptPaths, ...componentPaths]);
+    if (allowed.size > 0 && !pathsMatchAllowed(normFp, allowed)) {
+        return null;
+    }
+    const content = findExcerptContent(normFp, ctx.excerptsByNormPath);
+    if (content != null) {
+        const lineCount = Math.max(1, content.split(/\r?\n/).length);
+        let start = Math.max(1, Math.floor(ref.startLine));
+        let end = Math.max(start, Math.floor(ref.endLine));
+        start = Math.min(start, lineCount);
+        end = Math.min(end, lineCount);
+        if (end < start)
+            end = start;
+        return { ...ref, filePath: ref.filePath.trim(), startLine: start, endLine: end };
+    }
+    return { ...ref, filePath: ref.filePath.trim() };
+}
+function evidenceRefKey(ref) {
+    return `${(0, scan_paths_1.normScanPath)(ref.filePath)}|${ref.startLine}|${ref.endLine}|${ref.reason}`;
+}
+function flattenPropertyEvidence(map) {
+    const seen = new Set();
+    const out = [];
+    for (const refs of Object.values(map)) {
+        for (const ref of refs) {
+            const k = evidenceRefKey(ref);
+            if (seen.has(k))
+                continue;
+            seen.add(k);
+            out.push(ref);
+        }
+    }
+    return out;
+}
+function isPlaceholderPropertyValue(value) {
+    if (value === null || value === undefined)
+        return true;
+    if (typeof value === "string") {
+        const normalized = value.trim().toLowerCase();
+        return normalized === "" || normalized === "none";
+    }
+    if (Array.isArray(value))
+        return value.length === 0;
+    return false;
+}
+function sanitizeSetProperties(raw) {
+    const sanitized = {};
+    for (const [key, value] of Object.entries(raw)) {
+        if (key === "inference_status")
+            continue;
+        if (isPlaceholderPropertyValue(value))
+            continue;
+        sanitized[key] = value;
+    }
+    return sanitized;
+}
+const UI_DATA_CATEGORY_ALLOWED = new Set([
+    "personal_identifiers",
+    "contact_information",
+    "financial_information",
+    "health_data",
+    "biometric_data",
+    "location_data",
+    "behavioral_data",
+    "employment_data",
+    "education_data",
+    "special_category_data",
+    "criminal_records",
+    "demographic_data",
+    "online_identifiers",
+    "device_identifiers",
+    "usage_data",
+    "aggregated_analytics",
+    "financial_data",
+    "geolocation_gps",
+    "geolocation_ip",
+    "photos_selfies",
+    "professional_data",
+    "sensitive_personal_data",
+    "special_categories",
+    "user_identity_direct",
+    "user_identity_indirect",
+    "other",
+]);
+const UI_DATA_CATEGORY_ALIASES = {
+    personal_data: "user_identity_direct",
+    pii: "user_identity_direct",
+    personally_identifiable_information: "user_identity_direct",
+    document_files: "other",
+    document_metadata: "other",
+    file_content: "other",
+    file_metadata: "other",
+    email_address: "contact_information",
+    credit_card_data: "financial_data",
+    payment_data: "financial_data",
+    phi: "health_data",
+    ip_address: "geolocation_ip",
+    device_id: "device_identifiers",
+};
+const UI_PROCESSING_PURPOSE_ALLOWED = new Set([
+    "authentication",
+    "service_provision",
+    "customer_support",
+    "payment_processing",
+    "fraud_prevention",
+    "analytics",
+    "marketing",
+    "advertising",
+    "development_testing",
+    "debugging",
+    "security_monitoring",
+    "compliance",
+    "audit",
+    "data_migration",
+    "backup_recovery",
+    "research",
+    "infrastructure_management",
+    "hosting_services",
+    "data_storage",
+    "email_delivery",
+    "security",
+    "user_insights",
+    "performance_monitoring",
+    "customer_management",
+    "sales_automation",
+    "legal_compliance",
+    "transactional_emails",
+    "marketing_communications",
+    "backup_services",
+    "disaster_recovery",
+    "other",
+]);
+const UI_PROCESSING_PURPOSE_ALIASES = {
+    legal: "legal_compliance",
+    legal_basis: "legal_compliance",
+    provisioning: "service_provision",
+    document_processing: "service_provision",
+    storage: "data_storage",
+    backup: "backup_services",
+};
+function normalizeToken(value) {
+    return value
+        .trim()
+        .toLowerCase()
+        .replace(/[\s-]+/g, "_")
+        .replace(/[^a-z0-9_]/g, "");
+}
+function toStringArray(raw) {
+    if (raw == null)
+        return [];
+    if (Array.isArray(raw))
+        return raw.map((v) => String(v)).map((v) => v.trim()).filter(Boolean);
+    if (typeof raw === "string") {
+        return raw
+            .split(",")
+            .map((v) => v.trim())
+            .filter(Boolean);
+    }
+    return [String(raw).trim()].filter(Boolean);
+}
+function mapToAllowed(values, allowed, aliases) {
+    const out = [];
+    const seen = new Set();
+    for (const value of values) {
+        const token = normalizeToken(value);
+        if (!token)
+            continue;
+        const mapped = allowed.has(token) ? token : aliases[token] ?? "other";
+        if (!allowed.has(mapped))
+            continue;
+        if (!seen.has(mapped)) {
+            seen.add(mapped);
+            out.push(mapped);
+        }
+    }
+    return out;
+}
+function normalizeThirdPartySetPropertiesForUi(input) {
+    const normalized = { ...input };
+    const evidenceSourceByKey = {};
+    if (Object.prototype.hasOwnProperty.call(normalized, "data_categories_received")) {
+        const values = toStringArray(normalized.data_categories_received);
+        const mapped = mapToAllowed(values, UI_DATA_CATEGORY_ALLOWED, UI_DATA_CATEGORY_ALIASES);
+        if (mapped.length > 0)
+            normalized.data_categories_received = mapped;
+        else
+            delete normalized.data_categories_received;
+    }
+    if (Object.prototype.hasOwnProperty.call(normalized, "processing_purpose")) {
+        const values = toStringArray(normalized.processing_purpose);
+        const mapped = mapToAllowed(values, UI_PROCESSING_PURPOSE_ALLOWED, UI_PROCESSING_PURPOSE_ALIASES);
+        if (mapped.length > 0)
+            normalized.processing_purpose = mapped;
+        else
+            delete normalized.processing_purpose;
+    }
+    if (!Object.prototype.hasOwnProperty.call(normalized, "cloud_provider") &&
+        Object.prototype.hasOwnProperty.call(normalized, "cloud_services_used")) {
+        const services = toStringArray(normalized.cloud_services_used).map(normalizeToken);
+        const provider = services.find((s) => s === "aws" || s === "amazon" || s === "amazon_web_services")
+            ? "AWS"
+            : services.find((s) => s === "gcp" || s === "google" || s === "google_cloud")
+                ? "GCP"
+                : services.find((s) => s === "azure" || s === "microsoft_azure")
+                    ? "Azure"
+                    : undefined;
+        if (provider) {
+            normalized.cloud_provider = provider;
+            evidenceSourceByKey.cloud_provider = "cloud_services_used";
+        }
+    }
+    return { normalized, evidenceSourceByKey };
+}
+function parseAgentFromPrompt(prompt) {
+    try {
+        const o = JSON.parse(prompt);
+        if (o.agent && AGENTS.has(o.agent)) {
+            return o.agent;
+        }
+    }
+    catch {
+        /* user message may not be JSON */
+    }
+    return "propertyAgent";
+}
+function aiDebugEnabled() {
+    return (0, scan_env_1.isScanAiDebugEnabled)();
+}
+function normalizeEvidence(raw) {
+    if (!Array.isArray(raw))
+        return [];
+    const out = [];
+    for (const x of raw) {
+        if (!x || typeof x !== "object")
+            continue;
+        const o = x;
+        const filePath = typeof o.filePath === "string" ? o.filePath.trim() : "";
+        const startLine = typeof o.startLine === "number" && Number.isFinite(o.startLine)
+            ? Math.max(1, Math.floor(o.startLine))
+            : 1;
+        const endLine = typeof o.endLine === "number" && Number.isFinite(o.endLine)
+            ? Math.max(startLine, Math.floor(o.endLine))
+            : startLine;
+        const reason = typeof o.reason === "string" ? o.reason.trim() : "";
+        if (!filePath || !reason)
+            continue;
+        out.push({ filePath, startLine, endLine, reason });
+    }
+    return out;
+}
+function normalizeConfidence(raw) {
+    if (!raw || typeof raw !== "object")
+        return null;
+    const o = raw;
+    const score = typeof o.score === "number" && Number.isFinite(o.score) ? o.score : NaN;
+    if (score < 0 || score > 1)
+        return null;
+    const band = o.band === "high" || o.band === "medium" || o.band === "low" ? o.band : "medium";
+    return { score, band };
+}
+function normalizeComponentPatch(p, ct, conf, defaults, validationCtx, reportDrop) {
+    const targetComponentId = typeof p.targetComponentId === "string" ? p.targetComponentId.trim() : "";
+    if (!targetComponentId) {
+        reportDrop?.("component_patch_missing_targetComponentId", p);
+        return null;
+    }
+    const setProperties = p.setProperties;
+    if (typeof setProperties !== "object" ||
+        setProperties === null ||
+        Array.isArray(setProperties)) {
+        reportDrop?.("component_patch_invalid_setProperties_object", p);
+        return null;
+    }
+    const peRaw = p.propertyEvidence;
+    if (typeof peRaw !== "object" ||
+        peRaw === null ||
+        Array.isArray(peRaw)) {
+        reportDrop?.("component_patch_invalid_propertyEvidence_object", p);
+        return null;
+    }
+    const sanitizedSetProperties = sanitizeSetProperties(setProperties);
+    const thirdPartyNormalized = ct === "third_party"
+        ? normalizeThirdPartySetPropertiesForUi(sanitizedSetProperties)
+        : { normalized: sanitizedSetProperties, evidenceSourceByKey: {} };
+    const normalizedSetProperties = thirdPartyNormalized.normalized;
+    const evidenceSourceByKey = thirdPartyNormalized.evidenceSourceByKey;
+    const propKeys = Object.keys(normalizedSetProperties);
+    if (propKeys.length === 0) {
+        reportDrop?.("component_patch_setProperties_empty_after_sanitize", p);
+        return null;
+    }
+    const propertyEvidenceRecord = peRaw;
+    const keptProps = {};
+    const keptEvidence = {};
+    for (const key of propKeys) {
+        const sourceKey = evidenceSourceByKey[key] ?? key;
+        const refs = normalizeEvidence(propertyEvidenceRecord[sourceKey]);
+        if (refs.length === 0) {
+            reportDrop?.(`component_patch_property_${key}_missing_or_invalid_evidence`, {
+                targetComponentId,
+                key,
+            });
+            continue;
+        }
+        const adjustedRefs = [];
+        let allRefsValid = true;
+        for (const ref of refs) {
+            const adj = normalizeEvidenceRefForContext(ref, targetComponentId, validationCtx);
+            if (!adj) {
+                allRefsValid = false;
+                reportDrop?.(`component_patch_property_${key}_evidence_path_not_allowed`, {
+                    targetComponentId,
+                    key,
+                    evidenceFilePath: ref.filePath,
+                });
+                break;
+            }
+            adjustedRefs.push(adj);
+        }
+        if (!allRefsValid)
+            continue;
+        keptProps[key] = normalizedSetProperties[key];
+        keptEvidence[key] = adjustedRefs;
+    }
+    if (Object.keys(keptProps).length === 0) {
+        reportDrop?.("component_patch_all_properties_dropped_by_evidence_validation", {
+            targetComponentId,
+            attemptedPropertyKeys: propKeys,
+        });
+        return null;
+    }
+    const evidence = flattenPropertyEvidence(keptEvidence);
+    if (evidence.length === 0) {
+        reportDrop?.("component_patch_flattened_evidence_empty", {
+            targetComponentId,
+            keptPropertyKeys: Object.keys(keptProps),
+        });
+        return null;
+    }
+    return {
+        kind: "component_patch",
+        targetComponentId,
+        candidateType: ct,
+        setSubType: typeof p.setSubType === "string" && p.setSubType.trim()
+            ? p.setSubType.trim()
+            : undefined,
+        setDescription: typeof p.setDescription === "string" && p.setDescription.trim()
+            ? p.setDescription.trim()
+            : undefined,
+        setProperties: keptProps,
+        propertyEvidence: keptEvidence,
+        confidence: conf,
+        evidence,
+        provider: defaults.provider,
+        model: defaults.model,
+        agent: defaults.agent,
+    };
+}
+function normalizeProposal(raw, defaults, validationCtx, reportDrop) {
+    if (!raw || typeof raw !== "object") {
+        reportDrop?.("proposal_not_object", raw);
+        return null;
+    }
+    const p = raw;
+    const kind = p.kind;
+    const candidateType = p.candidateType;
+    if (typeof candidateType !== "string" ||
+        !CANDIDATE_TYPES.has(candidateType)) {
+        reportDrop?.("proposal_invalid_candidateType", p);
+        return null;
+    }
+    const ct = candidateType;
+    const conf = normalizeConfidence(p.confidence);
+    if (!conf) {
+        reportDrop?.("proposal_invalid_confidence", p);
+        return null;
+    }
+    if (kind === "component_patch") {
+        return normalizeComponentPatch(p, ct, conf, defaults, validationCtx, reportDrop);
+    }
+    const evidence = normalizeEvidence(p.evidence);
+    if (evidence.length === 0) {
+        reportDrop?.("flow_patch_missing_or_invalid_evidence", p);
+        return null;
+    }
+    if (kind === "flow_patch") {
+        const targetFlowId = typeof p.targetFlowId === "string" && p.targetFlowId.trim()
+            ? p.targetFlowId.trim()
+            : undefined;
+        const sourceComponentId = typeof p.sourceComponentId === "string" ? p.sourceComponentId.trim() : "";
+        const targetComponentId = typeof p.targetComponentId === "string" ? p.targetComponentId.trim() : "";
+        if (!targetFlowId && (!sourceComponentId || !targetComponentId)) {
+            reportDrop?.("flow_patch_missing_target_and_endpoints", p);
+            return null;
+        }
+        let setType;
+        if (typeof p.setType === "string" && FLOW_TYPES.has(p.setType)) {
+            setType = p.setType;
+        }
+        return {
+            kind: "flow_patch",
+            candidateType: ct,
+            targetFlowId,
+            insertIfMissing: p.insertIfMissing === true,
+            sourceComponentId: sourceComponentId || undefined,
+            targetComponentId: targetComponentId || undefined,
+            setType,
+            setDirection: p.setDirection === "forward" || p.setDirection === "reverse"
+                ? p.setDirection
+                : undefined,
+            setMethod: typeof p.setMethod === "string" && p.setMethod.trim()
+                ? p.setMethod.trim()
+                : undefined,
+            setEndpoint: typeof p.setEndpoint === "string" && p.setEndpoint.trim()
+                ? p.setEndpoint.trim()
+                : undefined,
+            setDescription: typeof p.setDescription === "string" && p.setDescription.trim()
+                ? p.setDescription.trim()
+                : undefined,
+            confidence: conf,
+            evidence,
+            provider: defaults.provider,
+            model: defaults.model,
+            agent: defaults.agent,
+        };
+    }
+    reportDrop?.("proposal_unknown_kind", p);
+    return null;
+}
+function previewJson(value, maxChars) {
+    try {
+        const serialized = JSON.stringify(value);
+        if (!serialized)
+            return "<empty>";
+        if (typeof maxChars !== "number" || !Number.isFinite(maxChars) || maxChars <= 0) {
+            return serialized;
+        }
+        return serialized.length > maxChars
+            ? `${serialized.slice(0, maxChars)}...<truncated>`
+            : serialized;
+    }
+    catch {
+        return "<unserializable>";
+    }
+}
+function coercePropertyEvidenceShape(proposal) {
+    if (!proposal || typeof proposal !== "object" || Array.isArray(proposal)) {
+        return proposal;
+    }
+    const p = proposal;
+    const propertyEvidence = p.propertyEvidence;
+    if (!propertyEvidence || typeof propertyEvidence !== "object" || Array.isArray(propertyEvidence)) {
+        return proposal;
+    }
+    const pe = propertyEvidence;
+    let changed = false;
+    const coerced = {};
+    for (const [key, value] of Object.entries(pe)) {
+        if (Array.isArray(value)) {
+            coerced[key] = value;
+            continue;
+        }
+        if (value && typeof value === "object") {
+            coerced[key] = [value];
+            changed = true;
+            continue;
+        }
+        coerced[key] = value;
+    }
+    if (!changed)
+        return proposal;
+    return { ...p, propertyEvidence: coerced };
+}
+function strictParseAndNormalizeProposals(raw, defaults, options) {
+    const dropReasonCounts = new Map();
+    const dropReasonSamples = new Map();
+    const reportDrop = (reason, rawValue) => {
+        dropReasonCounts.set(reason, (dropReasonCounts.get(reason) ?? 0) + 1);
+        if (!dropReasonSamples.has(reason) && rawValue !== undefined) {
+            dropReasonSamples.set(reason, previewJson(rawValue, 400));
+        }
+    };
+    const coerceRaw = (() => {
+        if (!raw || typeof raw !== "object" || Array.isArray(raw))
+            return raw;
+        const o = raw;
+        const proposals = o.proposals;
+        if (Array.isArray(proposals)) {
+            return {
+                ...o,
+                proposals: proposals.map((item) => coercePropertyEvidenceShape(item)),
+            };
+        }
+        if (!proposals || typeof proposals !== "object") {
+            return raw;
+        }
+        // Anthropic occasionally returns `proposals` as an object keyed by index.
+        // Coerce this shape into the expected array before strict schema validation.
+        const base = {
+            ...o,
+            proposals: Object.values(proposals),
+        };
+        const arr = base.proposals;
+        return Array.isArray(arr)
+            ? { ...base, proposals: arr.map((item) => coercePropertyEvidenceShape(item)) }
+            : base;
+    })();
+    const validated = openai_proposals_response_schema_1.openAiProposalsResponseSchema.safeParse(coerceRaw);
+    if (!validated.success) {
+        if (aiDebugEnabled()) {
+            const rawObj = raw && typeof raw === "object" && !Array.isArray(raw)
+                ? raw
+                : null;
+            const coercedObj = coerceRaw && typeof coerceRaw === "object" && !Array.isArray(coerceRaw)
+                ? coerceRaw
+                : null;
+            const rawProposals = rawObj?.proposals;
+            const coercedProposals = coercedObj?.proposals;
+            console.warn(`[dataparade-ai] ${options.debugLabel}: schema validation failed for provider response.`, validated.error.flatten());
+            console.warn(`[dataparade-ai] ${options.debugLabel}: response shape debug`, {
+                rawType: Array.isArray(raw) ? "array" : typeof raw,
+                rawKeys: rawObj ? Object.keys(rawObj).slice(0, 20) : [],
+                rawProposalsType: Array.isArray(rawProposals) ? "array" : typeof rawProposals,
+                coercedProposalsType: Array.isArray(coercedProposals)
+                    ? "array"
+                    : typeof coercedProposals,
+                coercedProposalsLength: Array.isArray(coercedProposals)
+                    ? coercedProposals.length
+                    : undefined,
+            });
+            console.warn(`[dataparade-ai] ${options.debugLabel}: raw response preview ${previewJson(raw)}`);
+            console.warn(`[dataparade-ai] ${options.debugLabel}: coerced response preview ${previewJson(coerceRaw)}`);
+        }
+        return [];
+    }
+    const validationCtx = options.userPrompt
+        ? buildProposalsValidationContext(options.userPrompt)
+        : null;
+    const out = [];
+    const totalValidated = validated.data.proposals.length;
+    for (const item of validated.data.proposals) {
+        const proposal = normalizeProposal(item, defaults, validationCtx, reportDrop);
+        if (proposal)
+            out.push(proposal);
+    }
+    if (aiDebugEnabled()) {
+        const dropped = totalValidated - out.length;
+        console.warn(`[dataparade-ai] ${options.debugLabel}: normalization summary validated=${totalValidated} kept=${out.length} dropped=${dropped}`);
+        if (dropped > 0) {
+            const sortedReasons = [...dropReasonCounts.entries()].sort((a, b) => b[1] - a[1]);
+            console.warn(`[dataparade-ai] ${options.debugLabel}: normalization drop reasons`, Object.fromEntries(sortedReasons));
+            for (const [reason] of sortedReasons.slice(0, 8)) {
+                const sample = dropReasonSamples.get(reason);
+                if (sample) {
+                    console.warn(`[dataparade-ai] ${options.debugLabel}: sample for ${reason}: ${sample}`);
+                }
+            }
+        }
+        if (out.length > 0) {
+            console.warn(`[dataparade-ai] ${options.debugLabel}: first normalized proposal preview ${previewJson(out[0])}`);
+        }
+    }
+    return out;
+}