@realtimex/folio 0.1.2

package/dist/api/src/services/PolicyEngine.js

@@ -0,0 +1,1353 @@
+import { createLogger } from "../utils/logger.js";
+import { SDKService } from "./SDKService.js";
+import { PolicyLoader } from "./PolicyLoader.js";
+import { GoogleSheetsService } from "./GoogleSheetsService.js";
+import { PolicyLearningService } from "./PolicyLearningService.js";
+import { Actuator } from "../utils/Actuator.js";
+import { extractLlmResponse, normalizeLlmContent, previewLlmText } from "../utils/llmResponse.js";
+import { DEFAULT_BASELINE_FIELDS } from "./BaselineConfigService.js";
+const logger = createLogger("PolicyEngine");
+/**
+ * Helper to build LLM message content. If the text contains the VLM marker
+ * generated by IngestionService, it casts the payload to an OpenAI-compatible
+ * Vision array structure so the underlying SDK bridge can transmit the image.
+ */
+function extractVlmPayload(text) {
+    const marker = text.match(/\[VLM_IMAGE_DATA:(data:[^;]+;base64,[^\]]+)\]/);
+    if (!marker)
+        return null;
+    const markerText = marker[0];
+    const supplementalText = text.replace(markerText, "").trim().slice(0, 4000);
+    return {
+        imageDataUrl: marker[1],
+        supplementalText,
+    };
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function buildMessageContent(prompt, text, textFirst = false) {
+    const vlmPayload = extractVlmPayload(text);
+    if (vlmPayload) {
+        const textPrompt = vlmPayload.supplementalText
+            ? `${prompt}\n\nSupplemental extracted fields:\n${vlmPayload.supplementalText}`
+            : prompt;
+        return [
+            { type: "text", text: textPrompt },
+            { type: "image_url", image_url: { url: vlmPayload.imageDataUrl } }
+        ];
+    }
+    // Standard text payload
+    return textFirst
+        ? `Document text:\n\n${text.trim().slice(0, 8000)}\n\n${prompt}`
+        : `${prompt}\n\nDocument text:\n${text.trim().slice(0, 8000)}`;
+}
+/**
+ * Robustly extracts and parses a JSON object from an LLM string response.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function parseLlmJson(raw) {
+    const rawText = normalizeLlmContent(raw).trim();
+    if (!rawText)
+        return null;
+    const jsonMatch = rawText.match(/```(?:json)?\s*([\s\S]*?)```/) ?? rawText.match(/(\{[\s\S]*\})/);
+    let jsonStr = jsonMatch ? (jsonMatch[1] ?? jsonMatch[0]) : rawText;
+    if (!jsonStr)
+        return null;
+    // Sanitize smart quotes to standard double quotes
+    jsonStr = jsonStr.replace(/[\u2018\u2019]/g, "'").replace(/[\u201C\u201D\u201E\u201F\u2033\u2036]/g, '"');
+    try {
+        return JSON.parse(jsonStr);
+    }
+    catch {
+        return null;
+    }
+}
+function hasText(value) {
+    return typeof value === "string" && value.trim().length > 0;
+}
+function stripTrailingPunctuation(value) {
+    return value.replace(/[),.;]+$/g, "");
+}
+function hasGidInReference(value) {
+    return hasText(value) && /(?:[?#]gid=\d+)/i.test(value);
+}
+function isDefaultSheetOneRange(value) {
+    if (!hasText(value))
+        return false;
+    const trimmed = value.trim();
+    const bang = trimmed.indexOf("!");
+    const sheetRef = (bang >= 0 ? trimmed.slice(0, bang) : trimmed).trim();
+    const normalized = /^'.*'$/.test(sheetRef)
+        ? sheetRef.slice(1, -1).replace(/''/g, "'").trim().toLowerCase()
+        : sheetRef.toLowerCase();
+    return normalized === "sheet1";
+}
+function shouldPreserveRangeHint(hints) {
+    if (!hasText(hints.range))
+        return false;
+    if (hasGidInReference(hints.sheetReference) && isDefaultSheetOneRange(hints.range)) {
+        return false;
+    }
+    return true;
+}
+function normalizeTemplateFieldKey(value) {
+    const normalized = value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "_")
+        .replace(/^_+|_+$/g, "");
+    return normalized || "value";
+}
+function inferTemplateFieldType(header) {
+    const normalized = normalizeTemplateFieldKey(header);
+    if (/(^|_)(date|due_date|invoice_date|receipt_date|service_date|posted_date)(_|$)/.test(normalized)) {
+        return "date";
+    }
+    if (/(^|_)(amount|total|subtotal|tax|price|cost|balance|fee|vat|discount|paid)(_|$)/.test(normalized)) {
+        return "currency";
+    }
+    if (/(^|_)(qty|quantity|count|units)(_|$)/.test(normalized)) {
+        return "number";
+    }
+    return "string";
+}
+function buildTemplateFieldHints(headers) {
+    const keyCounts = new Map();
+    return headers.map((header) => {
+        const baseKey = normalizeTemplateFieldKey(header);
+        const seen = keyCounts.get(baseKey) ?? 0;
+        keyCounts.set(baseKey, seen + 1);
+        const key = seen === 0 ? baseKey : `${baseKey}_${seen + 1}`;
+        return {
+            header,
+            key,
+            type: inferTemplateFieldType(header),
+        };
+    });
+}
+async function resolveSheetTemplateContext(hints, opts) {
+    if (!hasText(hints.sheetReference) || !hasText(opts.userId)) {
+        return {};
+    }
+    const preferredRange = shouldPreserveRangeHint(hints) ? hints.range : undefined;
+    const templateResult = await GoogleSheetsService.resolveTemplate(opts.userId, hints.sheetReference, preferredRange, opts.supabase);
+    if (!templateResult.success) {
+        return { warning: templateResult.error || "Failed to resolve Google Sheet template headers." };
+    }
+    const headers = (templateResult.headers ?? []).map((header) => header.trim()).filter(Boolean);
+    if (headers.length === 0) {
+        return { warning: "Google Sheet template has no headers in row 1." };
+    }
+    const fields = buildTemplateFieldHints(headers);
+    return {
+        context: {
+            spreadsheetReference: hints.sheetReference,
+            spreadsheetId: templateResult.spreadsheetId ?? hints.sheetReference,
+            range: templateResult.range ?? preferredRange ?? "Sheet1",
+            headers,
+            fields,
+        },
+    };
+}
+function applySheetTemplateContext(policy, template) {
+    if (!template || !policy || typeof policy !== "object" || !policy.spec || typeof policy.spec !== "object") {
+        return policy;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const spec = policy.spec;
+    if (!Array.isArray(spec.actions)) {
+        spec.actions = [];
+    }
+    const appendAction = spec.actions.find((action) => action?.type === "append_to_google_sheet");
+    if (appendAction) {
+        if (!hasText(appendAction.spreadsheet_id) && !hasText(appendAction.spreadsheet_url)) {
+            appendAction.spreadsheet_id = template.spreadsheetReference;
+        }
+        if (!hasText(appendAction.range) || isDefaultSheetOneRange(appendAction.range)) {
+            appendAction.range = template.range;
+        }
+    }
+    else {
+        spec.actions.push({
+            type: "append_to_google_sheet",
+            spreadsheet_id: template.spreadsheetReference,
+            range: template.range,
+        });
+    }
+    const existingExtract = Array.isArray(spec.extract) ? spec.extract : [];
+    const existingByKey = new Map();
+    for (const field of existingExtract) {
+        if (!field || typeof field.key !== "string")
+            continue;
+        const key = normalizeTemplateFieldKey(field.key);
+        if (!existingByKey.has(key)) {
+            existingByKey.set(key, field);
+        }
+    }
+    const usedKeys = new Set();
+    const orderedExtract = template.fields.map((templateField) => {
+        const existing = existingByKey.get(templateField.key);
+        usedKeys.add(templateField.key);
+        if (existing) {
+            return {
+                ...existing,
+                key: templateField.key,
+                type: existing.type ?? templateField.type,
+                description: existing.description?.trim() || `Extract value for Google Sheet column "${templateField.header}".`,
+            };
+        }
+        return {
+            key: templateField.key,
+            type: templateField.type,
+            description: `Extract value for Google Sheet column "${templateField.header}".`,
+        };
+    });
+    const extras = existingExtract.filter((field) => !usedKeys.has(normalizeTemplateFieldKey(field.key)));
+    spec.extract = [...orderedExtract, ...extras];
+    return policy;
+}
+function extractSynthesisTargetHints(description) {
+    const hints = {};
+    const gidMatch = description.match(/\b(?:gid|sheetid|sheet_id)\s*[:=]\s*(\d+)\b/i);
+    const gid = gidMatch?.[1];
+    const urlMatch = description.match(/https?:\/\/docs\.google\.com\/spreadsheets\/d\/[^\s)]+/i);
+    if (urlMatch?.[0]) {
+        const cleaned = stripTrailingPunctuation(urlMatch[0]);
+        // eslint-disable-next-line no-useless-escape
+        if (gid && !/[\?#]gid=\d+/i.test(cleaned)) {
+            hints.sheetReference = `${cleaned}#gid=${gid}`;
+        }
+        else {
+            hints.sheetReference = cleaned;
+        }
+    }
+    else {
+        const sheetIdMatch = description.match(/google\s*sheet(?:s)?(?:\s*(?:id|:))?[^A-Za-z0-9_-]*([A-Za-z0-9-_]{20,})/i);
+        if (sheetIdMatch?.[1]) {
+            hints.sheetReference = gid
+                ? `https://docs.google.com/spreadsheets/d/${sheetIdMatch[1]}/edit#gid=${gid}`
+                : sheetIdMatch[1];
+        }
+    }
+    const rangeMatch = description.match(/\brange\s*(?:is|=|:)?\s*([A-Za-z0-9_'"-]+![A-Za-z]+\d*(?::[A-Za-z]+\d*)?)/i);
+    if (rangeMatch?.[1]) {
+        hints.range = rangeMatch[1].trim();
+    }
+    const driveMatch = description.match(/\b(?:google\s*drive|gdrive|gdriver)?\s*folder(?:\s*id)?\s*(?:is|=|:)?\s*([A-Za-z0-9_-]{20,})\b/i);
+    if (driveMatch?.[1]) {
+        hints.driveFolderId = driveMatch[1];
+    }
+    return hints;
+}
+function applySynthesisTargetHints(policy, hints) {
+    if (!policy || typeof policy !== "object" || !policy.spec || typeof policy.spec !== "object") {
+        return policy;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const spec = policy.spec;
+    if (!Array.isArray(spec.actions)) {
+        spec.actions = [];
+    }
+    const actions = spec.actions;
+    if (hasText(hints.driveFolderId)) {
+        const driveAction = actions.find((action) => action?.type === "copy_to_gdrive");
+        if (driveAction) {
+            if (!hasText(driveAction.destination)) {
+                driveAction.destination = hints.driveFolderId;
+            }
+        }
+        else {
+            actions.push({
+                type: "copy_to_gdrive",
+                destination: hints.driveFolderId,
+            });
+        }
+    }
+    if (hasText(hints.sheetReference)) {
+        const sheetAction = actions.find((action) => action?.type === "append_to_google_sheet");
+        const preserveRangeHint = shouldPreserveRangeHint(hints);
+        if (sheetAction) {
+            if (!hasText(sheetAction.spreadsheet_id) && !hasText(sheetAction.spreadsheet_url)) {
+                sheetAction.spreadsheet_id = hints.sheetReference;
+            }
+            if (preserveRangeHint && !hasText(sheetAction.range)) {
+                sheetAction.range = hints.range;
+            }
+        }
+        else {
+            actions.push({
+                type: "append_to_google_sheet",
+                spreadsheet_id: hints.sheetReference,
+                ...(preserveRangeHint ? { range: hints.range } : {}),
+            });
+        }
+    }
+    return policy;
+}
+// ─── Matcher ────────────────────────────────────────────────────────────────
+async function evaluateCondition(condition, doc, trace, settings = {}) {
+    const sdk = SDKService.getSDK();
+    if (condition.type === "keyword") {
+        const values = Array.isArray(condition.value) ? condition.value : [condition.value ?? ""];
+        const text = condition.case_sensitive ? doc.text : doc.text.toLowerCase();
+        return values.some((v) => {
+            const needle = condition.case_sensitive ? v : v.toLowerCase();
+            return text.includes(needle);
+        });
+    }
+    if (condition.type === "filename") {
+        const values = Array.isArray(condition.value) ? condition.value : [condition.value ?? ""];
+        const name = condition.case_sensitive ? doc.filePath : doc.filePath.toLowerCase();
+        return values.some((v) => {
+            const needle = condition.case_sensitive ? v : v.toLowerCase();
+            return name.includes(needle);
+        });
+    }
+    if (condition.type === "file_type" || condition.type === "mime_type") {
+        const ext = doc.filePath.split(".").pop()?.toLowerCase() ?? "";
+        // MIME subtype → extension exceptions where they differ
+        const MIME_TO_EXT = { plain: "txt", markdown: "md", "x-markdown": "md" };
+        const values = Array.isArray(condition.value) ? condition.value : [condition.value ?? ""];
+        return values.some((v) => {
+            const normalized = v.toLowerCase().replace(/^\./, "");
+            // Direct extension match: "pdf" or ".pdf"
+            if (normalized === ext)
+                return true;
+            // MIME type match: "application/pdf" → subtype "pdf"
+            if (normalized.includes("/")) {
+                const subtype = normalized.split("/").pop() ?? "";
+                return (MIME_TO_EXT[subtype] ?? subtype) === ext;
+            }
+            return false;
+        });
+    }
+    if (condition.type === "llm_verify" || condition.type === "semantic") {
+        if (!sdk)
+            return false;
+        // For semantic conditions, treat the value(s) as the verification prompt if no explicit prompt is set
+        const prompt = condition.prompt
+            ?? (Array.isArray(condition.value) ? condition.value.join("; ") : condition.value)
+            ?? "";
+        if (!prompt)
+            return false;
+        trace.push({ timestamp: new Date().toISOString(), step: `Evaluating ${condition.type} condition`, details: { prompt } });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Policy Matching", { action: `Evaluating ${condition.type} condition`, prompt }, doc.supabase);
+        try {
+            const { provider, model } = await SDKService.resolveChatProvider(settings);
+            trace.push({
+                timestamp: new Date().toISOString(),
+                step: "LLM request (condition verify)",
+                details: {
+                    provider,
+                    model,
+                    condition_type: condition.type,
+                    prompt_preview: prompt.slice(0, 180),
+                    vision_payload: doc.text.includes("[VLM_IMAGE_DATA:")
+                }
+            });
+            Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Policy Matching", {
+                action: "LLM request (condition verify)",
+                provider,
+                model,
+                condition_type: condition.type,
+                prompt_preview: prompt.slice(0, 180),
+                vision_payload: doc.text.includes("[VLM_IMAGE_DATA:")
+            }, doc.supabase);
+            const result = await sdk.llm.chat([
+                {
+                    role: "system",
+                    content: "You are a document classifier. Answer with a single JSON object: { \"result\": true/false, \"confidence\": 0.0-1.0 }"
+                },
+                {
+                    role: "user",
+                    content: buildMessageContent(`Question: ${prompt}`, doc.text, true)
+                }
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ], { provider, model });
+            const raw = extractLlmResponse(result);
+            trace.push({
+                timestamp: new Date().toISOString(),
+                step: "LLM response (condition verify)",
+                details: {
+                    provider,
+                    model,
+                    raw_length: raw.length,
+                    raw_preview: previewLlmText(raw),
+                }
+            });
+            Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Policy Matching", {
+                action: "LLM response (condition verify)",
+                provider,
+                model,
+                raw_length: raw.length,
+                raw_preview: previewLlmText(raw),
+            }, doc.supabase);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const parsed = parseLlmJson(raw);
+            if (parsed && typeof parsed === "object" && typeof parsed.result === "boolean") {
+                const threshold = condition.confidence_threshold ?? 0.8;
+                const passed = parsed.result === true && (parsed.confidence ?? 1) >= threshold;
+                trace.push({ timestamp: new Date().toISOString(), step: `${condition.type} result`, details: { parsed, passed } });
+                Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Policy Matching", { action: `${condition.type} result`, parsed, passed }, doc.supabase);
+                return passed;
+            }
+        }
+        catch (err) {
+            logger.warn(`${condition.type} condition failed`, { err });
+        }
+        return false;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    logger.warn(`Unknown condition type "${condition.type}" — skipping`);
+    return false;
+}
+async function matchPolicy(policy, doc, trace, settings = {}) {
+    const { strategy, conditions } = policy.spec.match;
+    trace.push({ timestamp: new Date().toISOString(), step: `Evaluating policy rules`, details: { policyId: policy.metadata.id, strategy, conditionsCount: conditions.length } });
+    Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Policy Matching", { action: "Evaluating policy rules", policyId: policy.metadata.id, strategy, conditionsCount: conditions.length }, doc.supabase);
+    if (strategy === "ALL") {
+        for (const cond of conditions) {
+            if (!(await evaluateCondition(cond, doc, trace, settings))) {
+                trace.push({ timestamp: new Date().toISOString(), step: `Match failed on condition`, details: { condition: cond } });
+                Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Policy Matching", { action: "Match failed on condition", condition: cond }, doc.supabase);
+                return false;
+            }
+        }
+        trace.push({ timestamp: new Date().toISOString(), step: `All conditions matched`, details: { policyId: policy.metadata.id } });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Policy Matching", { action: "All conditions matched", policyId: policy.metadata.id }, doc.supabase);
+        return true;
+    }
+    // ANY strategy
+    for (const cond of conditions) {
+        if (await evaluateCondition(cond, doc, trace, settings)) {
+            trace.push({ timestamp: new Date().toISOString(), step: `Condition matched (ANY strategy)`, details: { condition: cond } });
+            Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Policy Matching", { action: "Condition matched (ANY strategy)", condition: cond }, doc.supabase);
+            return true;
+        }
+    }
+    trace.push({ timestamp: new Date().toISOString(), step: `No conditions matched (ANY strategy)` });
+    Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Policy Matching", { action: "No conditions matched (ANY strategy)" }, doc.supabase);
+    return false;
+}
+// ─── Extractor ───────────────────────────────────────────────────────────────
+async function extractData(fields, doc, trace, settings = {}) {
+    const sdk = SDKService.getSDK();
+    if (!sdk || fields.length === 0)
+        return {};
+    trace.push({ timestamp: new Date().toISOString(), step: "Starting data extraction", details: { fieldsCount: fields.length } });
+    Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Data Extraction", { action: "Starting data extraction", fieldsCount: fields.length }, doc.supabase);
+    const { provider, model } = await SDKService.resolveChatProvider(settings);
+    const fieldDescriptions = fields
+        .map((f) => `- "${f.key}" (${f.type}): ${f.description}${f.required ? " [REQUIRED]" : ""}`)
+        .join("\n");
+    const prompt = `Extract the following fields from the document. Return ONLY a valid JSON object with the field keys and their extracted values. Use null for fields that cannot be found.
+
+Fields to extract:
+${fieldDescriptions}`;
+    try {
+        const isVlmPayload = doc.text.startsWith("[VLM_IMAGE_DATA:");
+        const mixedPrompt = isVlmPayload
+            ? `You are a precise data extraction engine. Return only valid JSON.\n\n${prompt}`
+            : prompt;
+        trace.push({
+            timestamp: new Date().toISOString(),
+            step: "LLM request (data extraction)",
+            details: {
+                provider,
+                model,
+                fields_count: fields.length,
+                vision_payload: isVlmPayload,
+            }
+        });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Data Extraction", {
+            action: "LLM request (data extraction)",
+            provider,
+            model,
+            fields_count: fields.length,
+            vision_payload: isVlmPayload,
+        }, doc.supabase);
+        const result = await sdk.llm.chat(isVlmPayload
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ? [{ role: "user", content: buildMessageContent(mixedPrompt, doc.text) }]
+            : [
+                { role: "system", content: "You are a precise data extraction engine. Return only valid JSON." },
+                { role: "user", content: buildMessageContent(prompt, doc.text) }
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ], { provider, model });
+        const raw = extractLlmResponse(result);
+        trace.push({
+            timestamp: new Date().toISOString(),
+            step: "LLM response (data extraction)",
+            details: {
+                provider,
+                model,
+                raw_length: raw.length,
+                raw_preview: previewLlmText(raw),
+            }
+        });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Data Extraction", {
+            action: "LLM response (data extraction)",
+            provider,
+            model,
+            raw_length: raw.length,
+            raw_preview: previewLlmText(raw),
+        }, doc.supabase);
+        const parsed = parseLlmJson(raw);
+        if (parsed) {
+            trace.push({ timestamp: new Date().toISOString(), step: "Data extracted successfully", details: { extractedKeys: Object.keys(parsed) } });
+            Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Data Extraction", { action: "Data extracted successfully", extractedKeys: Object.keys(parsed), raw_response: parsed }, doc.supabase);
+            return parsed;
+        }
+        else {
+            logger.warn("Data extraction returned unparseable JSON", { raw: raw.slice(0, 300) });
+            trace.push({ timestamp: new Date().toISOString(), step: "Data extraction failed", details: { error: "Unparseable JSON from model" } });
+            Actuator.logEvent(doc.ingestionId, doc.userId, "error", "Data Extraction", { action: "Data extraction unparseable", raw_response: raw.slice(0, 300) }, doc.supabase);
+        }
+    }
+    catch (err) {
+        logger.error("Data extraction failed", { err });
+        trace.push({ timestamp: new Date().toISOString(), step: "Data extraction failed", details: { error: String(err) } });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "error", "Data Extraction", { action: "Data extraction failed", error: String(err) }, doc.supabase);
+    }
+    return {};
+}
+function valueToPromptPreview(value) {
+    if (value == null)
+        return "null";
+    if (typeof value === "string")
+        return value;
+    if (typeof value === "number" || typeof value === "boolean")
+        return String(value);
+    try {
+        return JSON.stringify(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+function hasMeaningfulValue(value) {
+    if (value == null)
+        return false;
+    if (typeof value === "string")
+        return value.trim().length > 0;
+    if (typeof value === "number")
+        return Number.isFinite(value);
+    if (typeof value === "boolean")
+        return true;
+    if (Array.isArray(value))
+        return value.some((item) => hasMeaningfulValue(item));
+    if (typeof value === "object")
+        return Object.values(value).some((item) => hasMeaningfulValue(item));
+    return true;
+}
+function removeDuplicateOrEmptyEnrichmentFields(enrichment, contractData) {
+    const existingKeys = new Set(Object.keys(contractData).map((key) => normalizeTemplateFieldKey(key)));
+    const cleaned = {};
+    for (const [key, value] of Object.entries(enrichment)) {
+        const normalizedKey = normalizeTemplateFieldKey(key);
+        if (existingKeys.has(normalizedKey))
+            continue;
+        if (!hasMeaningfulValue(value))
+            continue;
+        cleaned[key] = value;
+    }
+    return cleaned;
+}
+async function extractEnrichmentData(doc, contractData, trace, settings = {}) {
+    const sdk = SDKService.getSDK();
+    if (!sdk || Object.keys(contractData).length === 0)
+        return {};
+    const { provider, model } = await SDKService.resolveChatProvider(settings);
+    const knownBlock = Object.entries(contractData)
+        .slice(0, 80)
+        .map(([key, value]) => `- "${key}": ${valueToPromptPreview(value).slice(0, 300)}`)
+        .join("\n");
+    const prompt = `You already extracted the core template fields below.
+Return ONLY a valid JSON object with ADDITIONAL useful fields found in the document (do not repeat existing keys).
+
+Already extracted fields:
+${knownBlock || "- none"}
+
+Add only high-confidence extra signals useful for downstream automation or analytics, such as:
+- line_items: array of { description, quantity, unit_price, total }
+- tax_breakdown, discounts, payment_details
+- merchant_metadata, customer_metadata
+- notes or compliance identifiers
+
+Rules:
+- Return {} if no reliable extras exist.
+- Keep keys in snake_case.
+- Do not overwrite or repeat existing keys.
+- JSON only. No markdown or explanations.`;
+    try {
+        trace.push({
+            timestamp: new Date().toISOString(),
+            step: "LLM request (enrichment extraction)",
+            details: {
+                provider,
+                model,
+                known_fields_count: Object.keys(contractData).length,
+            },
+        });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Data Extraction", {
+            action: "LLM request (enrichment extraction)",
+            provider,
+            model,
+            known_fields_count: Object.keys(contractData).length,
+        }, doc.supabase);
+        const isVlmPayload = doc.text.startsWith("[VLM_IMAGE_DATA:");
+        const mixedPrompt = isVlmPayload
+            ? `You are a precise data extraction engine. Return only valid JSON.\n\n${prompt}`
+            : prompt;
+        const result = await sdk.llm.chat(isVlmPayload
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ? [{ role: "user", content: buildMessageContent(mixedPrompt, doc.text) }]
+            : [
+                { role: "system", content: "You are a precise data extraction engine. Return only valid JSON." },
+                { role: "user", content: buildMessageContent(prompt, doc.text) },
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ], { provider, model });
+        const raw = extractLlmResponse(result);
+        trace.push({
+            timestamp: new Date().toISOString(),
+            step: "LLM response (enrichment extraction)",
+            details: {
+                provider,
+                model,
+                raw_length: raw.length,
+                raw_preview: previewLlmText(raw),
+            },
+        });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Data Extraction", {
+            action: "LLM response (enrichment extraction)",
+            provider,
+            model,
+            raw_length: raw.length,
+            raw_preview: previewLlmText(raw),
+        }, doc.supabase);
+        const parsed = parseLlmJson(raw);
+        if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+            trace.push({
+                timestamp: new Date().toISOString(),
+                step: "Enrichment extraction skipped",
+                details: { reason: "Unparseable enrichment JSON" },
+            });
+            return {};
+        }
+        const cleaned = removeDuplicateOrEmptyEnrichmentFields(parsed, contractData);
+        trace.push({
+            timestamp: new Date().toISOString(),
+            step: "Enrichment extraction complete",
+            details: {
+                extracted_keys: Object.keys(cleaned),
+                extracted_count: Object.keys(cleaned).length,
+            },
+        });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Data Extraction", {
+            action: "Enrichment extraction complete",
+            extracted_keys: Object.keys(cleaned),
+            extracted_count: Object.keys(cleaned).length,
+        }, doc.supabase);
+        return cleaned;
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        trace.push({
+            timestamp: new Date().toISOString(),
+            step: "Enrichment extraction failed",
+            details: { error: msg },
+        });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "error", "Data Extraction", {
+            action: "Enrichment extraction failed",
+            error: msg,
+        }, doc.supabase);
+        return {};
+    }
+}
+function attachEnrichment(contractData, enrichment) {
+    if (Object.keys(enrichment).length === 0) {
+        return contractData;
+    }
+    return {
+        ...contractData,
+        _enrichment: enrichment,
+    };
+}
+function toActionData(data) {
+    const normalized = {};
+    for (const [key, value] of Object.entries(data)) {
+        if (value == null) {
+            normalized[key] = null;
+            continue;
+        }
+        if (typeof value === "string" || typeof value === "number") {
+            normalized[key] = value;
+            continue;
+        }
+        if (typeof value === "boolean") {
+            normalized[key] = value ? "true" : "false";
+            continue;
+        }
+        try {
+            normalized[key] = JSON.stringify(value);
+        }
+        catch {
+            normalized[key] = String(value);
+        }
+    }
+    return normalized;
+}
+async function executeMatchedPolicy(policy, doc, trace, settings, baselineEntities, matchSource = "rules") {
+    trace.push({
+        timestamp: new Date().toISOString(),
+        step: "Matched policy selected",
+        details: {
+            policyId: policy.metadata.id,
+            policyName: policy.metadata.name,
+            matchSource,
+        }
+    });
+    Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Policy Matching", {
+        action: "Matched policy selected",
+        policyId: policy.metadata.id,
+        policyName: policy.metadata.name,
+        matchSource,
+    }, doc.supabase);
+    const extractedData = await extractData(policy.spec.extract ?? [], doc, trace, settings);
+    const hasSheetAppendAction = (policy.spec.actions ?? []).some((action) => action.type === "append_to_google_sheet");
+    const enrichmentData = hasSheetAppendAction
+        ? await extractEnrichmentData(doc, extractedData, trace, settings)
+        : {};
+    const extractedForStorage = attachEnrichment(extractedData, enrichmentData);
+    const missingRequired = (policy.spec.extract ?? [])
+        .filter((f) => f.required && extractedData[f.key] == null)
+        .map((f) => f.key);
+    if (missingRequired.length > 0) {
+        trace.push({ timestamp: new Date().toISOString(), step: "Missing required fields", details: { missingRequired } });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "error", "Data Extraction", { action: "Missing required fields", missingRequired }, doc.supabase);
+        return {
+            filePath: doc.filePath,
+            matchedPolicy: policy.metadata.id,
+            extractedData: extractedForStorage,
+            actionsExecuted: [],
+            status: "error",
+            error: `Missing required fields: ${missingRequired.join(", ")}`,
+            trace,
+        };
+    }
+    const actuatorResult = await Actuator.execute(doc.ingestionId, doc.userId, policy.spec.actions ?? [], toActionData({ ...baselineEntities, ...extractedData }), { path: doc.filePath, name: doc.filePath.split('/').pop() || doc.filePath }, policy.spec.extract ?? [], doc.supabase);
+    trace.push(...actuatorResult.trace);
+    return {
+        filePath: doc.filePath,
+        matchedPolicy: policy.metadata.id,
+        extractedData: extractedForStorage,
+        actionsExecuted: actuatorResult.actionsExecuted,
+        status: "matched",
+        error: actuatorResult.errors[0],
+        trace,
+    };
+}
+// ─── Engine ──────────────────────────────────────────────────────────────────
+export class PolicyEngine {
+    /**
+     * Run a document through the policy pipeline.
+     * Returns the first matched policy result, or the fallback.
+     */
+    static async process(doc, settings = {}, baselineEntities = {}) {
+        logger.info(`Processing document: ${doc.filePath}`);
+        const policies = await PolicyLoader.load();
+        const globalTrace = [{ timestamp: new Date().toISOString(), step: "Loaded policies", details: { count: policies.length } }];
+        Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Triage", { action: "Loaded policies", count: policies.length }, doc.supabase);
+        for (const policy of policies) {
+            try {
+                const matched = await matchPolicy(policy, doc, globalTrace, settings);
+                if (!matched)
+                    continue;
+                logger.info(`Matched policy: ${policy.metadata.id} (priority: ${policy.metadata.priority})`);
+                return executeMatchedPolicy(policy, doc, globalTrace, settings, baselineEntities, "rules");
+            }
+            catch (err) {
+                logger.error(`Error evaluating policy ${policy.metadata.id}`, { err });
+            }
+        }
+        // Fallback: Inbox Zero
+        globalTrace.push({ timestamp: new Date().toISOString(), step: "No policy matched - routed to fallback" });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Triage", { action: "No policy matched - routed to fallback" }, doc.supabase);
+        logger.info(`No policy matched — routing to fallback`);
+        return {
+            filePath: doc.filePath,
+            matchedPolicy: null,
+            extractedData: {},
+            actionsExecuted: ["Moved to /_Needs_Review"],
+            status: "fallback",
+            trace: globalTrace
+        };
+    }
+    /**
+     * Same as process() but uses a pre-loaded list of policies.
+     * Used by IngestionService so user-scoped policies are evaluated.
+     */
+    static async processWithPolicies(doc, policies, settings = {}, baselineEntities = {}, opts = {}) {
+        logger.info(`Processing document with ${policies.length} policies: ${doc.filePath}`);
+        const globalTrace = [{ timestamp: new Date().toISOString(), step: "Loaded user policies", details: { count: policies.length } }];
+        Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Triage", { action: "Loaded user policies", count: policies.length }, doc.supabase);
+        const forcedPolicyId = opts.forcedPolicyId?.trim();
+        for (const policy of policies) {
+            if (forcedPolicyId && policy.metadata.id !== forcedPolicyId) {
+                continue;
+            }
+            try {
+                const isManualOverride = forcedPolicyId === policy.metadata.id;
+                const matched = isManualOverride ? true : await matchPolicy(policy, doc, globalTrace, settings);
+                if (!matched)
+                    continue;
+                if (isManualOverride) {
+                    globalTrace.push({
+                        timestamp: new Date().toISOString(),
+                        step: "Manual override policy selected",
+                        details: { policyId: policy.metadata.id, policyName: policy.metadata.name },
+                    });
+                    Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Policy Matching", {
+                        action: "Manual override policy selected",
+                        policyId: policy.metadata.id,
+                        policyName: policy.metadata.name,
+                    }, doc.supabase);
+                }
+                logger.info(`Matched policy: ${policy.metadata.id}${isManualOverride ? " (manual override)" : ""}`);
+                return executeMatchedPolicy(policy, doc, globalTrace, settings, baselineEntities, isManualOverride ? "manual_override" : "rules");
+            }
+            catch (err) {
+                logger.error(`Error evaluating policy ${policy.metadata.id}`, { err });
+            }
+        }
+        const allowLearnedFallback = opts.allowLearnedFallback !== false && !forcedPolicyId;
+        if (allowLearnedFallback && doc.supabase && policies.length > 0) {
+            try {
+                const learningText = doc.text.replace(/\[VLM_IMAGE_DATA:[^\]]+\]/g, "");
+                const learned = await PolicyLearningService.resolveLearnedCandidate({
+                    supabase: doc.supabase,
+                    userId: doc.userId,
+                    policyIds: policies.map((policy) => policy.metadata.id),
+                    filePath: doc.filePath,
+                    baselineEntities,
+                    documentText: learningText,
+                });
+                if (learned.candidate) {
+                    const learnedPolicy = policies.find((policy) => policy.metadata.id === learned.candidate?.policyId);
+                    if (learnedPolicy) {
+                        globalTrace.push({
+                            timestamp: new Date().toISOString(),
+                            step: "Learned policy candidate selected",
+                            details: {
+                                policyId: learned.candidate.policyId,
+                                score: learned.candidate.score,
+                                support: learned.candidate.support,
+                                topCandidates: learned.diagnostics.topCandidates,
+                            },
+                        });
+                        Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Policy Matching", {
+                            action: "Learned policy candidate selected",
+                            policyId: learned.candidate.policyId,
+                            score: learned.candidate.score,
+                            support: learned.candidate.support,
+                            topCandidates: learned.diagnostics.topCandidates,
+                        }, doc.supabase);
+                        logger.info(`Matched policy via learned fallback: ${learned.candidate.policyId} (score=${learned.candidate.score.toFixed(3)}, support=${learned.candidate.support})`);
+                        return executeMatchedPolicy(learnedPolicy, doc, globalTrace, settings, baselineEntities, "learned");
+                    }
+                }
+                globalTrace.push({
+                    timestamp: new Date().toISOString(),
+                    step: "Learned fallback analyzed",
+                    details: learned.diagnostics,
+                });
+                Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Policy Matching", {
+                    action: "Learned fallback analyzed",
+                    ...learned.diagnostics,
+                }, doc.supabase);
+            }
+            catch (learningError) {
+                logger.warn("Learned fallback lookup failed", { error: learningError });
+            }
+        }
+        globalTrace.push({ timestamp: new Date().toISOString(), step: "No policy matched - routed to fallback" });
+        Actuator.logEvent(doc.ingestionId, doc.userId, "info", "Triage", { action: "No policy matched - routed to fallback" }, doc.supabase);
+        return {
+            filePath: doc.filePath,
+            matchedPolicy: null,
+            extractedData: {},
+            actionsExecuted: [],
+            status: "fallback",
+            trace: globalTrace
+        };
+    }
+    /**
+     * Stage 1 of the optimised pipeline: extract baseline entities from a document
+     * using the user's active baseline config (or the built-in defaults).
+     *
+     * The result is always persisted on the ingestion record regardless of whether
+     * any policy ultimately matches — every document leaves the Fast Path with
+     * structured entities attached.
+     *
+     * Returns the extracted entity map plus a list of field keys the model flagged
+     * as uncertain or absent, which are later used by the confidence-gating logic
+     * to decide whether a targeted deep call is worth firing.
+     */
+    static async extractBaseline(doc, config, settings = {}) {
+        const sdk = SDKService.getSDK();
+        if (!sdk) {
+            logger.warn("SDK unavailable — skipping baseline extraction");
+            return { entities: {}, uncertain_fields: [], tags: [] };
+        }
+        let fields = (config.fields ?? DEFAULT_BASELINE_FIELDS).filter((f) => f.enabled);
+        // Always include suggested_filename so auto_rename actions have an AI-generated name
+        // even when the user's saved baseline config predates this field.
+        if (!fields.some((f) => f.key === "suggested_filename")) {
+            const suggestedField = DEFAULT_BASELINE_FIELDS.find((f) => f.key === "suggested_filename");
+            if (suggestedField)
+                fields = [...fields, suggestedField];
+        }
+        if (fields.length === 0)
+            return { entities: {}, uncertain_fields: [], tags: [] };
+        const { provider, model } = await SDKService.resolveChatProvider(settings);
+        const fieldList = fields
+            .map((f) => `- "${f.key}" (${f.type}): ${f.description}`)
+            .join("\n");
+        const contextBlock = config.context?.trim()
+            ? `\nAdditional context about this user's documents:\n${config.context.trim()}\n`
+            : "";
+        const systemPrompt = `You are a precise document entity extractor.${contextBlock}\n` +
+            `Return ONLY a valid JSON object with three keys:\n` +
+            `  "entities": an object containing each requested field (use null for absent fields),\n` +
+            `  "uncertain_fields": an array of field keys you are not confident about,\n` +
+            `  "tags": an array of 3-6 lowercase semantic labels that best classify this document ` +
+            `(e.g. "invoice", "utility", "tax-deductible", "receipt", "2025", "insurance"). ` +
+            `Include the calendar year if clearly present. Prefer hyphenated multi-word tags.\n` +
+            `No markdown, no explanation — only the JSON object.`;
+        const userPrompt = `Extract the following fields from the document:\n${fieldList}`;
+        const isVlmPayload = doc.text.startsWith("[VLM_IMAGE_DATA:");
+        const mixedPrompt = isVlmPayload ? `${systemPrompt}\n\n${userPrompt}` : userPrompt;
+        try {
+            Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Baseline Extraction", {
+                action: "LLM request (baseline extraction)",
+                provider,
+                model,
+                fields_count: fields.length,
+                vision_payload: isVlmPayload,
+            }, doc.supabase);
+            const result = await sdk.llm.chat(isVlmPayload
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                ? [{ role: "user", content: buildMessageContent(mixedPrompt, doc.text) }]
+                : [
+                    { role: "system", content: systemPrompt },
+                    { role: "user", content: buildMessageContent(userPrompt, doc.text) },
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                ], { provider, model });
+            const raw = extractLlmResponse(result);
+            Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Baseline Extraction", {
+                action: "LLM response (baseline extraction)",
+                provider,
+                model,
+                raw_length: raw.length,
+                raw_preview: previewLlmText(raw),
+            }, doc.supabase);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const parsed = parseLlmJson(raw);
+            if (!parsed) {
+                logger.warn("Baseline extraction returned unparseable JSON", { raw: raw.slice(0, 300) });
+                Actuator.logEvent(doc.ingestionId, doc.userId, "error", "Baseline Extraction", { action: "Baseline extraction unparseable", raw_response: raw.slice(0, 300) }, doc.supabase);
+                return { entities: {}, uncertain_fields: [], tags: [] };
+            }
+            const entities = parsed.entities ?? parsed;
+            const uncertain_fields = Array.isArray(parsed.uncertain_fields)
+                ? parsed.uncertain_fields
+                : [];
+            const tags = Array.isArray(parsed.tags)
+                ? parsed.tags.map((t) => String(t).toLowerCase().trim()).filter(Boolean)
+                : [];
+            logger.info(`Baseline extraction complete — ${Object.keys(entities).length} fields, ${uncertain_fields.length} uncertain, ${tags.length} tags`);
+            Actuator.logEvent(doc.ingestionId, doc.userId, "analysis", "Baseline Extraction", { action: "Baseline extraction complete", fields: Object.keys(entities).length, uncertain: uncertain_fields.length, tags, extracted: entities }, doc.supabase);
+            return { entities, uncertain_fields, tags };
+        }
+        catch (err) {
+            logger.error("Baseline extraction failed", { err });
+            Actuator.logEvent(doc.ingestionId, doc.userId, "error", "Baseline Extraction", { action: "Baseline extraction failed", error: String(err) }, doc.supabase);
+            return { entities: {}, uncertain_fields: [], tags: [] };
+        }
+    }
+    /**
+     * Suggest a baseline config (context + fields) from a workflow description.
+     * Returns a draft { context, fields } the user can review before saving.
+     */
+    static async suggestBaseline(description, currentFields, opts = {}) {
+        const sdk = SDKService.getSDK();
+        if (!sdk) {
+            const msg = "SDK not available for baseline suggestion";
+            logger.warn(msg);
+            return { suggestion: null, error: msg };
+        }
+        const defaults = await SDKService.getDefaultChatProvider();
+        const provider = opts.provider || defaults.provider;
+        const model = opts.model || defaults.model;
+        logger.info(`Suggesting baseline config via ${provider}/${model}`);
+        // Summarise the current field keys so the LLM knows what already exists
+        const existingKeys = currentFields.map((f) => f.key).join(", ");
+        const systemPrompt = `You are a document intelligence expert helping configure a baseline extraction schema for Folio, a local document automation tool.
+
+Given a description of the user's workflow, return ONLY a valid JSON object with this exact shape (no markdown, no backticks, no explanation):
+{
+  "context": "one or two sentences injected into the LLM extraction prompt — describe document types, languages, vendors, or any domain detail that helps the model",
+  "fields": [
+    { "key": "snake_case_key", "type": "string|number|date|currency|string[]", "description": "what to extract and why", "enabled": true, "is_default": false }
+  ]
+}
+
+Rules:
+- "context" must be a single concise string (≤ 3 sentences). Focus on what makes these documents distinctive.
+- "fields" must only contain CUSTOM fields the user should ADD — do not repeat any of the existing field keys: ${existingKeys}
+- Each custom field needs a clear key (snake_case, no spaces), the most precise type, and a description that doubles as a hint to the extraction model.
+- Suggest between 2 and 6 custom fields — quality over quantity.
+- Return an empty fields array if no meaningful custom fields apply.`;
+        try {
+            if (opts.userId) {
+                Actuator.logEvent(null, opts.userId, "analysis", "Configuration", {
+                    action: "LLM request (baseline suggestion)",
+                    provider,
+                    model,
+                    description_preview: description.slice(0, 180),
+                    current_fields_count: currentFields.length,
+                }, opts.supabase);
+            }
+            const result = await sdk.llm.chat([
+                { role: "system", content: systemPrompt },
+                { role: "user", content: `My workflow: ${description}` }
+            ], { provider, model });
+            const raw = extractLlmResponse(result);
+            if (opts.userId) {
+                Actuator.logEvent(null, opts.userId, "analysis", "Configuration", {
+                    action: "LLM response (baseline suggestion)",
+                    provider,
+                    model,
+                    raw_length: raw.length,
+                    raw_preview: previewLlmText(raw),
+                }, opts.supabase);
+            }
+            if (!raw)
+                return { suggestion: null, error: "LLM returned empty response" };
+            const parsed = parseLlmJson(raw);
+            if (!parsed) {
+                logger.error("JSON parse failed for baseline suggestion", { raw: raw.slice(0, 300) });
+                return { suggestion: null, error: "LLM response was not valid JSON" };
+            }
+            if (typeof parsed.context !== "string" || !Array.isArray(parsed.fields)) {
+                return { suggestion: null, error: "LLM response did not match expected shape" };
+            }
+            // Ensure all suggested fields are marked as custom
+            parsed.fields = parsed.fields.map((f) => ({ ...f, is_default: false, enabled: true }));
+            logger.info(`Baseline suggestion: context length=${parsed.context.length}, custom fields=${parsed.fields.length}`);
+            return { suggestion: parsed };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            logger.error("Baseline suggestion failed", { err });
+            if (opts.userId) {
+                Actuator.logEvent(null, opts.userId, "error", "Configuration", {
+                    action: "LLM baseline suggestion failed",
+                    provider,
+                    model,
+                    error: msg,
+                }, opts.supabase);
+            }
+            return { suggestion: null, error: msg };
+        }
+    }
+    /**
+     * Suggest a refinement draft for an existing policy based on a specific
+     * ingestion that the user expected to match.
+     */
+    static async suggestPolicyRefinement(currentPolicy, docContext, opts = {}) {
+        const sdk = SDKService.getSDK();
+        if (!sdk) {
+            const msg = "SDK not available for policy refinement";
+            logger.warn(msg);
+            return { policy: null, rationale: [], error: msg };
+        }
+        const defaults = await SDKService.getDefaultChatProvider();
+        const provider = opts.provider || defaults.provider;
+        const model = opts.model || defaults.model;
+        const compactTrace = (docContext.trace ?? [])
+            .slice(-12)
+            .map((entry) => ({
+            step: entry.step,
+            details: entry.details ?? null,
+        }));
+        const evidence = {
+            filename: docContext.filename,
+            mime_type: docContext.mimeType ?? null,
+            status: docContext.status ?? null,
+            summary: docContext.summary ?? null,
+            tags: docContext.tags ?? [],
+            extracted: docContext.extracted ?? {},
+            recent_trace: compactTrace,
+        };
+        const systemPrompt = `You are a Folio Policy Engine expert. Refine an existing policy so it better matches a target document while minimizing regressions.
+
+Return ONLY valid JSON with this exact shape:
+{
+  "policy": <full FolioPolicy object>,
+  "rationale": ["short reason 1", "short reason 2"]
+}
+
+Rules:
+- Keep metadata.id EXACTLY unchanged.
+- Preserve existing action targets unless the evidence clearly requires a correction.
+- Prefer tightening/adding deterministic match conditions (keyword/file_type) over broadening.
+- Keep extraction fields useful; do not remove important existing fields without a reason.
+- Ensure output remains a valid Folio policy schema.`;
+        const userPrompt = `Current policy:\n${JSON.stringify(currentPolicy, null, 2)}\n\n` +
+            `Target document evidence:\n${JSON.stringify(evidence, null, 2)}\n\n` +
+            `Produce a refined policy draft and rationale.`;
+        try {
+            if (opts.userId) {
+                Actuator.logEvent(docContext.ingestionId, opts.userId, "analysis", "Policy Synthesis", {
+                    action: "LLM request (policy refinement)",
+                    provider,
+                    model,
+                    policy_id: currentPolicy.metadata.id,
+                }, opts.supabase);
+            }
+            const result = await sdk.llm.chat([
+                { role: "system", content: systemPrompt },
+                { role: "user", content: userPrompt },
+            ], { provider, model });
+            const raw = extractLlmResponse(result);
+            if (opts.userId) {
+                Actuator.logEvent(docContext.ingestionId, opts.userId, "analysis", "Policy Synthesis", {
+                    action: "LLM response (policy refinement)",
+                    provider,
+                    model,
+                    raw_length: raw.length,
+                    raw_preview: previewLlmText(raw),
+                }, opts.supabase);
+            }
+            if (!raw.trim()) {
+                return { policy: null, rationale: [], error: "LLM returned empty refinement response" };
+            }
+            const parsed = parseLlmJson(raw);
+            if (!parsed) {
+                return { policy: null, rationale: [], error: "LLM refinement response was not valid JSON" };
+            }
+            const parsedPolicy = parsed.policy ?? parsed;
+            if (!parsedPolicy || typeof parsedPolicy !== "object") {
+                return { policy: null, rationale: [], error: "Refinement response did not include a policy draft" };
+            }
+            const repairedPolicy = {
+                ...currentPolicy,
+                ...parsedPolicy,
+                apiVersion: "folio/v1",
+                kind: "Policy",
+                metadata: {
+                    ...currentPolicy.metadata,
+                    ...(parsedPolicy.metadata ?? {}),
+                    id: currentPolicy.metadata.id,
+                    version: parsedPolicy.metadata?.version ?? currentPolicy.metadata.version ?? "1.0.0",
+                    enabled: parsedPolicy.metadata?.enabled ?? currentPolicy.metadata.enabled ?? true,
+                    priority: parsedPolicy.metadata?.priority ?? currentPolicy.metadata.priority,
+                    name: parsedPolicy.metadata?.name ?? currentPolicy.metadata.name,
+                    description: parsedPolicy.metadata?.description ?? currentPolicy.metadata.description,
+                },
+                spec: {
+                    ...currentPolicy.spec,
+                    ...(parsedPolicy.spec ?? {}),
+                    match: parsedPolicy.spec?.match ?? currentPolicy.spec.match,
+                    extract: Array.isArray(parsedPolicy.spec?.extract)
+                        ? parsedPolicy.spec.extract
+                        : currentPolicy.spec.extract,
+                    actions: Array.isArray(parsedPolicy.spec?.actions) && parsedPolicy.spec.actions.length > 0
+                        ? parsedPolicy.spec.actions
+                        : currentPolicy.spec.actions,
+                },
+            };
+            if (!PolicyLoader.validate(repairedPolicy)) {
+                return { policy: null, rationale: [], error: "Refined policy draft did not pass schema validation" };
+            }
+            const rationale = Array.isArray(parsed.rationale)
+                ? parsed.rationale.map((item) => String(item).trim()).filter(Boolean).slice(0, 6)
+                : [];
+            return {
+                policy: repairedPolicy,
+                rationale,
+            };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            logger.error("Policy refinement suggestion failed", { err });
+            if (opts.userId) {
+                Actuator.logEvent(docContext.ingestionId, opts.userId, "error", "Policy Synthesis", {
+                    action: "LLM policy refinement failed",
+                    provider,
+                    model,
+                    error: msg,
+                    policy_id: currentPolicy.metadata.id,
+                }, opts.supabase);
+            }
+            return { policy: null, rationale: [], error: msg };
+        }
+    }
+    /**
+     * Synthesize a FolioPolicy from a natural language description using the LLM.
+     */
+    static async synthesizeFromNL(description, opts = {}) {
+        const sdk = SDKService.getSDK();
+        if (!sdk) {
+            const msg = "SDK not available for policy synthesis";
+            logger.warn(msg);
+            return { policy: null, error: msg };
+        }
+        // Use explicitly provided provider/model, else fall back to SDK defaults
+        const defaults = await SDKService.getDefaultChatProvider();
+        const provider = opts.provider || defaults.provider;
+        const model = opts.model || defaults.model;
+        logger.info(`Synthesizing policy via ${provider}/${model}`);
+        const targetHints = extractSynthesisTargetHints(description);
+        const synthesisWarnings = [];
+        let sheetTemplateContext;
+        if (hasText(targetHints.sheetReference) && hasText(opts.userId)) {
+            try {
+                const templateResolution = await resolveSheetTemplateContext(targetHints, {
+                    userId: opts.userId,
+                    supabase: opts.supabase,
+                });
+                sheetTemplateContext = templateResolution.context;
+                if (templateResolution.warning) {
+                    synthesisWarnings.push(`Google Sheet template note: ${templateResolution.warning}`);
+                }
+                if (sheetTemplateContext && opts.userId) {
+                    Actuator.logEvent(null, opts.userId, "analysis", "Policy Synthesis", {
+                        action: "Resolved Google Sheet template for synthesis",
+                        spreadsheet_id: sheetTemplateContext.spreadsheetId,
+                        range: sheetTemplateContext.range,
+                        headers_count: sheetTemplateContext.headers.length,
+                        headers: sheetTemplateContext.headers,
+                    }, opts.supabase);
+                }
+            }
+            catch (templateErr) {
+                const templateMsg = templateErr instanceof Error ? templateErr.message : String(templateErr);
+                synthesisWarnings.push(`Google Sheet template note: ${templateMsg}`);
+                logger.warn("Failed to resolve Google Sheet template for synthesis", { error: templateErr });
+            }
+        }
+        const systemPrompt = `You are a Folio Policy Engine expert. Convert natural language descriptions into a valid FolioPolicy JSON object.
+
+Return ONLY a valid JSON object with this exact shape (no markdown, no backticks):
+{
+  "apiVersion": "folio/v1",
+  "kind": "Policy",
+  "metadata": { "id": "kebab-case-id", "name": "Human Name", "version": "1.0.0", "description": "Brief description", "priority": 100, "tags": ["tag1"], "enabled": true },
+  "spec": {
+    "match": { "strategy": "ALL", "conditions": [{ "type": "keyword", "value": ["keyword1", "keyword2"], "case_sensitive": false }] },
+    "extract": [{ "key": "field_name", "type": "string", "description": "what to extract", "required": true }],
+    "actions": [{ "type": "copy", "destination": "/path/to/folder" }]
+  }
+}
+
+Supported action types include:
+- copy, rename, auto_rename, copy_to_gdrive, append_to_google_sheet, log_csv, notify, webhook
+- For append_to_google_sheet use:
+  { "type": "append_to_google_sheet", "spreadsheet_id": "<sheet-id-or-url>", "columns": ["{date}","{issuer}"] }
+- columns is optional; if omitted, runtime auto-maps extracted fields to sheet headers dynamically.
+- range is optional; only include it when user explicitly requires a specific tab/range.
+- If spreadsheet_id is a Google Sheets URL with gid, omit range unless user explicitly provided one.
+- Never omit user-provided external targets (folder IDs, spreadsheet URLs/IDs); preserve explicit ranges unless the provided range is only a generic Sheet1 fallback alongside a gid URL.`;
+        const preserveRangeHint = shouldPreserveRangeHint(targetHints);
+        const sheetTemplateFieldsPreview = sheetTemplateContext
+            ? sheetTemplateContext.fields
+                .slice(0, 40)
+                .map((field) => `  - "${field.header}" -> key "${field.key}" (${field.type})`)
+                .join("\n")
+            : "";
+        const hasMoreTemplateFields = !!sheetTemplateContext && sheetTemplateContext.fields.length > 40;
+        const sheetTemplateGuidanceBlock = sheetTemplateContext
+            ? `\n\nGoogle Sheet template context (authoritative for this policy):
+- spreadsheet_id: ${sheetTemplateContext.spreadsheetReference}
+- resolved range: ${sheetTemplateContext.range}
+- headers (${sheetTemplateContext.fields.length}):
+${sheetTemplateFieldsPreview}${hasMoreTemplateFields ? "\n  - ... (truncated)" : ""}
+
+When template context is present:
+- Ensure spec.extract includes keys for these headers (same key names as listed).
+- Keep append_to_google_sheet action targeting this spreadsheet_id.
+- Use range "${sheetTemplateContext.range}" unless the user explicitly asks for a different valid range.`
+            : "";
+        const targetHintsBlock = [
+            hasText(targetHints.driveFolderId) ? `- Google Drive Folder ID: ${targetHints.driveFolderId}` : null,
+            hasText(targetHints.sheetReference) ? `- Google Sheet: ${targetHints.sheetReference}` : null,
+            preserveRangeHint ? `- Preferred range: ${targetHints.range}` : null,
+        ].filter(Boolean).join("\n");
+        const synthesisRequest = targetHintsBlock
+            ? `Create a policy for: ${description}\n\nRequired target values to preserve exactly:\n${targetHintsBlock}${sheetTemplateGuidanceBlock}`
+            : `Create a policy for: ${description}`;
+        try {
+            if (opts.userId) {
+                Actuator.logEvent(null, opts.userId, "analysis", "Policy Synthesis", {
+                    action: "LLM request (policy synthesis)",
+                    provider,
+                    model,
+                    description_preview: description.slice(0, 180),
+                }, opts.supabase);
+            }
+            const result = await sdk.llm.chat([
+                { role: "system", content: systemPrompt },
+                { role: "user", content: synthesisRequest }
+            ], { provider, model });
+            // Log the entire result to discover the SDK response schema
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            logger.info(`Full SDK result keys: ${Object.keys(result).join(", ")}`);
+            logger.info(`Full SDK result: ${JSON.stringify(result).slice(0, 1000)}`);
+            // SDK response shape: { success: true, response: { content: "..." } }
+            const raw = extractLlmResponse(result);
+            if (opts.userId) {
+                Actuator.logEvent(null, opts.userId, "analysis", "Policy Synthesis", {
+                    action: "LLM response (policy synthesis)",
+                    provider,
+                    model,
+                    raw_length: raw.length,
+                    raw_preview: previewLlmText(raw),
+                }, opts.supabase);
+            }
+            logger.info(`Synthesis raw response (first 500 chars): ${raw.slice(0, 500)}`);
+            if (!raw) {
+                return { policy: null, error: "LLM returned empty response", raw };
+            }
+            const parsed = parseLlmJson(raw);
+            if (!parsed) {
+                logger.error("JSON parse failed", { raw: raw.slice(0, 300) });
+                return { policy: null, error: "LLM response was not valid JSON", raw };
+            }
+            let repaired = applySynthesisTargetHints(parsed, targetHints);
+            repaired = applySheetTemplateContext(repaired, sheetTemplateContext);
+            if (PolicyLoader.validate(repaired)) {
+                return {
+                    policy: repaired,
+                    ...(synthesisWarnings.length > 0 ? { error: synthesisWarnings.join(" ") } : {}),
+                };
+            }
+            // Return as draft even if validation fails — let the UI show a preview
+            logger.warn("Synthesized policy failed strict validation, returning as draft");
+            synthesisWarnings.push("Policy schema may be incomplete — please review before saving");
+            return { policy: repaired, error: synthesisWarnings.join(" ") };
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            logger.error("Policy synthesis failed", { err });
+            if (opts.userId) {
+                Actuator.logEvent(null, opts.userId, "error", "Policy Synthesis", {
+                    action: "LLM policy synthesis failed",
+                    provider,
+                    model,
+                    error: msg,
+                }, opts.supabase);
+            }
+            return { policy: null, error: msg };
+        }
+    }
+}