@realtimex/folio 0.1.11 → 0.1.13

package/dist/api/src/services/IngestionService.js

@@ -1,5 +1,9 @@
 import fs from "fs/promises";
+import { execFile } from "child_process";
+import os from "os";
+import path from "path";
 import { PDFParse } from "pdf-parse";
+import { promisify } from "util";
 import { createLogger } from "../utils/logger.js";
 import { PolicyLoader } from "./PolicyLoader.js";
 import { PolicyEngine } from "./PolicyEngine.js";
@@ -11,6 +15,7 @@ import { RAGService } from "./RAGService.js";
 import { SDKService } from "./SDKService.js";
 import { ModelCapabilityService } from "./ModelCapabilityService.js";
 const logger = createLogger("IngestionService");
+const execFileAsync = promisify(execFile);
 /**
  * Multi-signal classifier that decides whether pdf-parse extracted enough
  * real text to skip GPU OCR and go straight to the local LLM (Fast Path).
@@ -51,6 +56,17 @@ function isPdfTextExtractable(pdfData) {
     return true;
 }
 export class IngestionService {
+    static FAST_EXTS = ["txt", "md", "csv", "json"];
+    static IMAGE_EXTS = ["png", "jpg", "jpeg", "webp"];
+    static IMAGE_REENCODE_TIMEOUT_MS = 15000;
+    static IMAGE_REENCODE_RETRY_ENABLED = (process.env.FOLIO_VLM_IMAGE_REENCODE_RETRY_ENABLED ?? "true").toLowerCase() !== "false";
+    static IMAGE_REENCODE_RETRY_METRICS = {
+        attempted: 0,
+        succeeded: 0,
+        failed: 0,
+        skipped_disabled: 0,
+        skipped_unavailable: 0,
+    };
     static NON_IDEMPOTENT_ACTION_TYPES = new Set([
         "append_to_google_sheet",
         "webhook",
@@ -85,10 +101,10 @@ export class IngestionService {
         return String(value);
     }
     static buildVlmSemanticText(opts) {
-        const { filename, finalStatus, policyName, extracted, tags } = opts;
+        const { filename, finalStatus, policyName, extracted, tags, modality } = opts;
         const lines = [
             `Document filename: ${filename}`,
-            "Document source: VLM image extraction",
+            `Document source: VLM ${modality} extraction`,
             `Processing status: ${finalStatus}`,
         ];
         if (policyName) {
@@ -134,6 +150,7 @@ export class IngestionService {
             policyName: opts.policyName,
             extracted: opts.extracted,
             tags: opts.tags,
+            modality: opts.modality,
         });
         const details = {
             synthetic_chars: syntheticText.length,
@@ -160,6 +177,76 @@ export class IngestionService {
         });
         return details;
     }
+    static buildVlmPayloadMarker(modality, dataUrl) {
+        const prefix = modality === "pdf" ? "VLM_PDF_DATA" : "VLM_IMAGE_DATA";
+        return `[${prefix}:${dataUrl}]`;
+    }
+    static async fileToDataUrl(filePath, mimeType) {
+        const buffer = await fs.readFile(filePath);
+        const base64 = buffer.toString("base64");
+        return `data:${mimeType};base64,${base64}`;
+    }
+    static errorToMessage(error) {
+        if (error instanceof Error)
+            return error.message;
+        if (typeof error === "string")
+            return error;
+        if (error && typeof error === "object") {
+            const candidate = error;
+            if (typeof candidate.message === "string")
+                return candidate.message;
+        }
+        return String(error ?? "");
+    }
+    static isInvalidModelError(error) {
+        const message = this.errorToMessage(error).toLowerCase();
+        return message.includes("invalid model");
+    }
+    static async reencodeImageToPngDataUrl(filePath) {
+        const tempOutputPath = path.join(os.tmpdir(), `folio-vlm-reencode-${Date.now()}-${Math.random().toString(16).slice(2)}.png`);
+        try {
+            await execFileAsync("sips", ["-s", "format", "png", filePath, "--out", tempOutputPath], {
+                timeout: this.IMAGE_REENCODE_TIMEOUT_MS,
+                maxBuffer: 1024 * 1024,
+            });
+            const pngBuffer = await fs.readFile(tempOutputPath);
+            return `data:image/png;base64,${pngBuffer.toString("base64")}`;
+        }
+        catch {
+            return null;
+        }
+        finally {
+            await fs.unlink(tempOutputPath).catch(() => undefined);
+        }
+    }
+    static async maybeBuildImageRetryMarker(opts) {
+        if (!this.isInvalidModelError(opts.error))
+            return null;
+        if (!this.IMAGE_REENCODE_RETRY_ENABLED) {
+            this.bumpImageReencodeRetryMetric("skipped_disabled", opts);
+            logger.info(`VLM ${opts.phase} retry skipped for ${opts.filename}: re-encode retry disabled (${opts.provider}/${opts.model}).`);
+            return null;
+        }
+        const retryDataUrl = await this.reencodeImageToPngDataUrl(opts.filePath);
+        if (!retryDataUrl) {
+            this.bumpImageReencodeRetryMetric("skipped_unavailable", opts);
+            logger.warn(`VLM ${opts.phase} retry skipped for ${opts.filename}: image re-encode unavailable (${opts.provider}/${opts.model}).`);
+            return null;
+        }
+        logger.warn(`VLM ${opts.phase} failed for ${opts.filename} with invalid model. Retrying once with re-encoded image payload (${opts.provider}/${opts.model}).`);
+        return this.buildVlmPayloadMarker("image", retryDataUrl);
+    }
+    static bumpImageReencodeRetryMetric(outcome, meta) {
+        this.IMAGE_REENCODE_RETRY_METRICS[outcome] += 1;
+        logger.info("VLM image re-encode retry metric", {
+            outcome,
+            phase: meta.phase,
+            provider: meta.provider,
+            model: meta.model,
+            filename: meta.filename,
+            counters: { ...this.IMAGE_REENCODE_RETRY_METRICS },
+        });
+    }
     /**
      * Ingest a document using Hybrid Routing Architecture.
      */
@@ -217,32 +304,31 @@ export class IngestionService {
         Actuator.logEvent(ingestion.id, userId, "info", "Triage", { action: "Ingestion started", source, filename, fileSize, is_high_intent: true }, supabase);
         // 2. Document Triage
         let isFastPath = false;
-        let isVlmFastPath = false;
+        let isMultimodalFastPath = false;
+        let multimodalModality = null;
         let extractionContent = content;
         const ext = filename.toLowerCase().split('.').pop() || '';
-        const fastExts = ['txt', 'md', 'csv', 'json'];
-        const imageExts = ['png', 'jpg', 'jpeg', 'webp'];
         // Pre-fetch settings to decide whether we should attempt VLM.
         const { data: triageSettingsRow } = await supabase
             .from("user_settings")
             .select("llm_provider, llm_model, embedding_provider, embedding_model, vision_model_capabilities")
             .eq("user_id", userId)
             .maybeSingle();
-        const visionResolution = ModelCapabilityService.resolveVisionSupport(triageSettingsRow);
-        const llmModel = visionResolution.model;
-        const llmProvider = visionResolution.provider;
-        if (fastExts.includes(ext)) {
+        const imageResolution = ModelCapabilityService.resolveVisionSupport(triageSettingsRow, "image");
+        const pdfResolution = ModelCapabilityService.resolveVisionSupport(triageSettingsRow, "pdf");
+        const llmModel = imageResolution.model;
+        const llmProvider = imageResolution.provider;
+        if (this.FAST_EXTS.includes(ext)) {
             isFastPath = true;
         }
-        else if (imageExts.includes(ext) && visionResolution.shouldAttempt) {
+        else if (this.IMAGE_EXTS.includes(ext) && imageResolution.shouldAttempt) {
             try {
-                const buffer = await fs.readFile(filePath);
-                const base64 = buffer.toString('base64');
                 const mimeTypeActual = mimeType || `image/${ext === 'jpg' ? 'jpeg' : ext}`;
-                // Special marker for PolicyEngine
-                extractionContent = `[VLM_IMAGE_DATA:data:${mimeTypeActual};base64,${base64}]`;
+                const dataUrl = await this.fileToDataUrl(filePath, mimeTypeActual);
+                extractionContent = this.buildVlmPayloadMarker("image", dataUrl);
                 isFastPath = true;
-                isVlmFastPath = true;
+                isMultimodalFastPath = true;
+                multimodalModality = "image";
                 logger.info(`Smart Triage: Image ${filename} routed to Fast Path using native VLM (${llmModel}).`);
                 Actuator.logEvent(ingestion.id, userId, "info", "Triage", { action: "VLM Fast Path selected", type: ext, model: llmModel }, supabase);
             }
@@ -250,7 +336,7 @@ export class IngestionService {
                 logger.warn(`Failed to read VLM image ${filename}. Routing to Heavy Path.`, { err });
             }
         }
-        else if (imageExts.includes(ext)) {
+        else if (this.IMAGE_EXTS.includes(ext)) {
             logger.info(`Smart Triage: Image ${filename} kept on Heavy Path because ${llmProvider}/${llmModel} is marked vision-unsupported.`);
             Actuator.logEvent(ingestion.id, userId, "info", "Triage", {
                 action: "VLM skipped (model marked unsupported)",
@@ -270,9 +356,30 @@ export class IngestionService {
                     logger.info(`Smart Triage: PDF ${filename} passed text quality check (${pdfData.pages.filter(p => p.text.trim().length > 30).length}/${pdfData.total} pages with text). Routing to Fast Path.`);
                     Actuator.logEvent(ingestion.id, userId, "info", "Triage", { action: "Smart Triage passed", type: "pdf", fast_path: true }, supabase);
                 }
+                else if (pdfResolution.shouldAttempt) {
+                    // Reuse the already-loaded parse buffer; avoid a second readFile in fileToDataUrl.
+                    const dataUrl = `data:application/pdf;base64,${buffer.toString("base64")}`;
+                    extractionContent = this.buildVlmPayloadMarker("pdf", dataUrl);
+                    isFastPath = true;
+                    isMultimodalFastPath = true;
+                    multimodalModality = "pdf";
+                    logger.info(`Smart Triage: PDF ${filename} routed to multimodal Fast Path using native VLM (${llmModel}).`);
+                    Actuator.logEvent(ingestion.id, userId, "info", "Triage", {
+                        action: "VLM Fast Path selected",
+                        type: "pdf",
+                        modality: "pdf",
+                        model: llmModel,
+                    }, supabase);
+                }
                 else {
-                    logger.info(`Smart Triage: PDF ${filename} failed text quality check. Routing to Heavy Path.`);
-                    Actuator.logEvent(ingestion.id, userId, "info", "Triage", { action: "Smart Triage failed", type: "pdf", fast_path: false }, supabase);
+                    logger.info(`Smart Triage: PDF ${filename} kept on Heavy Path because ${llmProvider}/${llmModel} is marked PDF-unsupported.`);
+                    Actuator.logEvent(ingestion.id, userId, "info", "Triage", {
+                        action: "VLM skipped (model marked unsupported)",
+                        type: "pdf",
+                        modality: "pdf",
+                        model: llmModel,
+                        provider: llmProvider,
+                    }, supabase);
                 }
             }
             catch (err) {
@@ -296,117 +403,184 @@ export class IngestionService {
                     embedding_provider: processingSettingsRow.data?.embedding_provider ?? undefined,
                     embedding_model: processingSettingsRow.data?.embedding_model ?? undefined,
                 };
-                const doc = { filePath: filePath, text: extractionContent, ingestionId: ingestion.id, userId, supabase };
-                // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                const baselineTrace = [];
-                // Fire and forget Semantic Embedding Storage
-                RAGService.chunkAndEmbed(ingestion.id, userId, doc.text, supabase, embedSettings).catch(err => {
-                    logger.error(`RAG embedding failed for ${ingestion.id}`, err);
-                });
-                // 4. Stage 1: Baseline extraction (always runs, LLM call 1 of max 2)
-                baselineTrace.push({
-                    timestamp: new Date().toISOString(),
-                    step: "LLM request (baseline extraction)",
-                    details: {
-                        provider: llmSettings.llm_provider ?? llmProvider,
-                        model: llmSettings.llm_model ?? llmModel,
-                        mode: isVlmFastPath ? "vision" : "text",
+                const resolvedProvider = llmSettings.llm_provider ?? llmProvider;
+                const resolvedModel = llmSettings.llm_model ?? llmModel;
+                const runFastPathAttempt = async (attemptContent, attemptType) => {
+                    const doc = { filePath: filePath, text: attemptContent, ingestionId: ingestion.id, userId, supabase };
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    const baselineTrace = [];
+                    // Fire and forget Semantic Embedding Storage
+                    RAGService.chunkAndEmbed(ingestion.id, userId, doc.text, supabase, embedSettings).catch(err => {
+                        logger.error(`RAG embedding failed for ${ingestion.id}`, err);
+                    });
+                    // 4. Stage 1: Baseline extraction (always runs, LLM call 1 of max 2)
+                    baselineTrace.push({
+                        timestamp: new Date().toISOString(),
+                        step: "LLM request (baseline extraction)",
+                        details: {
+                            provider: resolvedProvider,
+                            model: resolvedModel,
+                            mode: isMultimodalFastPath
+                                ? `vision:${multimodalModality ?? "image"}${attemptType === "reencoded_image_retry" ? ":reencoded" : ""}`
+                                : "text",
+                        }
+                    });
+                    const baselineResult = await PolicyEngine.extractBaseline(doc, { context: baselineConfig?.context, fields: baselineConfig?.fields }, llmSettings);
+                    const baselineEntities = baselineResult.entities;
+                    const autoTags = baselineResult.tags;
+                    baselineTrace.push({
+                        timestamp: new Date().toISOString(),
+                        step: "LLM response (baseline extraction)",
+                        details: {
+                            entities_count: Object.keys(baselineEntities).length,
+                            uncertain_count: baselineResult.uncertain_fields.length,
+                            tags_count: autoTags.length,
+                        }
+                    });
+                    // Enrich the document with extracted entities so policy keyword/semantic
+                    // conditions can match against semantic field values (e.g. document_type:
+                    // "invoice") even when those exact words don't appear in the raw text.
+                    const entityLines = Object.entries(baselineEntities)
+                        .filter(([, v]) => v != null)
+                        .map(([k, v]) => `${k}: ${Array.isArray(v) ? v.join(", ") : String(v)}`);
+                    const enrichedDoc = entityLines.length > 0
+                        ? { ...doc, text: doc.text + "\n\n[Extracted fields]\n" + entityLines.join("\n") }
+                        : doc;
+                    // 5. Stage 2: Policy matching + policy-specific field extraction
+                    let result;
+                    if (userPolicies.length > 0) {
+                        result = await PolicyEngine.processWithPolicies(enrichedDoc, userPolicies, llmSettings, baselineEntities);
                     }
-                });
-                const baselineResult = await PolicyEngine.extractBaseline(doc, { context: baselineConfig?.context, fields: baselineConfig?.fields }, llmSettings);
-                const baselineEntities = baselineResult.entities;
-                const autoTags = baselineResult.tags;
-                baselineTrace.push({
-                    timestamp: new Date().toISOString(),
-                    step: "LLM response (baseline extraction)",
-                    details: {
-                        entities_count: Object.keys(baselineEntities).length,
-                        uncertain_count: baselineResult.uncertain_fields.length,
-                        tags_count: autoTags.length,
+                    else {
+                        result = await PolicyEngine.process(enrichedDoc, llmSettings, baselineEntities);
                     }
-                });
-                // Enrich the document with extracted entities so policy keyword/semantic
-                // conditions can match against semantic field values (e.g. document_type:
-                // "invoice") even when those exact words don't appear in the raw text.
-                const entityLines = Object.entries(baselineEntities)
-                    .filter(([, v]) => v != null)
-                    .map(([k, v]) => `${k}: ${Array.isArray(v) ? v.join(", ") : String(v)}`);
-                const enrichedDoc = entityLines.length > 0
-                    ? { ...doc, text: doc.text + "\n\n[Extracted fields]\n" + entityLines.join("\n") }
-                    : doc;
-                // 5. Stage 2: Policy matching + policy-specific field extraction
-                let result;
-                if (userPolicies.length > 0) {
-                    result = await PolicyEngine.processWithPolicies(enrichedDoc, userPolicies, llmSettings, baselineEntities);
+                    const policyName = userPolicies.find((p) => p.metadata.id === result.matchedPolicy)?.metadata.name;
+                    const finalStatus = result.status === "fallback" ? "no_match" : result.status;
+                    // Merge: baseline entities are the foundation; policy-specific fields
+                    // are overlaid on top so more precise extractions take precedence.
+                    const mergedExtracted = { ...baselineEntities, ...result.extractedData };
+                    let finalTrace = [...baselineTrace, ...(result.trace || [])];
+                    const { data: updatedIngestion } = await supabase
+                        .from("ingestions")
+                        .update({
+                        status: finalStatus,
+                        policy_id: result.matchedPolicy,
+                        policy_name: policyName,
+                        extracted: mergedExtracted,
+                        actions_taken: result.actionsExecuted,
+                        trace: finalTrace,
+                        tags: autoTags,
+                        baseline_config_id: baselineConfig?.id ?? null,
+                    })
+                        .eq("id", ingestion.id)
+                        .select()
+                        .single();
+                    if (isMultimodalFastPath && multimodalModality) {
+                        const embeddingMeta = this.queueVlmSemanticEmbedding({
+                            ingestionId: ingestion.id,
+                            userId,
+                            filename,
+                            finalStatus,
+                            policyName,
+                            extracted: mergedExtracted,
+                            tags: autoTags,
+                            modality: multimodalModality,
+                            supabase,
+                            embedSettings,
+                        });
+                        finalTrace = [
+                            ...finalTrace,
+                            {
+                                timestamp: new Date().toISOString(),
+                                step: "Queued synthetic VLM embedding",
+                                details: embeddingMeta,
+                            }
+                        ];
+                        await supabase
+                            .from("ingestions")
+                            .update({ trace: finalTrace })
+                            .eq("id", ingestion.id);
+                    }
+                    if (isMultimodalFastPath && multimodalModality) {
+                        await ModelCapabilityService.learnVisionSuccess({
+                            supabase,
+                            userId,
+                            provider: resolvedProvider,
+                            model: resolvedModel,
+                            modality: multimodalModality,
+                        });
+                    }
+                    return updatedIngestion;
+                };
+                let terminalError = null;
+                try {
+                    return await runFastPathAttempt(extractionContent, "primary");
                 }
-                else {
-                    result = await PolicyEngine.process(enrichedDoc, llmSettings, baselineEntities);
+                catch (primaryErr) {
+                    terminalError = primaryErr;
                 }
-                const policyName = userPolicies.find((p) => p.metadata.id === result.matchedPolicy)?.metadata.name;
-                const finalStatus = result.status === "fallback" ? "no_match" : result.status;
-                // Merge: baseline entities are the foundation; policy-specific fields
-                // are overlaid on top so more precise extractions take precedence.
-                const mergedExtracted = { ...baselineEntities, ...result.extractedData };
-                let finalTrace = [...baselineTrace, ...(result.trace || [])];
-                const { data: updatedIngestion } = await supabase
-                    .from("ingestions")
-                    .update({
-                    status: finalStatus,
-                    policy_id: result.matchedPolicy,
-                    policy_name: policyName,
-                    extracted: mergedExtracted,
-                    actions_taken: result.actionsExecuted,
-                    trace: finalTrace,
-                    tags: autoTags,
-                    baseline_config_id: baselineConfig?.id ?? null,
-                })
-                    .eq("id", ingestion.id)
-                    .select()
-                    .single();
-                if (isVlmFastPath) {
-                    const embeddingMeta = this.queueVlmSemanticEmbedding({
-                        ingestionId: ingestion.id,
-                        userId,
+                if (isMultimodalFastPath && multimodalModality === "image") {
+                    const retryMarker = await this.maybeBuildImageRetryMarker({
+                        error: terminalError,
+                        filePath,
                         filename,
-                        finalStatus,
-                        policyName,
-                        extracted: mergedExtracted,
-                        tags: autoTags,
-                        supabase,
-                        embedSettings,
+                        provider: resolvedProvider,
+                        model: resolvedModel,
+                        phase: "ingest",
                     });
-                    finalTrace = [
-                        ...finalTrace,
-                        {
-                            timestamp: new Date().toISOString(),
-                            step: "Queued synthetic VLM embedding",
-                            details: embeddingMeta,
+                    if (retryMarker) {
+                        this.bumpImageReencodeRetryMetric("attempted", {
+                            phase: "ingest",
+                            provider: resolvedProvider,
+                            model: resolvedModel,
+                            filename,
+                        });
+                        Actuator.logEvent(ingestion.id, userId, "info", "Processing", {
+                            action: "Retrying VLM with re-encoded image payload",
+                            provider: resolvedProvider,
+                            model: resolvedModel,
+                        }, supabase);
+                        try {
+                            const retryResult = await runFastPathAttempt(retryMarker, "reencoded_image_retry");
+                            this.bumpImageReencodeRetryMetric("succeeded", {
+                                phase: "ingest",
+                                provider: resolvedProvider,
+                                model: resolvedModel,
+                                filename,
+                            });
+                            Actuator.logEvent(ingestion.id, userId, "analysis", "Processing", {
+                                action: "VLM re-encoded image retry succeeded",
+                                provider: resolvedProvider,
+                                model: resolvedModel,
+                            }, supabase);
+                            return retryResult;
                         }
-                    ];
-                    await supabase
-                        .from("ingestions")
-                        .update({ trace: finalTrace })
-                        .eq("id", ingestion.id);
-                }
-                if (isVlmFastPath) {
-                    await ModelCapabilityService.learnVisionSuccess({
-                        supabase,
-                        userId,
-                        provider: llmSettings.llm_provider ?? llmProvider,
-                        model: llmSettings.llm_model ?? llmModel,
-                    });
+                        catch (retryErr) {
+                            this.bumpImageReencodeRetryMetric("failed", {
+                                phase: "ingest",
+                                provider: resolvedProvider,
+                                model: resolvedModel,
+                                filename,
+                            });
+                            Actuator.logEvent(ingestion.id, userId, "error", "Processing", {
+                                action: "VLM re-encoded image retry failed",
+                                provider: resolvedProvider,
+                                model: resolvedModel,
+                                error: this.errorToMessage(retryErr),
+                            }, supabase);
+                            terminalError = retryErr;
+                        }
+                    }
                 }
-                return updatedIngestion;
-            }
-            catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
-                if (isVlmFastPath) {
+                const msg = this.errorToMessage(terminalError);
+                if (isMultimodalFastPath && multimodalModality) {
                     const learnedState = await ModelCapabilityService.learnVisionFailure({
                         supabase,
                         userId,
-                        provider: llmProvider,
-                        model: llmModel,
-                        error: err,
+                        provider: resolvedProvider,
+                        model: resolvedModel,
+                        error: terminalError,
+                        modality: multimodalModality,
                     });
                     logger.warn(`VLM extraction failed for ${filename}. Falling back to Heavy Path. Error: ${msg}`);
                     Actuator.logEvent(ingestion.id, userId, "error", "Processing", {
@@ -428,6 +602,17 @@ export class IngestionService {
                     return updatedIngestion;
                 }
             }
+            catch (err) {
+                const msg = this.errorToMessage(err);
+                Actuator.logEvent(ingestion.id, userId, "error", "Processing", { error: msg }, supabase);
+                const { data: updatedIngestion } = await supabase
+                    .from("ingestions")
+                    .update({ status: "error", error_message: msg })
+                    .eq("id", ingestion.id)
+                    .select()
+                    .single();
+                return updatedIngestion;
+            }
         }
         // 4. Heavy Path (Delegate to RealTimeX)
         const { error: rtxErr } = await supabase
@@ -477,31 +662,31 @@ export class IngestionService {
         if (!filePath)
             throw new Error("No storage path found for this ingestion");
         let isFastPath = false;
-        let isVlmFastPath = false;
+        let isMultimodalFastPath = false;
+        let multimodalModality = null;
         let extractionContent = "";
         const ext = filename.toLowerCase().split('.').pop() || '';
-        const fastExts = ['txt', 'md', 'csv', 'json'];
-        const imageExts = ['png', 'jpg', 'jpeg', 'webp'];
         const { data: triageSettingsRow } = await supabase
             .from("user_settings")
             .select("llm_provider, llm_model, embedding_provider, embedding_model, vision_model_capabilities")
             .eq("user_id", userId)
             .maybeSingle();
-        const visionResolution = ModelCapabilityService.resolveVisionSupport(triageSettingsRow);
-        const llmModel = visionResolution.model;
-        const llmProvider = visionResolution.provider;
-        if (fastExts.includes(ext)) {
+        const imageResolution = ModelCapabilityService.resolveVisionSupport(triageSettingsRow, "image");
+        const pdfResolution = ModelCapabilityService.resolveVisionSupport(triageSettingsRow, "pdf");
+        const llmModel = imageResolution.model;
+        const llmProvider = imageResolution.provider;
+        if (this.FAST_EXTS.includes(ext)) {
             isFastPath = true;
             extractionContent = await fs.readFile(filePath, "utf-8");
         }
-        else if (imageExts.includes(ext) && visionResolution.shouldAttempt) {
+        else if (this.IMAGE_EXTS.includes(ext) && imageResolution.shouldAttempt) {
             try {
-                const buffer = await fs.readFile(filePath);
-                const base64 = buffer.toString('base64');
                 const mimeTypeActual = `image/${ext === 'jpg' ? 'jpeg' : ext}`;
-                extractionContent = `[VLM_IMAGE_DATA:data:${mimeTypeActual};base64,${base64}]`;
+                const dataUrl = await this.fileToDataUrl(filePath, mimeTypeActual);
+                extractionContent = this.buildVlmPayloadMarker("image", dataUrl);
                 isFastPath = true;
-                isVlmFastPath = true;
+                isMultimodalFastPath = true;
+                multimodalModality = "image";
                 logger.info(`Smart Triage: Re-run image ${filename} routed to Fast Path using native VLM (${llmModel}).`);
                 Actuator.logEvent(ingestionId, userId, "info", "Triage", { action: "VLM Fast Path selected", type: ext, model: llmModel }, supabase);
             }
@@ -509,7 +694,7 @@ export class IngestionService {
                 logger.warn(`Failed to read VLM image ${filename} during rerun. Routing to Heavy Path.`, { err });
             }
         }
-        else if (imageExts.includes(ext)) {
+        else if (this.IMAGE_EXTS.includes(ext)) {
             logger.info(`Smart Triage: Re-run image ${filename} kept on Heavy Path because ${llmProvider}/${llmModel} is marked vision-unsupported.`);
             Actuator.logEvent(ingestionId, userId, "info", "Triage", {
                 action: "VLM skipped (model marked unsupported)",
@@ -527,10 +712,34 @@ export class IngestionService {
                     isFastPath = true;
                     extractionContent = pdfData.text;
                 }
-                // eslint-disable-next-line @typescript-eslint/no-unused-vars
+                else if (pdfResolution.shouldAttempt) {
+                    // Reuse the already-loaded parse buffer; avoid a second readFile in fileToDataUrl.
+                    const dataUrl = `data:application/pdf;base64,${buffer.toString("base64")}`;
+                    extractionContent = this.buildVlmPayloadMarker("pdf", dataUrl);
+                    isFastPath = true;
+                    isMultimodalFastPath = true;
+                    multimodalModality = "pdf";
+                    logger.info(`Smart Triage: Re-run PDF ${filename} routed to multimodal Fast Path using native VLM (${llmModel}).`);
+                    Actuator.logEvent(ingestionId, userId, "info", "Triage", {
+                        action: "VLM Fast Path selected",
+                        type: "pdf",
+                        modality: "pdf",
+                        model: llmModel,
+                    }, supabase);
+                }
+                else {
+                    logger.info(`Smart Triage: Re-run PDF ${filename} kept on Heavy Path because ${llmProvider}/${llmModel} is marked PDF-unsupported.`);
+                    Actuator.logEvent(ingestionId, userId, "info", "Triage", {
+                        action: "VLM skipped (model marked unsupported)",
+                        type: "pdf",
+                        modality: "pdf",
+                        model: llmModel,
+                        provider: llmProvider
+                    }, supabase);
+                }
             }
             catch (err) {
-                // ignore
+                logger.warn(`Failed to parse PDF ${filename} during rerun. Routing to Heavy Path.`, { err });
             }
         }
         if (isFastPath) {
@@ -547,44 +756,47 @@ export class IngestionService {
                 embedding_provider: processingSettingsRow.data?.embedding_provider ?? undefined,
                 embedding_model: processingSettingsRow.data?.embedding_model ?? undefined,
             };
-            const doc = { filePath, text: extractionContent, ingestionId, userId, supabase };
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const baselineTrace = [];
-            // Fire and forget Semantic Embedding Storage for re-runs
-            RAGService.chunkAndEmbed(ingestionId, userId, doc.text, supabase, embedSettings).catch(err => {
-                logger.error(`RAG embedding failed during rerun for ${ingestionId}`, err);
-            });
-            baselineTrace.push({
-                timestamp: new Date().toISOString(),
-                step: "LLM request (baseline extraction)",
-                details: {
-                    provider: llmSettings.llm_provider ?? llmProvider,
-                    model: llmSettings.llm_model ?? llmModel,
-                    mode: isVlmFastPath ? "vision" : "text",
-                }
-            });
-            const baselineResult = await PolicyEngine.extractBaseline(doc, { context: baselineConfig?.context, fields: baselineConfig?.fields }, llmSettings);
-            const baselineEntities = baselineResult.entities;
-            const autoTags = baselineResult.tags;
-            baselineTrace.push({
-                timestamp: new Date().toISOString(),
-                step: "LLM response (baseline extraction)",
-                details: {
-                    entities_count: Object.keys(baselineEntities).length,
-                    uncertain_count: baselineResult.uncertain_fields.length,
-                    tags_count: autoTags.length,
-                }
-            });
-            const entityLines = Object.entries(baselineEntities)
-                .filter(([, v]) => v != null)
-                .map(([k, v]) => `${k}: ${Array.isArray(v) ? v.join(", ") : String(v)}`);
-            const enrichedDoc = entityLines.length > 0
-                ? { ...doc, text: doc.text + "\n\n[Extracted fields]\n" + entityLines.join("\n") }
-                : doc;
-            let finalStatus = "no_match";
-            let result;
-            let policyName;
-            try {
+            const resolvedProvider = llmSettings.llm_provider ?? llmProvider;
+            const resolvedModel = llmSettings.llm_model ?? llmModel;
+            const runFastPathAttempt = async (attemptContent, attemptType) => {
+                const doc = { filePath, text: attemptContent, ingestionId, userId, supabase };
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const baselineTrace = [];
+                // Fire and forget Semantic Embedding Storage for re-runs
+                RAGService.chunkAndEmbed(ingestionId, userId, doc.text, supabase, embedSettings).catch(err => {
+                    logger.error(`RAG embedding failed during rerun for ${ingestionId}`, err);
+                });
+                baselineTrace.push({
+                    timestamp: new Date().toISOString(),
+                    step: "LLM request (baseline extraction)",
+                    details: {
+                        provider: resolvedProvider,
+                        model: resolvedModel,
+                        mode: isMultimodalFastPath
+                            ? `vision:${multimodalModality ?? "image"}${attemptType === "reencoded_image_retry" ? ":reencoded" : ""}`
+                            : "text",
+                    }
+                });
+                const baselineResult = await PolicyEngine.extractBaseline(doc, { context: baselineConfig?.context, fields: baselineConfig?.fields }, llmSettings);
+                const baselineEntities = baselineResult.entities;
+                const autoTags = baselineResult.tags;
+                baselineTrace.push({
+                    timestamp: new Date().toISOString(),
+                    step: "LLM response (baseline extraction)",
+                    details: {
+                        entities_count: Object.keys(baselineEntities).length,
+                        uncertain_count: baselineResult.uncertain_fields.length,
+                        tags_count: autoTags.length,
+                    }
+                });
+                const entityLines = Object.entries(baselineEntities)
+                    .filter(([, v]) => v != null)
+                    .map(([k, v]) => `${k}: ${Array.isArray(v) ? v.join(", ") : String(v)}`);
+                const enrichedDoc = entityLines.length > 0
+                    ? { ...doc, text: doc.text + "\n\n[Extracted fields]\n" + entityLines.join("\n") }
+                    : doc;
+                let finalStatus = "no_match";
+                let result;
                 const forcedPolicyId = opts.forcedPolicyId?.trim();
                 const activePolicies = forcedPolicyId
                     ? userPolicies.filter((policy) => policy.metadata.id === forcedPolicyId)
@@ -601,7 +813,7 @@ export class IngestionService {
                 else {
                     result = await PolicyEngine.process(enrichedDoc, llmSettings, baselineEntities);
                 }
-                policyName = result.matchedPolicy ? activePolicies.find((p) => p.metadata.id === result.matchedPolicy)?.metadata.name : undefined;
+                const policyName = result.matchedPolicy ? activePolicies.find((p) => p.metadata.id === result.matchedPolicy)?.metadata.name : undefined;
                 finalStatus = result.status === "fallback" ? "no_match" : result.status;
                 const mergedExtracted = { ...baselineEntities, ...result.extractedData };
                 // Preserve any human-added tags; merge with freshly generated auto-tags.
@@ -626,7 +838,7 @@ export class IngestionService {
                     baseline_config_id: baselineConfig?.id ?? null,
                 })
                     .eq("id", ingestionId);
-                if (isVlmFastPath) {
+                if (isMultimodalFastPath && multimodalModality) {
                     const embeddingMeta = this.queueVlmSemanticEmbedding({
                         ingestionId,
                         userId,
@@ -635,6 +847,7 @@ export class IngestionService {
                         policyName,
                         extracted: mergedExtracted,
                         tags: mergedTags,
+                        modality: multimodalModality,
                         supabase,
                         embedSettings,
                     });
@@ -651,38 +864,98 @@ export class IngestionService {
                         .update({ trace: rerunTrace })
                         .eq("id", ingestionId);
                 }
-                if (isVlmFastPath) {
+                if (isMultimodalFastPath && multimodalModality) {
                     await ModelCapabilityService.learnVisionSuccess({
                         supabase,
                         userId,
-                        provider: llmSettings.llm_provider ?? llmProvider,
-                        model: llmSettings.llm_model ?? llmModel,
+                        provider: resolvedProvider,
+                        model: resolvedModel,
+                        modality: multimodalModality,
                     });
                 }
                 return finalStatus === "matched";
+            };
+            let terminalError = null;
+            try {
+                return await runFastPathAttempt(extractionContent, "primary");
             }
-            catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
-                if (isVlmFastPath) {
-                    const learnedState = await ModelCapabilityService.learnVisionFailure({
-                        supabase,
-                        userId,
-                        provider: llmProvider,
-                        model: llmModel,
-                        error: err,
+            catch (primaryErr) {
+                terminalError = primaryErr;
+            }
+            if (isMultimodalFastPath && multimodalModality === "image") {
+                const retryMarker = await this.maybeBuildImageRetryMarker({
+                    error: terminalError,
+                    filePath,
+                    filename,
+                    provider: resolvedProvider,
+                    model: resolvedModel,
+                    phase: "rerun",
+                });
+                if (retryMarker) {
+                    this.bumpImageReencodeRetryMetric("attempted", {
+                        phase: "rerun",
+                        provider: resolvedProvider,
+                        model: resolvedModel,
+                        filename,
                     });
-                    logger.warn(`VLM extraction failed during rerun for ${filename}. Falling back to Heavy Path. Error: ${msg}`);
-                    Actuator.logEvent(ingestionId, userId, "error", "Processing", {
-                        action: "VLM Failed, Fallback to Heavy",
-                        error: msg,
-                        learned_state: learnedState,
+                    Actuator.logEvent(ingestionId, userId, "info", "Processing", {
+                        action: "Retrying VLM with re-encoded image payload",
+                        provider: resolvedProvider,
+                        model: resolvedModel,
                     }, supabase);
-                    isFastPath = false; // Trigger heavy path fallthrough
-                }
-                else {
-                    throw err; // Re-throw to caller
+                    try {
+                        const retryResult = await runFastPathAttempt(retryMarker, "reencoded_image_retry");
+                        this.bumpImageReencodeRetryMetric("succeeded", {
+                            phase: "rerun",
+                            provider: resolvedProvider,
+                            model: resolvedModel,
+                            filename,
+                        });
+                        Actuator.logEvent(ingestionId, userId, "analysis", "Processing", {
+                            action: "VLM re-encoded image retry succeeded",
+                            provider: resolvedProvider,
+                            model: resolvedModel,
+                        }, supabase);
+                        return retryResult;
+                    }
+                    catch (retryErr) {
+                        this.bumpImageReencodeRetryMetric("failed", {
+                            phase: "rerun",
+                            provider: resolvedProvider,
+                            model: resolvedModel,
+                            filename,
+                        });
+                        Actuator.logEvent(ingestionId, userId, "error", "Processing", {
+                            action: "VLM re-encoded image retry failed",
+                            provider: resolvedProvider,
+                            model: resolvedModel,
+                            error: this.errorToMessage(retryErr),
+                        }, supabase);
+                        terminalError = retryErr;
+                    }
                 }
             }
+            const msg = this.errorToMessage(terminalError);
+            if (isMultimodalFastPath && multimodalModality) {
+                const learnedState = await ModelCapabilityService.learnVisionFailure({
+                    supabase,
+                    userId,
+                    provider: resolvedProvider,
+                    model: resolvedModel,
+                    error: terminalError,
+                    modality: multimodalModality,
+                });
+                logger.warn(`VLM extraction failed during rerun for ${filename}. Falling back to Heavy Path. Error: ${msg}`);
+                Actuator.logEvent(ingestionId, userId, "error", "Processing", {
+                    action: "VLM Failed, Fallback to Heavy",
+                    error: msg,
+                    learned_state: learnedState,
+                }, supabase);
+                isFastPath = false; // Trigger heavy path fallthrough
+            }
+            else {
+                throw terminalError instanceof Error ? terminalError : new Error(msg); // Re-throw to caller
+            }
         }
         // Re-delegate to rtx_activities
         await supabase