@realtimex/folio 0.1.10 → 0.1.12

package/dist/api/src/services/ModelCapabilityService.js

@@ -2,45 +2,78 @@ import { createLogger } from "../utils/logger.js";
 import { SDKService } from "./SDKService.js";
 const logger = createLogger("ModelCapabilityService");
 export class ModelCapabilityService {
-    static SUPPORTED_TTL_DAYS = 180;
-    static UNSUPPORTED_TTL_DAYS = 30;
-    static resolveVisionSupport(settingsRow) {
+    static SUPPORTED_TTL_MS = 180 * 24 * 60 * 60 * 1000;
+    static UNSUPPORTED_TTL_MS = 30 * 24 * 60 * 60 * 1000;
+    static PENDING_UNSUPPORTED_TTL_MS = 24 * 60 * 60 * 1000;
+    static UNSUPPORTED_CONFIRMATION_WINDOW_MS = 24 * 60 * 60 * 1000;
+    static UNSUPPORTED_CONFIRMATION_FAILURES = 2;
+    static UNSUPPORTED_SCORE_THRESHOLD = 3;
+    static resolveVisionSupport(settingsRow, modality = "image") {
         const provider = (settingsRow?.llm_provider || SDKService.DEFAULT_LLM_PROVIDER).trim();
         const model = (settingsRow?.llm_model || SDKService.DEFAULT_LLM_MODEL).trim();
-        const state = this.getVisionState(settingsRow?.vision_model_capabilities, provider, model);
+        const state = this.getVisionState(settingsRow?.vision_model_capabilities, provider, model, modality);
         return {
             provider,
             model,
+            modality,
             state,
             shouldAttempt: state !== "unsupported",
         };
     }
-    static getVisionState(rawMap, provider, model) {
+    static getVisionState(rawMap, provider, model, modality = "image") {
         const map = this.normalizeCapabilityMap(rawMap);
-        const entry = map[this.capabilityKey(provider, model)];
-        if (!entry)
+        const entry = map[this.capabilityKey(provider, model, modality)];
+        if (!entry || this.isExpired(entry))
+            return "unknown";
+        if (entry.state === "pending_unsupported")
             return "unknown";
-        if (entry.expires_at) {
-            const expiryTs = Date.parse(entry.expires_at);
-            if (Number.isFinite(expiryTs) && expiryTs <= Date.now()) {
-                return "unknown";
-            }
-        }
         return entry.state;
     }
     static async learnVisionSuccess(opts) {
         await this.writeCapability({
             ...opts,
+            modality: opts.modality ?? "image",
             state: "supported",
             reason: "vision_request_succeeded",
-            ttlDays: this.SUPPORTED_TTL_DAYS,
+            ttlMs: this.SUPPORTED_TTL_MS,
         });
     }
     static async learnVisionFailure(opts) {
-        const classification = this.classifyVisionFailure(opts.error);
+        const modality = opts.modality ?? "image";
+        const classification = this.classifyVisionFailure({
+            error: opts.error,
+            provider: opts.provider,
+            modality,
+        });
         if (!classification.isCapabilityError) {
-            logger.info(`Vision failure for ${opts.provider}/${opts.model} treated as transient; leaving capability unknown`, {
+            logger.info(`Vision failure for ${opts.provider}/${opts.model} (${modality}) treated as non-capability; leaving capability unknown`, {
                 reason: classification.reason,
+                score: classification.score,
+                evidence: classification.evidence,
+            });
+            return "unknown";
+        }
+        const map = await this.readCapabilityMap(opts.supabase, opts.userId);
+        if (!map) {
+            return "unknown";
+        }
+        const key = this.capabilityKey(opts.provider, opts.model, modality);
+        const now = new Date();
+        const failureCount = this.nextFailureCount(map[key], now.getTime());
+        if (failureCount < this.UNSUPPORTED_CONFIRMATION_FAILURES) {
+            await this.writeCapability({
+                supabase: opts.supabase,
+                userId: opts.userId,
+                provider: opts.provider,
+                model: opts.model,
+                modality,
+                state: "pending_unsupported",
+                reason: "capability_signal_pending_confirmation",
+                ttlMs: this.PENDING_UNSUPPORTED_TTL_MS,
+                preloadedMap: map,
+                failureCount,
+                lastFailureAt: now.toISOString(),
+                evidence: classification.evidence,
             });
             return "unknown";
         }
@@ -49,43 +82,76 @@ export class ModelCapabilityService {
             userId: opts.userId,
             provider: opts.provider,
             model: opts.model,
+            modality,
             state: "unsupported",
             reason: classification.reason,
-            ttlDays: this.UNSUPPORTED_TTL_DAYS,
+            ttlMs: this.UNSUPPORTED_TTL_MS,
+            preloadedMap: map,
+            failureCount,
+            lastFailureAt: now.toISOString(),
+            evidence: classification.evidence,
         });
         return "unsupported";
     }
-    static async writeCapability(opts) {
-        const { supabase, userId, provider, model, state, reason, ttlDays } = opts;
-        const { data, error: readErr } = await supabase
+    static async readCapabilityMap(supabase, userId) {
+        const { data, error } = await supabase
             .from("user_settings")
             .select("vision_model_capabilities")
             .eq("user_id", userId)
             .maybeSingle();
-        if (readErr) {
-            logger.warn("Failed to read user_settings for model capability write", { userId, readErr });
+        if (error) {
+            logger.warn("Failed to read user_settings for model capability", { userId, error });
+            return null;
+        }
+        return this.normalizeCapabilityMap(data?.vision_model_capabilities);
+    }
+    static async persistCapabilityMap(supabase, userId, map) {
+        const { error } = await supabase
+            .from("user_settings")
+            .upsert({
+            user_id: userId,
+            vision_model_capabilities: map,
+        }, { onConflict: "user_id" });
+        if (error) {
+            logger.warn("Failed to persist model capability state", { userId, error });
+            return false;
+        }
+        return true;
+    }
+    static async writeCapability(opts) {
+        const { supabase, userId, provider, model, modality, state, reason, ttlMs, preloadedMap, failureCount, lastFailureAt, evidence, } = opts;
+        const map = preloadedMap ?? (await this.readCapabilityMap(supabase, userId));
+        if (!map) {
             return;
         }
-        const map = this.normalizeCapabilityMap(data?.vision_model_capabilities);
         const now = new Date();
-        const expiresAt = new Date(now.getTime() + ttlDays * 24 * 60 * 60 * 1000).toISOString();
-        map[this.capabilityKey(provider, model)] = {
+        const key = this.capabilityKey(provider, model, modality);
+        const nextEntry = {
             state,
             learned_at: now.toISOString(),
-            expires_at: expiresAt,
+            expires_at: new Date(now.getTime() + ttlMs).toISOString(),
             reason,
         };
-        const { error: writeErr } = await supabase
-            .from("user_settings")
-            .upsert({
-            user_id: userId,
-            vision_model_capabilities: map,
-        }, { onConflict: "user_id" });
-        if (writeErr) {
-            logger.warn("Failed to persist model capability state", { userId, provider, model, state, writeErr });
+        if (typeof failureCount === "number" && Number.isFinite(failureCount) && failureCount > 0) {
+            nextEntry.failure_count = Math.floor(failureCount);
+        }
+        if (typeof lastFailureAt === "string") {
+            nextEntry.last_failure_at = lastFailureAt;
+        }
+        if (Array.isArray(evidence) && evidence.length > 0) {
+            nextEntry.evidence = evidence.slice(0, 5);
+        }
+        map[key] = nextEntry;
+        const persisted = await this.persistCapabilityMap(supabase, userId, map);
+        if (!persisted) {
             return;
         }
-        logger.info(`Updated model capability for ${provider}/${model}: ${state}`, { reason, ttlDays });
+        logger.info(`Updated model capability for ${provider}/${model} (${modality}): ${state}`, {
+            reason,
+            ttlMs,
+            failureCount,
+            evidence: nextEntry.evidence,
+        });
     }
     static normalizeCapabilityMap(rawMap) {
         if (!rawMap || typeof rawMap !== "object" || Array.isArray(rawMap)) {
@@ -97,62 +163,212 @@ export class ModelCapabilityService {
             if (!value || typeof value !== "object" || Array.isArray(value)) {
                 continue;
             }
-            const state = String(value.state || "");
-            if (state !== "supported" && state !== "unsupported") {
+            const record = value;
+            const state = String(record.state || "");
+            if (state !== "supported" && state !== "unsupported" && state !== "pending_unsupported") {
                 continue;
             }
-            const learnedAt = value.learned_at;
-            const expiresAt = value.expires_at;
-            const reason = value.reason;
-            normalized[key] = {
+            const learnedAt = record.learned_at;
+            const expiresAt = record.expires_at;
+            const reason = record.reason;
+            const failureCount = record.failure_count;
+            const lastFailureAt = record.last_failure_at;
+            const evidence = record.evidence;
+            const normalizedEntry = {
                 state,
                 learned_at: typeof learnedAt === "string" ? learnedAt : new Date(0).toISOString(),
                 expires_at: typeof expiresAt === "string" ? expiresAt : undefined,
                 reason: typeof reason === "string" ? reason : undefined,
             };
+            if (typeof failureCount === "number" && Number.isFinite(failureCount) && failureCount > 0) {
+                normalizedEntry.failure_count = Math.floor(failureCount);
+            }
+            if (typeof lastFailureAt === "string") {
+                normalizedEntry.last_failure_at = lastFailureAt;
+            }
+            if (Array.isArray(evidence)) {
+                normalizedEntry.evidence = evidence
+                    .filter((item) => typeof item === "string")
+                    .map((item) => item.trim())
+                    .filter((item) => item.length > 0)
+                    .slice(0, 5);
+            }
+            normalized[key] = normalizedEntry;
         }
         return normalized;
     }
-    static capabilityKey(provider, model) {
+    static capabilityBaseKey(provider, model) {
         return `${provider.toLowerCase().trim()}:${model.toLowerCase().trim()}`;
     }
-    static classifyVisionFailure(error) {
-        const message = this.errorToMessage(error).toLowerCase();
-        if (!message)
-            return { isCapabilityError: false, reason: "empty_error" };
-        const hardCapabilityHints = [
-            "does not support images",
-            "model does not support image",
-            "invalid model", // e.g. text-only models fed image payloads in realtimexai provider
-        ];
-        if (hardCapabilityHints.some((hint) => message.includes(hint))) {
-            return { isCapabilityError: true, reason: "capability_mismatch" };
-        }
-        const documentSpecificHints = [
-            "image_url",
-            "vision",
-            "multimodal",
-            "multi-modal",
-            "unsupported content type",
-            "unsupported message content",
-            "invalid content type",
-            "invalid image",
-            "unrecognized content type",
-            "image too large",
-            "base64",
-        ];
-        if (documentSpecificHints.some((hint) => message.includes(hint))) {
-            return { isCapabilityError: false, reason: "document_specific_failure" };
+    static capabilityKey(provider, model, modality = "image") {
+        const base = this.capabilityBaseKey(provider, model);
+        if (modality === "image")
+            return base;
+        return `${base}:${modality}`;
+    }
+    static isExpired(entry) {
+        if (!entry.expires_at)
+            return false;
+        const expiryTs = Date.parse(entry.expires_at);
+        return Number.isFinite(expiryTs) && expiryTs <= Date.now();
+    }
+    static nextFailureCount(entry, nowTs) {
+        if (!entry || entry.state !== "pending_unsupported" || this.isExpired(entry)) {
+            return 1;
+        }
+        const lastFailureTs = entry.last_failure_at ? Date.parse(entry.last_failure_at) : Number.NaN;
+        if (!Number.isFinite(lastFailureTs)) {
+            return 1;
+        }
+        if (nowTs - lastFailureTs > this.UNSUPPORTED_CONFIRMATION_WINDOW_MS) {
+            return 1;
+        }
+        const currentCount = typeof entry.failure_count === "number" && Number.isFinite(entry.failure_count)
+            ? Math.max(1, Math.floor(entry.failure_count))
+            : 1;
+        return currentCount + 1;
+    }
+    static classifyVisionFailure(opts) {
+        const signal = this.extractVisionFailureSignal(opts.error);
+        if (!signal.message && signal.codes.size === 0 && signal.statusCodes.size === 0) {
+            return { isCapabilityError: false, reason: "empty_error", score: 0, evidence: [] };
+        }
+        const transientEvidence = this.matchTransientOrAuth(signal);
+        if (transientEvidence.length > 0) {
+            return {
+                isCapabilityError: false,
+                reason: "transient_or_auth",
+                score: 0,
+                evidence: transientEvidence,
+            };
+        }
+        const documentEvidence = this.matchDocumentSpecific(signal, opts.modality);
+        if (documentEvidence.length > 0) {
+            return {
+                isCapabilityError: false,
+                reason: "document_specific_failure",
+                score: 0,
+                evidence: documentEvidence,
+            };
+        }
+        const capability = this.scoreCapabilitySignal(signal, opts.provider, opts.modality);
+        if (capability.score >= this.UNSUPPORTED_SCORE_THRESHOLD) {
+            return {
+                isCapabilityError: true,
+                reason: "capability_mismatch",
+                score: capability.score,
+                evidence: capability.evidence,
+            };
+        }
+        if (capability.score > 0) {
+            return {
+                isCapabilityError: false,
+                reason: "insufficient_capability_evidence",
+                score: capability.score,
+                evidence: capability.evidence,
+            };
         }
-        const transientHints = [
+        return {
+            isCapabilityError: false,
+            reason: "unknown_error_class",
+            score: 0,
+            evidence: [],
+        };
+    }
+    static extractVisionFailureSignal(error) {
+        const messages = new Set();
+        const statusCodes = new Set();
+        const codes = new Set();
+        const pushMessage = (value) => {
+            if (typeof value !== "string")
+                return;
+            const normalized = value.trim().toLowerCase();
+            if (normalized)
+                messages.add(normalized);
+        };
+        const pushStatus = (value) => {
+            const parsed = typeof value === "number" ? value : Number(value);
+            if (!Number.isFinite(parsed) || parsed <= 0)
+                return;
+            statusCodes.add(Math.floor(parsed));
+        };
+        const pushCode = (value) => {
+            if (typeof value !== "string")
+                return;
+            const normalized = value.trim().toLowerCase();
+            if (!normalized)
+                return;
+            codes.add(normalized);
+            codes.add(normalized.replace(/[\s.-]+/g, "_"));
+        };
+        pushMessage(this.errorToMessage(error));
+        const queue = [{ value: error, depth: 0 }];
+        const visited = new Set();
+        while (queue.length > 0) {
+            const current = queue.shift();
+            if (!current || current.depth > 2) {
+                continue;
+            }
+            const { value, depth } = current;
+            if (!value || typeof value !== "object") {
+                continue;
+            }
+            if (visited.has(value)) {
+                continue;
+            }
+            visited.add(value);
+            const candidate = value;
+            pushMessage(candidate.message);
+            pushMessage(candidate.details);
+            pushMessage(candidate.error_description);
+            pushMessage(candidate.detail);
+            if (typeof candidate.error === "string") {
+                pushMessage(candidate.error);
+            }
+            pushStatus(candidate.status);
+            pushStatus(candidate.statusCode);
+            pushCode(candidate.code);
+            pushCode(candidate.type);
+            if (typeof candidate.error === "object") {
+                const nested = candidate.error;
+                pushCode(nested.code);
+                pushCode(nested.type);
+                pushStatus(nested.status);
+                pushMessage(nested.message);
+            }
+            for (const key of ["response", "data", "error", "cause"]) {
+                if (candidate[key] !== undefined) {
+                    queue.push({ value: candidate[key], depth: depth + 1 });
+                }
+            }
+        }
+        return {
+            message: Array.from(messages).join(" | "),
+            statusCodes,
+            codes,
+        };
+    }
+    static matchTransientOrAuth(signal) {
+        const statusMatches = Array.from(signal.statusCodes).filter((status) => [401, 403, 408, 429, 500, 502, 503, 504].includes(status));
+        const codeMatches = this.matchCodes(signal.codes, [
+            "timeout",
+            "timed_out",
+            "rate_limit",
+            "too_many_requests",
+            "temporarily_unavailable",
+            "service_unavailable",
+            "network_error",
+            "connection_error",
+            "unauthorized",
+            "forbidden",
+            "invalid_api_key",
+            "insufficient_quota",
+        ]);
+        const messageMatches = this.matchMessage(signal.message, [
             "timeout",
             "timed out",
             "rate limit",
             "too many requests",
-            "429",
-            "503",
-            "502",
-            "504",
             "service unavailable",
             "temporar",
             "network",
@@ -160,11 +376,239 @@ export class ModelCapabilityService {
             "unauthorized",
             "forbidden",
             "invalid api key",
+            "insufficient quota",
+            "overloaded",
+        ]);
+        return [
+            ...statusMatches.map((status) => `status:${status}`),
+            ...codeMatches.map((match) => `code:${match}`),
+            ...messageMatches.map((match) => `msg:${match}`),
+        ];
+    }
+    static matchDocumentSpecific(signal, modality) {
+        const imageCodeHints = [
+            "image_too_large",
+            "invalid_base64",
+            "invalid_image",
+            "invalid_image_data",
+            "malformed_image",
+            "invalid_image_url",
+            "image_decode_failed",
+        ];
+        const imageMessageHints = [
+            "image too large",
+            "invalid base64",
+            "malformed image",
+            "invalid image data",
+            "unable to decode image",
+            "failed to decode image",
+            "invalid image url",
+        ];
+        const pdfCodeHints = [
+            "invalid_pdf",
+            "malformed_pdf",
+            "corrupt_pdf",
+            "encrypted_pdf",
+            "password_protected_pdf",
+            "pdf_parse_error",
+            "file_too_large",
         ];
-        if (transientHints.some((hint) => message.includes(hint))) {
-            return { isCapabilityError: false, reason: "transient_or_auth" };
+        const pdfMessageHints = [
+            "invalid pdf",
+            "malformed pdf",
+            "corrupt pdf",
+            "encrypted pdf",
+            "password protected pdf",
+            "failed to parse pdf",
+            "unable to parse pdf",
+            "pdf is corrupted",
+            "pdf too large",
+            "file too large",
+        ];
+        const codeMatches = this.matchCodes(signal.codes, modality === "pdf" ? pdfCodeHints : imageCodeHints);
+        const messageMatches = this.matchMessage(signal.message, modality === "pdf" ? pdfMessageHints : imageMessageHints);
+        const statusMatches = Array.from(signal.statusCodes).filter((status) => {
+            if (status === 413)
+                return true;
+            if (status === 415 || status === 422) {
+                return codeMatches.length > 0 || messageMatches.length > 0;
+            }
+            return false;
+        });
+        return [
+            ...statusMatches.map((status) => `status:${status}`),
+            ...codeMatches.map((match) => `code:${match}`),
+            ...messageMatches.map((match) => `msg:${match}`),
+        ];
+    }
+    static scoreCapabilitySignal(signal, provider, modality) {
+        const evidence = [];
+        let score = 0;
+        const explicitCapabilityCodes = this.matchCodes(signal.codes, modality === "pdf"
+            ? [
+                "pdf_not_supported",
+                "unsupported_pdf_input",
+                "unsupported_document_input",
+                "unsupported_file_input",
+                "input_file_not_supported",
+                "unsupported_file_type",
+                "model_not_document_capable",
+            ]
+            : [
+                "vision_not_supported",
+                "unsupported_vision",
+                "model_not_vision_capable",
+                "image_not_supported",
+                "unsupported_message_content",
+                "unsupported_content_type_for_model",
+                "unsupported_image_input",
+                "invalid_model_for_vision",
+            ]);
+        if (explicitCapabilityCodes.length > 0) {
+            score += 3;
+            evidence.push(...explicitCapabilityCodes.map((match) => `code:${match}`));
+        }
+        const highPrecisionMessageMatches = this.matchMessage(signal.message, modality === "pdf"
+            ? [
+                "this model does not support pdf",
+                "model does not support pdf",
+                "pdf is not supported for this model",
+                "file input is not supported for this model",
+                "input_file is not supported",
+                "unsupported file type: application/pdf",
+                "application/pdf is not supported for this model",
+            ]
+            : [
+                "does not support images",
+                "does not support image inputs",
+                "model does not support image",
+                "this model cannot process images",
+                "text-only model",
+                "images are not supported for this model",
+                "vision is not supported for this model",
+                "vision is not supported",
+                "vision not supported",
+                "image_url is only supported by certain models",
+            ]);
+        if (highPrecisionMessageMatches.length > 0) {
+            score += 3;
+            evidence.push(...highPrecisionMessageMatches.map((match) => `msg:${match}`));
+        }
+        const providerSpecificMatches = this.matchMessage(signal.message, this.providerCapabilityHints(provider, modality));
+        if (providerSpecificMatches.length > 0) {
+            score += 2;
+            evidence.push(...providerSpecificMatches.map((match) => `provider:${match}`));
+        }
+        const weakCapabilityHints = this.matchMessage(signal.message, modality === "pdf"
+            ? [
+                "pdf input",
+                "pdf support",
+                "pdf not supported",
+                "application/pdf",
+                "input_file",
+                "file input",
+                "document input",
+                "unsupported file type",
+                "unsupported content type",
+                "invalid content type",
+            ]
+            : [
+                "vision",
+                "unsupported content type",
+                "unsupported message content",
+                "invalid content type",
+                "unrecognized content type",
+                "image_url",
+                "multimodal",
+                "multi-modal",
+            ]);
+        const hasClientValidationStatus = Array.from(signal.statusCodes).some((status) => [400, 415, 422].includes(status));
+        if (weakCapabilityHints.length > 0 && hasClientValidationStatus) {
+            score += 1;
+            evidence.push(...weakCapabilityHints.map((match) => `weak:${match}`));
+        }
+        if (Array.from(signal.statusCodes).some((status) => status === 400 || status === 422)) {
+            score += 1;
+            evidence.push("status:client_validation");
+        }
+        return {
+            score,
+            evidence: Array.from(new Set(evidence)).slice(0, 8),
+        };
+    }
+    static providerCapabilityHints(provider, modality) {
+        const normalized = provider.toLowerCase().trim();
+        if (modality === "pdf") {
+            if (normalized.includes("openai")) {
+                return [
+                    "input_file is not supported",
+                    "unsupported file type: application/pdf",
+                    "application/pdf is not supported for this model",
+                ];
+            }
+            if (normalized.includes("anthropic")) {
+                return [
+                    "pdf is not supported for this model",
+                    "file input is not supported for this model",
+                ];
+            }
+            if (normalized.includes("google") || normalized.includes("gemini")) {
+                return [
+                    "unsupported document input",
+                    "pdf input is not supported",
+                ];
+            }
+            if (normalized.includes("realtimex")) {
+                return [
+                    "unsupported file input",
+                    "invalid model",
+                ];
+            }
+            return [];
+        }
+        if (normalized.includes("openai")) {
+            return [
+                "image_url is only supported by certain models",
+                "this model does not support image inputs",
+            ];
+        }
+        if (normalized.includes("anthropic")) {
+            return [
+                "only some claude models support vision",
+                "images are not supported for this model",
+            ];
+        }
+        if (normalized.includes("google") || normalized.includes("gemini")) {
+            return [
+                "model does not support multimodal input",
+                "unsupported input modality",
+            ];
+        }
+        if (normalized.includes("realtimex")) {
+            return [
+                "invalid model",
+                "text-only model",
+            ];
+        }
+        return [];
+    }
+    static matchMessage(message, hints) {
+        if (!message)
+            return [];
+        return hints.filter((hint) => message.includes(hint));
+    }
+    static matchCodes(codes, hints) {
+        const matches = [];
+        for (const code of codes) {
+            const normalizedCode = code.replace(/[\s.-]+/g, "_");
+            for (const hint of hints) {
+                if (normalizedCode === hint || normalizedCode.includes(hint)) {
+                    matches.push(code);
+                    break;
+                }
+            }
         }
-        return { isCapabilityError: false, reason: "unknown_error_class" };
+        return matches;
     }
     static errorToMessage(error) {
         if (error instanceof Error)