@aexol/spectral 0.9.12 → 0.9.14

package/dist/extensions/spectral-vision-fallback.d.ts

@@ -1,22 +1,17 @@
 /**
  * Spectral Vision Extension
  *
- * Automatically switches to a vision-capable model for image processing
- * when images are attached. This allows non-vision models to "see" images
- * and saves main agent context by using a potentially smaller/cheaper
- * vision model for image understanding.
+ * Converts attached images into text analysis before the main agent call.
  *
- * Logic:
- *   - If the main model supports vision natively, images pass through
- *     directly — no interception, no text descriptions.
- *   - When the main model does NOT support vision, falls back to the
- *     admin-configured default vision model (isVisionDefault flag from
- *     backend via SettingsManager) for image descriptions.
- *   - If no admin default is configured, falls back to the best
- *     available vision-capable model by provider priority.
- *
- * Hooks into the `context` event to intercept ALL images before they reach
- * the LLM (covers user-attached images and tool-result images alike).
+ * Pipeline:
+ *   - If the current main model can process images, use it as the analyzer.
+ *   - Otherwise use the admin-configured default vision model.
+ *   - Otherwise use the best available authenticated image-capable model.
+ *   - Replace image blocks with the text analysis before persistence when
+ *     possible (`message_end`), so image data is deleted from conversation
+ *     history and is not re-analyzed on later turns.
+ *   - Keep a `context` hook as a defense-in-depth fallback for older sessions
+ *     or messages that entered context before the replacement hook existed.
  */
 import type { ExtensionAPI } from "../sdk/coding-agent/index.js";
 export default function spectralVisionExtension(ext: ExtensionAPI): void;

package/dist/extensions/spectral-vision-fallback.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"spectral-vision-fallback.d.ts","sourceRoot":"","sources":["../../src/extensions/spectral-vision-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAuNjE,MAAM,CAAC,OAAO,UAAU,uBAAuB,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAmFvE"}
+{"version":3,"file":"spectral-vision-fallback.d.ts","sourceRoot":"","sources":["../../src/extensions/spectral-vision-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAsWjE,MAAM,CAAC,OAAO,UAAU,uBAAuB,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CA6DvE"}

package/dist/extensions/spectral-vision-fallback.js

@@ -1,27 +1,47 @@
 /**
  * Spectral Vision Extension
  *
- * Automatically switches to a vision-capable model for image processing
- * when images are attached. This allows non-vision models to "see" images
- * and saves main agent context by using a potentially smaller/cheaper
- * vision model for image understanding.
+ * Converts attached images into text analysis before the main agent call.
  *
- * Logic:
- *   - If the main model supports vision natively, images pass through
- *     directly — no interception, no text descriptions.
- *   - When the main model does NOT support vision, falls back to the
- *     admin-configured default vision model (isVisionDefault flag from
- *     backend via SettingsManager) for image descriptions.
- *   - If no admin default is configured, falls back to the best
- *     available vision-capable model by provider priority.
- *
- * Hooks into the `context` event to intercept ALL images before they reach
- * the LLM (covers user-attached images and tool-result images alike).
+ * Pipeline:
+ *   - If the current main model can process images, use it as the analyzer.
+ *   - Otherwise use the admin-configured default vision model.
+ *   - Otherwise use the best available authenticated image-capable model.
+ *   - Replace image blocks with the text analysis before persistence when
+ *     possible (`message_end`), so image data is deleted from conversation
+ *     history and is not re-analyzed on later turns.
+ *   - Keep a `context` hook as a defense-in-depth fallback for older sessions
+ *     or messages that entered context before the replacement hook existed.
  */
 import { streamSimple } from "../sdk/ai/index.js";
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
+const EXTENSION_NAME = "spectral-vision";
+const IMAGE_DESCRIPTION_PATTERN = /^\[Image analysis from /;
+const MAX_CACHE_ENTRIES = 500;
+function withImageInput(model) {
+    if (model.input.includes("image"))
+        return model;
+    return {
+        ...model,
+        input: [...model.input, "image"],
+    };
+}
+function sameModel(a, b) {
+    return !!a && !!b && a.provider === b.provider && a.id === b.id;
+}
+function getCurrentImageAnalyzer(ctx) {
+    const currentModel = ctx.model;
+    if (!currentModel)
+        return undefined;
+    if (!ctx.modelRegistry.hasConfiguredAuth(currentModel))
+        return undefined;
+    if (currentModel.input.includes("image")) {
+        return { model: withImageInput(currentModel), source: "main" };
+    }
+    return undefined;
+}
 /**
  * Check if the SettingsManager has an admin-configured default vision model
  * with working auth. Returns the model if found and auth-ready, or undefined.
@@ -36,75 +56,126 @@ function getAdminVisionModel(ctx) {
         return undefined;
     const allModels = ctx.modelRegistry.getAll();
     const match = allModels.find((m) => m.provider === defaultVisionProvider && m.id === defaultVisionModel);
-    if (!match || !match.input.includes("image"))
+    if (!match)
         return undefined;
     if (!ctx.modelRegistry.hasConfiguredAuth(match))
         return undefined;
-    return match;
+    // Trust the admin's explicit choice: if they marked this model as the
+    // default vision model, it should be used even when the backend's
+    // capabilities data doesn't report image support.
+    return withImageInput(match);
 }
-/**
- * Resolve the best vision model to use for image descriptions.
- *
- * Priority:
- *   1. Current main model if it supports images — passthrough, no interception
- *   2. Admin-configured default vision model (isVisionDefault from backend)
- *   3. Best available vision model by provider priority
- *   4. First available vision model
- */
-function resolveVisionModel(ctx) {
-    // 1. If the main model supports vision natively, skip interception entirely
-    //    and let the main model process images directly.
-    const currentModel = ctx.model;
-    if (currentModel?.input.includes("image") && ctx.modelRegistry.hasConfiguredAuth(currentModel)) {
-        process.stderr.write(`[spectral-vision] Main model supports vision — passing images through natively (${currentModel.provider}/${currentModel.id})\n`);
-        return undefined;
-    }
-    // 2. Admin-configured default vision model (fallback for non-vision main models)
+function getFallbackVisionModel(ctx, exclude) {
     const adminModel = getAdminVisionModel(ctx);
-    if (adminModel) {
-        process.stderr.write(`[spectral-vision] Using admin-configured default: ${adminModel.provider}/${adminModel.id}\n`);
-        return adminModel;
+    if (adminModel && !sameModel(adminModel, exclude)) {
+        return { model: adminModel, source: "default" };
     }
-    // 3. Fall back to best available vision model by provider priority
     const allModels = ctx.modelRegistry.getAll();
-    const visionModels = allModels.filter((m) => m.input.includes("image") && ctx.modelRegistry.hasConfiguredAuth(m));
+    const visionModels = allModels.filter((m) => m.input.includes("image") &&
+        ctx.modelRegistry.hasConfiguredAuth(m) &&
+        !sameModel(m, exclude));
     if (visionModels.length === 0)
         return undefined;
     const providerPriority = ["anthropic", "openai", "google", "openrouter"];
     for (const provider of providerPriority) {
         const match = visionModels.find((m) => m.provider === provider);
-        if (match) {
-            process.stderr.write(`[spectral-vision] Using fallback vision model: ${match.provider}/${match.id}\n`);
-            return match;
-        }
+        if (match)
+            return { model: match, source: "fallback" };
     }
-    // 4. First available vision model
-    const fallback = visionModels[0];
-    process.stderr.write(`[spectral-vision] Using first available vision model: ${fallback.provider}/${fallback.id}\n`);
-    return fallback;
+    return { model: visionModels[0], source: "fallback" };
+}
+function resolvePrimaryAnalyzer(ctx) {
+    const currentAnalyzer = getCurrentImageAnalyzer(ctx);
+    if (currentAnalyzer) {
+        process.stderr.write(`[${EXTENSION_NAME}] Using main model for image analysis: ${currentAnalyzer.model.provider}/${currentAnalyzer.model.id}\n`);
+        return currentAnalyzer;
+    }
+    const fallback = getFallbackVisionModel(ctx);
+    if (fallback) {
+        const label = fallback.source === "default" ? "admin-configured default" : "fallback vision model";
+        process.stderr.write(`[${EXTENSION_NAME}] Using ${label}: ${fallback.model.provider}/${fallback.model.id}\n`);
+        return fallback;
+    }
+    return undefined;
 }
 /** Check if any content block in an array is an image */
 function hasImageContent(content) {
     return Array.isArray(content) && content.some((c) => c?.type === "image");
 }
+function removeImageContent(content) {
+    return content.filter((block) => block.type === "text");
+}
 /** Count images in a message */
 function countImages(msg) {
     if (!Array.isArray(msg.content))
         return 0;
     return msg.content.filter((c) => c.type === "image").length;
 }
+function isProcessableImageMessage(msg) {
+    return (msg.role === "user" || msg.role === "toolResult") && Array.isArray(msg.content) && hasImageContent(msg.content);
+}
+/**
+ * Check if a message has already been processed by our vision fallback.
+ */
+function isAlreadyDescribed(msg) {
+    if (!Array.isArray(msg.content))
+        return false;
+    return msg.content.some((c) => c.type === "text" && IMAGE_DESCRIPTION_PATTERN.test(String(c.text)));
+}
+function contextTextFor(content) {
+    return content
+        .filter((c) => c.type === "text")
+        .map((c) => c.text)
+        .join(" ")
+        .substring(0, 500);
+}
+function imageMessageCacheKey(msg) {
+    const content = Array.isArray(msg.content) ? msg.content : [];
+    const text = content
+        .filter((c) => c.type === "text")
+        .map((c) => String(c.text))
+        .join("\n")
+        .slice(0, 1000);
+    const images = content
+        .filter((c) => c.type === "image")
+        .map((c) => `${c.mimeType ?? "image"}:${String(c.data ?? "").length}:${String(c.data ?? "").slice(0, 64)}`)
+        .join("|");
+    return `${msg.role}:${msg.timestamp ?? "no-ts"}:${text}:${images}`;
+}
+function rememberAnalysis(cache, key, content) {
+    cache.set(key, content);
+    if (cache.size <= MAX_CACHE_ENTRIES)
+        return;
+    const first = cache.keys().next().value;
+    if (first)
+        cache.delete(first);
+}
+function omittedContent(msg, reason) {
+    const textBlocks = Array.isArray(msg.content)
+        ? msg.content.filter((c) => c.type === "text")
+        : [];
+    const imageCount = countImages(msg);
+    return [
+        ...textBlocks,
+        {
+            type: "text",
+            text: `[${imageCount} image(s) omitted — ${reason}]`,
+        },
+    ];
+}
 // ---------------------------------------------------------------------------
 // Core: call vision model to describe images
 // ---------------------------------------------------------------------------
 async function describeImages(visionModel, content, contextText, ctx) {
+    const model = withImageInput(visionModel);
     const userContent = [
         {
             type: "text",
-            text: `Please describe the following image(s). ` +
+            text: `Please analyze the following image(s). ` +
                 (contextText ? `Context: ${contextText}. ` : "") +
                 "Focus on what is visually visible — text content, UI elements, diagrams, " +
-                "code structure, layout, colors, etc. Be concise but thorough. " +
-                "If multiple images are provided, describe each one separately with a heading.",
+                "code structure, layout, colors, errors, and any details needed to answer the user's request. " +
+                "Be concise but thorough. If multiple images are provided, analyze each one separately with a heading.",
         },
     ];
     const textPrefixBlocks = [];
@@ -120,115 +191,141 @@ async function describeImages(visionModel, content, contextText, ctx) {
     }
     if (imageCount === 0)
         return content;
-    const auth = await ctx.modelRegistry.getApiKeyAndHeaders(visionModel);
+    const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
     if (!auth.ok) {
-        process.stderr.write(`[spectral-vision] No API key for vision model ${visionModel.provider}/${visionModel.id}\n`);
+        throw new Error(`No API key for vision model ${model.provider}/${model.id}: ${auth.error}`);
+    }
+    const result = await streamSimple(model, {
+        systemPrompt: "You are an image analysis assistant. Analyze images accurately and concisely. Return only the visual analysis, not commentary about being an assistant.",
+        messages: [{ role: "user", content: userContent, timestamp: Date.now() }],
+        tools: [],
+    }, {
+        apiKey: auth.apiKey,
+        headers: auth.headers,
+    });
+    let description = "";
+    for await (const event of result) {
+        if (event.type === "text_delta") {
+            description += event.delta;
+        }
+    }
+    const textNote = description.trim()
+        ? `[Image analysis from ${model.provider}/${model.id} (${imageCount} image(s)):\n${description.trim()}\n]`
+        : `[${imageCount} image(s) — vision model returned empty analysis]`;
+    try {
+        ctx.ui.notify(`[${EXTENSION_NAME}] Analyzed ${imageCount} image(s) with ${model.provider}/${model.id}`, "info");
+    }
+    catch {
+        // UI context may not be available in headless modes.
+    }
+    return [
+        ...textPrefixBlocks,
+        { type: "text", text: textNote },
+    ];
+}
+async function analyzeMessageImages(msg, ctx, cache) {
+    const key = imageMessageCacheKey(msg);
+    const cached = cache.get(key);
+    if (cached)
+        return cached;
+    if (isAlreadyDescribed(msg)) {
+        const withoutImages = removeImageContent(msg.content);
+        rememberAnalysis(cache, key, withoutImages);
+        return withoutImages;
+    }
+    const primary = resolvePrimaryAnalyzer(ctx);
+    if (!primary) {
+        const content = omittedContent(msg, "no authenticated vision-capable model is configured");
+        rememberAnalysis(cache, key, content);
         return content;
     }
+    const contextText = contextTextFor(msg.content);
+    const imageCount = countImages(msg);
     try {
-        const result = await streamSimple(visionModel, {
-            systemPrompt: "You are an image description assistant. Describe images accurately and concisely.",
-            messages: [{ role: "user", content: userContent, timestamp: Date.now() }],
-            tools: [],
-        }, {
-            apiKey: auth.apiKey,
-            headers: auth.headers,
-        });
-        let description = "";
-        for await (const event of result) {
-            if (event.type === "text_delta") {
-                description += event.delta;
-            }
-        }
-        const textNote = description.trim()
-            ? `[Image description from ${visionModel.provider}/${visionModel.id} (${imageCount} image(s)):\n${description}\n]`
-            : `[${imageCount} image(s) — vision model returned empty description]`;
-        // Notify the frontend that the vision extension analyzed the image(s)
-        try {
-            ctx.ui.notify(`[spectral-vision] Analyzed ${imageCount} image(s) with ${visionModel.provider}/${visionModel.id}`, "info");
-        }
-        catch {
-            // UI context may not be available in headless modes
-        }
-        return [
-            ...textPrefixBlocks,
-            { type: "text", text: textNote },
-        ];
+        const described = await describeImages(primary.model, msg.content, contextText, ctx);
+        const withoutImages = removeImageContent(described);
+        rememberAnalysis(cache, key, withoutImages);
+        return withoutImages;
     }
     catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        process.stderr.write(`[spectral-vision] Vision call failed: ${msg}\n`);
-        // Notify frontend about the failure
+        const message = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`[${EXTENSION_NAME}] Image analysis with ${primary.model.provider}/${primary.model.id} failed: ${message}\n`);
+        const fallback = primary.source === "main" ? getFallbackVisionModel(ctx, primary.model) : undefined;
+        if (fallback) {
+            process.stderr.write(`[${EXTENSION_NAME}] Retrying image analysis with ${fallback.model.provider}/${fallback.model.id}\n`);
+            try {
+                const described = await describeImages(fallback.model, msg.content, contextText, ctx);
+                const withoutImages = removeImageContent(described);
+                rememberAnalysis(cache, key, withoutImages);
+                return withoutImages;
+            }
+            catch (fallbackErr) {
+                const fallbackMessage = fallbackErr instanceof Error ? fallbackErr.message : String(fallbackErr);
+                process.stderr.write(`[${EXTENSION_NAME}] Fallback image analysis with ${fallback.model.provider}/${fallback.model.id} failed: ${fallbackMessage}\n`);
+                try {
+                    ctx.ui.notify(`[${EXTENSION_NAME}] Failed to analyze ${imageCount} image(s): ${fallbackMessage}`, "warning");
+                }
+                catch {
+                    // UI context may not be available in headless modes.
+                }
+                const content = omittedContent(msg, `vision analysis failed: ${fallbackMessage}`);
+                rememberAnalysis(cache, key, content);
+                return content;
+            }
+        }
         try {
-            ctx.ui.notify(`[spectral-vision] Failed to analyze ${imageCount} image(s): ${msg}`, "warning");
+            ctx.ui.notify(`[${EXTENSION_NAME}] Failed to analyze ${imageCount} image(s): ${message}`, "warning");
         }
         catch {
-            // UI context may not be available in headless modes
+            // UI context may not be available in headless modes.
         }
-        return [
-            ...textPrefixBlocks,
-            {
-                type: "text",
-                text: `[${imageCount} image(s) omitted — vision fallback failed: ${msg}]`,
-            },
-        ];
+        const content = omittedContent(msg, `vision analysis failed: ${message}`);
+        rememberAnalysis(cache, key, content);
+        return content;
     }
 }
 // ---------------------------------------------------------------------------
 // Extension entry point
 // ---------------------------------------------------------------------------
 export default function spectralVisionExtension(ext) {
-    let visionModel;
+    const analysisCache = new Map();
     ext.on("session_start", (_event, ctx) => {
-        visionModel = resolveVisionModel(ctx);
-        if (visionModel) {
-            process.stderr.write(`[spectral-vision] Ready — using ${visionModel.provider}/${visionModel.id} for image descriptions.\n`);
-        }
-        else if (ctx.model?.input.includes("image")) {
-            process.stderr.write(`[spectral-vision] Ready — main model ${ctx.model.provider}/${ctx.model.id} handles vision natively, passthrough enabled.\n`);
+        const analyzer = resolvePrimaryAnalyzer(ctx);
+        if (analyzer) {
+            const label = analyzer.source === "main" ? "main model" : analyzer.source === "default" ? "default vision model" : "fallback vision model";
+            process.stderr.write(`[${EXTENSION_NAME}] Ready — ${label} ${analyzer.model.provider}/${analyzer.model.id} will analyze images before agent calls.\n`);
         }
         else {
-            process.stderr.write("[spectral-vision] No vision model with auth configured. Image description disabled.\n");
+            process.stderr.write(`[${EXTENSION_NAME}] No vision model with auth configured. Images will be removed with an explanatory note.\n`);
         }
     });
+    // Primary path: replace user/tool-result image blocks before AgentSession
+    // persists the message. This deletes raw base64 image data from conversation
+    // history and prevents repeat analysis on later turns.
+    ext.on("message_end", async (event, ctx) => {
+        const msg = event.message;
+        if (!isProcessableImageMessage(msg))
+            return;
+        process.stderr.write(`[${EXTENSION_NAME}] Analyzing ${countImages(msg)} image(s) before persisting ${msg.role} message...\n`);
+        const content = await analyzeMessageImages(msg, ctx, analysisCache);
+        return { message: { ...msg, content } };
+    });
+    // Defense-in-depth path: handle images still present in context from older
+    // sessions or from messages that bypassed message_end replacement. Results are
+    // cached by message/image fingerprint so old images are not re-analyzed every
+    // turn; cached analysis is substituted instead.
     ext.on("context", async (event, ctx) => {
         const messages = event.messages;
-        // Check if any message contains images
-        let totalImages = 0;
-        for (const msg of messages) {
-            totalImages += countImages(msg);
-        }
-        if (totalImages === 0)
+        const imageMessages = messages.filter(isProcessableImageMessage);
+        if (imageMessages.length === 0)
             return;
-        // Re-resolve vision model each time images are encountered.
-        // If the main model supports vision natively, resolveVisionModel
-        // returns undefined and we pass images through without interception.
-        visionModel = resolveVisionModel(ctx);
-        if (!visionModel) {
-            // Check if we're in passthrough mode (main model has native vision)
-            if (ctx.model?.input.includes("image")) {
-                process.stderr.write(`[spectral-vision] Passthrough — ${totalImages} image(s) flowing to main model natively\n`);
-            }
-            else {
-                process.stderr.write("[spectral-vision] No vision model available, images will be omitted\n");
-            }
-            return;
-        }
-        process.stderr.write(`[spectral-vision] Describing ${totalImages} image(s) with ${visionModel.provider}/${visionModel.id}...\n`);
-        // Process each message — intercept both user messages and tool results
-        // that contain images (e.g. read tool returning a screenshot).
+        const totalImages = imageMessages.reduce((sum, msg) => sum + countImages(msg), 0);
+        process.stderr.write(`[${EXTENSION_NAME}] Context contains ${totalImages} image(s); replacing with analysis text before provider request...\n`);
         const processed = await Promise.all(messages.map(async (msg) => {
-            if ((msg.role !== "user" && msg.role !== "toolResult") || !Array.isArray(msg.content) || !hasImageContent(msg.content)) {
+            if (!isProcessableImageMessage(msg))
                 return msg;
-            }
-            // Extract context from preceding text blocks
-            const textBlocks = msg.content
-                .filter((c) => c.type === "text")
-                .map((c) => c.text)
-                .join(" ");
-            const contextText = textBlocks.substring(0, 500); // limit context
-            const described = await describeImages(visionModel, msg.content, contextText, ctx);
-            return { ...msg, content: described };
+            const content = await analyzeMessageImages(msg, ctx, analysisCache);
+            return { ...msg, content };
         }));
         return { messages: processed };
     });

package/dist/memory/hooks/compaction-hook.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compaction-hook.d.ts","sourceRoot":"","sources":["../../../src/memory/hooks/compaction-hook.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAoB,MAAM,iCAAiC,CAAC;AAsBtF,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AA6B7C,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CA2VhF"}
+{"version":3,"file":"compaction-hook.d.ts","sourceRoot":"","sources":["../../../src/memory/hooks/compaction-hook.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAA+C,MAAM,iCAAiC,CAAC;AAsBjH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAmE7C,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CA8XhF"}