open-agents-ai 0.185.28 → 0.185.29

package/dist/index.js

@@ -26415,29 +26415,18 @@ If you're stuck, try a completely different approach. Do NOT repeat what failed
               }
               let imageRecovered = false;
               if (/invalid image|image.*invalid|image_url.*unsupported|does not support.*image|image.*not supported/i.test(errMsg)) {
-                this.emit({
-                  type: "status",
-                  content: `Backend rejected image \u2014 stripping images and retrying`,
-                  timestamp: (/* @__PURE__ */ new Date()).toISOString()
-                });
-                for (const msg2 of messages) {
-                  if (Array.isArray(msg2.content)) {
-                    const textParts = msg2.content.filter((p) => p.type !== "image_url");
-                    if (textParts.length > 0) {
-                      msg2.content = textParts.length === 1 && textParts[0].text ? textParts[0].text : textParts;
-                    } else {
-                      msg2.content = "[Image was here but backend doesn't support images]";
-                    }
+                imageRecovered = await this._recoverFromImageError(messages, chatRequest, turn);
+                if (imageRecovered) {
+                  try {
+                    const imgRetry = this.options.streamEnabled && this.hasStreamingSupport() ? await this.streamingRequest(chatRequest, turn) : await this.backend.chatCompletion(chatRequest);
+                    response = imgRetry;
+                  } catch (imgRetryErr) {
+                    const msg2 = imgRetryErr instanceof Error ? imgRetryErr.message : String(imgRetryErr);
+                    this.emit({ type: "error", content: `Retry after image fallback also failed: ${msg2}`, timestamp: (/* @__PURE__ */ new Date()).toISOString() });
+                    imageRecovered = false;
+                    break;
                   }
-                }
-                chatRequest.messages = messages;
-                try {
-                  const imgRetry = this.options.streamEnabled && this.hasStreamingSupport() ? await this.streamingRequest(chatRequest, turn) : await this.backend.chatCompletion(chatRequest);
-                  response = imgRetry;
-                  imageRecovered = true;
-                } catch (imgRetryErr) {
-                  const msg2 = imgRetryErr instanceof Error ? imgRetryErr.message : String(imgRetryErr);
-                  this.emit({ type: "error", content: `Retry without images also failed: ${msg2}`, timestamp: (/* @__PURE__ */ new Date()).toISOString() });
+                } else {
                   break;
                 }
               }
@@ -28475,6 +28464,172 @@ ${transcript}`
           return true;
         return false;
       }
+      /**
+       * Graceful image error recovery chain:
+       *   1. Downconvert images (resize to ≤512px, JPEG compress) and retry inline
+       *   2. Describe images via moondream/Ollama vision → replace image_url with text description
+       *   3. Last resort: strip images, keep text context
+       *
+       * Mutates messages in-place. Returns true if messages were successfully transformed.
+       */
+      async _recoverFromImageError(messages, chatRequest, turn) {
+        const imageEntries = [];
+        for (let mi = 0; mi < messages.length; mi++) {
+          const msg = messages[mi];
+          if (!Array.isArray(msg.content))
+            continue;
+          for (let pi = 0; pi < msg.content.length; pi++) {
+            const part = msg.content[pi];
+            if (part.type === "image_url" && part.image_url?.url) {
+              imageEntries.push({ msgIdx: mi, partIdx: pi, dataUrl: part.image_url.url });
+            }
+          }
+        }
+        if (imageEntries.length === 0)
+          return false;
+        this.emit({ type: "status", content: `Image rejected \u2014 trying downconversion (${imageEntries.length} image(s))...`, timestamp: (/* @__PURE__ */ new Date()).toISOString() });
+        let downconverted = false;
+        try {
+          for (const entry of imageEntries) {
+            const { dataUrl } = entry;
+            if (!dataUrl.startsWith("data:"))
+              continue;
+            const commaIdx = dataUrl.indexOf(",");
+            if (commaIdx < 0)
+              continue;
+            const rawBase64 = dataUrl.slice(commaIdx + 1);
+            const buffer = Buffer.from(rawBase64, "base64");
+            let resizedBase64 = null;
+            try {
+              const { execSync: execSync35 } = await import("node:child_process");
+              const { writeFileSync: writeFileSync30, readFileSync: readFileSync45, unlinkSync: unlinkSync13 } = await import("node:fs");
+              const { join: join77 } = await import("node:path");
+              const { tmpdir: tmpdir11 } = await import("node:os");
+              const tmpIn = join77(tmpdir11(), `oa_img_in_${Date.now()}.png`);
+              const tmpOut = join77(tmpdir11(), `oa_img_out_${Date.now()}.jpg`);
+              writeFileSync30(tmpIn, buffer);
+              execSync35(`python3 -c "
+from PIL import Image
+img = Image.open('${tmpIn}')
+img.thumbnail((512, 512), Image.LANCZOS)
+img = img.convert('RGB')
+img.save('${tmpOut}', 'JPEG', quality=75)
+"`, { timeout: 1e4, stdio: "pipe" });
+              const resizedBuf = readFileSync45(tmpOut);
+              resizedBase64 = `data:image/jpeg;base64,${resizedBuf.toString("base64")}`;
+              try {
+                unlinkSync13(tmpIn);
+              } catch {
+              }
+              try {
+                unlinkSync13(tmpOut);
+              } catch {
+              }
+            } catch {
+            }
+            if (resizedBase64) {
+              const msg = messages[entry.msgIdx];
+              const parts = msg.content;
+              parts[entry.partIdx] = { type: "image_url", image_url: { url: resizedBase64 } };
+              downconverted = true;
+            }
+          }
+        } catch {
+        }
+        if (downconverted) {
+          chatRequest.messages = messages;
+          this.emit({ type: "status", content: `Downconverted images to 512px JPEG \u2014 retrying`, timestamp: (/* @__PURE__ */ new Date()).toISOString() });
+          return true;
+        }
+        this.emit({ type: "status", content: `Downconversion unavailable \u2014 describing images via vision model...`, timestamp: (/* @__PURE__ */ new Date()).toISOString() });
+        const ollamaHost = process.env["OLLAMA_HOST"] || "http://127.0.0.1:11434";
+        let described = false;
+        for (const entry of imageEntries) {
+          const { dataUrl } = entry;
+          if (!dataUrl.startsWith("data:"))
+            continue;
+          const commaIdx = dataUrl.indexOf(",");
+          if (commaIdx < 0)
+            continue;
+          const rawBase64 = dataUrl.slice(commaIdx + 1);
+          try {
+            const model = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+            let res = await fetch(`${ollamaHost}/api/generate`, {
+              method: "POST",
+              headers: { "Content-Type": "application/json" },
+              body: JSON.stringify({
+                model,
+                prompt: "Describe this image in detail. Include text content, UI elements, code, errors, and any relevant visual information.",
+                images: [rawBase64],
+                stream: false
+              }),
+              signal: AbortSignal.timeout(6e4)
+            });
+            if (!res.ok && model === "moondream" && res.status === 404) {
+              this.emit({ type: "status", content: `Pulling moondream vision model...`, timestamp: (/* @__PURE__ */ new Date()).toISOString() });
+              try {
+                const { execSync: execSync35 } = await import("node:child_process");
+                execSync35("ollama pull moondream", { timeout: 3e5, stdio: "pipe" });
+                res = await fetch(`${ollamaHost}/api/generate`, {
+                  method: "POST",
+                  headers: { "Content-Type": "application/json" },
+                  body: JSON.stringify({
+                    model,
+                    prompt: "Describe this image in detail. Include text content, UI elements, code, errors, and any relevant visual information.",
+                    images: [rawBase64],
+                    stream: false
+                  }),
+                  signal: AbortSignal.timeout(6e4)
+                });
+              } catch {
+              }
+            }
+            if (res.ok) {
+              const data = await res.json();
+              const description = data.response?.trim();
+              if (description && description.length > 20) {
+                const msg = messages[entry.msgIdx];
+                const parts = msg.content;
+                parts[entry.partIdx] = {
+                  type: "text",
+                  text: `[Image description from vision model]:
+${description}`
+                };
+                described = true;
+                this.emit({ type: "status", content: `Image described (${description.length} chars) \u2014 replacing inline`, timestamp: (/* @__PURE__ */ new Date()).toISOString() });
+              }
+            }
+          } catch {
+          }
+        }
+        if (described) {
+          for (const msg of messages) {
+            if (Array.isArray(msg.content)) {
+              const parts = msg.content;
+              const allText = parts.every((p) => p.type === "text");
+              if (allText && parts.length === 1 && parts[0].text) {
+                msg.content = parts[0].text;
+              }
+            }
+          }
+          chatRequest.messages = messages;
+          this.emit({ type: "status", content: `Images replaced with descriptions \u2014 retrying`, timestamp: (/* @__PURE__ */ new Date()).toISOString() });
+          return true;
+        }
+        this.emit({ type: "status", content: `No vision model available \u2014 stripping images (text context preserved)`, timestamp: (/* @__PURE__ */ new Date()).toISOString() });
+        for (const msg of messages) {
+          if (Array.isArray(msg.content)) {
+            const textParts = msg.content.filter((p) => p.type !== "image_url");
+            if (textParts.length > 0) {
+              msg.content = textParts.length === 1 && textParts[0].text ? textParts[0].text : textParts;
+            } else {
+              msg.content = "[Image was provided but could not be processed \u2014 no vision model available]";
+            }
+          }
+        }
+        chatRequest.messages = messages;
+        return true;
+      }
       /**
        * Retry a failed model request up to 3 times with exponential backoff.
        * Returns the response on success, or null if all retries failed.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.185.28",
+  "version": "0.185.29",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",