open-agents-ai 0.187.592 → 0.187.594

package/dist/index.js

@@ -503992,6 +503992,13 @@ ${cameras.join("\n")}`,
         }
         const data = readFileSync29(filePath);
         const sizeKB = Math.round(data.length / 1024);
+        const outputPath = userOutputPath ?? join55(process.cwd(), ".oa", "camera-captures", `capture-${Date.now()}.jpg`);
+        if (!userOutputPath) {
+          mkdirSync14(join55(process.cwd(), ".oa", "camera-captures"), {
+            recursive: true
+          });
+          writeFileSync15(outputPath, data);
+        }
         if (userOutputPath) {
           return {
             success: true,
@@ -504005,12 +504012,13 @@ Saved to: ${userOutputPath}`,
         } catch {
         }
         const base642 = data.toString("base64");
+        const display = `Captured ${resolution} frame from ${source} (${sizeKB}KB JPEG).
+Saved to: ${outputPath}`;
         return {
           success: true,
-          output: `Captured ${resolution} frame from ${source} (${sizeKB}KB JPEG).
-
-Base64 image data (use with vision tools):
-data:image/jpeg;base64,${base642}`,
+          output: display,
+          llmContent: `${display}
+[IMAGE_BASE64:image/jpeg:${base642}]`,
           durationMs: performance.now() - start2
         };
       }
@@ -527374,9 +527382,11 @@ RECOVERY: cd to the directory containing '${file}', run a plain install with no
 });
 
 // packages/orchestrator/dist/agenticRunner.js
-import { existsSync as _fsExistsSync, readFileSync as _fsReadFileSync, writeFileSync as _fsWriteFileSync, mkdirSync as _fsMkdirSync } from "node:fs";
+import { existsSync as _fsExistsSync, readFileSync as _fsReadFileSync, writeFileSync as _fsWriteFileSync, unlinkSync as _fsUnlinkSync, mkdirSync as _fsMkdirSync } from "node:fs";
+import { execFile as _execFile } from "node:child_process";
 import { createHash as _createHash } from "node:crypto";
 import { join as _pathJoin } from "node:path";
+import { tmpdir as _osTmpdir } from "node:os";
 import { homedir as _osHomedir } from "node:os";
 import { z as z15 } from "zod";
 function repairJson(raw) {
@@ -531233,19 +531243,22 @@ TASK: ${task}` : task;
           web_fetch: 4,
           list_directory: 12,
           find_files: 10,
-          grep_search: 12
+          grep_search: 12,
+          camera_capture: 3
         } : loopTier === "medium" ? {
           web_search: 10,
           web_fetch: 8,
           list_directory: 18,
           find_files: 14,
-          grep_search: 18
+          grep_search: 18,
+          camera_capture: 4
         } : {
           web_search: 20,
           web_fetch: 15,
           list_directory: 30,
           find_files: 20,
-          grep_search: 30
+          grep_search: 30,
+          camera_capture: 5
         };
         for (const [tool, budget] of Object.entries(toolBudgets)) {
           toolCallBudget.set(tool, budget);
@@ -532289,45 +532302,7 @@ ${_staleSamples.join("\n")}` : ``,
           }
           while (this.pendingUserMessages.length > 0) {
             const userMsg = this.pendingUserMessages.shift();
-            const imagePattern = /\[IMAGE_BASE64:([^:]+):([^\]]+)\]/;
-            const imgMatch = userMsg.match(imagePattern);
-            if (imgMatch) {
-              const mime = imgMatch[1];
-              const base642 = imgMatch[2];
-              const textContent = userMsg.replace(imagePattern, "").trim();
-              const parts = [];
-              if (textContent) {
-                parts.push({
-                  type: "text",
-                  text: `[User added context]: ${textContent}
-
-Describe what you see and integrate this into your current approach.`
-                });
-              } else {
-                parts.push({
-                  type: "text",
-                  text: "[User shared an image]. Describe what you see and integrate this into your current approach."
-                });
-              }
-              parts.push({
-                type: "image_url",
-                image_url: { url: `data:${mime};base64,${base642}` }
-              });
-              messages2.push({ role: "user", content: parts });
-            } else {
-              messages2.push({
-                role: "user",
-                content: `[User added context]: ${userMsg}
-
-Integrate this guidance into your current approach. Continue working on the task.`
-              });
-            }
-            this.emit({
-              type: "user_interrupt",
-              content: userMsg.replace(/\[IMAGE_BASE64:[^\]]+\]/, "[image]").slice(0, 200),
-              turn,
-              timestamp: (/* @__PURE__ */ new Date()).toISOString()
-            });
+            await this.appendInjectedUserMessage(userMsg, messages2, turn);
           }
           {
             const maybeReminder = this.getTodoReminderContent(turn);
@@ -532648,12 +532623,28 @@ ${memoryLines.join("\n")}`
             timeoutMs: this.options.requestTimeoutMs
           };
           {
+            const _imgB64Pat = /\[IMAGE_BASE64:[^\]]+\]/g;
             const ctxChars = compacted.reduce((s2, m2) => {
-              let c9 = typeof m2.content === "string" ? m2.content.length : 100;
+              let c9 = 0;
+              let imgCount = 0;
+              if (typeof m2.content === "string") {
+                const imgMatches = m2.content.match(_imgB64Pat);
+                imgCount = imgMatches ? imgMatches.length : 0;
+                c9 = m2.content.replace(_imgB64Pat, "").length;
+              } else if (Array.isArray(m2.content)) {
+                for (const p2 of m2.content) {
+                  if (p2.type === "text" && p2.text)
+                    c9 += p2.text.length;
+                  else if (p2.type === "image_url")
+                    imgCount++;
+                }
+              } else {
+                c9 = 100;
+              }
               if (m2.tool_calls)
                 for (const tc of m2.tool_calls)
                   c9 += tc.function.arguments?.length ?? 0;
-              return s2 + c9;
+              return s2 + c9 + imgCount * 1500 * 4;
             }, 0);
             const estTokens = Math.ceil(ctxChars / 4);
             const limits = this.contextLimits();
@@ -532867,13 +532858,31 @@ ${memoryLines.join("\n")}`
           const choiceContent = response.choices[0]?.message?.content ?? "";
           const choiceArgs = response.choices[0]?.message?.toolCalls?.map((tc) => JSON.stringify(tc.arguments)).join("") ?? "";
           estimatedTokens += Math.ceil((choiceContent.length + choiceArgs.length) / 4);
+          const IMAGE_TOKEN_ESTIMATE = 1500;
+          const imageBase64Pattern = /\[IMAGE_BASE64:[^\]]+\]/g;
           const estimatedContextTokens = Math.ceil(compacted.reduce((sum, m2) => {
-            let chars = typeof m2.content === "string" ? m2.content.length : 100;
+            let chars = 0;
+            let imageCount = 0;
+            if (typeof m2.content === "string") {
+              const imageMatches = m2.content.match(imageBase64Pattern);
+              imageCount = imageMatches ? imageMatches.length : 0;
+              chars = m2.content.replace(imageBase64Pattern, "").length;
+            } else if (Array.isArray(m2.content)) {
+              for (const part of m2.content) {
+                if (part.type === "text" && part.text) {
+                  chars += part.text.length;
+                } else if (part.type === "image_url") {
+                  imageCount++;
+                }
+              }
+            } else {
+              chars = 100;
+            }
             if (m2.tool_calls) {
               for (const tc of m2.tool_calls)
                 chars += tc.function.arguments?.length ?? 0;
             }
-            return sum + chars;
+            return sum + chars + imageCount * IMAGE_TOKEN_ESTIMATE * 4;
           }, 0) / 4);
           this.emit({
             type: "token_usage",
@@ -534466,6 +534475,9 @@ Respond with EXACTLY this structure before your next tool call:
                   };
                 }
               }
+              if (result.success) {
+                result = await this.offloadEmbeddedImageResult(result, tc.name, turn);
+              }
               let output = this.normalizeToolOutput(result, tc.name, tc.arguments, turn);
               if (!result.success && (this.options.modelTier === "small" || this.options.modelTier === "medium")) {
                 const recovery = this.buildRecoveryGuidance(tc.name, result.error ?? "", tc.arguments);
@@ -535327,45 +535339,7 @@ You have ${this.options.maxTurns} more turns. Continue making progress. Call tas
             }
             while (this.pendingUserMessages.length > 0) {
               const userMsg = this.pendingUserMessages.shift();
-              const imagePattern = /\[IMAGE_BASE64:([^:]+):([^\]]+)\]/;
-              const imgMatch = userMsg.match(imagePattern);
-              if (imgMatch) {
-                const mime = imgMatch[1];
-                const base642 = imgMatch[2];
-                const textContent = userMsg.replace(imagePattern, "").trim();
-                const parts = [];
-                if (textContent) {
-                  parts.push({
-                    type: "text",
-                    text: `[User added context]: ${textContent}
-
-Describe what you see and integrate this into your current approach.`
-                  });
-                } else {
-                  parts.push({
-                    type: "text",
-                    text: "[User shared an image]. Describe what you see and integrate this into your current approach."
-                  });
-                }
-                parts.push({
-                  type: "image_url",
-                  image_url: { url: `data:${mime};base64,${base642}` }
-                });
-                messages2.push({ role: "user", content: parts });
-              } else {
-                messages2.push({
-                  role: "user",
-                  content: `[User added context]: ${userMsg}
-
-Integrate this guidance into your current approach. Continue working on the task.`
-                });
-              }
-              this.emit({
-                type: "user_interrupt",
-                content: userMsg.replace(/\[IMAGE_BASE64:[^\]]+\]/, "[image]").slice(0, 200),
-                turn,
-                timestamp: (/* @__PURE__ */ new Date()).toISOString()
-              });
+              await this.appendInjectedUserMessage(userMsg, messages2, turn);
             }
             let compactedMsgs;
             if (this._pendingCompaction) {
@@ -535440,13 +535414,29 @@ Integrate this guidance into your current approach. Continue working on the task
             const choiceContent2 = response.choices[0]?.message?.content ?? "";
             const choiceArgs2 = response.choices[0]?.message?.toolCalls?.map((tc) => JSON.stringify(tc.arguments)).join("") ?? "";
             estimatedTokens += Math.ceil((choiceContent2.length + choiceArgs2.length) / 4);
+            const _bfImgPat = /\[IMAGE_BASE64:[^\]]+\]/g;
             const bfEstCtx = Math.ceil(compactedMsgs.reduce((sum, m2) => {
-              let chars = typeof m2.content === "string" ? m2.content.length : 100;
+              let chars = 0;
+              let imgCount = 0;
+              if (typeof m2.content === "string") {
+                const imgMatches = m2.content.match(_bfImgPat);
+                imgCount = imgMatches ? imgMatches.length : 0;
+                chars = m2.content.replace(_bfImgPat, "").length;
+              } else if (Array.isArray(m2.content)) {
+                for (const p2 of m2.content) {
+                  if (p2.type === "text" && p2.text)
+                    chars += p2.text.length;
+                  else if (p2.type === "image_url")
+                    imgCount++;
+                }
+              } else {
+                chars = 100;
+              }
               if (m2.tool_calls) {
                 for (const tc of m2.tool_calls)
                   chars += tc.function.arguments?.length ?? 0;
               }
-              return sum + chars;
+              return sum + chars + imgCount * 1500 * 4;
             }, 0) / 4);
             this.emit({
               type: "token_usage",
@@ -536874,18 +536864,241 @@ ${tail}`;
         }
         return folded;
       }
+      async appendInjectedUserMessage(userMsg, messages2, turn) {
+        const imagePattern = /\[IMAGE_BASE64:([^:]+):([^\]]+)\]/;
+        const imgMatch = userMsg.match(imagePattern);
+        if (imgMatch) {
+          const mime = imgMatch[1];
+          const base642 = imgMatch[2];
+          const textContent = userMsg.replace(imagePattern, "").trim();
+          await this.appendOffloadedImageMessage(messages2, mime, base642, textContent, turn);
+        } else {
+          messages2.push({
+            role: "user",
+            content: `[User added context]: ${userMsg}
+
+Integrate this guidance into your current approach. Continue working on the task.`
+          });
+        }
+        this.emit({
+          type: "user_interrupt",
+          content: userMsg.replace(/\[IMAGE_BASE64:[^\]]+\]/, "[image]").slice(0, 200),
+          turn,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        });
+      }
+      async appendOffloadedImageMessage(messages2, mime, base642, textContent, turn) {
+        const imageUrl = `data:${mime};base64,${base642}`;
+        this.emit({
+          type: "status",
+          content: "Image received; offloading visual analysis outside main context",
+          turn,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        });
+        const analysis = await this.analyzeImageDataForContext(mime, base642, textContent);
+        if (analysis.contextBlock) {
+          const userPrefix = textContent ? `[User added context]: ${textContent}
+
+` : "[User shared an image]. ";
+          messages2.push({
+            role: "user",
+            content: userPrefix + analysis.contextBlock + "\n\nIntegrate this visual information into your current approach."
+          });
+          this.emit({
+            type: "status",
+            content: "Image analysis added as text; base64 excluded from main context",
+            turn,
+            timestamp: (/* @__PURE__ */ new Date()).toISOString()
+          });
+          return;
+        }
+        const reason = analysis.errorReason || "vision and OCR returned no text";
+        this.emit({
+          type: "status",
+          content: `Image offload unavailable (${reason}); falling back to inline image`,
+          turn,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        });
+        this.appendInlineImageMessage(messages2, imageUrl, textContent);
+      }
+      async offloadEmbeddedImageResult(result, toolName, turn) {
+        const modelSource = result.llmContent ?? result.output;
+        const image = this.extractFirstEmbeddedImage(modelSource);
+        if (!image)
+          return result;
+        this.emit({
+          type: "status",
+          content: `${toolName}: offloading embedded image analysis outside main context`,
+          turn,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        });
+        const analysis = await this.analyzeImageDataForContext(image.mime, image.base64, image.textWithoutImage.slice(0, 2e3));
+        const imageNote = analysis.contextBlock ? `${analysis.contextBlock}
+
+Use this image analysis. Do not repeat ${toolName} with the same arguments unless the scene has changed.` : `[Embedded image data omitted from model context; ${analysis.errorReason || "vision and OCR returned no text"}. Use any saved image path above with vision/image_read if further inspection is needed.]`;
+        return {
+          ...result,
+          llmContent: `${image.textWithoutImage.trim()}
+
+${imageNote}`.trim()
+        };
+      }
+      extractFirstEmbeddedImage(text) {
+        const markerPattern = /\[IMAGE_BASE64:([^:\]]+):([^\]]+)\]/;
+        const markerMatch = text.match(markerPattern);
+        if (markerMatch) {
+          const mime2 = markerMatch[1];
+          const base643 = markerMatch[2];
+          return {
+            mime: mime2,
+            base64: base643,
+            textWithoutImage: text.replace(markerPattern, `[image data omitted: ${mime2}, ${base643.length} base64 chars]`).trim()
+          };
+        }
+        const dataUrlPattern = /data:(image\/[a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=]+)/;
+        const dataUrlMatch = text.match(dataUrlPattern);
+        if (!dataUrlMatch)
+          return null;
+        const mime = dataUrlMatch[1];
+        const base642 = dataUrlMatch[2];
+        return {
+          mime,
+          base64: base642,
+          textWithoutImage: text.replace(dataUrlPattern, `[image data omitted: ${mime}, ${base642.length} base64 chars]`).trim()
+        };
+      }
+      async analyzeImageDataForContext(mime, base642, textContent) {
+        const imageUrl = `data:${mime};base64,${base642}`;
+        const tmpImgPath = this.writeTempImageForOcr(mime, base642);
+        const [visionOutcome, ocrOutcome] = await Promise.allSettled([
+          this.describeImageViaVisionSubagent(imageUrl, textContent),
+          tmpImgPath ? this.extractImageOcrText(tmpImgPath) : Promise.resolve("")
+        ]);
+        const visionDesc = visionOutcome.status === "fulfilled" ? visionOutcome.value.trim() : "";
+        const ocrText = ocrOutcome.status === "fulfilled" ? ocrOutcome.value.trim() : "";
+        const sections = [];
+        if (visionDesc)
+          sections.push(`[Image analysis]: ${visionDesc}`);
+        if (ocrText)
+          sections.push(`[OCR extracted text]: ${ocrText}`);
+        if (sections.length > 0)
+          return { contextBlock: sections.join("\n\n") };
+        const errorReason = visionOutcome.status === "rejected" ? String(visionOutcome.reason?.message ?? visionOutcome.reason) : void 0;
+        return { contextBlock: "", errorReason };
+      }
+      async describeImageViaVisionSubagent(imageUrl, textContent) {
+        const visionMessages = [
+          {
+            role: "system",
+            content: "You are a visual analysis sub-agent. Describe the image in detail, including visible text, UI elements, code, diagrams, errors, and other task-relevant visual details. Be thorough but concise."
+          },
+          {
+            role: "user",
+            content: [
+              {
+                type: "text",
+                text: textContent ? `Context from user: ${textContent}
+
+Describe what you see in this image.` : "Describe what you see in this image in detail."
+              },
+              { type: "image_url", image_url: { url: imageUrl } }
+            ]
+          }
+        ];
+        const result = await this.backend.chatCompletion({
+          messages: visionMessages,
+          tools: [],
+          temperature: 0.3,
+          maxTokens: 2048,
+          timeoutMs: 3e4,
+          think: false
+        });
+        return result.choices[0]?.message?.content ?? "";
+      }
+      appendInlineImageMessage(messages2, imageUrl, textContent) {
+        const parts = [
+          {
+            type: "text",
+            text: textContent ? `[User added context]: ${textContent}
+
+Describe what you see and integrate this into your current approach.` : "[User shared an image]. Describe what you see and integrate this into your current approach."
+          },
+          { type: "image_url", image_url: { url: imageUrl } }
+        ];
+        messages2.push({ role: "user", content: parts });
+      }
+      writeTempImageForOcr(mime, base642) {
+        try {
+          const ext = this.imageExtensionForMime(mime);
+          const id = _createHash("sha256").update(`${process.pid}:${Date.now()}:`).update(base642.slice(0, 4096)).digest("hex").slice(0, 16);
+          const tmpImgPath = _pathJoin(_osTmpdir(), `oa-img-${id}.${ext}`);
+          _fsWriteFileSync(tmpImgPath, Buffer.from(base642, "base64"));
+          return tmpImgPath;
+        } catch {
+          return null;
+        }
+      }
+      imageExtensionForMime(mime) {
+        const normalized = mime.toLowerCase();
+        if (normalized.includes("jpeg") || normalized.includes("jpg"))
+          return "jpg";
+        if (normalized.includes("webp"))
+          return "webp";
+        if (normalized.includes("gif"))
+          return "gif";
+        if (normalized.includes("bmp"))
+          return "bmp";
+        if (normalized.includes("tiff"))
+          return "tif";
+        return "png";
+      }
+      async extractImageOcrText(tmpImgPath) {
+        try {
+          const stdout = await new Promise((resolve44, reject) => {
+            _execFile("tesseract", [tmpImgPath, "stdout"], {
+              encoding: "utf8",
+              timeout: 15e3,
+              maxBuffer: 2 * 1024 * 1024
+            }, (err, out) => {
+              if (err) {
+                reject(err);
+                return;
+              }
+              resolve44(out);
+            });
+          });
+          return stdout.trim();
+        } catch {
+          return "";
+        } finally {
+          try {
+            _fsUnlinkSync(tmpImgPath);
+          } catch {
+          }
+        }
+      }
       // -------------------------------------------------------------------------
       // Context compaction
       // -------------------------------------------------------------------------
       async compactMessages(messages2, strategy = "default", force = false) {
         if (messages2.length < 3)
           return messages2;
+        const _compImgPat = /\[IMAGE_BASE64:[^\]]+\]/g;
         const totalChars = messages2.reduce((sum, m2) => {
           let chars = 0;
-          if (typeof m2.content === "string")
-            chars += m2.content.length;
-          else if (Array.isArray(m2.content)) {
-            chars += m2.content.reduce((s2, p2) => s2 + (p2.text?.length || 0) + (p2.image_url ? 1e3 : 0), 0);
+          if (typeof m2.content === "string") {
+            const imgMatches = m2.content.match(_compImgPat);
+            const imgCount = imgMatches ? imgMatches.length : 0;
+            chars += m2.content.replace(_compImgPat, "").length + imgCount * 1500 * 4;
+          } else if (Array.isArray(m2.content)) {
+            chars += m2.content.reduce((s2, p2) => {
+              if (p2.type === "text" && p2.text)
+                return s2 + p2.text.length;
+              if (p2.type === "image_url" && p2.image_url?.url) {
+                return s2 + 1500 * 4;
+              }
+              return s2;
+            }, 0);
           }
           if (m2.tool_calls) {
             for (const tc of m2.tool_calls) {
@@ -536923,7 +537136,22 @@ ${tail}`;
           let budgetCut = messages2.length;
           for (let i2 = messages2.length - 1; i2 >= headEndIdx; i2--) {
             const msg = messages2[i2];
-            const msgChars = typeof msg.content === "string" ? msg.content.length : 100;
+            let msgChars = 0;
+            if (typeof msg.content === "string") {
+              const _mImgPat = /\[IMAGE_BASE64:[^\]]+\]/g;
+              const imgMatches = msg.content.match(_mImgPat);
+              const imgCount = imgMatches ? imgMatches.length : 0;
+              msgChars = msg.content.replace(_mImgPat, "").length + imgCount * 1500 * 4;
+            } else if (Array.isArray(msg.content)) {
+              for (const p2 of msg.content) {
+                if (p2.type === "text" && p2.text)
+                  msgChars += p2.text.length;
+                else if (p2.type === "image_url")
+                  msgChars += 1500 * 4;
+              }
+            } else {
+              msgChars = 100;
+            }
             const toolCallChars = (msg.tool_calls || []).reduce((s2, tc) => s2 + (tc.function?.arguments?.length || 0) + (tc.function?.name?.length || 0), 0);
             const msgTokens = Math.ceil((msgChars + toolCallChars) / 4) + 10;
             if (accumulated + msgTokens > tailTokenBudget && messages2.length - i2 >= 4) {
@@ -537032,7 +537260,25 @@ ${tail}`;
         const strategyLabel = strategy !== "default" ? ` (${strategy})` : "";
         const forceLabel = force ? " [manual]" : "";
         const preTokens = Math.ceil(totalChars / 4);
-        const postChars = combinedSummary.length + recent.reduce((s2, m2) => s2 + (typeof m2.content === "string" ? m2.content.length : 100), 0) + head.reduce((s2, m2) => s2 + (typeof m2.content === "string" ? m2.content.length : 100), 0);
+        const _postImgPat = /\[IMAGE_BASE64:[^\]]+\]/g;
+        const _estimateMsgChars = (m2) => {
+          if (typeof m2.content === "string") {
+            const imgMatches = m2.content.match(_postImgPat);
+            const imgCount = imgMatches ? imgMatches.length : 0;
+            return m2.content.replace(_postImgPat, "").length + imgCount * 1500 * 4;
+          } else if (Array.isArray(m2.content)) {
+            let c9 = 0;
+            for (const p2 of m2.content) {
+              if (p2.type === "text" && p2.text)
+                c9 += p2.text.length;
+              else if (p2.type === "image_url")
+                c9 += 1500 * 4;
+            }
+            return c9;
+          }
+          return 100;
+        };
+        const postChars = combinedSummary.length + recent.reduce((s2, m2) => s2 + _estimateMsgChars(m2), 0) + head.reduce((s2, m2) => s2 + _estimateMsgChars(m2), 0);
         const postTokens = Math.ceil(postChars / 4);
         const savedTokens = preTokens - postTokens;
         this.emit({
@@ -537220,13 +537466,29 @@ ${content.slice(0, 8e3)}
         }
         const ctxWindow = this.options.contextWindowSize;
         if (ctxWindow > 0) {
+          const _safetyImgPat = /\[IMAGE_BASE64:[^\]]+\]/g;
           const estimateResult = (msgs) => msgs.reduce((sum, m2) => {
-            let chars = typeof m2.content === "string" ? m2.content.length : 100;
+            let chars = 0;
+            let imgCount = 0;
+            if (typeof m2.content === "string") {
+              const imgMatches = m2.content.match(_safetyImgPat);
+              imgCount = imgMatches ? imgMatches.length : 0;
+              chars = m2.content.replace(_safetyImgPat, "").length;
+            } else if (Array.isArray(m2.content)) {
+              for (const p2 of m2.content) {
+                if (p2.type === "text" && p2.text)
+                  chars += p2.text.length;
+                else if (p2.type === "image_url")
+                  imgCount++;
+              }
+            } else {
+              chars = 100;
+            }
             if (m2.tool_calls) {
               for (const tc of m2.tool_calls)
                 chars += tc.function.arguments?.length ?? 0;
             }
-            return sum + chars;
+            return sum + chars + imgCount * 1500 * 4;
           }, 0) / 4;
           const safetyTarget = Math.floor(ctxWindow * 0.65);
           let trimmedRecent = [...filteredRecent];
@@ -538340,7 +538602,49 @@ ${transcript}`
           "pascal_case",
           "screaming_snake_case",
           "dot_notation",
-          "title_case"
+          "title_case",
+          // Conditionally-registered tools documented in the base system prompt
+          // as available but only registered in TUI/desktop mode, NOT telegram:
+          "background_run",
+          // TUI-only: background shell execution
+          "task_status",
+          // TUI-only: check background task status
+          "task_output",
+          // TUI-only: read background task output
+          "task_stop",
+          // TUI-only: kill background task
+          "skill_list",
+          // TUI-only: discover available skills
+          "skill_execute",
+          // TUI-only: load and run a skill
+          "skill_build",
+          // TUI-only: generate a new skill
+          "desktop_click",
+          // TUI-only: click UI element by description
+          "desktop_describe",
+          // TUI-only: screenshot + describe desktop
+          "repl_exec",
+          // TUI-only: persistent Python REPL
+          "cron_agent",
+          // TUI-only: scheduled agent tasks
+          "scheduler",
+          // TUI-only: OS-level cron scheduling
+          "reminder",
+          // TUI-only: cross-session reminders
+          "agenda",
+          // TUI-only: attention directives
+          "priority_classify",
+          // TUI-only: task priority classification
+          "priority_delegate",
+          // TUI-only: delegate to sub-agent by priority
+          "create_tool",
+          // TUI-only: create custom tool
+          "manage_tools",
+          // TUI-only: list/inspect/delete custom tools
+          "sub_agent",
+          // TUI-only: delegate sub-task (telegram uses full_sub_agent)
+          "nexus"
+          // TUI-only: P2P networking
         ]);
         for (const tool of this.tools.values()) {
           const props = tool.parameters?.properties;
@@ -591700,6 +592004,36 @@ function normalizeTelegramMedia(message2) {
   }
   return void 0;
 }
+function telegramMediaIsImage(media) {
+  if (media.type === "photo") return true;
+  if (media.mimeType?.toLowerCase().startsWith("image/")) return true;
+  return /\.(png|jpe?g|gif|webp|bmp|tiff?)$/i.test(media.fileName ?? "");
+}
+function telegramImageExtension(media) {
+  const fileName = media.fileName ?? "";
+  const dotIdx = fileName.lastIndexOf(".");
+  if (dotIdx >= 0) {
+    const ext = fileName.slice(dotIdx).toLowerCase();
+    if (/^\.(png|jpe?g|gif|webp|bmp|tiff?)$/.test(ext)) return ext;
+  }
+  const mime = media.mimeType?.toLowerCase() ?? "";
+  if (mime.includes("png")) return ".png";
+  if (mime.includes("webp")) return ".webp";
+  if (mime.includes("gif")) return ".gif";
+  if (mime.includes("bmp")) return ".bmp";
+  if (mime.includes("tiff")) return ".tif";
+  return ".jpg";
+}
+function telegramImageMime(media) {
+  if (media.mimeType?.toLowerCase().startsWith("image/")) return media.mimeType;
+  const ext = telegramImageExtension(media);
+  if (ext === ".png") return "image/png";
+  if (ext === ".webp") return "image/webp";
+  if (ext === ".gif") return "image/gif";
+  if (ext === ".bmp") return "image/bmp";
+  if (ext === ".tif" || ext === ".tiff") return "image/tiff";
+  return "image/jpeg";
+}
 function normalizeTelegramUpdate(update2) {
   const sourceUpdateType = update2.guest_message ? "guest_message" : update2.message ? "message" : null;
   if (!sourceUpdateType) return null;
@@ -592558,12 +592892,22 @@ Join: ${newUrl}`);
         }
         const existing = this.subAgents.get(sessionKey);
         if (existing && !existing.aborted) {
-          this.recordChatHistory(sessionKey, { role: "user", text: msg.text, mode: "steering" });
+          let steeringText = msg.text;
+          if (msg.media) {
+            const mediaContext = await this.processMedia(msg);
+            if (mediaContext) {
+              steeringText += `
+
+[Media attached — processed content below]
+${mediaContext}`;
+            }
+          }
+          this.recordChatHistory(sessionKey, { role: "user", text: steeringText, mode: "steering" });
           if (existing.runner) {
-            existing.runner.injectUserMessage(msg.text);
+            existing.runner.injectUserMessage(steeringText);
             this.tuiWrite(() => renderTelegramSubAgentEvent(msg.username, "mid-conversation steering injected"));
           } else {
-            existing.pendingMessages.push(msg.text);
+            existing.pendingMessages.push(steeringText);
             this.tuiWrite(() => renderTelegramSubAgentEvent(msg.username, `queued (${existing.pendingMessages.length} pending)`));
           }
           return;
@@ -593258,8 +593602,9 @@ Todo/session id: ${sessionContext.sessionId}` : `Telegram ${isGroup ? "group" :
       async processMedia(msg) {
         if (!msg.media) return "";
         const { type, fileId, fileUniqueId, mimeType, caption } = msg.media;
+        const isImageMedia = telegramMediaIsImage(msg.media);
         let ext = ".bin";
-        if (type === "photo") ext = ".jpg";
+        if (isImageMedia) ext = telegramImageExtension(msg.media);
         else if (type === "audio" || type === "voice") ext = ".ogg";
         else if (type === "video" || type === "video_note" || type === "live_photo") ext = ".mp4";
         else if (msg.media.fileName) {
@@ -593290,23 +593635,27 @@ Todo/session id: ${sessionContext.sessionId}` : `Telegram ${isGroup ? "group" :
           username: msg.username
         });
         let description = `[${type}${caption ? `: ${caption}` : ""}]`;
-        if (type === "photo") {
+        if (isImageMedia) {
           let visionContext = "";
           try {
             const { runVisionIngress: runVisionIngress2, formatImageContextPrefix: formatImageContextPrefix2 } = await Promise.resolve().then(() => (init_vision_ingress(), vision_ingress_exports));
             const ingressResult = await runVisionIngress2(
-              { path: localPath, buffer: Buffer.from(""), mime: "image/png" },
-              ""
+              {
+                path: localPath,
+                buffer: readFileSync84(localPath),
+                mime: telegramImageMime(msg.media)
+              },
+              this.agentConfig?.model ?? ""
             );
             visionContext = formatImageContextPrefix2(ingressResult);
             cacheEntry.extractedContent = ingressResult.contextBlock;
           } catch {
           }
           if (visionContext) {
-            description = `[Photo received: ${localPath}${caption ? ` — caption: "${caption}"` : ""}
+            description = `[Image received: ${localPath}${caption ? ` — caption: "${caption}"` : ""}
 ${visionContext}]`;
           } else {
-            description = `[Photo received and saved to ${localPath}${caption ? ` — caption: "${caption}"` : ""}. You can use image_read or vision tools to analyze it if available.]`;
+            description = `[Image received and saved to ${localPath}${caption ? ` — caption: "${caption}"` : ""}. You can use image_read or vision tools to analyze it if available.]`;
           }
           try {
             await fetch("http://127.0.0.1:11435/v1/memory/ingest", {
@@ -593317,8 +593666,7 @@ ${visionContext}]`;
             });
           } catch {
           }
-        }
-        if (type === "audio" || type === "voice") {
+        } else if (type === "audio" || type === "voice") {
           let transcription = null;
           try {
             const { getListenEngine: getListenEngine2 } = await Promise.resolve().then(() => (init_listen(), listen_exports));
@@ -593344,12 +593692,10 @@ ${visionContext}]`;
             });
           } catch {
           }
-        }
-        if (type === "video" || type === "video_note" || type === "live_photo") {
+        } else if (type === "video" || type === "video_note" || type === "live_photo") {
           const label = type === "live_photo" ? "Live photo" : "Video";
           description = `[${label} received and saved to ${localPath}${caption ? ` — caption: "${caption}"` : ""}.]`;
-        }
-        if (type === "document") {
+        } else if (type === "document") {
           description = `[Document received: ${msg.media.fileName || "unnamed"}${mimeType ? ` (${mimeType})` : ""}, saved to ${localPath}${caption ? ` — caption: "${caption}"` : ""}.]`;
         }
         cacheEntry.extractedContent = description;

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "open-agents-ai",
-  "version": "0.187.592",
+  "version": "0.187.594",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "open-agents-ai",
-      "version": "0.187.592",
+      "version": "0.187.594",
       "hasInstallScript": true,
       "license": "CC-BY-NC-4.0",
       "dependencies": {
@@ -2049,9 +2049,9 @@
       }
     },
     "node_modules/aiwg": {
-      "version": "2026.5.4",
-      "resolved": "https://registry.npmjs.org/aiwg/-/aiwg-2026.5.4.tgz",
-      "integrity": "sha512-/10XfF6pD+7/I945vx1uhh37+N4NIp1NscGJUEAJAMwVVrHXdqZ4UotCfLBp6dnwOI3tI5jfg3zWJkr1yhUPOw==",
+      "version": "2026.5.5",
+      "resolved": "https://registry.npmjs.org/aiwg/-/aiwg-2026.5.5.tgz",
+      "integrity": "sha512-bRMCp3qOAgZycb1Pyahmx3CgrMVtKU5gQhYpyY23r6B/LhNOP/+t0SHRuvPN4bOVOVsec/qKpma9OvQnlqrp0w==",
       "license": "MIT",
       "dependencies": {
         "@modelcontextprotocol/sdk": "^1.24.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.187.592",
+  "version": "0.187.594",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",