open-agents-ai 0.187.592 → 0.187.593

package/dist/index.js

@@ -527374,9 +527374,11 @@ RECOVERY: cd to the directory containing '${file}', run a plain install with no
 });
 
 // packages/orchestrator/dist/agenticRunner.js
-import { existsSync as _fsExistsSync, readFileSync as _fsReadFileSync, writeFileSync as _fsWriteFileSync, mkdirSync as _fsMkdirSync } from "node:fs";
+import { existsSync as _fsExistsSync, readFileSync as _fsReadFileSync, writeFileSync as _fsWriteFileSync, unlinkSync as _fsUnlinkSync, mkdirSync as _fsMkdirSync } from "node:fs";
+import { execFile as _execFile } from "node:child_process";
 import { createHash as _createHash } from "node:crypto";
 import { join as _pathJoin } from "node:path";
+import { tmpdir as _osTmpdir } from "node:os";
 import { homedir as _osHomedir } from "node:os";
 import { z as z15 } from "zod";
 function repairJson(raw) {
@@ -532289,45 +532291,7 @@ ${_staleSamples.join("\n")}` : ``,
           }
           while (this.pendingUserMessages.length > 0) {
             const userMsg = this.pendingUserMessages.shift();
-            const imagePattern = /\[IMAGE_BASE64:([^:]+):([^\]]+)\]/;
-            const imgMatch = userMsg.match(imagePattern);
-            if (imgMatch) {
-              const mime = imgMatch[1];
-              const base642 = imgMatch[2];
-              const textContent = userMsg.replace(imagePattern, "").trim();
-              const parts = [];
-              if (textContent) {
-                parts.push({
-                  type: "text",
-                  text: `[User added context]: ${textContent}
-
-Describe what you see and integrate this into your current approach.`
-                });
-              } else {
-                parts.push({
-                  type: "text",
-                  text: "[User shared an image]. Describe what you see and integrate this into your current approach."
-                });
-              }
-              parts.push({
-                type: "image_url",
-                image_url: { url: `data:${mime};base64,${base642}` }
-              });
-              messages2.push({ role: "user", content: parts });
-            } else {
-              messages2.push({
-                role: "user",
-                content: `[User added context]: ${userMsg}
-
-Integrate this guidance into your current approach. Continue working on the task.`
-              });
-            }
-            this.emit({
-              type: "user_interrupt",
-              content: userMsg.replace(/\[IMAGE_BASE64:[^\]]+\]/, "[image]").slice(0, 200),
-              turn,
-              timestamp: (/* @__PURE__ */ new Date()).toISOString()
-            });
+            await this.appendInjectedUserMessage(userMsg, messages2, turn);
           }
           {
             const maybeReminder = this.getTodoReminderContent(turn);
@@ -532648,12 +532612,28 @@ ${memoryLines.join("\n")}`
             timeoutMs: this.options.requestTimeoutMs
           };
           {
+            const _imgB64Pat = /\[IMAGE_BASE64:[^\]]+\]/g;
             const ctxChars = compacted.reduce((s2, m2) => {
-              let c9 = typeof m2.content === "string" ? m2.content.length : 100;
+              let c9 = 0;
+              let imgCount = 0;
+              if (typeof m2.content === "string") {
+                const imgMatches = m2.content.match(_imgB64Pat);
+                imgCount = imgMatches ? imgMatches.length : 0;
+                c9 = m2.content.replace(_imgB64Pat, "").length;
+              } else if (Array.isArray(m2.content)) {
+                for (const p2 of m2.content) {
+                  if (p2.type === "text" && p2.text)
+                    c9 += p2.text.length;
+                  else if (p2.type === "image_url")
+                    imgCount++;
+                }
+              } else {
+                c9 = 100;
+              }
               if (m2.tool_calls)
                 for (const tc of m2.tool_calls)
                   c9 += tc.function.arguments?.length ?? 0;
-              return s2 + c9;
+              return s2 + c9 + imgCount * 1500 * 4;
             }, 0);
             const estTokens = Math.ceil(ctxChars / 4);
             const limits = this.contextLimits();
@@ -532867,13 +532847,31 @@ ${memoryLines.join("\n")}`
           const choiceContent = response.choices[0]?.message?.content ?? "";
           const choiceArgs = response.choices[0]?.message?.toolCalls?.map((tc) => JSON.stringify(tc.arguments)).join("") ?? "";
           estimatedTokens += Math.ceil((choiceContent.length + choiceArgs.length) / 4);
+          const IMAGE_TOKEN_ESTIMATE = 1500;
+          const imageBase64Pattern = /\[IMAGE_BASE64:[^\]]+\]/g;
           const estimatedContextTokens = Math.ceil(compacted.reduce((sum, m2) => {
-            let chars = typeof m2.content === "string" ? m2.content.length : 100;
+            let chars = 0;
+            let imageCount = 0;
+            if (typeof m2.content === "string") {
+              const imageMatches = m2.content.match(imageBase64Pattern);
+              imageCount = imageMatches ? imageMatches.length : 0;
+              chars = m2.content.replace(imageBase64Pattern, "").length;
+            } else if (Array.isArray(m2.content)) {
+              for (const part of m2.content) {
+                if (part.type === "text" && part.text) {
+                  chars += part.text.length;
+                } else if (part.type === "image_url") {
+                  imageCount++;
+                }
+              }
+            } else {
+              chars = 100;
+            }
             if (m2.tool_calls) {
               for (const tc of m2.tool_calls)
                 chars += tc.function.arguments?.length ?? 0;
             }
-            return sum + chars;
+            return sum + chars + imageCount * IMAGE_TOKEN_ESTIMATE * 4;
           }, 0) / 4);
           this.emit({
             type: "token_usage",
@@ -535327,45 +535325,7 @@ You have ${this.options.maxTurns} more turns. Continue making progress. Call tas
             }
             while (this.pendingUserMessages.length > 0) {
               const userMsg = this.pendingUserMessages.shift();
-              const imagePattern = /\[IMAGE_BASE64:([^:]+):([^\]]+)\]/;
-              const imgMatch = userMsg.match(imagePattern);
-              if (imgMatch) {
-                const mime = imgMatch[1];
-                const base642 = imgMatch[2];
-                const textContent = userMsg.replace(imagePattern, "").trim();
-                const parts = [];
-                if (textContent) {
-                  parts.push({
-                    type: "text",
-                    text: `[User added context]: ${textContent}
-
-Describe what you see and integrate this into your current approach.`
-                  });
-                } else {
-                  parts.push({
-                    type: "text",
-                    text: "[User shared an image]. Describe what you see and integrate this into your current approach."
-                  });
-                }
-                parts.push({
-                  type: "image_url",
-                  image_url: { url: `data:${mime};base64,${base642}` }
-                });
-                messages2.push({ role: "user", content: parts });
-              } else {
-                messages2.push({
-                  role: "user",
-                  content: `[User added context]: ${userMsg}
-
-Integrate this guidance into your current approach. Continue working on the task.`
-                });
-              }
-              this.emit({
-                type: "user_interrupt",
-                content: userMsg.replace(/\[IMAGE_BASE64:[^\]]+\]/, "[image]").slice(0, 200),
-                turn,
-                timestamp: (/* @__PURE__ */ new Date()).toISOString()
-              });
+              await this.appendInjectedUserMessage(userMsg, messages2, turn);
             }
             let compactedMsgs;
             if (this._pendingCompaction) {
@@ -535440,13 +535400,29 @@ Integrate this guidance into your current approach. Continue working on the task
             const choiceContent2 = response.choices[0]?.message?.content ?? "";
             const choiceArgs2 = response.choices[0]?.message?.toolCalls?.map((tc) => JSON.stringify(tc.arguments)).join("") ?? "";
             estimatedTokens += Math.ceil((choiceContent2.length + choiceArgs2.length) / 4);
+            const _bfImgPat = /\[IMAGE_BASE64:[^\]]+\]/g;
             const bfEstCtx = Math.ceil(compactedMsgs.reduce((sum, m2) => {
-              let chars = typeof m2.content === "string" ? m2.content.length : 100;
+              let chars = 0;
+              let imgCount = 0;
+              if (typeof m2.content === "string") {
+                const imgMatches = m2.content.match(_bfImgPat);
+                imgCount = imgMatches ? imgMatches.length : 0;
+                chars = m2.content.replace(_bfImgPat, "").length;
+              } else if (Array.isArray(m2.content)) {
+                for (const p2 of m2.content) {
+                  if (p2.type === "text" && p2.text)
+                    chars += p2.text.length;
+                  else if (p2.type === "image_url")
+                    imgCount++;
+                }
+              } else {
+                chars = 100;
+              }
               if (m2.tool_calls) {
                 for (const tc of m2.tool_calls)
                   chars += tc.function.arguments?.length ?? 0;
               }
-              return sum + chars;
+              return sum + chars + imgCount * 1500 * 4;
             }, 0) / 4);
             this.emit({
               type: "token_usage",
@@ -536874,18 +536850,187 @@ ${tail}`;
         }
         return folded;
       }
+      async appendInjectedUserMessage(userMsg, messages2, turn) {
+        const imagePattern = /\[IMAGE_BASE64:([^:]+):([^\]]+)\]/;
+        const imgMatch = userMsg.match(imagePattern);
+        if (imgMatch) {
+          const mime = imgMatch[1];
+          const base642 = imgMatch[2];
+          const textContent = userMsg.replace(imagePattern, "").trim();
+          await this.appendOffloadedImageMessage(messages2, mime, base642, textContent, turn);
+        } else {
+          messages2.push({
+            role: "user",
+            content: `[User added context]: ${userMsg}
+
+Integrate this guidance into your current approach. Continue working on the task.`
+          });
+        }
+        this.emit({
+          type: "user_interrupt",
+          content: userMsg.replace(/\[IMAGE_BASE64:[^\]]+\]/, "[image]").slice(0, 200),
+          turn,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        });
+      }
+      async appendOffloadedImageMessage(messages2, mime, base642, textContent, turn) {
+        const imageUrl = `data:${mime};base64,${base642}`;
+        this.emit({
+          type: "status",
+          content: "Image received; offloading visual analysis outside main context",
+          turn,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        });
+        const tmpImgPath = this.writeTempImageForOcr(mime, base642);
+        const [visionOutcome, ocrOutcome] = await Promise.allSettled([
+          this.describeImageViaVisionSubagent(imageUrl, textContent),
+          tmpImgPath ? this.extractImageOcrText(tmpImgPath) : Promise.resolve("")
+        ]);
+        const visionDesc = visionOutcome.status === "fulfilled" ? visionOutcome.value.trim() : "";
+        const ocrText = ocrOutcome.status === "fulfilled" ? ocrOutcome.value.trim() : "";
+        if (visionDesc || ocrText) {
+          const sections = [];
+          if (visionDesc)
+            sections.push(`[Image analysis]: ${visionDesc}`);
+          if (ocrText)
+            sections.push(`[OCR extracted text]: ${ocrText}`);
+          const userPrefix = textContent ? `[User added context]: ${textContent}
+
+` : "[User shared an image]. ";
+          messages2.push({
+            role: "user",
+            content: userPrefix + sections.join("\n\n") + "\n\nIntegrate this visual information into your current approach."
+          });
+          this.emit({
+            type: "status",
+            content: "Image analysis added as text; base64 excluded from main context",
+            turn,
+            timestamp: (/* @__PURE__ */ new Date()).toISOString()
+          });
+          return;
+        }
+        const reason = visionOutcome.status === "rejected" ? String(visionOutcome.reason?.message ?? visionOutcome.reason) : "vision and OCR returned no text";
+        this.emit({
+          type: "status",
+          content: `Image offload unavailable (${reason}); falling back to inline image`,
+          turn,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        });
+        this.appendInlineImageMessage(messages2, imageUrl, textContent);
+      }
+      async describeImageViaVisionSubagent(imageUrl, textContent) {
+        const visionMessages = [
+          {
+            role: "system",
+            content: "You are a visual analysis sub-agent. Describe the image in detail, including visible text, UI elements, code, diagrams, errors, and other task-relevant visual details. Be thorough but concise."
+          },
+          {
+            role: "user",
+            content: [
+              {
+                type: "text",
+                text: textContent ? `Context from user: ${textContent}
+
+Describe what you see in this image.` : "Describe what you see in this image in detail."
+              },
+              { type: "image_url", image_url: { url: imageUrl } }
+            ]
+          }
+        ];
+        const result = await this.backend.chatCompletion({
+          messages: visionMessages,
+          tools: [],
+          temperature: 0.3,
+          maxTokens: 2048,
+          timeoutMs: 3e4,
+          think: false
+        });
+        return result.choices[0]?.message?.content ?? "";
+      }
+      appendInlineImageMessage(messages2, imageUrl, textContent) {
+        const parts = [
+          {
+            type: "text",
+            text: textContent ? `[User added context]: ${textContent}
+
+Describe what you see and integrate this into your current approach.` : "[User shared an image]. Describe what you see and integrate this into your current approach."
+          },
+          { type: "image_url", image_url: { url: imageUrl } }
+        ];
+        messages2.push({ role: "user", content: parts });
+      }
+      writeTempImageForOcr(mime, base642) {
+        try {
+          const ext = this.imageExtensionForMime(mime);
+          const id = _createHash("sha256").update(`${process.pid}:${Date.now()}:`).update(base642.slice(0, 4096)).digest("hex").slice(0, 16);
+          const tmpImgPath = _pathJoin(_osTmpdir(), `oa-img-${id}.${ext}`);
+          _fsWriteFileSync(tmpImgPath, Buffer.from(base642, "base64"));
+          return tmpImgPath;
+        } catch {
+          return null;
+        }
+      }
+      imageExtensionForMime(mime) {
+        const normalized = mime.toLowerCase();
+        if (normalized.includes("jpeg") || normalized.includes("jpg"))
+          return "jpg";
+        if (normalized.includes("webp"))
+          return "webp";
+        if (normalized.includes("gif"))
+          return "gif";
+        if (normalized.includes("bmp"))
+          return "bmp";
+        if (normalized.includes("tiff"))
+          return "tif";
+        return "png";
+      }
+      async extractImageOcrText(tmpImgPath) {
+        try {
+          const stdout = await new Promise((resolve44, reject) => {
+            _execFile("tesseract", [tmpImgPath, "stdout"], {
+              encoding: "utf8",
+              timeout: 15e3,
+              maxBuffer: 2 * 1024 * 1024
+            }, (err, out) => {
+              if (err) {
+                reject(err);
+                return;
+              }
+              resolve44(out);
+            });
+          });
+          return stdout.trim();
+        } catch {
+          return "";
+        } finally {
+          try {
+            _fsUnlinkSync(tmpImgPath);
+          } catch {
+          }
+        }
+      }
       // -------------------------------------------------------------------------
       // Context compaction
       // -------------------------------------------------------------------------
       async compactMessages(messages2, strategy = "default", force = false) {
         if (messages2.length < 3)
           return messages2;
+        const _compImgPat = /\[IMAGE_BASE64:[^\]]+\]/g;
         const totalChars = messages2.reduce((sum, m2) => {
           let chars = 0;
-          if (typeof m2.content === "string")
-            chars += m2.content.length;
-          else if (Array.isArray(m2.content)) {
-            chars += m2.content.reduce((s2, p2) => s2 + (p2.text?.length || 0) + (p2.image_url ? 1e3 : 0), 0);
+          if (typeof m2.content === "string") {
+            const imgMatches = m2.content.match(_compImgPat);
+            const imgCount = imgMatches ? imgMatches.length : 0;
+            chars += m2.content.replace(_compImgPat, "").length + imgCount * 1500 * 4;
+          } else if (Array.isArray(m2.content)) {
+            chars += m2.content.reduce((s2, p2) => {
+              if (p2.type === "text" && p2.text)
+                return s2 + p2.text.length;
+              if (p2.type === "image_url" && p2.image_url?.url) {
+                return s2 + 1500 * 4;
+              }
+              return s2;
+            }, 0);
           }
           if (m2.tool_calls) {
             for (const tc of m2.tool_calls) {
@@ -536923,7 +537068,22 @@ ${tail}`;
           let budgetCut = messages2.length;
           for (let i2 = messages2.length - 1; i2 >= headEndIdx; i2--) {
             const msg = messages2[i2];
-            const msgChars = typeof msg.content === "string" ? msg.content.length : 100;
+            let msgChars = 0;
+            if (typeof msg.content === "string") {
+              const _mImgPat = /\[IMAGE_BASE64:[^\]]+\]/g;
+              const imgMatches = msg.content.match(_mImgPat);
+              const imgCount = imgMatches ? imgMatches.length : 0;
+              msgChars = msg.content.replace(_mImgPat, "").length + imgCount * 1500 * 4;
+            } else if (Array.isArray(msg.content)) {
+              for (const p2 of msg.content) {
+                if (p2.type === "text" && p2.text)
+                  msgChars += p2.text.length;
+                else if (p2.type === "image_url")
+                  msgChars += 1500 * 4;
+              }
+            } else {
+              msgChars = 100;
+            }
             const toolCallChars = (msg.tool_calls || []).reduce((s2, tc) => s2 + (tc.function?.arguments?.length || 0) + (tc.function?.name?.length || 0), 0);
             const msgTokens = Math.ceil((msgChars + toolCallChars) / 4) + 10;
             if (accumulated + msgTokens > tailTokenBudget && messages2.length - i2 >= 4) {
@@ -537032,7 +537192,25 @@ ${tail}`;
         const strategyLabel = strategy !== "default" ? ` (${strategy})` : "";
         const forceLabel = force ? " [manual]" : "";
         const preTokens = Math.ceil(totalChars / 4);
-        const postChars = combinedSummary.length + recent.reduce((s2, m2) => s2 + (typeof m2.content === "string" ? m2.content.length : 100), 0) + head.reduce((s2, m2) => s2 + (typeof m2.content === "string" ? m2.content.length : 100), 0);
+        const _postImgPat = /\[IMAGE_BASE64:[^\]]+\]/g;
+        const _estimateMsgChars = (m2) => {
+          if (typeof m2.content === "string") {
+            const imgMatches = m2.content.match(_postImgPat);
+            const imgCount = imgMatches ? imgMatches.length : 0;
+            return m2.content.replace(_postImgPat, "").length + imgCount * 1500 * 4;
+          } else if (Array.isArray(m2.content)) {
+            let c9 = 0;
+            for (const p2 of m2.content) {
+              if (p2.type === "text" && p2.text)
+                c9 += p2.text.length;
+              else if (p2.type === "image_url")
+                c9 += 1500 * 4;
+            }
+            return c9;
+          }
+          return 100;
+        };
+        const postChars = combinedSummary.length + recent.reduce((s2, m2) => s2 + _estimateMsgChars(m2), 0) + head.reduce((s2, m2) => s2 + _estimateMsgChars(m2), 0);
         const postTokens = Math.ceil(postChars / 4);
         const savedTokens = preTokens - postTokens;
         this.emit({
@@ -537220,13 +537398,29 @@ ${content.slice(0, 8e3)}
         }
         const ctxWindow = this.options.contextWindowSize;
         if (ctxWindow > 0) {
+          const _safetyImgPat = /\[IMAGE_BASE64:[^\]]+\]/g;
           const estimateResult = (msgs) => msgs.reduce((sum, m2) => {
-            let chars = typeof m2.content === "string" ? m2.content.length : 100;
+            let chars = 0;
+            let imgCount = 0;
+            if (typeof m2.content === "string") {
+              const imgMatches = m2.content.match(_safetyImgPat);
+              imgCount = imgMatches ? imgMatches.length : 0;
+              chars = m2.content.replace(_safetyImgPat, "").length;
+            } else if (Array.isArray(m2.content)) {
+              for (const p2 of m2.content) {
+                if (p2.type === "text" && p2.text)
+                  chars += p2.text.length;
+                else if (p2.type === "image_url")
+                  imgCount++;
+              }
+            } else {
+              chars = 100;
+            }
             if (m2.tool_calls) {
               for (const tc of m2.tool_calls)
                 chars += tc.function.arguments?.length ?? 0;
             }
-            return sum + chars;
+            return sum + chars + imgCount * 1500 * 4;
           }, 0) / 4;
           const safetyTarget = Math.floor(ctxWindow * 0.65);
           let trimmedRecent = [...filteredRecent];
@@ -538340,7 +538534,49 @@ ${transcript}`
           "pascal_case",
           "screaming_snake_case",
           "dot_notation",
-          "title_case"
+          "title_case",
+          // Conditionally-registered tools documented in the base system prompt
+          // as available but only registered in TUI/desktop mode, NOT telegram:
+          "background_run",
+          // TUI-only: background shell execution
+          "task_status",
+          // TUI-only: check background task status
+          "task_output",
+          // TUI-only: read background task output
+          "task_stop",
+          // TUI-only: kill background task
+          "skill_list",
+          // TUI-only: discover available skills
+          "skill_execute",
+          // TUI-only: load and run a skill
+          "skill_build",
+          // TUI-only: generate a new skill
+          "desktop_click",
+          // TUI-only: click UI element by description
+          "desktop_describe",
+          // TUI-only: screenshot + describe desktop
+          "repl_exec",
+          // TUI-only: persistent Python REPL
+          "cron_agent",
+          // TUI-only: scheduled agent tasks
+          "scheduler",
+          // TUI-only: OS-level cron scheduling
+          "reminder",
+          // TUI-only: cross-session reminders
+          "agenda",
+          // TUI-only: attention directives
+          "priority_classify",
+          // TUI-only: task priority classification
+          "priority_delegate",
+          // TUI-only: delegate to sub-agent by priority
+          "create_tool",
+          // TUI-only: create custom tool
+          "manage_tools",
+          // TUI-only: list/inspect/delete custom tools
+          "sub_agent",
+          // TUI-only: delegate sub-task (telegram uses full_sub_agent)
+          "nexus"
+          // TUI-only: P2P networking
         ]);
         for (const tool of this.tools.values()) {
           const props = tool.parameters?.properties;

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "open-agents-ai",
-  "version": "0.187.592",
+  "version": "0.187.593",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "open-agents-ai",
-      "version": "0.187.592",
+      "version": "0.187.593",
       "hasInstallScript": true,
       "license": "CC-BY-NC-4.0",
       "dependencies": {
@@ -2049,9 +2049,9 @@
       }
     },
     "node_modules/aiwg": {
-      "version": "2026.5.4",
-      "resolved": "https://registry.npmjs.org/aiwg/-/aiwg-2026.5.4.tgz",
-      "integrity": "sha512-/10XfF6pD+7/I945vx1uhh37+N4NIp1NscGJUEAJAMwVVrHXdqZ4UotCfLBp6dnwOI3tI5jfg3zWJkr1yhUPOw==",
+      "version": "2026.5.5",
+      "resolved": "https://registry.npmjs.org/aiwg/-/aiwg-2026.5.5.tgz",
+      "integrity": "sha512-bRMCp3qOAgZycb1Pyahmx3CgrMVtKU5gQhYpyY23r6B/LhNOP/+t0SHRuvPN4bOVOVsec/qKpma9OvQnlqrp0w==",
       "license": "MIT",
       "dependencies": {
         "@modelcontextprotocol/sdk": "^1.24.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.187.592",
+  "version": "0.187.593",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",