open-agents-ai 0.26.0 → 0.27.0

package/dist/index.js

@@ -6635,9 +6635,18 @@ var init_vision = __esm({
         },
         required: ["image"]
       };
+      /** Active Ollama model name — used as vision fallback if it has vision capability */
+      _activeModel = "";
+      /** Whether the active model has vision capability */
+      _activeModelHasVision = false;
       constructor(workingDir) {
         this.workingDir = workingDir;
       }
+      /** Set the active Ollama model and whether it supports vision */
+      setActiveModel(model, hasVision) {
+        this._activeModel = model;
+        this._activeModelHasVision = hasVision;
+      }
       async execute(args) {
         const start = performance.now();
         const rawPath = args["image"];
@@ -6738,7 +6747,8 @@ Coordinates are normalized (0-1). Multiply by image width/height for pixel value
       }
       async tryOllamaVision(buffer, filename, action, prompt, length, start) {
         const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
-        const model = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+        const envModel = process.env["OLLAMA_VISION_MODEL"];
+        const model = envModel || (this._activeModelHasVision && this._activeModel ? this._activeModel : "moondream");
         const imageBase64 = buffer.toString("base64");
         let ollamaPrompt;
         switch (action) {
@@ -6965,7 +6975,16 @@ var init_desktop_click = __esm({
     DesktopClickTool = class {
       workingDir;
       name = "desktop_click";
-      description = "Click on a UI element identified by natural language description. Takes a screenshot, uses Moondream vision to find the described element, then clicks at that location using xdotool (Linux) or cliclick (macOS). Example: desktop_click({ target: 'the Save button' })";
+      description = "Click on a UI element identified by natural language description. Takes a screenshot, uses vision to find the described element, then clicks at that location using xdotool (Linux) or cliclick (macOS). Example: desktop_click({ target: 'the Save button' })";
+      /** Active Ollama model name — used as vision fallback if it has vision capability */
+      _activeModel = "";
+      /** Whether the active model has vision capability */
+      _activeModelHasVision = false;
+      /** Set the active Ollama model and whether it supports vision */
+      setActiveModel(model, hasVision) {
+        this._activeModel = model;
+        this._activeModelHasVision = hasVision;
+      }
       parameters = {
         type: "object",
         properties: {
@@ -7050,7 +7069,8 @@ var init_desktop_click = __esm({
           if (!visionWorked) {
             try {
               const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
-              const ollamaModel = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+              const envModel = process.env["OLLAMA_VISION_MODEL"];
+              const ollamaModel = envModel || (this._activeModelHasVision && this._activeModel ? this._activeModel : "moondream");
               const imageBase64 = readFileSync11(screenshotPath).toString("base64");
               const res = await fetch(`${ollamaHost}/api/generate`, {
                 method: "POST",
@@ -7148,7 +7168,16 @@ Screenshot: ${screenshotPath}`,
     DesktopDescribeTool = class {
       workingDir;
       name = "desktop_describe";
-      description = "Take a screenshot and describe what's on the desktop using Moondream vision. Optionally ask a specific question about the screen contents. Use this to become aware of the desktop environment.";
+      description = "Take a screenshot and describe what's on the desktop using vision. Optionally ask a specific question about the screen contents. Use this to become aware of the desktop environment.";
+      /** Active Ollama model name — used as vision fallback if it has vision capability */
+      _activeModel = "";
+      /** Whether the active model has vision capability */
+      _activeModelHasVision = false;
+      /** Set the active Ollama model and whether it supports vision */
+      setActiveModel(model, hasVision) {
+        this._activeModel = model;
+        this._activeModelHasVision = hasVision;
+      }
       parameters = {
         type: "object",
         properties: {
@@ -7214,7 +7243,8 @@ ${caption}`);
           if (!visionWorked) {
             try {
               const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
-              const ollamaModel = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+              const envModel = process.env["OLLAMA_VISION_MODEL"];
+              const ollamaModel = envModel || (this._activeModelHasVision && this._activeModel ? this._activeModel : "moondream");
               const imageBase64 = imageBuffer.toString("base64");
               const ollamaPrompt = question || "Describe what you see on this desktop screenshot in detail. Include visible applications, windows, text, and UI elements.";
               const res = await fetch(`${ollamaHost}/api/generate`, {
@@ -12783,6 +12813,52 @@ async function queryContextSize(baseUrl, modelName, apiKey) {
     return ollamaSize;
   return queryOpenAIContextSize(baseUrl, modelName, apiKey);
 }
+async function queryModelCapabilities(baseUrl, modelName) {
+  const caps = { vision: false, toolUse: false, thinking: false };
+  try {
+    const normalized = normalizeBaseUrl(baseUrl);
+    const res = await fetch(`${normalized}/api/show`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ name: modelName }),
+      signal: AbortSignal.timeout(1e4)
+    });
+    if (!res.ok)
+      return caps;
+    const data = await res.json();
+    if (Array.isArray(data.capabilities)) {
+      if (data.capabilities.includes("vision"))
+        caps.vision = true;
+      if (data.capabilities.includes("tools"))
+        caps.toolUse = true;
+      if (data.capabilities.includes("thinking"))
+        caps.thinking = true;
+    }
+    if (data.model_info) {
+      for (const key of Object.keys(data.model_info)) {
+        const k = key.toLowerCase();
+        if (k.includes("vision.block_count") || k.includes("clip.") || k.includes("image_token_id") || k.includes("projector") || k.includes("vision.embedding_length")) {
+          const val = data.model_info[key];
+          if (val !== null && val !== void 0 && val !== 0 && val !== "") {
+            caps.vision = true;
+          }
+        }
+      }
+    }
+    const nameLower = modelName.toLowerCase();
+    if (/qwen3|qwen2\.5|llama3\.[13]|mistral|mixtral|command-r|gemma3|devstral|deepseek/.test(nameLower)) {
+      caps.toolUse = true;
+    }
+    if (data.template) {
+      if (data.template.includes("<think>") || data.template.includes("thinking")) {
+        caps.thinking = true;
+      }
+    }
+    return caps;
+  } catch {
+    return caps;
+  }
+}
 function formatBytes(bytes) {
   if (bytes < 1024)
     return `${bytes} B`;
@@ -14685,6 +14761,63 @@ function isFirstRun() {
     return true;
   }
 }
+async function ensureVisionDeps(onInfo) {
+  const log = onInfo ?? (() => {
+  });
+  try {
+    execSync13("which tesseract", { stdio: "pipe", timeout: 3e3 });
+  } catch {
+    log("Installing tesseract-ocr...");
+    try {
+      const cmds = [
+        "sudo -n apt-get install -y tesseract-ocr 2>/dev/null",
+        "sudo -n dnf install -y tesseract 2>/dev/null",
+        "sudo -n pacman -S --noconfirm tesseract 2>/dev/null",
+        "brew install tesseract 2>/dev/null"
+      ];
+      let installed = false;
+      for (const cmd of cmds) {
+        try {
+          execSync13(cmd, { stdio: "pipe", timeout: 6e4 });
+          installed = true;
+          break;
+        } catch {
+        }
+      }
+      if (installed)
+        log("Tesseract installed.");
+      else
+        log("Could not auto-install tesseract (install manually: sudo apt install tesseract-ocr)");
+    } catch {
+    }
+  }
+  try {
+    execSync13("which moondream-station", { stdio: "pipe", timeout: 3e3 });
+  } catch {
+    log("Installing moondream-station...");
+    try {
+      const pipCmds = [
+        "pip3 install moondream-station 2>/dev/null",
+        "pip install moondream-station 2>/dev/null",
+        "python3 -m pip install moondream-station 2>/dev/null"
+      ];
+      let installed = false;
+      for (const cmd of pipCmds) {
+        try {
+          execSync13(cmd, { stdio: "pipe", timeout: 12e4 });
+          installed = true;
+          break;
+        } catch {
+        }
+      }
+      if (installed)
+        log("moondream-station installed.");
+      else
+        log("Could not auto-install moondream-station (install manually: pip install moondream-station)");
+    } catch {
+    }
+  }
+}
 function expandedModelName(baseModel) {
   return `open-agents-${baseModel.replace(":", "-").replace(/\./g, "")}`;
 }
@@ -15484,6 +15617,10 @@ async function switchModel(query, ctx, local = false) {
         ctx.setContextWindowSize(ctxSize);
       }
     }
+    if (ctx.setCapabilities) {
+      const caps = await queryModelCapabilities(ctx.config.backendUrl, finalModel);
+      ctx.setCapabilities(caps);
+    }
   } catch (err) {
     renderError(`Failed to switch model: ${err instanceof Error ? err.message : String(err)}`);
   }
@@ -18666,6 +18803,18 @@ var init_status_bar = __esm({
       setContextWindowSize(size) {
         this.metrics.contextWindowSize = size;
       }
+      /** Model capabilities — shown as emoji indicators on the status bar */
+      _caps = {
+        vision: false,
+        toolUse: false,
+        thinking: false
+      };
+      /** Update model capability indicators */
+      setCapabilities(caps) {
+        this._caps = caps;
+        if (this.active)
+          this.renderFooterPreserveCursor();
+      }
       /** Update token metrics from a token_usage event */
       updateMetrics(update) {
         if (update.promptTokens !== void 0)
@@ -18822,7 +18971,15 @@ var init_status_bar = __esm({
           const countdown = this._countdown > 0 ? c2.dim(` ${this._countdown}s`) : "";
           recordingLabel = pipe + dot + pastel2(210, " REC") + countdown;
         }
-        return ` ${tokInLabel}${pipe}${tokOutLabel}${pipe}${ctxLabel}${costLabel}${recordingLabel}`;
+        const capParts = [];
+        if (this._caps.vision)
+          capParts.push("\u{1F441}");
+        if (this._caps.toolUse)
+          capParts.push("\u{1F527}");
+        if (this._caps.thinking)
+          capParts.push("\u{1F9E0}");
+        const capsLabel = capParts.length > 0 ? pipe + pastel2(183, capParts.join(" ")) : "";
+        return ` ${tokInLabel}${pipe}${tokOutLabel}${pipe}${ctxLabel}${costLabel}${capsLabel}${recordingLabel}`;
       }
       // -------------------------------------------------------------------------
       // Private
@@ -19157,7 +19314,7 @@ Use task_status("${taskId}") or task_output("${taskId}") to check progress.`
     }
   };
 }
-function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce, statusBar, sudoCallback, costTracker, onComplete, taskType, contextWindowSize) {
+function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce, statusBar, sudoCallback, costTracker, onComplete, taskType, contextWindowSize, modelCaps) {
   const modelTier = getModelTier(config.model);
   const projectCtx = buildProjectContext(repoRoot, taskStores?.contextStores);
   let dynamicContext = formatContextForPrompt(projectCtx, modelTier);
@@ -19190,6 +19347,12 @@ function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce
       }
     }
   }
+  const hasVision = modelCaps?.vision ?? false;
+  for (const tool of tools) {
+    if ("setActiveModel" in tool && typeof tool.setActiveModel === "function") {
+      tool.setActiveModel(config.model, hasVision);
+    }
+  }
   runner.registerTools(tools);
   const filesTouched = /* @__PURE__ */ new Set();
   const toolSequence = [];
@@ -19489,11 +19652,29 @@ async function startInteractive(config, repoPath) {
     }
   }).catch(() => {
   });
+  let resolvedCaps = {
+    vision: false,
+    toolUse: false,
+    thinking: false
+  };
+  queryModelCapabilities(config.backendUrl, config.model).then((caps) => {
+    resolvedCaps = caps;
+    statusBar.setCapabilities(caps);
+  }).catch(() => {
+  });
   const provider = detectProvider(config.backendUrl);
   const costTracker = new CostTracker(provider.id);
   const sessionMetrics = new SessionMetrics();
   const workEvaluator = new WorkEvaluator();
   ensureTranscribeCliBackground();
+  ensureVisionDeps((msg) => {
+    if (statusBar?.isActive) {
+      statusBar.beginContentWrite();
+      renderInfo(msg);
+      statusBar.endContentWrite();
+    }
+  }).catch(() => {
+  });
   const voiceEngine = new VoiceEngine();
   const streamRenderer = new StreamRenderer();
   if (savedSettings.voice) {
@@ -19791,6 +19972,10 @@ async function startInteractive(config, repoPath) {
       resolvedContextWindowSize = size;
       statusBar.setContextWindowSize(size);
     },
+    setCapabilities: (caps) => {
+      resolvedCaps = caps;
+      statusBar.setCapabilities(caps);
+    },
     hasActiveTask: () => activeTask !== null,
     abortTask() {
       if (!activeTask)
@@ -19946,7 +20131,7 @@ Execute this skill now. Follow the behavioral guidance above.`;
             toolPatternStore: toolPatternStore ?? void 0
           }, bruteForceEnabled, statusBar, handleSudoRequest, costTracker, (summary) => {
             lastCompletedSummary = summary;
-          }, currentTaskType, resolvedContextWindowSize);
+          }, currentTaskType, resolvedContextWindowSize, resolvedCaps);
           activeTask = task;
           showPrompt();
           await task.promise;
@@ -20048,7 +20233,7 @@ Summarize or analyze this transcription as appropriate.`;
         toolPatternStore: toolPatternStore ?? void 0
       }, bruteForceEnabled, statusBar, handleSudoRequest, costTracker, (summary) => {
         lastCompletedSummary = summary;
-      }, currentTaskType, resolvedContextWindowSize);
+      }, currentTaskType, resolvedContextWindowSize, resolvedCaps);
       activeTask = task;
       showPrompt();
       await task.promise;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.26.0",
+  "version": "0.27.0",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",