open-agents-ai 0.26.0 → 0.28.0

package/dist/index.js

@@ -4019,7 +4019,7 @@ function ensureCommand(command) {
     const result2 = {
       available: false,
       installed: false,
-      error: `No supported package manager found. Install ${command} manually.`
+      error: `No supported package manager detected \u2014 ${command} unavailable.`
     };
     _cache.set(command, result2);
     return result2;
@@ -4029,7 +4029,7 @@ function ensureCommand(command) {
     const result2 = {
       available: false,
       installed: false,
-      error: `No package for ${command} on ${pm}. Install manually.`
+      error: `No package mapping for ${command} on ${pm} \u2014 cannot auto-install.`
     };
     _cache.set(command, result2);
     return result2;
@@ -6474,6 +6474,17 @@ async function probeStation(endpoint) {
   }
 }
 function findStationBinary() {
+  const oaVenvPython = join15(process.env["HOME"] || "/root", ".open-agents", "venv", "bin", "python");
+  if (existsSync11(oaVenvPython)) {
+    try {
+      execSync10(`${JSON.stringify(oaVenvPython)} -c "import moondream_station"`, { stdio: "pipe", timeout: 5e3 });
+      return oaVenvPython;
+    } catch {
+    }
+  }
+  const oaVenvBin = join15(process.env["HOME"] || "/root", ".open-agents", "venv", "bin", "moondream-station");
+  if (existsSync11(oaVenvBin))
+    return oaVenvBin;
   const thisDir = dirname4(fileURLToPath(import.meta.url));
   const localVenvPaths = [
     resolve15(thisDir, "../../../../.moondream-venv/bin/python"),
@@ -6635,9 +6646,18 @@ var init_vision = __esm({
         },
         required: ["image"]
       };
+      /** Active Ollama model name — used as vision fallback if it has vision capability */
+      _activeModel = "";
+      /** Whether the active model has vision capability */
+      _activeModelHasVision = false;
       constructor(workingDir) {
         this.workingDir = workingDir;
       }
+      /** Set the active Ollama model and whether it supports vision */
+      setActiveModel(model, hasVision) {
+        this._activeModel = model;
+        this._activeModelHasVision = hasVision;
+      }
       async execute(args) {
         const start = performance.now();
         const rawPath = args["image"];
@@ -6738,7 +6758,8 @@ Coordinates are normalized (0-1). Multiply by image width/height for pixel value
       }
       async tryOllamaVision(buffer, filename, action, prompt, length, start) {
         const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
-        const model = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+        const envModel = process.env["OLLAMA_VISION_MODEL"];
+        const model = envModel || (this._activeModelHasVision && this._activeModel ? this._activeModel : "moondream");
         const imageBase64 = buffer.toString("base64");
         let ollamaPrompt;
         switch (action) {
@@ -6965,7 +6986,16 @@ var init_desktop_click = __esm({
     DesktopClickTool = class {
       workingDir;
       name = "desktop_click";
-      description = "Click on a UI element identified by natural language description. Takes a screenshot, uses Moondream vision to find the described element, then clicks at that location using xdotool (Linux) or cliclick (macOS). Example: desktop_click({ target: 'the Save button' })";
+      description = "Click on a UI element identified by natural language description. Takes a screenshot, uses vision to find the described element, then clicks at that location using xdotool (Linux) or cliclick (macOS). Example: desktop_click({ target: 'the Save button' })";
+      /** Active Ollama model name — used as vision fallback if it has vision capability */
+      _activeModel = "";
+      /** Whether the active model has vision capability */
+      _activeModelHasVision = false;
+      /** Set the active Ollama model and whether it supports vision */
+      setActiveModel(model, hasVision) {
+        this._activeModel = model;
+        this._activeModelHasVision = hasVision;
+      }
       parameters = {
         type: "object",
         properties: {
@@ -7050,7 +7080,8 @@ var init_desktop_click = __esm({
           if (!visionWorked) {
             try {
               const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
-              const ollamaModel = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+              const envModel = process.env["OLLAMA_VISION_MODEL"];
+              const ollamaModel = envModel || (this._activeModelHasVision && this._activeModel ? this._activeModel : "moondream");
               const imageBase64 = readFileSync11(screenshotPath).toString("base64");
               const res = await fetch(`${ollamaHost}/api/generate`, {
                 method: "POST",
@@ -7148,7 +7179,16 @@ Screenshot: ${screenshotPath}`,
     DesktopDescribeTool = class {
       workingDir;
       name = "desktop_describe";
-      description = "Take a screenshot and describe what's on the desktop using Moondream vision. Optionally ask a specific question about the screen contents. Use this to become aware of the desktop environment.";
+      description = "Take a screenshot and describe what's on the desktop using vision. Optionally ask a specific question about the screen contents. Use this to become aware of the desktop environment.";
+      /** Active Ollama model name — used as vision fallback if it has vision capability */
+      _activeModel = "";
+      /** Whether the active model has vision capability */
+      _activeModelHasVision = false;
+      /** Set the active Ollama model and whether it supports vision */
+      setActiveModel(model, hasVision) {
+        this._activeModel = model;
+        this._activeModelHasVision = hasVision;
+      }
       parameters = {
         type: "object",
         properties: {
@@ -7214,7 +7254,8 @@ ${caption}`);
           if (!visionWorked) {
             try {
               const ollamaHost = process.env["OLLAMA_HOST"] || "http://localhost:11434";
-              const ollamaModel = process.env["OLLAMA_VISION_MODEL"] || "moondream";
+              const envModel = process.env["OLLAMA_VISION_MODEL"];
+              const ollamaModel = envModel || (this._activeModelHasVision && this._activeModel ? this._activeModel : "moondream");
               const imageBase64 = imageBuffer.toString("base64");
               const ollamaPrompt = question || "Describe what you see on this desktop screenshot in detail. Include visible applications, windows, text, and UI elements.";
               const res = await fetch(`${ollamaHost}/api/generate`, {
@@ -12783,6 +12824,52 @@ async function queryContextSize(baseUrl, modelName, apiKey) {
     return ollamaSize;
   return queryOpenAIContextSize(baseUrl, modelName, apiKey);
 }
+async function queryModelCapabilities(baseUrl, modelName) {
+  const caps = { vision: false, toolUse: false, thinking: false };
+  try {
+    const normalized = normalizeBaseUrl(baseUrl);
+    const res = await fetch(`${normalized}/api/show`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ name: modelName }),
+      signal: AbortSignal.timeout(1e4)
+    });
+    if (!res.ok)
+      return caps;
+    const data = await res.json();
+    if (Array.isArray(data.capabilities)) {
+      if (data.capabilities.includes("vision"))
+        caps.vision = true;
+      if (data.capabilities.includes("tools"))
+        caps.toolUse = true;
+      if (data.capabilities.includes("thinking"))
+        caps.thinking = true;
+    }
+    if (data.model_info) {
+      for (const key of Object.keys(data.model_info)) {
+        const k = key.toLowerCase();
+        if (k.includes("vision.block_count") || k.includes("clip.") || k.includes("image_token_id") || k.includes("projector") || k.includes("vision.embedding_length")) {
+          const val = data.model_info[key];
+          if (val !== null && val !== void 0 && val !== 0 && val !== "") {
+            caps.vision = true;
+          }
+        }
+      }
+    }
+    const nameLower = modelName.toLowerCase();
+    if (/qwen3|qwen2\.5|llama3\.[13]|mistral|mixtral|command-r|gemma3|devstral|deepseek/.test(nameLower)) {
+      caps.toolUse = true;
+    }
+    if (data.template) {
+      if (data.template.includes("<think>") || data.template.includes("thinking")) {
+        caps.thinking = true;
+      }
+    }
+    return caps;
+  } catch {
+    return caps;
+  }
+}
 function formatBytes(bytes) {
   if (bytes < 1024)
     return `${bytes} B`;
@@ -14685,6 +14772,221 @@ function isFirstRun() {
     return true;
   }
 }
+function hasCmd(cmd) {
+  try {
+    execSync13(`which ${cmd}`, { stdio: "pipe", timeout: 3e3 });
+    return true;
+  } catch {
+    return false;
+  }
+}
+function detectPkgManager() {
+  if (hasCmd("apt-get"))
+    return "apt";
+  if (hasCmd("dnf"))
+    return "dnf";
+  if (hasCmd("pacman"))
+    return "pacman";
+  if (hasCmd("brew"))
+    return "brew";
+  return null;
+}
+function getVenvDir() {
+  return join22(homedir8(), ".open-agents", "venv");
+}
+function hasVenvModule() {
+  try {
+    execSync13("python3 -m venv --help", { stdio: "pipe", timeout: 5e3 });
+    return true;
+  } catch {
+    return false;
+  }
+}
+function ensureVenv(log) {
+  const venvDir = getVenvDir();
+  const venvPip = join22(venvDir, "bin", "pip");
+  if (existsSync15(venvPip))
+    return venvDir;
+  log("Creating Python venv for vision deps...");
+  if (!hasCmd("python3")) {
+    log("python3 not found \u2014 cannot create venv.");
+    return null;
+  }
+  if (!hasVenvModule()) {
+    log("python3 venv module not available \u2014 venv creation skipped.");
+    return null;
+  }
+  try {
+    mkdirSync7(join22(homedir8(), ".open-agents"), { recursive: true });
+    execSync13(`python3 -m venv "${venvDir}"`, { stdio: "pipe", timeout: 3e4 });
+    execSync13(`"${join22(venvDir, "bin", "pip")}" install --upgrade pip`, {
+      stdio: "pipe",
+      timeout: 6e4
+    });
+    log("Python venv created at ~/.open-agents/venv");
+    return venvDir;
+  } catch (err) {
+    log(`Failed to create venv: ${err instanceof Error ? err.message : String(err)}`);
+    return null;
+  }
+}
+function trySudoPasswordless(cmd, timeoutMs = 12e4) {
+  try {
+    execSync13(`sudo -n ${cmd}`, {
+      stdio: "pipe",
+      timeout: timeoutMs,
+      env: { ...process.env, DEBIAN_FRONTEND: "noninteractive" }
+    });
+    return true;
+  } catch {
+    return false;
+  }
+}
+function runWithSudo(cmd, password, timeoutMs = 12e4) {
+  try {
+    execSync13(`sudo -S ${cmd}`, {
+      input: password + "\n",
+      stdio: ["pipe", "pipe", "pipe"],
+      timeout: timeoutMs,
+      env: { ...process.env, DEBIAN_FRONTEND: "noninteractive" }
+    });
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function sudoInstall(cmd, getSudoPassword, log, cachedPasswordRef, timeoutMs = 12e4) {
+  if (cachedPasswordRef.value) {
+    if (runWithSudo(cmd, cachedPasswordRef.value, timeoutMs))
+      return true;
+  }
+  if (trySudoPasswordless(cmd, timeoutMs))
+    return true;
+  const pw = await getSudoPassword();
+  if (pw) {
+    cachedPasswordRef.value = pw;
+    if (runWithSudo(cmd, pw, timeoutMs))
+      return true;
+    log("Authentication failed \u2014 please re-enter password.");
+    const pw2 = await getSudoPassword();
+    if (pw2) {
+      cachedPasswordRef.value = pw2;
+      if (runWithSudo(cmd, pw2, timeoutMs))
+        return true;
+    }
+  }
+  return false;
+}
+async function ensureVisionDeps(onInfo, getSudoPassword) {
+  const log = onInfo ?? (() => {
+  });
+  const cachedPasswordRef = { value: null };
+  const getPassword = getSudoPassword ?? (() => Promise.resolve(null));
+  if (!hasCmd("tesseract")) {
+    const pm2 = detectPkgManager();
+    if (pm2) {
+      const pkgMap = {
+        apt: { cmd: "apt-get install -y tesseract-ocr", needsSudo: true },
+        dnf: { cmd: "dnf install -y tesseract", needsSudo: true },
+        pacman: { cmd: "pacman -S --noconfirm tesseract", needsSudo: true },
+        brew: { cmd: "brew install tesseract", needsSudo: false }
+      };
+      const pkg = pkgMap[pm2];
+      if (pkg.needsSudo) {
+        log("Installing tesseract-ocr...");
+        const ok = await sudoInstall(pkg.cmd, getPassword, log, cachedPasswordRef);
+        if (ok && hasCmd("tesseract")) {
+          log("tesseract-ocr installed successfully.");
+        } else {
+          log("tesseract-ocr could not be installed \u2014 OCR features will be limited.");
+        }
+      } else {
+        log("Installing tesseract-ocr...");
+        try {
+          execSync13(pkg.cmd, { stdio: "pipe", timeout: 12e4 });
+          if (hasCmd("tesseract")) {
+            log("tesseract-ocr installed successfully.");
+          } else {
+            log("tesseract-ocr install completed but binary not found \u2014 OCR features will be limited.");
+          }
+        } catch {
+          log("tesseract-ocr could not be installed \u2014 OCR features will be limited.");
+        }
+      }
+    } else {
+      log("No supported package manager detected \u2014 tesseract OCR unavailable.");
+    }
+  }
+  const pm = detectPkgManager();
+  if (!hasCmd("pip3") && !hasCmd("pip") && pm) {
+    const pipCmds = {
+      apt: "apt-get install -y python3-pip",
+      dnf: "dnf install -y python3-pip"
+    };
+    const pipCmd = pipCmds[pm];
+    if (pipCmd) {
+      log("Installing python3-pip...");
+      const ok = await sudoInstall(pipCmd, getPassword, log, cachedPasswordRef);
+      if (!ok) {
+        log("python3-pip could not be installed \u2014 moondream-station may be unavailable.");
+      }
+    }
+  }
+  if (hasCmd("python3") && !hasVenvModule() && pm) {
+    const venvCmds = {
+      apt: () => {
+        try {
+          const pyVer = execSync13(`python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"`, { encoding: "utf8", stdio: "pipe", timeout: 5e3 }).trim();
+          return `apt-get install -y python3-venv python${pyVer}-venv`;
+        } catch {
+          return "apt-get install -y python3-venv";
+        }
+      },
+      dnf: () => "dnf install -y python3-venv"
+    };
+    const cmdFn = venvCmds[pm];
+    if (cmdFn) {
+      const cmd = cmdFn();
+      if (cmd) {
+        log("Installing python3-venv...");
+        const ok = await sudoInstall(cmd, getPassword, log, cachedPasswordRef);
+        if (!ok) {
+          log("python3-venv could not be installed \u2014 moondream-station may be unavailable.");
+        }
+      }
+    }
+  }
+  const venvDir = getVenvDir();
+  const venvBin = join22(venvDir, "bin");
+  const venvMoondream = join22(venvBin, "moondream-station");
+  if (hasCmd("moondream-station") || existsSync15(venvMoondream)) {
+    return;
+  }
+  const venv = ensureVenv(log);
+  if (!venv) {
+    log("Python venv unavailable \u2014 moondream-station will not be installed.");
+    return;
+  }
+  const venvPip = join22(venvBin, "pip");
+  log("Installing moondream-station in ~/.open-agents/venv...");
+  try {
+    execSync13(`"${venvPip}" install moondream-station`, { stdio: "pipe", timeout: 3e5 });
+    if (existsSync15(venvMoondream)) {
+      log("moondream-station installed successfully.");
+    } else {
+      try {
+        const check = execSync13(`"${venvPip}" show moondream-station`, { encoding: "utf8", stdio: "pipe", timeout: 5e3 });
+        if (check.includes("moondream")) {
+          log("moondream-station package installed.");
+        }
+      } catch {
+        log("moondream-station install completed.");
+      }
+    }
+  } catch (err) {
+    log(`moondream-station install failed: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
 function expandedModelName(baseModel) {
   return `open-agents-${baseModel.replace(":", "-").replace(/\./g, "")}`;
 }
@@ -15484,6 +15786,10 @@ async function switchModel(query, ctx, local = false) {
         ctx.setContextWindowSize(ctxSize);
       }
     }
+    if (ctx.setCapabilities) {
+      const caps = await queryModelCapabilities(ctx.config.backendUrl, finalModel);
+      ctx.setCapabilities(caps);
+    }
   } catch (err) {
     renderError(`Failed to switch model: ${err instanceof Error ? err.message : String(err)}`);
   }
@@ -18666,6 +18972,18 @@ var init_status_bar = __esm({
       setContextWindowSize(size) {
         this.metrics.contextWindowSize = size;
       }
+      /** Model capabilities — shown as emoji indicators on the status bar */
+      _caps = {
+        vision: false,
+        toolUse: false,
+        thinking: false
+      };
+      /** Update model capability indicators */
+      setCapabilities(caps) {
+        this._caps = caps;
+        if (this.active)
+          this.renderFooterPreserveCursor();
+      }
       /** Update token metrics from a token_usage event */
       updateMetrics(update) {
         if (update.promptTokens !== void 0)
@@ -18822,7 +19140,15 @@ var init_status_bar = __esm({
           const countdown = this._countdown > 0 ? c2.dim(` ${this._countdown}s`) : "";
           recordingLabel = pipe + dot + pastel2(210, " REC") + countdown;
         }
-        return ` ${tokInLabel}${pipe}${tokOutLabel}${pipe}${ctxLabel}${costLabel}${recordingLabel}`;
+        const capParts = [];
+        if (this._caps.vision)
+          capParts.push("\u{1F441}");
+        if (this._caps.toolUse)
+          capParts.push("\u{1F527}");
+        if (this._caps.thinking)
+          capParts.push("\u{1F9E0}");
+        const capsLabel = capParts.length > 0 ? pipe + pastel2(183, capParts.join(" ")) : "";
+        return ` ${tokInLabel}${pipe}${tokOutLabel}${pipe}${ctxLabel}${costLabel}${capsLabel}${recordingLabel}`;
       }
       // -------------------------------------------------------------------------
       // Private
@@ -19157,7 +19483,7 @@ Use task_status("${taskId}") or task_output("${taskId}") to check progress.`
     }
   };
 }
-function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce, statusBar, sudoCallback, costTracker, onComplete, taskType, contextWindowSize) {
+function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce, statusBar, sudoCallback, costTracker, onComplete, taskType, contextWindowSize, modelCaps) {
   const modelTier = getModelTier(config.model);
   const projectCtx = buildProjectContext(repoRoot, taskStores?.contextStores);
   let dynamicContext = formatContextForPrompt(projectCtx, modelTier);
@@ -19190,6 +19516,12 @@ function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce
       }
     }
   }
+  const hasVision = modelCaps?.vision ?? false;
+  for (const tool of tools) {
+    if ("setActiveModel" in tool && typeof tool.setActiveModel === "function") {
+      tool.setActiveModel(config.model, hasVision);
+    }
+  }
   runner.registerTools(tools);
   const filesTouched = /* @__PURE__ */ new Set();
   const toolSequence = [];
@@ -19489,11 +19821,49 @@ async function startInteractive(config, repoPath) {
     }
   }).catch(() => {
   });
+  let resolvedCaps = {
+    vision: false,
+    toolUse: false,
+    thinking: false
+  };
+  queryModelCapabilities(config.backendUrl, config.model).then((caps) => {
+    resolvedCaps = caps;
+    statusBar.setCapabilities(caps);
+  }).catch(() => {
+  });
   const provider = detectProvider(config.backendUrl);
   const costTracker = new CostTracker(provider.id);
   const sessionMetrics = new SessionMetrics();
   const workEvaluator = new WorkEvaluator();
   ensureTranscribeCliBackground();
+  let depSudoResolver = null;
+  let depSudoPromptPending = false;
+  ensureVisionDeps((msg) => {
+    if (statusBar?.isActive) {
+      statusBar.beginContentWrite();
+      renderInfo(msg);
+      statusBar.endContentWrite();
+    }
+  }, () => new Promise((resolve19) => {
+    depSudoPromptPending = true;
+    depSudoResolver = (pw) => {
+      depSudoPromptPending = false;
+      depSudoResolver = null;
+      if (pw)
+        sessionSudoPassword = pw;
+      resolve19(pw);
+    };
+    if (statusBar?.isActive) {
+      statusBar.beginContentWrite();
+    }
+    process.stdout.write(`  ${c2.bold(c2.yellow("\u{1F511} Password needed for dependency install:"))}
+`);
+    process.stdout.write(`  ${c2.bold(c2.yellow("\u{1F511} Password:"))} `);
+    if (statusBar?.isActive) {
+      statusBar.endContentWrite();
+    }
+  })).catch(() => {
+  });
   const voiceEngine = new VoiceEngine();
   const streamRenderer = new StreamRenderer();
   if (savedSettings.voice) {
@@ -19791,6 +20161,10 @@ async function startInteractive(config, repoPath) {
       resolvedContextWindowSize = size;
       statusBar.setContextWindowSize(size);
     },
+    setCapabilities: (caps) => {
+      resolvedCaps = caps;
+      statusBar.setCapabilities(caps);
+    },
     hasActiveTask: () => activeTask !== null,
     abortTask() {
       if (!activeTask)
@@ -19885,6 +20259,14 @@ async function startInteractive(config, repoPath) {
     }, 50);
   });
   async function processLine(input) {
+    if (depSudoPromptPending && depSudoResolver) {
+      const pw = input.trim();
+      process.stdout.write(`\r\x1B[K  ${c2.dim("\u{1F511} Password received")}
+`);
+      depSudoResolver(pw || null);
+      showPrompt();
+      return;
+    }
     if (sudoPromptPending && activeTask) {
       sudoPromptPending = false;
       sessionSudoPassword = input;
@@ -19946,7 +20328,7 @@ Execute this skill now. Follow the behavioral guidance above.`;
             toolPatternStore: toolPatternStore ?? void 0
           }, bruteForceEnabled, statusBar, handleSudoRequest, costTracker, (summary) => {
             lastCompletedSummary = summary;
-          }, currentTaskType, resolvedContextWindowSize);
+          }, currentTaskType, resolvedContextWindowSize, resolvedCaps);
           activeTask = task;
           showPrompt();
           await task.promise;
@@ -20048,7 +20430,7 @@ Summarize or analyze this transcription as appropriate.`;
         toolPatternStore: toolPatternStore ?? void 0
       }, bruteForceEnabled, statusBar, handleSudoRequest, costTracker, (summary) => {
         lastCompletedSummary = summary;
-      }, currentTaskType, resolvedContextWindowSize);
+      }, currentTaskType, resolvedContextWindowSize, resolvedCaps);
       activeTask = task;
       showPrompt();
       await task.promise;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.26.0",
+  "version": "0.28.0",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",