micro-models-agent 1.6.0 → 1.7.0

package/CHANGELOG.md

@@ -4,6 +4,22 @@ All notable changes to Micro Models Agent (MMA) will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/).
 
+## [1.7.0] - 2026-09-22
+
+### Added
+
+- **Model capability resolution** (`src/modules/providers/capabilities.ts`, `src/config/types.ts`): a single source of truth for a model's `vision`/`tools` abilities, merged from user override (`config.modelCapabilities[model]`) → live native probe (LM Studio `/api/v1/models` `capabilities`, read by `probeModel` in `src/llm/model-loader.ts`) → provider preset. Exposed to tools via the live `ToolContext.capabilities` getter.
+- **First-class tool image channel** (`src/tools/types.ts` `ToolResult.images`, `src/core/agent/tool-images.ts`): tools can return images; the agent loop queues them onto the next user message (`queueToolImages`), keeping `role:"tool"` content text-only and portable.
+
+### Fixed
+
+- **Vision misdetection for non-`-vl` models** (`src/tools/browser.ts`): the `qwen3.5-9b`-style models report `vision: true` in LM Studio but the name-based `modelSupportsVision` heuristic returned `false`, so screenshots were silently downgraded to a text notice. Name matching is removed; the browser tool now reads `ctx.capabilities.vision`.
+- **Browser screenshots never reached the model** (`src/tools/browser.ts`): screenshots were embedded as base64 text inside the `role:"tool"` output, which backends never decode and the budget truncates. They are now delivered as `image_url` parts; text-only models get the snapshot instead.
+
+### Changed
+
+- **Browser module prompt** (`src/modules/browser/module.ts`): no longer asks the model to self-assess vision support — screenshot delivery is automatic.
+
 ## [1.6.0] - 2026-09-22
 
 ### Added

package/dist/main.js

@@ -2312,6 +2312,7 @@ var init_defaults = __esm(() => {
         description: "browser + file reading for visual tasks"
       }
     },
+    modelCapabilities: {},
     contextWindow: 32768,
     contextBudget: {
       systemPrompt: 0.1,
@@ -5118,7 +5119,7 @@ var init_domains = __esm(() => {
       "reasoning",
       "hallucination"
     ],
-    provider: ["provider", "orchestrator", "retry", "pricing"],
+    provider: ["provider", "orchestrator", "retry", "pricing", "modelCapabilities"],
     moe: ["moe", "experts"],
     security: ["security"],
     browser: ["browser"],
@@ -7547,7 +7548,7 @@ var init_factory = __esm(() => {
 });
 
 // src/llm/model-loader.ts
-async function getLoadedContextLength(baseUrl, model, timeoutMs = 3000) {
+async function probeModel(baseUrl, model, timeoutMs = 3000) {
   const base = baseUrl.replace(/\/$/, "").replace(/\/v1\/?$/, "");
   try {
     const response = await fetch(`${base}/api/v1/models`, {
@@ -7561,10 +7562,18 @@ async function getLoadedContextLength(baseUrl, model, timeoutMs = 3000) {
     if (!found)
       return null;
     const loadedInstance = (found.loaded_instances ?? [])[0];
-    const actual = loadedInstance?.config?.context_length ?? found.max_context_length;
-    if (typeof actual !== "number" || actual <= 0)
+    const contextLength = loadedInstance?.config?.context_length ?? found.max_context_length;
+    if (typeof contextLength !== "number" || contextLength <= 0)
       return null;
-    return { model: found.key ?? found.name ?? model, actual };
+    const raw = found.capabilities;
+    let capabilities;
+    if (raw && typeof raw.vision === "boolean") {
+      capabilities = { vision: raw.vision };
+      if (typeof raw.trained_for_tool_use === "boolean") {
+        capabilities.tools = raw.trained_for_tool_use;
+      }
+    }
+    return { model: found.key ?? found.name ?? model, contextLength, capabilities };
   } catch {
     return null;
   }
@@ -15436,6 +15445,15 @@ var init_tool_output = __esm(() => {
   init_i18n();
 });
 
+// src/core/agent/tool-images.ts
+function queueToolImages(sink, images) {
+  if (!sink || !images || images.length === 0)
+    return;
+  for (const image of images) {
+    sink.addPendingImage(image);
+  }
+}
+
 // src/core/agent/tool-batch.ts
 class ToolBatchExecutor {
   deps;
@@ -15565,6 +15583,7 @@ class ToolBatchExecutor {
         arguments: call.arguments
       });
       answeredToolCallIds.add(call.id);
+      queueToolImages(contextManager, result.images);
       const tokensAfterTool = contextManager.getEstimatedTokens();
       const costAfterTool = this.deps.costTracker.total;
       onTool?.({
@@ -21290,6 +21309,119 @@ var init_session = __esm(() => {
   init_i18n();
 });
 
+// src/llm/image-utils.ts
+var exports_image_utils = {};
+__export(exports_image_utils, {
+  readClipboardImage: () => readClipboardImage,
+  loadUrlAsDataUrl: () => loadUrlAsDataUrl,
+  loadFileAsDataUrl: () => loadFileAsDataUrl,
+  detectMime: () => detectMime,
+  bufferToDataUrl: () => bufferToDataUrl
+});
+import { readFileSync as readFileSync20 } from "fs";
+import { extname as extname4 } from "path";
+function detectMime(filePath) {
+  const ext = extname4(filePath).toLowerCase();
+  return MIME_MAP[ext] ?? "image/png";
+}
+async function readClipboardImage() {
+  if (typeof Bun !== "undefined" && typeof Bun.Image !== "undefined") {
+    try {
+      const img = Bun.Image.fromClipboard();
+      if (img) {
+        const buf = await img.resize(800, 800, { fit: "inside" }).jpeg({ quality: 60 }).buffer();
+        return Buffer.from(buf);
+      }
+    } catch {}
+  }
+  return readClipboardFallback();
+}
+async function readClipboardFallback() {
+  const { platform: platform7 } = await import("os");
+  const { execSync } = await import("child_process");
+  const { readFileSync: readFileSync21, unlinkSync: unlinkSync5 } = await import("fs");
+  const { join: join29 } = await import("path");
+  if (platform7() !== "linux")
+    return null;
+  const tmpPath = join29(process.env.TEMP || process.env.TMP || "/tmp", `mma-clip-${Date.now()}.png`);
+  const commands = [
+    `wl-paste --type image/png > "${tmpPath}" 2>/dev/null`,
+    `xclip -selection clipboard -t image/png -o > "${tmpPath}" 2>/dev/null`
+  ];
+  for (const cmd of commands) {
+    try {
+      execSync(cmd, { timeout: 5000 });
+      const buf = readFileSync21(tmpPath);
+      unlinkSync5(tmpPath);
+      if (buf.length > 0)
+        return buf;
+    } catch {}
+  }
+  try {
+    unlinkSync5(tmpPath);
+  } catch {}
+  return null;
+}
+async function loadFileAsDataUrl(filePath) {
+  const buf = readFileSync20(filePath);
+  if (typeof Bun !== "undefined" && typeof Bun.Image !== "undefined") {
+    try {
+      const img = new Bun.Image(buf);
+      const { width } = await img.metadata();
+      const targetWidth = Math.min(800, width || 800);
+      const dataUrl = await img.resize(targetWidth, 800, { fit: "inside" }).jpeg({ quality: 60 }).dataurl();
+      return { dataUrl };
+    } catch {}
+  }
+  const b64 = buf.toString("base64");
+  const ext = filePath.split(".").pop()?.toLowerCase();
+  const mime = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : ext === "webp" ? "image/webp" : "image/png";
+  return { dataUrl: `data:${mime};base64,${b64}` };
+}
+async function loadUrlAsDataUrl(url) {
+  const resp = await fetch(url);
+  if (!resp.ok) {
+    throw new Error(`Failed to fetch image: ${resp.status} ${resp.statusText}`);
+  }
+  const buf = Buffer.from(await resp.arrayBuffer());
+  if (typeof Bun !== "undefined" && typeof Bun.Image !== "undefined") {
+    try {
+      const img = new Bun.Image(buf);
+      const { width } = await img.metadata();
+      const targetWidth = Math.min(800, width || 800);
+      const dataUrl = await img.resize(targetWidth, 800, { fit: "inside" }).jpeg({ quality: 60 }).dataurl();
+      return { dataUrl };
+    } catch {}
+  }
+  const b64 = buf.toString("base64");
+  return { dataUrl: `data:image/png;base64,${b64}` };
+}
+async function bufferToDataUrl(buf) {
+  if (typeof Bun !== "undefined" && typeof Bun.Image !== "undefined") {
+    try {
+      const img = new Bun.Image(buf);
+      const { width } = await img.metadata();
+      const targetWidth = Math.min(800, width || 800);
+      const dataUrl = await img.resize(targetWidth, 800, { fit: "inside" }).jpeg({ quality: 60 }).dataurl();
+      return { dataUrl };
+    } catch {}
+  }
+  const b64 = buf.toString("base64");
+  return { dataUrl: `data:image/png;base64,${b64}` };
+}
+var MIME_MAP;
+var init_image_utils = __esm(() => {
+  MIME_MAP = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".bmp": "image/bmp",
+    ".svg": "image/svg+xml"
+  };
+});
+
 // src/tools/browser.ts
 import { join as join29 } from "path";
 import { mkdirSync as mkdirSync16 } from "fs";
@@ -21326,18 +21458,9 @@ function formatScreenshotForTextModel(snapshot) {
 
 ${snapshot}`;
 }
-function modelSupportsVision(model) {
-  const m = model.toLowerCase();
-  return /vision|\bvl\b|qwen.*vl|gpt-4|claude|gemini|llava|pixtral/.test(m);
-}
-function formatScreenshotResult(snapshot, screenshot, supportsVision) {
-  if (!supportsVision) {
-    return formatScreenshotForTextModel(snapshot);
-  }
-  const base642 = screenshot.toString("base64");
-  return `${snapshot}
-
-[Screenshot: data:image/png;base64,${base642}]`;
+async function screenshotToImagePart(screenshot) {
+  const { dataUrl } = await bufferToDataUrl(screenshot);
+  return { type: "image_url", image_url: { url: dataUrl } };
 }
 function createBrowserTool() {
   return {
@@ -21402,12 +21525,15 @@ function createBrowserTool() {
       }
       const s = getSession(ctx);
       const result = await serialize(() => s.execute(action, args));
-      const supportsVision = modelSupportsVision(ctx.config.model);
       if (result.screenshot) {
-        return {
-          success: result.success,
-          output: formatScreenshotResult(result.output, result.screenshot, supportsVision)
-        };
+        if (ctx.capabilities?.vision) {
+          return {
+            success: result.success,
+            output: result.output,
+            images: [await screenshotToImagePart(result.screenshot)]
+          };
+        }
+        return { success: result.success, output: formatScreenshotForTextModel(result.output) };
       }
       return { success: result.success, output: result.output };
     }
@@ -21426,125 +21552,13 @@ var session = null, sessionFactory, actionChain;
 var init_browser = __esm(() => {
   init_session();
   init_types();
+  init_image_utils();
   init_i18n();
   init_output();
   init_utils();
   actionChain = Promise.resolve();
 });
 
-// src/llm/image-utils.ts
-var exports_image_utils = {};
-__export(exports_image_utils, {
-  readClipboardImage: () => readClipboardImage,
-  loadUrlAsDataUrl: () => loadUrlAsDataUrl,
-  loadFileAsDataUrl: () => loadFileAsDataUrl,
-  detectMime: () => detectMime,
-  bufferToDataUrl: () => bufferToDataUrl
-});
-import { readFileSync as readFileSync20 } from "fs";
-import { extname as extname4 } from "path";
-function detectMime(filePath) {
-  const ext = extname4(filePath).toLowerCase();
-  return MIME_MAP[ext] ?? "image/png";
-}
-async function readClipboardImage() {
-  if (typeof Bun !== "undefined" && typeof Bun.Image !== "undefined") {
-    try {
-      const img = Bun.Image.fromClipboard();
-      if (img) {
-        const buf = await img.resize(800, 800, { fit: "inside" }).jpeg({ quality: 60 }).buffer();
-        return Buffer.from(buf);
-      }
-    } catch {}
-  }
-  return readClipboardFallback();
-}
-async function readClipboardFallback() {
-  const { platform: platform7 } = await import("os");
-  const { execSync } = await import("child_process");
-  const { readFileSync: readFileSync21, unlinkSync: unlinkSync5 } = await import("fs");
-  const { join: join30 } = await import("path");
-  if (platform7() !== "linux")
-    return null;
-  const tmpPath = join30(process.env.TEMP || process.env.TMP || "/tmp", `mma-clip-${Date.now()}.png`);
-  const commands = [
-    `wl-paste --type image/png > "${tmpPath}" 2>/dev/null`,
-    `xclip -selection clipboard -t image/png -o > "${tmpPath}" 2>/dev/null`
-  ];
-  for (const cmd of commands) {
-    try {
-      execSync(cmd, { timeout: 5000 });
-      const buf = readFileSync21(tmpPath);
-      unlinkSync5(tmpPath);
-      if (buf.length > 0)
-        return buf;
-    } catch {}
-  }
-  try {
-    unlinkSync5(tmpPath);
-  } catch {}
-  return null;
-}
-async function loadFileAsDataUrl(filePath) {
-  const buf = readFileSync20(filePath);
-  if (typeof Bun !== "undefined" && typeof Bun.Image !== "undefined") {
-    try {
-      const img = new Bun.Image(buf);
-      const { width } = await img.metadata();
-      const targetWidth = Math.min(800, width || 800);
-      const dataUrl = await img.resize(targetWidth, 800, { fit: "inside" }).jpeg({ quality: 60 }).dataurl();
-      return { dataUrl };
-    } catch {}
-  }
-  const b64 = buf.toString("base64");
-  const ext = filePath.split(".").pop()?.toLowerCase();
-  const mime = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : ext === "webp" ? "image/webp" : "image/png";
-  return { dataUrl: `data:${mime};base64,${b64}` };
-}
-async function loadUrlAsDataUrl(url) {
-  const resp = await fetch(url);
-  if (!resp.ok) {
-    throw new Error(`Failed to fetch image: ${resp.status} ${resp.statusText}`);
-  }
-  const buf = Buffer.from(await resp.arrayBuffer());
-  if (typeof Bun !== "undefined" && typeof Bun.Image !== "undefined") {
-    try {
-      const img = new Bun.Image(buf);
-      const { width } = await img.metadata();
-      const targetWidth = Math.min(800, width || 800);
-      const dataUrl = await img.resize(targetWidth, 800, { fit: "inside" }).jpeg({ quality: 60 }).dataurl();
-      return { dataUrl };
-    } catch {}
-  }
-  const b64 = buf.toString("base64");
-  return { dataUrl: `data:image/png;base64,${b64}` };
-}
-async function bufferToDataUrl(buf) {
-  if (typeof Bun !== "undefined" && typeof Bun.Image !== "undefined") {
-    try {
-      const img = new Bun.Image(buf);
-      const { width } = await img.metadata();
-      const targetWidth = Math.min(800, width || 800);
-      const dataUrl = await img.resize(targetWidth, 800, { fit: "inside" }).jpeg({ quality: 60 }).dataurl();
-      return { dataUrl };
-    } catch {}
-  }
-  const b64 = buf.toString("base64");
-  return { dataUrl: `data:image/png;base64,${b64}` };
-}
-var MIME_MAP;
-var init_image_utils = __esm(() => {
-  MIME_MAP = {
-    ".png": "image/png",
-    ".jpg": "image/jpeg",
-    ".jpeg": "image/jpeg",
-    ".gif": "image/gif",
-    ".webp": "image/webp",
-    ".bmp": "image/bmp",
-    ".svg": "image/svg+xml"
-  };
-});
-
 // src/tools/attach-image.ts
 import { existsSync as existsSync35 } from "fs";
 import { resolve as resolve17 } from "path";
@@ -25291,7 +25305,7 @@ class BrowserModule {
         "Workflow: open(url) → read snapshot → click/type on element numbers → read updated snapshot.",
         'Each snapshot includes a "Content" section with the visible page text, a "Console" section with browser messages, and network errors. Use these to understand the page, detect JS errors, and verify the result of your actions.',
         'Always start with "open" action. Use element numbers from snapshot for click/type targets.',
-        'Use "screenshot" action only if the model supports vision. Otherwise rely on the text snapshot.'
+        'The "screenshot" action is delivered automatically: a vision model receives the image, a text-only model receives the text snapshot instead. You do not need to decide — just call it and read what you get.'
       ].join(`
 `),
       priority: "normal",
@@ -27704,6 +27718,19 @@ var init_context_log = __esm(() => {
   SEP2 = "─".repeat(78);
 });
 
+// src/modules/providers/capabilities.ts
+function resolveModelCapabilities(preset, probe, override) {
+  const base = preset ? { vision: preset.vision, tools: preset.tools } : { ...DEFAULT_MODEL_CAPABILITIES };
+  return { ...base, ...probe ?? {}, ...override ?? {} };
+}
+var DEFAULT_MODEL_CAPABILITIES;
+var init_capabilities = __esm(() => {
+  DEFAULT_MODEL_CAPABILITIES = {
+    vision: false,
+    tools: false
+  };
+});
+
 // src/core/environment.ts
 import { existsSync as existsSync50, readFileSync as readFileSync32, readdirSync as readdirSync16 } from "fs";
 import { spawnSync as spawnSync3 } from "child_process";
@@ -28335,7 +28362,9 @@ function buildSystemInfo(config, baseDir, profileCompressed) {
   lines.push(`Runtime verification: when a bug is described in terms of runtime behavior (what happens when the app runs), reproduce it — run the app / build / browser tool — and confirm the fix against actual runtime output before calling "plan update step=N status=done". Check existing servers first with process_list/process_log: a stale dev server started BEFORE your change still runs the old pipeline and keeps failing even after the files on disk are fixed. Restart or kill it (process_kill) and verify the exact endpoint/port the user reported — a fresh server on a new port does NOT prove the user's error is gone.`);
   const sec = config.security;
   if (sec?.enabled) {
-    const secLines = [`SECURITY RESTRICTIONS (enforced — blocked calls return "[SECURITY BLOCKED]", do not retry them):`];
+    const secLines = [
+      `SECURITY RESTRICTIONS (enforced — blocked calls return "[SECURITY BLOCKED]", do not retry them):`
+    ];
     if (sec.bash?.enabled && sec.bash.blacklist.length > 0) {
       const cmds = sec.bash.blacklist.slice(0, 15).join(", ");
       secLines.push(`Forbidden bash commands: ${cmds}${sec.bash.blacklist.length > 15 ? ", …" : ""}.`);
@@ -28417,19 +28446,23 @@ async function bootstrap(configDir, projectDir, noAgentsMd, exitOnComplete, reas
       logger4.info(t("env.model_loaded", { model: config.model, seconds: String(loadResult.loadTime) }));
     }
   }
+  const capabilitiesRef = {};
   const contextProbePromise = (async () => {
     try {
-      const probe = await getLoadedContextLength(config.provider.baseUrl, config.model);
+      const probe = await probeModel(config.provider.baseUrl, config.model);
       if (!probe)
         return null;
-      if (probe.actual < config.contextWindow) {
-        logger4.logSilent("warn", `Context probe: model "${probe.model}" is loaded with ${probe.actual} tokens, but contextWindow is configured as ${config.contextWindow} — overflow/429 risk. Lower contextWindow or reload the model with a larger context.`);
-      } else if (probe.actual > config.contextWindow) {
-        logger4.logSilent("info", `Context probe: model "${probe.model}" supports ${probe.actual} tokens — consider "mma context ${probe.actual}" to use the full window.`);
+      if (probe.capabilities)
+        capabilitiesRef.probe = probe.capabilities;
+      const contextProbe = { model: probe.model, actual: probe.contextLength };
+      if (probe.contextLength < config.contextWindow) {
+        logger4.logSilent("warn", `Context probe: model "${probe.model}" is loaded with ${probe.contextLength} tokens, but contextWindow is configured as ${config.contextWindow} — overflow/429 risk. Lower contextWindow or reload the model with a larger context.`);
+      } else if (probe.contextLength > config.contextWindow) {
+        logger4.logSilent("info", `Context probe: model "${probe.model}" supports ${probe.contextLength} tokens — consider "mma context ${probe.contextLength}" to use the full window.`);
       } else {
-        logger4.logSilent("debug", `Context probe: configured contextWindow matches loaded context (${probe.actual})`);
+        logger4.logSilent("debug", `Context probe: configured contextWindow matches loaded context (${probe.contextLength})`);
       }
-      return probe;
+      return contextProbe;
     } catch {
       return null;
     }
@@ -28608,6 +28641,14 @@ async function bootstrap(configDir, projectDir, noAgentsMd, exitOnComplete, reas
     get: () => sessionManager,
     configurable: true
   });
+  Object.defineProperty(toolCtx, "capabilities", {
+    get: () => {
+      const cfg = configRef.current;
+      const spec = BUILTIN_PROVIDERS.find((p) => p.type === cfg.provider.type);
+      return resolveModelCapabilities(spec?.capabilities, capabilitiesRef.probe, cfg.modelCapabilities?.[cfg.model]);
+    },
+    configurable: true
+  });
   const toolExecutor = new ToolExecutor(toolRegistry, toolCtx, pluginManager);
   toolCtx.llmProvider = llmProvider;
   toolCtx.toolExecutor = toolExecutor;
@@ -28831,6 +28872,7 @@ var init_bootstrap = __esm(() => {
   init_i18n();
   init_agent();
   init_context_log();
+  init_capabilities();
   init_version();
   init_environment();
   init_utils();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "micro-models-agent",
-  "version": "1.6.0",
+  "version": "1.7.0",
   "description": "Micro Models Agent (MMA) — LLM agent harness for small models (Qwen3.5-9B, 32K-64K context)",
   "type": "module",
   "bin": {