@holoscript/holoscript-agent 2.1.3 → 2.1.5

package/dist/index.js

@@ -1082,13 +1082,13 @@ var MESH_TOOLS = [
   },
   {
     name: "vision_analyze",
-    description: "Analyze an image using the local Fara-7B vision model (Ollama on loopback). Reads the image file at `image_path`, sends it to fara:7b via the local Ollama API, and returns the model's text analysis. Counts as a productive tool call \u2014 use for GUI-grounding, visual QA, image captioning, or any task that requires perceiving image content. Only available on surfaces with a local Ollama instance running fara:7b.",
+    description: "Analyze an image using the local Fara-7B vision model (Ollama on loopback). Reads the image file at `image_path` (max 512KB \u2014 downscale larger images first), sends it to the vision model via the local Ollama API (env: HOLOSCRIPT_AGENT_VISION_MODEL), and returns the model's text analysis. Counts as a productive tool call \u2014 use for GUI-grounding, visual QA, image captioning, or any task that requires perceiving image content. Only available on surfaces with a local Ollama instance and HOLOSCRIPT_AGENT_LOCAL_LLM_BASE_URL set.",
     input_schema: {
       type: "object",
       properties: {
         image_path: {
           type: "string",
-          description: "Absolute path to the image file (png, jpg, webp, gif)"
+          description: "Absolute path to the image file (png, jpg, webp) \u2014 must be under 512KB"
         },
         prompt: {
           type: "string",
@@ -1096,7 +1096,7 @@ var MESH_TOOLS = [
         },
         model: {
           type: "string",
-          description: 'Ollama model tag to use (default: "fara:7b")'
+          description: "Ollama model tag override (default: HOLOSCRIPT_AGENT_VISION_MODEL env var)"
         }
       },
       required: ["image_path"]
@@ -1350,19 +1350,29 @@ ${truncated}`);
       const denied = checkReadAllowed(imagePath);
       if (denied) return errResult(use.id, `vision_analyze: ${denied}`);
       const prompt = String(use.input.prompt ?? "Describe this image in detail.");
-      const model = String(use.input.model ?? "fara:7b");
+      const model = String(
+        use.input.model ?? process.env.HOLOSCRIPT_AGENT_VISION_MODEL ?? "fara:7b"
+      );
       const ollamaBase = process.env.HOLOSCRIPT_AGENT_LOCAL_LLM_BASE_URL;
       if (!ollamaBase) {
         return errResult(
           use.id,
-          "vision_analyze: HOLOSCRIPT_AGENT_LOCAL_LLM_BASE_URL is not set \u2014 configure it to point to your local Ollama instance (e.g. http://holojetson.local:11434)"
+          "vision_analyze: HOLOSCRIPT_AGENT_LOCAL_LLM_BASE_URL is not set \u2014 configure it to point to your local Ollama instance"
         );
       }
+      const MAX_IMAGE_BYTES = 512e3;
       const TIMEOUT_MS = 12e4;
       const controller = new AbortController();
       const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
       try {
         const imageBytes = await readFile3(imagePath);
+        if (imageBytes.length > MAX_IMAGE_BYTES) {
+          clearTimeout(timer);
+          return errResult(
+            use.id,
+            `vision_analyze: image is ${Math.round(imageBytes.length / 1024)}KB \u2014 exceeds ${MAX_IMAGE_BYTES / 1024}KB limit. Downscale the image first (e.g. to 256\xD7256 or smaller) then retry vision_analyze.`
+          );
+        }
         const imageB64 = imageBytes.toString("base64");
         const res = await fetch(`${ollamaBase}/api/generate`, {
           method: "POST",
@@ -1837,6 +1847,44 @@ Call write_file NOW. Embed ALL data from the tool result above into the content.
       finalText = reResp.content;
       lastResponse = reResp;
     }
+    const WRITE_NAMES = /* @__PURE__ */ new Set(["write_file", "str_replace"]);
+    if (toolsCalled.has("vision_analyze") && ![...toolsCalled].some((n) => WRITE_NAMES.has(n)) && iters < MAX_TOOL_ITERS) {
+      iters++;
+      if (messages.length > 0 && messages[messages.length - 1].role === "assistant") {
+        messages.pop();
+      }
+      messages.push({
+        role: "user",
+        content: `vision_analyze returned a caption but you did NOT call write_file.
+Task: ${target.title}
+Output path: ${target.description.match(/path[:\s]+([^\s\n,]+\.json)/i)?.[1] ?? "see task description"}
+Call write_file NOW. Put the caption from vision_analyze into the JSON content field. Do NOT output text \u2014 your ONLY valid response is a write_file tool call.`
+      });
+      const vwResp = await provider.complete(
+        { messages, maxTokens: 8192, temperature: 0, tools: activeTools },
+        identity.llmModel
+      );
+      aggUsage = {
+        promptTokens: aggUsage.promptTokens + vwResp.usage.promptTokens,
+        completionTokens: aggUsage.completionTokens + vwResp.usage.completionTokens,
+        totalTokens: aggUsage.totalTokens + vwResp.usage.totalTokens
+      };
+      if (vwResp.finishReason === "tool_use" && vwResp.toolUses && vwResp.toolUses.length > 0) {
+        log({ ev: "vision-write-call", taskId: target.id, iter: iters, tools: vwResp.toolUses.map((t) => t.name) });
+        const vwProd = summarizeToolProductivity(vwResp.toolUses);
+        for (const n of vwProd.names) toolsCalled.add(n);
+        productiveCallCount += vwProd.productiveCount;
+        messages.push({ role: "assistant", content: vwResp.assistantBlocks ?? [] });
+        const vwResults = await Promise.all(
+          vwResp.toolUses.map(
+            (u) => runTool(u, { signReceipt: this.opts.signReceipt, addTask: (tasks2) => mesh.addTasks(tasks2) })
+          )
+        );
+        messages.push({ role: "user", content: vwResults });
+      }
+      finalText = vwResp.content;
+      lastResponse = vwResp;
+    }
     const durationMs = Date.now() - start;
     if (productiveCallCount === 0) {
       log({