@holoscript/holoscript-agent 2.1.1 → 2.1.3

package/dist/index.js

@@ -941,6 +941,8 @@ function isProductiveToolUse(use) {
       return true;
     case "str_replace":
       return true;
+    case "vision_analyze":
+      return true;
     default:
       return false;
   }
@@ -1077,6 +1079,28 @@ var MESH_TOOLS = [
       },
       required: ["title"]
     }
+  },
+  {
+    name: "vision_analyze",
+    description: "Analyze an image using the local Fara-7B vision model (Ollama on loopback). Reads the image file at `image_path`, sends it to fara:7b via the local Ollama API, and returns the model's text analysis. Counts as a productive tool call \u2014 use for GUI-grounding, visual QA, image captioning, or any task that requires perceiving image content. Only available on surfaces with a local Ollama instance running fara:7b.",
+    input_schema: {
+      type: "object",
+      properties: {
+        image_path: {
+          type: "string",
+          description: "Absolute path to the image file (png, jpg, webp, gif)"
+        },
+        prompt: {
+          type: "string",
+          description: 'Instruction for the vision model (default: "Describe this image in detail.")'
+        },
+        model: {
+          type: "string",
+          description: 'Ollama model tag to use (default: "fara:7b")'
+        }
+      },
+      required: ["image_path"]
+    }
   }
 ];
 function resolveActiveTools(brain, opts = {}) {
@@ -1320,6 +1344,49 @@ ${truncated}`);
       const result = await opts.addTask([{ title, description, priority: priority2, source, tags }]);
       return okResult(use.id, `delegate_task: posted "${title}" to board \u2014 ${result.added} task(s) added`);
     }
+    if (use.name === "vision_analyze") {
+      const imagePath = String(use.input.image_path ?? "").trim();
+      if (!imagePath) return errResult(use.id, "vision_analyze: image_path is required");
+      const denied = checkReadAllowed(imagePath);
+      if (denied) return errResult(use.id, `vision_analyze: ${denied}`);
+      const prompt = String(use.input.prompt ?? "Describe this image in detail.");
+      const model = String(use.input.model ?? "fara:7b");
+      const ollamaBase = process.env.HOLOSCRIPT_AGENT_LOCAL_LLM_BASE_URL;
+      if (!ollamaBase) {
+        return errResult(
+          use.id,
+          "vision_analyze: HOLOSCRIPT_AGENT_LOCAL_LLM_BASE_URL is not set \u2014 configure it to point to your local Ollama instance (e.g. http://holojetson.local:11434)"
+        );
+      }
+      const TIMEOUT_MS = 12e4;
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
+      try {
+        const imageBytes = await readFile3(imagePath);
+        const imageB64 = imageBytes.toString("base64");
+        const res = await fetch(`${ollamaBase}/api/generate`, {
+          method: "POST",
+          headers: { "content-type": "application/json" },
+          body: JSON.stringify({ model, prompt, images: [imageB64], stream: false }),
+          signal: controller.signal
+        });
+        clearTimeout(timer);
+        if (!res.ok) {
+          const text = await res.text();
+          return errResult(use.id, `vision_analyze: Ollama HTTP ${res.status}: ${text.slice(0, 500)}`);
+        }
+        const json = await res.json();
+        if (json.error) return errResult(use.id, `vision_analyze: model error \u2014 ${json.error}`);
+        return okResult(use.id, json.response ?? "");
+      } catch (err) {
+        clearTimeout(timer);
+        const msg = err instanceof Error ? err.message : String(err);
+        return errResult(
+          use.id,
+          msg.includes("abort") ? `vision_analyze: timed out after ${TIMEOUT_MS}ms` : `vision_analyze: ${msg}`
+        );
+      }
+    }
     return errResult(use.id, `unknown tool: ${use.name}`);
   } catch (err) {
     return errResult(use.id, err instanceof Error ? err.message : String(err));
@@ -1735,12 +1802,18 @@ var AgentRunner = class {
     }
     if (productiveCallCount === 0 && toolsCalled.size > 0 && iters < MAX_TOOL_ITERS) {
       iters++;
+      if (messages.length > 0 && messages[messages.length - 1].role === "assistant") {
+        messages.pop();
+      }
       messages.push({
         role: "user",
-        content: "You gathered data but did not write the task deliverable. Call write_file NOW with the exact output path from the task description. Embed all data you gathered into the write_file content field. Do NOT output text \u2014 your only valid response is a write_file tool call."
+        content: `You read data but did NOT call write_file. This is a TASK FAILURE unless you act now.
+Task: ${target.title}
+Required output path (from task description): ${target.description.match(/path[:\s]+([^\s\n,]+\.json)/i)?.[1] ?? "see task description"}
+Call write_file NOW. Embed ALL data from the tool result above into the content. Do NOT output any text \u2014 your ONLY valid response is a write_file tool call.`
       });
       const reResp = await provider.complete(
-        { messages, maxTokens: 8192, temperature: 0.4, tools: activeTools },
+        { messages, maxTokens: 8192, temperature: 0, tools: activeTools },
         identity.llmModel
       );
       aggUsage = {