open-agents-ai 0.22.3 → 0.23.1

package/dist/index.js

@@ -6961,20 +6961,45 @@ var init_desktop_click = __esm({
               durationMs: performance.now() - start
             };
           }
-          const { vl } = await import("moondream");
-          const apiKey = process.env["MOONDREAM_API_KEY"];
-          const endpoint = process.env["MOONDREAM_ENDPOINT"];
-          let client;
-          if (apiKey) {
-            client = new vl({ apiKey });
-          } else if (endpoint) {
-            client = new vl({ endpoint });
-          } else {
-            client = new vl({ endpoint: "http://localhost:2020/v1" });
+          let points = [];
+          let visionWorked = false;
+          try {
+            const { vl } = await import("moondream");
+            const apiKey = process.env["MOONDREAM_API_KEY"];
+            const endpoint = process.env["MOONDREAM_ENDPOINT"];
+            let client;
+            if (apiKey) {
+              client = new vl({ apiKey });
+            } else if (endpoint) {
+              client = new vl({ endpoint });
+            } else {
+              client = new vl({ endpoint: "http://localhost:2020/v1" });
+            }
+            const imageBuffer = readFileSync11(screenshotPath);
+            const pointResult = await client.point({ image: imageBuffer, object: target });
+            points = pointResult.points ?? [];
+            visionWorked = true;
+          } catch {
+          }
+          if (!visionWorked) {
+            const hints = [
+              `(Moondream vision not available \u2014 cannot locate "${target}" on screen)`,
+              `Screenshot saved: ${screenshotPath}`,
+              `Screen: ${dims.width}x${dims.height}`,
+              "",
+              "Use shell commands to interact with the desktop instead:",
+              "  xdotool search --name 'pattern' \u2014 find windows by title",
+              "  xdotool key 'ctrl+s' \u2014 send keyboard shortcuts",
+              "  xdotool mousemove X Y click 1 \u2014 click at known coordinates",
+              "  wmctrl -a 'window title' \u2014 activate a window by name",
+              "  xdg-open <url> \u2014 open a URL in the default browser"
+            ];
+            return {
+              success: true,
+              output: hints.join("\n"),
+              durationMs: performance.now() - start
+            };
           }
-          const imageBuffer = readFileSync11(screenshotPath);
-          const pointResult = await client.point({ image: imageBuffer, object: target });
-          const points = pointResult.points ?? [];
           if (points.length === 0) {
             return {
               success: false,
@@ -7066,37 +7091,62 @@ Screenshot: ${screenshotPath}`,
           captureScreenshot(screenshotPath);
           const dims = getImageDimensions2(screenshotPath);
           const imageBuffer = readFileSync11(screenshotPath);
-          const { vl } = await import("moondream");
-          const apiKey = process.env["MOONDREAM_API_KEY"];
-          const endpoint = process.env["MOONDREAM_ENDPOINT"];
-          let client;
-          if (apiKey) {
-            client = new vl({ apiKey });
-          } else if (endpoint) {
-            client = new vl({ endpoint });
-          } else {
-            client = new vl({ endpoint: "http://localhost:2020/v1" });
-          }
           const parts = [];
-          if (question) {
-            const result = await client.query({ image: imageBuffer, question });
-            const answer = typeof result.answer === "string" ? result.answer : "(streaming not supported)";
-            parts.push(`Q: ${question}`);
-            parts.push(`A: ${answer}`);
-          } else {
-            const result = await client.caption({ image: imageBuffer, length });
-            const caption = typeof result.caption === "string" ? result.caption : "(streaming not supported)";
-            parts.push(`Desktop description:
+          let visionWorked = false;
+          try {
+            const { vl } = await import("moondream");
+            const apiKey = process.env["MOONDREAM_API_KEY"];
+            const endpoint = process.env["MOONDREAM_ENDPOINT"];
+            let client;
+            if (apiKey) {
+              client = new vl({ apiKey });
+            } else if (endpoint) {
+              client = new vl({ endpoint });
+            } else {
+              client = new vl({ endpoint: "http://localhost:2020/v1" });
+            }
+            if (question) {
+              const result = await client.query({ image: imageBuffer, question });
+              const answer = typeof result.answer === "string" ? result.answer : "(streaming not supported)";
+              parts.push(`Q: ${question}`);
+              parts.push(`A: ${answer}`);
+            } else {
+              const result = await client.caption({ image: imageBuffer, length });
+              const caption = typeof result.caption === "string" ? result.caption : "(streaming not supported)";
+              parts.push(`Desktop description:
 ${caption}`);
+            }
+            visionWorked = true;
+          } catch {
+          }
+          if (!visionWorked) {
+            let ocrText = "";
+            try {
+              ocrText = execSync11(`tesseract ${JSON.stringify(screenshotPath)} stdout 2>/dev/null`, {
+                encoding: "utf8",
+                timeout: 15e3
+              }).trim();
+            } catch {
+            }
+            if (ocrText) {
+              parts.push("(Moondream vision not available \u2014 using OCR text extraction)");
+              parts.push(`
+Visible text on screen:
+${ocrText}`);
+            } else {
+              parts.push("(Moondream vision not available, OCR failed)");
+              parts.push("Screenshot captured but cannot describe contents.");
+              parts.push("Use shell commands to check window state instead:");
+              parts.push("  xdotool getactivewindow getwindowname  \u2014 get active window title");
+              parts.push("  wmctrl -l  \u2014 list all open windows");
+              parts.push("  xdg-open <url>  \u2014 open a URL in the default browser");
+            }
           }
           if (dims) {
             parts.push(`
 Screen: ${dims.width}x${dims.height}`);
           }
           parts.push(`Screenshot: ${screenshotPath}`);
-          const base64 = imageBuffer.toString("base64");
-          parts.push(`
-[IMAGE_BASE64:image/png:${base64}]`);
           return {
             success: true,
             output: parts.join("\n"),
@@ -9851,7 +9901,17 @@ var init_ralphLoop = __esm({
 });
 
 // packages/orchestrator/dist/agenticRunner.js
-var SYSTEM_PROMPT, AgenticRunner, OllamaAgenticBackend;
+function getSystemPromptForTier(tier) {
+  switch (tier) {
+    case "small":
+      return SYSTEM_PROMPT_SMALL;
+    case "medium":
+      return SYSTEM_PROMPT_MEDIUM;
+    default:
+      return SYSTEM_PROMPT;
+  }
+}
+var SYSTEM_PROMPT, SYSTEM_PROMPT_MEDIUM, SYSTEM_PROMPT_SMALL, AgenticRunner, OllamaAgenticBackend;
 var init_agenticRunner = __esm({
   "packages/orchestrator/dist/agenticRunner.js"() {
     "use strict";
@@ -10051,6 +10111,63 @@ If you notice you're performing the SAME multi-step sequence for the 3rd time or
 - batch_edit to apply multiple edits across files in one call (reduces turns)
 - Focus on error messages in shell output \u2014 skip verbose build logs
 - Don't read files you don't need to modify`;
+    SYSTEM_PROMPT_MEDIUM = `You are Open Agent, an AI coding agent with access to the local machine. You can read/write files, execute shell commands, search the web, and interact with any software. You solve tasks by using tools iteratively until complete.
+
+## NEVER REFUSE \u2014 ALWAYS ATTEMPT
+
+NEVER say "I can't do that". ALWAYS attempt the task using your tools. If a tool fails, try a different approach.
+
+## Tools
+
+- file_read: Read file contents (always read before editing)
+- file_write: Create or overwrite a file
+- file_edit: Precise string replacement (preferred over rewriting). old_string must be unique.
+- file_patch: Edit specific line ranges in large files
+- find_files: Find files by glob pattern
+- grep_search: Search file contents with regex
+- shell: Execute any shell command (tests, builds, git, npm, etc.)
+- list_directory: List files in a directory
+- web_search: Search the web
+- web_fetch: Fetch a web page's text
+- memory_read / memory_write: Persistent memory across sessions
+- task_complete: Signal task completion
+- batch_edit: Multiple edits across files in one call
+- skill_list / skill_execute: Discover and load specialized skills (use on-demand)
+
+## Workflow
+
+1. EXPLORE: Use find_files, grep_search, file_read to understand the codebase
+2. IMPLEMENT: Make changes with file_edit (preferred) or file_write
+3. VALIDATE: Run tests/build with shell. Read FULL output.
+4. FIX: If validation fails, fix the specific issue and re-validate
+5. ITERATE: Repeat until all tests pass. Do NOT give up.
+6. COMPLETE: Call task_complete when done
+
+## Rules
+
+- ALWAYS read a file before modifying it
+- ALWAYS run validation after changes
+- If tests fail, read the FULL error. Fix the exact issue.
+- Do NOT give up after failure. Iterate until it passes.
+- Use file_edit for small changes, not full file rewrites
+- You MUST call task_complete when done
+- Do NOT output long explanations. Focus on tool calls.`;
+    SYSTEM_PROMPT_SMALL = `You are a coding agent. You MUST call tools in EVERY response. NEVER reply with only text.
+
+Tools: file_read, file_write, file_edit, shell, task_complete, find_files, grep_search, web_search, web_fetch
+
+Steps:
+1. file_read the source files AND test files
+2. file_edit or file_write to make changes
+3. shell to run tests (npm test, etc.)
+4. If tests fail: read error, fix, retest
+5. task_complete when tests pass
+
+Rules:
+- ALWAYS call tools. NEVER just write text.
+- Read files before editing them.
+- Run tests after every change.
+- Call task_complete when done.`;
     AgenticRunner = class {
       backend;
       tools = /* @__PURE__ */ new Map();
@@ -10073,7 +10190,8 @@ If you notice you're performing the SAME multi-step sequence for the 3rd time or
           dynamicContext: options?.dynamicContext ?? "",
           streamEnabled: options?.streamEnabled ?? false,
           bruteForce: options?.bruteForce ?? true,
-          bruteForceMaxCycles: options?.bruteForceMaxCycles ?? 100
+          bruteForceMaxCycles: options?.bruteForceMaxCycles ?? 100,
+          modelTier: options?.modelTier ?? "large"
         };
       }
       /** Register a tool for the agent to use */
@@ -10151,9 +10269,10 @@ Respond with your assessment, then take action. Do NOT just say you'll continue
         const toolCallLog = [];
         this.aborted = false;
         this.pendingUserMessages.length = 0;
-        const systemPrompt = this.options.dynamicContext ? `${SYSTEM_PROMPT}
+        const basePrompt = getSystemPromptForTier(this.options.modelTier);
+        const systemPrompt = this.options.dynamicContext ? `${basePrompt}
 
-${this.options.dynamicContext}` : SYSTEM_PROMPT;
+${this.options.dynamicContext}` : basePrompt;
         const messages = [
           { role: "system", content: systemPrompt },
           { role: "user", content: context ? `${context}
@@ -15147,6 +15266,21 @@ import { existsSync as existsSync16, readFileSync as readFileSync13, readdirSync
 import { join as join23, basename as basename6 } from "node:path";
 import { execSync as execSync14 } from "node:child_process";
 import { homedir as homedir9, platform, release } from "node:os";
+function getModelTier(modelName) {
+  const m = modelName.toLowerCase();
+  const sizeMatch = m.match(/\b(\d+)b\b/);
+  if (sizeMatch) {
+    const size = parseInt(sizeMatch[1], 10);
+    if (size >= 30)
+      return "large";
+    if (size >= 8)
+      return "medium";
+    return "small";
+  }
+  if (/\b(small|mini|nano|tiny)\b/.test(m))
+    return "small";
+  return "large";
+}
 function loadProjectFiles(repoRoot) {
   const discovered = discoverContextFiles(repoRoot);
   if (discovered.length === 0)
@@ -15361,7 +15495,7 @@ function buildProjectContext(repoRoot, stores) {
     skillsSummary: buildSkillsSummary(discoverSkills(repoRoot))
   };
 }
-function formatContextForPrompt(ctx) {
+function formatContextForPrompt(ctx, modelTier = "large") {
   const sections = [];
   if (ctx.environment) {
     sections.push(`## Environment
@@ -15374,9 +15508,17 @@ ${ctx.environment}`);
 ${ctx.gitInfo}`);
   }
   if (ctx.projectMap) {
-    sections.push(`## Project Map
+    if (modelTier === "small") {
+    } else if (modelTier === "medium" && ctx.projectMap.length > 2e3) {
+      sections.push(`## Project Map (truncated)
+
+${ctx.projectMap.slice(0, 2e3)}
+...(use find_files/list_directory for full listing)`);
+    } else {
+      sections.push(`## Project Map
 
 ${ctx.projectMap}`);
+    }
   }
   if (ctx.projectInstructions) {
     sections.push(ctx.projectInstructions);
@@ -15388,17 +15530,19 @@ ${ctx.memoryContext}
 
 Use this context to avoid re-learning known patterns. Update with memory_write if you discover new insights.`);
   }
-  if (ctx.sessionHistory) {
-    sections.push(`## Session History
+  if (modelTier !== "small") {
+    if (ctx.sessionHistory) {
+      sections.push(`## Session History
 
 ${ctx.sessionHistory}`);
-  }
-  if (ctx.taskMemories) {
-    sections.push(`## Cross-Session Task Memory
+    }
+    if (ctx.taskMemories) {
+      sections.push(`## Cross-Session Task Memory
 
 ${ctx.taskMemories}
 
 Use this history to avoid re-doing completed work and to learn from past approaches.`);
+    }
   }
   if (ctx.failurePatterns) {
     sections.push(`## Known Failure Patterns
@@ -15407,14 +15551,14 @@ ${ctx.failurePatterns}
 
 Avoid approaches that led to these failures. If you encounter these errors, try a different strategy.`);
   }
-  if (ctx.patternSuggestions) {
+  if (modelTier === "large" && ctx.patternSuggestions) {
     sections.push(`## Tool Creation Suggestions
 
 ${ctx.patternSuggestions}
 
 These patterns have been repeated 3+ times. Consider using create_tool to automate them.`);
   }
-  if (ctx.skillsSummary) {
+  if (modelTier === "large" && ctx.skillsSummary) {
     sections.push(ctx.skillsSummary);
   }
   return sections.join("\n\n");
@@ -17813,8 +17957,9 @@ ${result.summary}`;
       /** Run a dream agent with appropriate tools */
       async runDreamAgent(prompt, toolMode, onEvent) {
         const backend = new OllamaAgenticBackend(this.config.backendUrl, this.config.model, this.config.apiKey);
+        const modelTier = getModelTier(this.config.model);
         const projectCtx = buildProjectContext(this.repoRoot);
-        const dynamicContext = formatContextForPrompt(projectCtx);
+        const dynamicContext = formatContextForPrompt(projectCtx, modelTier);
         const runner = new AgenticRunner(backend, {
           maxTurns: 20,
           maxTokens: 16384,
@@ -17822,8 +17967,9 @@ ${result.summary}`;
           // Slightly creative temperature for dreaming
           requestTimeoutMs: this.config.timeoutMs,
           taskTimeoutMs: this.config.timeoutMs * 3,
-          compactionThreshold: 4e4,
-          dynamicContext
+          compactionThreshold: modelTier === "small" ? 12e3 : modelTier === "medium" ? 24e3 : 4e4,
+          dynamicContext,
+          modelTier
         });
         const tools = this.buildDreamTools(toolMode);
         runner.registerTools(tools);
@@ -18769,12 +18915,14 @@ Use task_status("${taskId}") or task_output("${taskId}") to check progress.`
   };
 }
 function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce, statusBar, sudoCallback, costTracker, onComplete, taskType) {
+  const modelTier = getModelTier(config.model);
   const projectCtx = buildProjectContext(repoRoot, taskStores?.contextStores);
-  let dynamicContext = formatContextForPrompt(projectCtx);
-  if (taskType) {
+  let dynamicContext = formatContextForPrompt(projectCtx, modelTier);
+  if (taskType && modelTier !== "small") {
     dynamicContext += "\n\n" + buildTaskContext(taskType);
   }
   const backend = new OllamaAgenticBackend(config.backendUrl, config.model, config.apiKey);
+  const compactionThreshold = modelTier === "small" ? 12e3 : modelTier === "medium" ? 24e3 : 4e4;
   const runner = new AgenticRunner(backend, {
     maxTurns: 60,
     maxTokens: 16384,
@@ -18782,8 +18930,9 @@ function startTask(task, config, repoRoot, voice, stream, taskStores, bruteForce
     requestTimeoutMs: config.timeoutMs,
     taskTimeoutMs: 36e5,
     // 60 minutes — never give up prematurely
-    compactionThreshold: 4e4,
+    compactionThreshold,
     dynamicContext,
+    modelTier,
     streamEnabled: stream?.enabled ?? false,
     bruteForce: bruteForce ?? true,
     bruteForceMaxCycles: 100

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.22.3",
+  "version": "0.23.1",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",