open-agents-ai 0.34.2 → 0.34.4

package/README.md

@@ -29,7 +29,7 @@ An autonomous multi-turn tool-calling agent that reads your code, makes changes,
 
 ## Features
 
-- **35+ autonomous tools** — file I/O, shell, grep, web search/fetch/crawl, memory, sub-agents, background tasks, image/OCR, git, diagnostics, vision, desktop automation, structured files, code sandbox
+- **47 autonomous tools** — file I/O, shell, grep, web search/fetch/crawl, memory (read/write/search), sub-agents, background tasks, image/OCR/PDF, git, diagnostics, vision, desktop automation, structured files, code sandbox, transcription, skills
 - **Moondream vision** — see and interact with the desktop via Moondream VLM (caption, query, detect, point-and-click)
 - **Desktop automation** — vision-guided clicking: describe a UI element in natural language, the agent finds and clicks it
 - **Auto-install desktop deps** — screenshot, mouse, OCR, and image tools auto-install missing system packages (scrot, xdotool, tesseract, imagemagick) on first use
@@ -39,6 +39,7 @@ An autonomous multi-turn tool-calling agent that reads your code, makes changes,
 - **Dream Mode** — creative idle exploration modeled after real sleep architecture (NREM→REM cycles)
 - **Live Listen** — bidirectional voice communication with real-time Whisper transcription
 - **Neural TTS** — hear what the agent is doing via GLaDOS or Overwatch ONNX voices
+- **Human expert speed ratio** — real-time `Exp: Nx` gauge comparing agent speed to a leading human expert, calibrated across 47 tool baselines
 - **Cost tracking** — real-time token cost estimation for 15+ cloud providers
 - **Work evaluation** — LLM-as-judge scoring with task-type-specific rubrics
 - **Session metrics** — track turns, tool calls, tokens, files modified, tasks completed per session
@@ -371,7 +372,7 @@ Launch without arguments to enter the interactive REPL:
 oa
 ```
 
-The TUI features an animated multilingual phrase carousel, live metrics bar with pastel-colored labels (token in/out, context window usage), rotating tips, syntax-highlighted tool output, and dynamic terminal-width cropping.
+The TUI features an animated multilingual phrase carousel, live metrics bar with pastel-colored labels (token in/out, context window usage, human expert speed ratio, cost), rotating tips, syntax-highlighted tool output, and dynamic terminal-width cropping.
 
 ### Slash Commands
 
@@ -433,7 +434,7 @@ While the agent is working (shown by the `+` prompt), type to add context:
   ⎿  Edit: src/auth.ts
 ```
 
-## Tools (37)
+## Tools (47)
 
 | Tool | Description |
 |------|-------------|
@@ -447,9 +448,9 @@ While the agent is working (shown by the `+` prompt), type to add context:
 | **Search & Navigation** | |
 | `grep_search` | Search file contents with regex (ripgrep with grep fallback) |
 | `find_files` | Find files by glob pattern (excludes node_modules/.git) |
-| `codebase_map` | High-level project structure overview with directory tree |
+| `codebase_map` | High-level project structure overview with directory tree and language breakdown |
 | **Shell & Execution** | |
-| `shell` | Execute any shell command (non-interactive, CI=true) |
+| `shell` | Execute any shell command (non-interactive, CI=true, sudo support) |
 | `code_sandbox` | Isolated code execution (JS, Python, Bash, TS) in subprocess or Docker |
 | `background_run` | Run shell command in background, returns task ID |
 | `task_status` | Check background task status |
@@ -461,26 +462,34 @@ While the agent is working (shown by the `+` prompt), type to add context:
 | `web_crawl` | Multi-page web scraping with Crawlee/Playwright for deep documentation |
 | **Structured Data** | |
 | `structured_file` | Generate CSV, TSV, JSON, Markdown tables, Excel-compatible files |
-| `read_structured_file` | Parse CSV, TSV, JSON, Markdown tables with binary detection |
+| `structured_read` | Parse CSV, TSV, JSON, Markdown tables with binary format detection |
 | **Vision & Desktop** | |
 | `vision` | Moondream VLM — caption, query, detect, point on any image |
 | `desktop_click` | Vision-guided clicking: describe a UI element, agent finds and clicks it |
 | `desktop_describe` | Screenshot + Moondream caption/query for desktop awareness |
-| `image_read` | Read images (base64 + OCR) |
+| `image_read` | Read images (base64 + OCR metadata) |
 | `screenshot` | Capture screen/window/active window |
 | `ocr` | Extract text from images (Tesseract with multi-variant preprocessing) |
+| `ocr_image_advanced` | Advanced multi-variant OCR pipeline with preprocessing, multi-PSM, and confidence scoring |
 | `ocr_pdf` | Add searchable text layer to scanned/image PDFs |
-| `pdf_to_text` | Extract text from PDF using pdftotext (Poppler) |
-| `transcribe_file` | Transcribe audio/video to text (Whisper) |
+| `pdf_to_text` | Extract text from PDF using pdftotext (Poppler) with OCR fallback |
+| **Transcription** | |
+| `transcribe_file` | Transcribe local audio/video files to text (Whisper) |
+| `transcribe_url` | Download and transcribe audio/video from URLs |
 | **Memory & Knowledge** | |
-| `memory_read` | Read from persistent memory store by topic |
+| `memory_read` | Read from persistent memory store by topic and key |
 | `memory_write` | Store facts/patterns in persistent memory with provenance tracking |
+| `memory_search` | Semantic search across all memory entries by query |
 | `memex_retrieve` | Recover full tool output archived during context compaction by hash ID |
 | **Git & Diagnostics** | |
 | `diagnostic` | Lint/typecheck/test/build validation pipeline in one call |
 | `git_info` | Structured git status, log, diff, branch, staged/unstaged files |
-| **Agents & Skills** | |
-| `create_tool` | Create reusable custom tools from workflow at runtime |
+| **Agents & Delegation** | |
+| `sub_agent` | Delegate subtasks to independent agent instances (foreground or background) |
+| `explore_tools` | Meta-tool: discover and unlock additional tools on demand (for small models) |
+| `task_complete` | Signal task completion with summary |
+| **Custom Tools & Skills** | |
+| `create_tool` | Create reusable custom tools from workflow patterns at runtime |
 | `manage_tools` | List, inspect, delete custom tools |
 | `skill_list` | Discover available AIWG skills |
 | `skill_execute` | Run an AIWG skill |
@@ -510,13 +519,13 @@ Open Agents classifies models into three tiers and adapts its behavior according
 
 | Tier | Parameters | Base Tools | System Prompt | Compaction |
 |------|-----------|------------|---------------|------------|
-| **Large** (≥30B) | 70B, 122B | All 37 tools | Full (344 lines) | 40K threshold |
+| **Large** (≥30B) | 70B, 122B | All 47 tools | Full (344 lines) | 40K threshold |
 | **Medium** (8-29B) | 9B, 27B | 15 core tools | Condensed (100 lines) | 24K threshold |
 | **Small** (≤7B) | 4B, 1.5B | 6 base tools + explore_tools | Minimal (15 lines) | 12K threshold |
 
 ### Tool Nesting for Small Models
 
-Small models use an **explore_tools** meta-tool pattern inspired by hierarchical API retrieval research (ToolLLM, arXiv:2307.16789). Instead of presenting all 37 tools (which overwhelms small context windows), only 6 core tools are loaded initially:
+Small models use an **explore_tools** meta-tool pattern inspired by hierarchical API retrieval research (ToolLLM, arXiv:2307.16789). Instead of presenting all 47 tools (which overwhelms small context windows), only 6 core tools are loaded initially:
 
 - `file_read`, `file_write`, `file_edit`, `shell`, `task_complete`, `explore_tools`
 
@@ -549,6 +558,45 @@ All context-dependent values scale automatically with the actual context window
 
 Auto-downloads the ONNX voice model (~50MB) on first use. Install `espeak-ng` for best quality (`apt install espeak-ng` / `brew install espeak-ng`).
 
+## Human Expert Speed Ratio
+
+The status bar displays a real-time `Exp: Nx` gauge estimating how fast the agent is working relative to a leading human expert performing equivalent tasks.
+
+```
+In: 12,345 | Out: 4,567 | Ctx: 18,000/131,072 86% | Exp: 4.2x | Cost: $0.34
+                                                       ^^^^^^^^
+                                                    Agent is 4.2x faster
+                                                    than a human expert
+```
+
+### How It Works
+
+Each tool call maps to a calibrated expert baseline time — the estimated seconds a top-tier human developer would take to perform the equivalent operation manually:
+
+| Operation | Expert Time | Agent Equivalent |
+|-----------|-------------|-----------------|
+| Read a file | 12s | `file_read` |
+| Write a new file | 90s | `file_write` |
+| Make a precise edit | 25s | `file_edit` |
+| Grep search + scan results | 15s | `grep_search` |
+| Run a shell command | 20s | `shell` |
+| Web search + evaluate | 60s | `web_search` |
+| Survey codebase structure | 180s | `codebase_map` |
+
+Additional overhead per action:
+- **+5s context-switch** per tool call (expert switching between tools)
+- **+15s planning** per reasoning turn (expert thinking about next step)
+
+The ratio accumulates across all tasks in the session:
+
+```
+speedRatio = totalHumanExpertTime / totalAgentWallClockTime
+```
+
+Color coding: green (2x+ faster), yellow (1-2x, comparable), red (<1x, slower than expert).
+
+All 47 tools have calibrated baselines ranging from 3s (`task_stop`) to 180s (`codebase_map`). Unknown tools default to 20s.
+
 ## Cost Tracking & Session Metrics
 
 Real-time token cost estimation for cloud providers. The status bar shows running cost when using a paid endpoint.
@@ -717,7 +765,7 @@ The agent auto-detects the provider, normalizes the URL (strips `/v1/chat/comple
 
 ## Evaluation Suite
 
-33 evaluation tasks test the agent's autonomous capabilities across coding, web research, SDLC analysis, tool creation, and multi-file reasoning:
+40 evaluation tasks test the agent's autonomous capabilities across coding, web research, SDLC analysis, tool creation, multi-file reasoning, and memory systems:
 
 ```bash
 node eval/run-agentic.mjs                          # Run all tasks
@@ -754,15 +802,23 @@ node eval/run-agentic.mjs --model qwen2.5-coder:32b  # Different model
 | 31 | Web extractor bug fixes (3 bugs) | Multi-Bug Fix |
 | 32 | CSV pipeline across 3 files | Multi-File Tracking |
 | 33 | FSM bug fixes + factory implementation | State Machine |
+| 34 | Search pre-populated memories | Memory Search |
+| 35 | Analyze code, write to memory, cross-reference | Memory Cross-Reference |
+| 36 | Discover explore_tools, unlock grep_search | Explore Tools |
+| 37 | Analyze code patterns, store and recall from memory | Memory Store & Recall |
+| 38 | Read configs, write to multiple memory topics | Memory Multi-Topic |
+| 39 | Search pre-loaded memories across 3 topics | Memory Pre-Loaded Search |
+| 40 | Combined explore_tools + memory analysis pipeline | Explore + Memory |
 
-Tasks 31-33 are designed for small model (≤9B) evaluation using `file_edit` patterns instead of `file_write` to avoid JSON truncation issues with smaller models.
+Tasks 31-33 are designed for small model (≤9B) evaluation using `file_edit` patterns. Tasks 34-40 test the memory system (read/write/search) and tool discovery.
 
 ### Benchmark Results
 
 ```
-Qwen3.5-122B: 100% pass rate (30/30 tasks)
+Qwen3.5-122B: 100% pass rate (37/37 tasks, including memory tasks 34-40)
 Qwen3.5-27B:  100% pass rate (30/30 tasks)
 Qwen3.5-9B:   100% pass rate (tasks 31-33, file_edit-optimized)
+              71% pass rate (5/7 memory tasks 34-40)
 ```
 
 The eval runner includes model-tier-aware features: automatic tool set filtering, HTTP 500 recovery with file_edit hints, loop detection with tool banning, and tier-based output truncation.

package/dist/index.js

@@ -90,7 +90,7 @@ function setConfigValue(key, value) {
   const configPath = join(dir, "config.json");
   const existing = loadConfigFile();
   existing[key] = coerceConfigValue(key, value);
-  writeFileSync(configPath, JSON.stringify(existing, null, 2) + "\n", "utf8");
+  writeFileSync(configPath, JSON.stringify(existing, null, 2) + "\n", { encoding: "utf8", mode: 384 });
 }
 function coerceConfigValue(key, value) {
   const intKeys = /* @__PURE__ */ new Set(["maxRetries", "timeoutMs"]);
@@ -16181,6 +16181,17 @@ function initOaDirectory(repoRoot) {
   for (const sub of SUBDIRS) {
     mkdirSync6(join25(oaPath, sub), { recursive: true });
   }
+  try {
+    const gitignorePath = join25(repoRoot, ".gitignore");
+    const settingsPattern = ".oa/settings.json";
+    if (existsSync19(gitignorePath)) {
+      const content = readFileSync13(gitignorePath, "utf-8");
+      if (!content.includes(settingsPattern)) {
+        writeFileSync6(gitignorePath, content.trimEnd() + "\n" + settingsPattern + "\n", "utf-8");
+      }
+    }
+  } catch {
+  }
   return oaPath;
 }
 function hasOaDirectory(repoRoot) {
@@ -16201,7 +16212,7 @@ function saveProjectSettings(repoRoot, settings) {
   mkdirSync6(oaPath, { recursive: true });
   const existing = loadProjectSettings(repoRoot);
   const merged = { ...existing, ...settings };
-  writeFileSync6(join25(oaPath, "settings.json"), JSON.stringify(merged, null, 2) + "\n", "utf-8");
+  writeFileSync6(join25(oaPath, "settings.json"), JSON.stringify(merged, null, 2) + "\n", { encoding: "utf-8", mode: 384 });
 }
 function loadGlobalSettings() {
   const settingsPath = join25(homedir9(), ".open-agents", "settings.json");
@@ -16218,7 +16229,7 @@ function saveGlobalSettings(settings) {
   mkdirSync6(dir, { recursive: true });
   const existing = loadGlobalSettings();
   const merged = { ...existing, ...settings };
-  writeFileSync6(join25(dir, "settings.json"), JSON.stringify(merged, null, 2) + "\n", "utf-8");
+  writeFileSync6(join25(dir, "settings.json"), JSON.stringify(merged, null, 2) + "\n", { encoding: "utf-8", mode: 384 });
 }
 function resolveSettings(repoRoot) {
   const global = loadGlobalSettings();
@@ -16670,6 +16681,49 @@ function ask(rl, question) {
     rl.question(question, (answer) => resolve23(answer.trim()));
   });
 }
+function askSecret(rl, question) {
+  return new Promise((resolve23) => {
+    process.stdout.write(question);
+    let secret = "";
+    const stdin = process.stdin;
+    const hadRawMode = stdin.isRaw;
+    if (typeof stdin.setRawMode === "function") {
+      stdin.setRawMode(true);
+    }
+    stdin.resume();
+    const onData = (chunk) => {
+      const ch = chunk.toString("utf8");
+      for (const c3 of ch) {
+        if (c3 === "\r" || c3 === "\n") {
+          stdin.removeListener("data", onData);
+          if (typeof stdin.setRawMode === "function") {
+            stdin.setRawMode(hadRawMode ?? false);
+          }
+          process.stdout.write("\n");
+          resolve23(secret.trim());
+          return;
+        } else if (c3 === "") {
+          stdin.removeListener("data", onData);
+          if (typeof stdin.setRawMode === "function") {
+            stdin.setRawMode(hadRawMode ?? false);
+          }
+          process.stdout.write("\n");
+          resolve23("");
+          return;
+        } else if (c3 === "\x7F" || c3 === "\b") {
+          if (secret.length > 0) {
+            secret = secret.slice(0, -1);
+            process.stdout.write("\b \b");
+          }
+        } else if (c3.charCodeAt(0) >= 32) {
+          secret += c3;
+          process.stdout.write("*");
+        }
+      }
+    };
+    stdin.on("data", onData);
+  });
+}
 async function autoInstallOllama(rl) {
   const plat = platform();
   if (plat === "linux") {
@@ -16899,7 +16953,7 @@ async function promptForCustomEndpoint(config, rl) {
   ${c2.bold("Does this endpoint require an API key?")} (y/n) `);
   let apiKey = "";
   if (needsKey.toLowerCase() === "y" || needsKey.toLowerCase() === "yes") {
-    apiKey = await ask(rl, `  ${c2.bold("API key:")} `);
+    apiKey = await askSecret(rl, `  ${c2.bold("API key:")} `);
   }
   process.stdout.write(`
   ${c2.cyan("\u25CF")} Enter the model name for this endpoint.
@@ -18353,7 +18407,7 @@ async function handleEndpoint(arg, ctx, local = false) {
   process.stdout.write(`    ${c2.cyan("Type".padEnd(12))} ${backendType}
 `);
   if (apiKey) {
-    process.stdout.write(`    ${c2.cyan("Auth".padEnd(12))} Bearer ${apiKey.slice(0, 8)}...
+    process.stdout.write(`    ${c2.cyan("Auth".padEnd(12))} Bearer ${apiKey.slice(0, 4)}...
 `);
   } else {
     process.stdout.write(`    ${c2.cyan("Auth".padEnd(12))} ${provider.authRequired ? c2.yellow("none (may be required)") : "none"}
@@ -24654,12 +24708,18 @@ async function statusCommand(opts, config) {
       "OPEN_AGENTS_TIMEOUT_MS",
       "OPEN_AGENTS_DRY_RUN",
       "OPEN_AGENTS_VERBOSE",
-      "VLLM_BASE_URL"
+      "VLLM_BASE_URL",
+      "VLLM_API_KEY"
     ];
+    const sensitiveEnvVars = /* @__PURE__ */ new Set([
+      "OPEN_AGENTS_API_KEY",
+      "VLLM_API_KEY"
+    ]);
     for (const key of envVars) {
       const val = process.env[key];
       if (val !== void 0) {
-        printKeyValue(key, val.includes("key") || val.includes("KEY") ? "[redacted]" : val, 2);
+        const display = sensitiveEnvVars.has(key) ? val.length > 4 ? `[set \u2014 ${val.slice(0, 4)}...]` : val.length > 0 ? "[set]" : "[empty]" : val;
+        printKeyValue(key, display, 2);
       }
     }
   }
@@ -24777,6 +24837,12 @@ __export(config_exports, {
 import { join as join34, resolve as resolve22 } from "node:path";
 import { homedir as homedir13 } from "node:os";
 import { cwd as cwd3 } from "node:process";
+function redactIfSensitive(key, value) {
+  if (SENSITIVE_KEYS.has(key) && typeof value === "string" && value.length > 0) {
+    return value.length > 4 ? `[set \u2014 ${value.slice(0, 4)}...]` : "[set]";
+  }
+  return String(value);
+}
 function coerceForSettings(key, value) {
   if (INT_KEYS.has(key))
     return parseInt(value, 10);
@@ -24812,7 +24878,7 @@ function handleShow(opts, config) {
   if (projectKeys.length > 0) {
     printSection(`Project Overrides (.oa/settings.json)`);
     for (const [k, v] of projectKeys) {
-      printKeyValue(k, String(v), 2);
+      printKeyValue(k, redactIfSensitive(k, v), 2);
     }
   } else {
     printSection("Project Overrides");
@@ -24823,7 +24889,7 @@ function handleShow(opts, config) {
   if (globalKeys.length > 0) {
     printSection("Global Settings (~/.open-agents/settings.json)");
     for (const [k, v] of globalKeys) {
-      printKeyValue(k, String(v), 2);
+      printKeyValue(k, redactIfSensitive(k, v), 2);
     }
   }
   printSection("Config File");
@@ -24865,7 +24931,7 @@ function handleSet(opts, _config) {
       initOaDirectory(repoRoot);
       const coerced = coerceForSettings(key, value);
       saveProjectSettings(repoRoot, { [key]: coerced });
-      printSuccess(`Project override set: ${key} = ${value}`);
+      printSuccess(`Project override set: ${key} = ${redactIfSensitive(key, value)}`);
       printInfo(`Saved to ${join34(repoRoot, ".oa", "settings.json")}`);
       printInfo("This override applies only when running in this workspace.");
     } catch (err) {
@@ -24875,7 +24941,7 @@ function handleSet(opts, _config) {
   } else {
     try {
       setConfigValue(key, value);
-      printSuccess(`Config updated: ${key} = ${value}`);
+      printSuccess(`Config updated: ${key} = ${redactIfSensitive(key, value)}`);
       printInfo(`Saved to ~/.open-agents/config.json`);
       printInfo("Tip: Use --local to set project-specific overrides.");
     } catch (err) {
@@ -24894,7 +24960,7 @@ function handleKeys() {
   printInfo("  oa config set model qwen3.5:122b          # global default");
   printInfo("  oa config set model qwen3.5:122b --local  # this project only");
 }
-var CONFIG_KEYS, INT_KEYS, BOOL_KEYS;
+var CONFIG_KEYS, SENSITIVE_KEYS, INT_KEYS, BOOL_KEYS;
 var init_config3 = __esm({
   "packages/cli/dist/commands/config.js"() {
     "use strict";
@@ -24916,6 +24982,7 @@ var init_config3 = __esm({
       stream: "Enable real-time token streaming with pastel syntax highlighting (true/false)",
       bruteforce: "Brute-force mode: auto re-engage agent when turn limit hit (true/false)"
     };
+    SENSITIVE_KEYS = /* @__PURE__ */ new Set(["apiKey", "api_key", "secret", "password", "token"]);
     INT_KEYS = /* @__PURE__ */ new Set(["maxRetries", "timeoutMs"]);
     BOOL_KEYS = /* @__PURE__ */ new Set(["dryRun", "verbose", "voice", "stream", "bruteforce"]);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-agents-ai",
-  "version": "0.34.2",
+  "version": "0.34.4",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",