@poncho-ai/harness 0.14.2 → 0.15.1

package/.turbo/turbo-build.log

@@ -1,5 +1,5 @@
 
-> @poncho-ai/harness@0.14.2 build /Users/cesar/Dev/latitude/poncho-ai/packages/harness
+> @poncho-ai/harness@0.15.1 build /Users/cesar/Dev/latitude/poncho-ai/packages/harness
 > tsup src/index.ts --format esm --dts
 
 CLI Building entry: src/index.ts
@@ -7,8 +7,8 @@
 CLI tsup v8.5.1
 CLI Target: es2022
 ESM Build start
-ESM dist/index.js 177.01 KB
-ESM ⚡️ Build success in 73ms
+ESM dist/index.js 182.55 KB
+ESM ⚡️ Build success in 131ms
 DTS Build start
-DTS ⚡️ Build success in 3459ms
-DTS dist/index.d.ts 21.71 KB
+DTS ⚡️ Build success in 6605ms
+DTS dist/index.d.ts 22.34 KB

package/.turbo/turbo-test.log

@@ -1,18 +1,20 @@
 
-> @poncho-ai/harness@0.14.0 test /Users/cesar/Dev/latitude/poncho-ai/packages/harness
+> @poncho-ai/harness@0.14.2 test /Users/cesar/Dev/latitude/poncho-ai/packages/harness
 > vitest
 
 
  RUN  v1.6.1 /Users/cesar/Dev/latitude/poncho-ai/packages/harness
 
 [event] step:completed {"type":"step:completed","step":1,"duration":1}
+ ✓ test/telemetry.test.ts  (3 tests) 3ms
 [event] step:started {"type":"step:started","step":2}
- ✓ test/telemetry.test.ts  (3 tests) 11ms
- ✓ test/schema-converter.test.ts  (27 tests) 21ms
+ ✓ test/schema-converter.test.ts  (27 tests) 16ms
 stdout | test/mcp.test.ts > mcp bridge protocol transports > discovers and calls tools over streamable HTTP
 [poncho][mcp] {"event":"catalog.loaded","server":"remote","discoveredCount":1}
 [poncho][mcp] {"event":"tools.selected","requestedPatternCount":1,"registeredCount":1,"filteredByPolicyCount":0,"filteredByIntentCount":0}
 
+ ✓ test/agent-parser.test.ts  (10 tests) 17ms
+ ✓ test/memory.test.ts  (4 tests) 16ms
 stdout | test/mcp.test.ts > mcp bridge protocol transports > selects discovered tools by requested patterns
 [poncho][mcp] {"event":"catalog.loaded","server":"remote","discoveredCount":2}
 [poncho][mcp] {"event":"tools.selected","requestedPatternCount":1,"registeredCount":1,"filteredByPolicyCount":0,"filteredByIntentCount":1}
@@ -28,12 +30,10 @@
 [poncho][mcp] {"event":"catalog.loaded","server":"remote","discoveredCount":1}
 [poncho][mcp] {"event":"tools.selected","requestedPatternCount":1,"registeredCount":1,"filteredByPolicyCount":0,"filteredByIntentCount":0}
 
- ✓ test/mcp.test.ts  (6 tests) 516ms
- ✓ test/agent-parser.test.ts  (10 tests) 130ms
- ✓ test/memory.test.ts  (4 tests) 192ms
- ✓ test/state.test.ts  (5 tests) 294ms
- ✓ test/model-factory.test.ts  (4 tests) 8ms
- ✓ test/agent-identity.test.ts  (2 tests) 33ms
+ ✓ test/mcp.test.ts  (6 tests) 97ms
+ ✓ test/state.test.ts  (5 tests) 217ms
+ ✓ test/agent-identity.test.ts  (2 tests) 15ms
+ ✓ test/model-factory.test.ts  (4 tests) 3ms
 stdout | test/harness.test.ts > agent harness > registers default filesystem tools
 [poncho][mcp] {"event":"tools.cleared","reason":"initialize","requestedPatterns":[]}
 
@@ -57,6 +57,8 @@
 
 stdout | test/harness.test.ts > agent harness > refreshes skill metadata and tools in development mode
 [poncho][mcp] {"event":"tools.cleared","reason":"skills:changed","requestedPatterns":[]}
+
+stdout | test/harness.test.ts > agent harness > refreshes skill metadata and tools in development mode
 [poncho][mcp] {"event":"tools.cleared","reason":"activate:beta","requestedPatterns":[]}
 
 stdout | test/harness.test.ts > agent harness > prunes removed active skills after refresh in development mode
@@ -71,8 +73,6 @@
 
 stdout | test/harness.test.ts > agent harness > clears active skills when skill metadata changes in development mode
 [poncho][mcp] {"event":"tools.cleared","reason":"initialize","requestedPatterns":[]}
-
-stdout | test/harness.test.ts > agent harness > clears active skills when skill metadata changes in development mode
 [poncho][mcp] {"event":"tools.cleared","reason":"activate:alpha","requestedPatterns":[]}
 
 stdout | test/harness.test.ts > agent harness > clears active skills when skill metadata changes in development mode
@@ -98,6 +98,8 @@
 [poncho][mcp] {"event":"tools.cleared","reason":"initialize","requestedPatterns":[]}
 [poncho][mcp] {"event":"tools.selected","requestedPatternCount":1,"registeredCount":1,"filteredByPolicyCount":0,"filteredByIntentCount":1}
 [poncho][mcp] {"event":"tools.refreshed","reason":"activate:skill-a","requestedPatterns":["remote/a"],"registeredCount":1,"activeSkills":["skill-a"]}
+
+stdout | test/harness.test.ts > agent harness > registers MCP tools dynamically for stacked active skills and supports deactivation
 [poncho][mcp] {"event":"tools.selected","requestedPatternCount":2,"registeredCount":2,"filteredByPolicyCount":0,"filteredByIntentCount":0}
 [poncho][mcp] {"event":"tools.refreshed","reason":"activate:skill-b","requestedPatterns":["remote/a","remote/b"],"registeredCount":2,"activeSkills":["skill-a","skill-b"]}
 [poncho][mcp] {"event":"tools.selected","requestedPatternCount":1,"registeredCount":1,"filteredByPolicyCount":0,"filteredByIntentCount":1}
@@ -128,10 +130,10 @@
 [poncho][mcp] {"event":"tools.refreshed","reason":"activate:skill-slow","requestedPatterns":["remote/slow"],"registeredCount":1,"activeSkills":["skill-slow"]}
 [poncho][mcp] {"event":"tools.cleared","reason":"deactivate:skill-slow","requestedPatterns":[]}
 
- ✓ test/harness.test.ts  (25 tests) 429ms
+ ✓ test/harness.test.ts  (25 tests) 365ms
 
  Test Files  9 passed (9)
       Tests  86 passed (86)
-   Start at  10:02:54
-   Duration  4.19s (transform 2.16s, setup 0ms, collect 7.51s, tests 1.63s, environment 1ms, prepare 1.97s)
+   Start at  13:30:36
+   Duration  2.19s (transform 1.11s, setup 0ms, collect 2.81s, tests 749ms, environment 5ms, prepare 1.23s)
 

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @poncho-ai/harness
 
+## 0.15.1
+
+### Patch Changes
+
+- Fix browser session reconnection, tab lifecycle management, and web UI panel state handling.
+
+- Updated dependencies []:
+  - @poncho-ai/sdk@1.1.1
+
+## 0.15.0
+
+### Minor Changes
+
+- [`139ed89`](https://github.com/cesr/poncho-ai/commit/139ed89a5df2372cfb0a124c967f51f4d8158c3b) Thanks [@cesr](https://github.com/cesr)! - Add browser automation for Poncho agents with real-time viewport streaming, per-conversation tab management, interactive browser control in the web UI, and shared agent-level profiles for authentication persistence.
+
+### Patch Changes
+
+- Updated dependencies [[`139ed89`](https://github.com/cesr/poncho-ai/commit/139ed89a5df2372cfb0a124c967f51f4d8158c3b)]:
+  - @poncho-ai/sdk@1.1.0
+
 ## 0.14.2
 
 ### Patch Changes

package/dist/index.d.ts

@@ -316,6 +316,19 @@ interface PonchoConfig extends McpConfig {
     };
     /** Set to `false` to disable the built-in web UI (headless / API-only mode). */
     webUi?: false;
+    /** Enable browser automation tools. Set `true` for defaults, or provide config. */
+    browser?: boolean | {
+        viewport?: {
+            width?: number;
+            height?: number;
+        };
+        quality?: number;
+        everyNthFrame?: number;
+        profileDir?: string;
+        sessionName?: string;
+        executablePath?: string;
+        headless?: boolean;
+    };
 }
 declare const resolveStateConfig: (config: PonchoConfig | undefined) => StateConfig | undefined;
 declare const resolveMemoryConfig: (config: PonchoConfig | undefined) => MemoryConfig | undefined;
@@ -439,6 +452,8 @@ declare class AgentHarness {
     private readonly registeredMcpToolNames;
     private latitudeTelemetry?;
     private insideTelemetryCapture;
+    private _browserSession?;
+    private _browserMod?;
     private parsedAgent?;
     private mcpBridge?;
     private resolveToolAccess;
@@ -471,6 +486,10 @@ declare class AgentHarness {
     private registerSkillTools;
     private refreshSkillsIfChanged;
     initialize(): Promise<void>;
+    private initBrowserTools;
+    /** Conversation ID of the currently executing run (set during run, cleared after). */
+    private _currentRunConversationId?;
+    get browserSession(): unknown;
     shutdown(): Promise<void>;
     listTools(): ToolDefinition[];
     /**

package/dist/index.js

@@ -2884,6 +2884,30 @@ Since all fields have defaults, you only need to specify \`*Env\` when your env
 - For setup, skills, MCP, auth, storage, telemetry, or "how do I..." questions, proactively read \`README.md\` with \`read_file\` before answering.
 - Prefer quoting concrete commands and examples from \`README.md\` over guessing.
 - Keep edits minimal, preserve unrelated settings/code, and summarize what changed.`;
+function extractMediaFromToolOutput(output) {
+  const mediaItems = [];
+  function walk(node) {
+    if (node === null || node === void 0) return node;
+    if (Array.isArray(node)) return node.map(walk);
+    if (typeof node === "object") {
+      const obj = node;
+      if (obj.type === "file" && typeof obj.data === "string" && typeof obj.mediaType === "string" && obj.mediaType.startsWith("image/")) {
+        mediaItems.push({
+          type: "media",
+          data: obj.data,
+          mediaType: obj.mediaType
+        });
+        return { type: "file", mediaType: obj.mediaType, filename: obj.filename ?? "image", _stripped: true };
+      }
+      const out = {};
+      for (const [k, v] of Object.entries(obj)) out[k] = walk(v);
+      return out;
+    }
+    return node;
+  }
+  const strippedOutput = walk(output);
+  return { mediaItems, strippedOutput };
+}
 var AgentHarness = class {
   workingDir;
   environment;
@@ -2900,6 +2924,8 @@ var AgentHarness = class {
   registeredMcpToolNames = /* @__PURE__ */ new Set();
   latitudeTelemetry;
   insideTelemetryCapture = false;
+  _browserSession;
+  _browserMod;
   parsedAgent;
   mcpBridge;
   resolveToolAccess(toolName) {
@@ -3219,6 +3245,13 @@ var AgentHarness = class {
         })
       );
     }
+    if (config?.browser) {
+      await this.initBrowserTools(config).catch((e) => {
+        console.warn(
+          `[poncho][browser] Failed to load browser tools: ${e instanceof Error ? e.message : String(e)}`
+        );
+      });
+    }
     await bridge.startLocalServers();
     await bridge.discoverTools();
     await this.refreshMcpTools("initialize");
@@ -3257,7 +3290,61 @@ var AgentHarness = class {
       );
     }
   }
+  async initBrowserTools(config) {
+    const spec = ["@poncho-ai", "browser"].join("/");
+    let browserMod;
+    try {
+      const { existsSync } = await import("fs");
+      const { join, dirname: dirname5 } = await import("path");
+      const { pathToFileURL: pathToFileURL2 } = await import("url");
+      let searchDir = this.workingDir;
+      let entryPath;
+      for (; ; ) {
+        const candidate = join(searchDir, "node_modules", "@poncho-ai", "browser", "dist", "index.js");
+        if (existsSync(candidate)) {
+          entryPath = candidate;
+          break;
+        }
+        const parent = dirname5(searchDir);
+        if (parent === searchDir) break;
+        searchDir = parent;
+      }
+      if (!entryPath) throw new Error("not installed");
+      browserMod = await import(pathToFileURL2(entryPath).href);
+    } catch {
+      throw new Error(
+        `browser: true is set in poncho.config but @poncho-ai/browser is not installed.
+  Run: pnpm add @poncho-ai/browser`
+      );
+    }
+    this._browserMod = browserMod;
+    const browserCfg = typeof config.browser === "object" ? config.browser : {};
+    const agentId = this.parsedAgent?.frontmatter.id ?? this.parsedAgent?.frontmatter.name ?? "default";
+    const session = new browserMod.BrowserSession(`poncho-${agentId}`, browserCfg);
+    this._browserSession = session;
+    const tools = browserMod.createBrowserTools(
+      () => session,
+      () => this._currentRunConversationId ?? "__default__"
+    );
+    for (const tool of tools) {
+      if (this.isToolEnabled(tool.name)) {
+        this.registerIfMissing(tool);
+      }
+    }
+  }
+  /** Conversation ID of the currently executing run (set during run, cleared after). */
+  _currentRunConversationId;
+  get browserSession() {
+    return this._browserSession;
+  }
   async shutdown() {
+    if (this._browserSession) {
+      try {
+        await this._browserSession.close();
+      } catch {
+      }
+      this._browserSession = void 0;
+    }
     await this.mcpBridge?.stopLocalServers();
     if (this.latitudeTelemetry) {
       await this.latitudeTelemetry.shutdown().catch((err) => {
@@ -3343,6 +3430,7 @@ var AgentHarness = class {
       await this.initialize();
     }
     await this.refreshSkillsIfChanged();
+    this._currentRunConversationId = input.conversationId;
     const agent = this.parsedAgent;
     const runId = `run_${randomUUID3()}`;
     const start = now();
@@ -3402,6 +3490,25 @@ ${boundedMainMemory.trim()}` : "";
       agentId: agent.frontmatter.id ?? agent.frontmatter.name,
       contextWindow
     });
+    const browserEventQueue = [];
+    const browserCleanups = [];
+    const browserSession = this._browserSession;
+    const conversationId = input.conversationId ?? "__default__";
+    if (browserSession) {
+      browserCleanups.push(
+        browserSession.onFrame(conversationId, (frame) => {
+          browserEventQueue.push({ type: "browser:frame", data: frame.data, width: frame.width, height: frame.height });
+        }),
+        browserSession.onStatus(conversationId, (status) => {
+          browserEventQueue.push({ type: "browser:status", ...status });
+        })
+      );
+    }
+    const drainBrowserEvents = function* () {
+      while (browserEventQueue.length > 0) {
+        yield browserEventQueue.shift();
+      }
+    };
     if (input.task != null) {
       if (input.files && input.files.length > 0) {
         const parts = [
@@ -3447,6 +3554,7 @@ ${boundedMainMemory.trim()}` : "";
     let transientStepRetryCount = 0;
     for (let step = 1; step <= maxSteps; step += 1) {
       try {
+        yield* drainBrowserEvents();
         if (isCancelled()) {
           yield emitCancellation();
           return;
@@ -3498,6 +3606,11 @@ ${boundedMainMemory.trim()}` : "";
         }
         const convertMessage = async (msg) => {
           if (msg.role === "tool") {
+            const meta = msg.metadata;
+            const rich = meta?._richToolResults;
+            if (rich && rich.length > 0) {
+              return [{ role: "tool", content: rich }];
+            }
             const textContent = typeof msg.content === "string" ? msg.content : getTextContent(msg);
             try {
               const parsed = JSON.parse(textContent);
@@ -3860,6 +3973,7 @@ ${textContent}` };
           abortSignal: input.abortSignal
         };
         const toolResultsForModel = [];
+        const richToolResults = [];
         const approvedCalls = [];
         for (const call of toolCalls) {
           if (isCancelled()) {
@@ -3953,6 +4067,12 @@ ${textContent}` };
               tool_name: result2.tool,
               content: `Tool error: ${result2.error}`
             });
+            richToolResults.push({
+              type: "tool-result",
+              toolCallId: result2.callId,
+              toolName: result2.tool,
+              output: { type: "json", value: { error: result2.error } }
+            });
           } else {
             span?.end({ result: { value: result2.output ?? null, isError: false } });
             yield pushEvent({
@@ -3961,12 +4081,34 @@ ${textContent}` };
               output: result2.output,
               duration: now() - batchStart
             });
+            const { mediaItems, strippedOutput } = extractMediaFromToolOutput(result2.output);
             toolResultsForModel.push({
               type: "tool_result",
               tool_use_id: result2.callId,
               tool_name: result2.tool,
-              content: JSON.stringify(result2.output ?? null)
+              content: JSON.stringify(strippedOutput ?? null)
             });
+            if (mediaItems.length > 0) {
+              richToolResults.push({
+                type: "tool-result",
+                toolCallId: result2.callId,
+                toolName: result2.tool,
+                output: {
+                  type: "content",
+                  value: [
+                    { type: "text", text: JSON.stringify(strippedOutput ?? null) },
+                    ...mediaItems
+                  ]
+                }
+              });
+            } else {
+              richToolResults.push({
+                type: "tool-result",
+                toolCallId: result2.callId,
+                toolName: result2.tool,
+                output: { type: "json", value: result2.output ?? null }
+              });
+            }
           }
         }
         const assistantContent = toolCalls.length > 0 ? JSON.stringify({
@@ -3982,10 +4124,11 @@ ${textContent}` };
           content: assistantContent,
           metadata: { timestamp: now(), id: randomUUID3(), step }
         });
+        const toolMsgMeta = { timestamp: now(), id: randomUUID3(), step, _richToolResults: richToolResults };
         messages.push({
           role: "tool",
           content: JSON.stringify(toolResultsForModel),
-          metadata: { timestamp: now(), id: randomUUID3(), step }
+          metadata: toolMsgMeta
         });
         yield pushEvent({
           type: "step:completed",
@@ -4040,6 +4183,8 @@ ${textContent}` };
         }
       });
     }
+    yield* drainBrowserEvents();
+    for (const cleanup of browserCleanups) cleanup();
   }
   async executeTools(calls, context) {
     return this.dispatcher.executeBatch(calls, context);
@@ -5133,6 +5278,15 @@ var createConversationStore = (config, options) => {
 };
 
 // src/telemetry.ts
+var MAX_FIELD_LENGTH = 200;
+function sanitizeEventForLog(event) {
+  return JSON.stringify(event, (_key, value) => {
+    if (typeof value === "string" && value.length > MAX_FIELD_LENGTH) {
+      return `${value.slice(0, 80)}...[${value.length} chars]`;
+    }
+    return value;
+  });
+}
 var TelemetryEmitter = class {
   config;
   constructor(config) {
@@ -5149,7 +5303,7 @@ var TelemetryEmitter = class {
     if (this.config?.otlp) {
       await this.sendOtlp(event);
     }
-    process.stdout.write(`[event] ${event.type} ${JSON.stringify(event)}
+    process.stdout.write(`[event] ${event.type} ${sanitizeEventForLog(event)}
 `);
   }
   async sendOtlp(event) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@poncho-ai/harness",
-  "version": "0.14.2",
+  "version": "0.15.1",
   "description": "Agent execution runtime - conversation loop, tool dispatch, streaming",
   "repository": {
     "type": "git",
@@ -31,7 +31,7 @@
     "redis": "^5.10.0",
     "yaml": "^2.4.0",
     "zod": "^3.22.0",
-    "@poncho-ai/sdk": "1.0.3"
+    "@poncho-ai/sdk": "1.1.1"
   },
   "devDependencies": {
     "@types/mustache": "^4.2.6",

package/src/config.ts

@@ -114,6 +114,18 @@ export interface PonchoConfig extends McpConfig {
   };
   /** Set to `false` to disable the built-in web UI (headless / API-only mode). */
   webUi?: false;
+  /** Enable browser automation tools. Set `true` for defaults, or provide config. */
+  browser?:
+    | boolean
+    | {
+        viewport?: { width?: number; height?: number };
+        quality?: number;
+        everyNthFrame?: number;
+        profileDir?: string;
+        sessionName?: string;
+        executablePath?: string;
+        headless?: boolean;
+      };
 }
 
 const resolveTtl = (

package/src/harness.ts

@@ -435,6 +435,49 @@ Since all fields have defaults, you only need to specify \`*Env\` when your env
 - Prefer quoting concrete commands and examples from \`README.md\` over guessing.
 - Keep edits minimal, preserve unrelated settings/code, and summarize what changed.`;
 
+/**
+ * Detect FileContentPart objects ({ type:"file", data, mediaType }) in a tool
+ * output value and split them into:
+ *  - `mediaItems` – items suitable for the AI SDK multi-part `content` output
+ *    (images become proper vision tokens, not base64 text).
+ *  - `strippedOutput` – the original output with base64 `data` fields replaced
+ *    by a short placeholder so the stored conversation stays small.
+ */
+function extractMediaFromToolOutput(output: unknown): {
+  mediaItems: Array<{ type: "media"; data: string; mediaType: string }>;
+  strippedOutput: unknown;
+} {
+  const mediaItems: Array<{ type: "media"; data: string; mediaType: string }> = [];
+
+  function walk(node: unknown): unknown {
+    if (node === null || node === undefined) return node;
+    if (Array.isArray(node)) return node.map(walk);
+    if (typeof node === "object") {
+      const obj = node as Record<string, unknown>;
+      if (
+        obj.type === "file" &&
+        typeof obj.data === "string" &&
+        typeof obj.mediaType === "string" &&
+        (obj.mediaType as string).startsWith("image/")
+      ) {
+        mediaItems.push({
+          type: "media",
+          data: obj.data as string,
+          mediaType: obj.mediaType as string,
+        });
+        return { type: "file", mediaType: obj.mediaType, filename: obj.filename ?? "image", _stripped: true };
+      }
+      const out: Record<string, unknown> = {};
+      for (const [k, v] of Object.entries(obj)) out[k] = walk(v);
+      return out;
+    }
+    return node;
+  }
+
+  const strippedOutput = walk(output);
+  return { mediaItems, strippedOutput };
+}
+
 export class AgentHarness {
   private readonly workingDir: string;
   private readonly environment: HarnessOptions["environment"];
@@ -451,6 +494,11 @@ export class AgentHarness {
   private readonly registeredMcpToolNames = new Set<string>();
   private latitudeTelemetry?: LatitudeTelemetry;
   private insideTelemetryCapture = false;
+  private _browserSession?: unknown;
+  private _browserMod?: {
+    createBrowserTools: (getSession: () => unknown, getConversationId: () => string) => ToolDefinition[];
+    BrowserSession: new (sessionId: string, config: Record<string, unknown>) => unknown;
+  };
 
   private parsedAgent?: ParsedAgent;
   private mcpBridge?: LocalMcpBridge;
@@ -816,6 +864,16 @@ export class AgentHarness {
         }),
       );
     }
+
+    if (config?.browser) {
+      await this.initBrowserTools(config)
+        .catch((e) => {
+          console.warn(
+            `[poncho][browser] Failed to load browser tools: ${e instanceof Error ? e.message : String(e)}`,
+          );
+        });
+    }
+
     await bridge.startLocalServers();
     await bridge.discoverTools();
     await this.refreshMcpTools("initialize");
@@ -856,7 +914,68 @@ export class AgentHarness {
     }
   }
 
+  private async initBrowserTools(config: PonchoConfig): Promise<void> {
+    const spec = ["@poncho-ai", "browser"].join("/");
+    let browserMod: {
+      createBrowserTools: (getSession: () => unknown, getConversationId: () => string) => ToolDefinition[];
+      BrowserSession: new (sessionId: string, cfg?: Record<string, unknown>) => unknown;
+    };
+    try {
+      // Resolve from the agent project's node_modules (not the harness dist
+      // location).  Walk up from workingDir the same way Node's resolution
+      // algorithm does, then dynamically import the ESM entry point.
+      const { existsSync } = await import("node:fs");
+      const { join, dirname } = await import("node:path");
+      const { pathToFileURL } = await import("node:url");
+
+      let searchDir = this.workingDir;
+      let entryPath: string | undefined;
+      for (;;) {
+        const candidate = join(searchDir, "node_modules", "@poncho-ai", "browser", "dist", "index.js");
+        if (existsSync(candidate)) { entryPath = candidate; break; }
+        const parent = dirname(searchDir);
+        if (parent === searchDir) break;
+        searchDir = parent;
+      }
+      if (!entryPath) throw new Error("not installed");
+      browserMod = await import(pathToFileURL(entryPath).href);
+    } catch {
+      throw new Error(
+        `browser: true is set in poncho.config but @poncho-ai/browser is not installed.\n` +
+        `  Run: pnpm add @poncho-ai/browser`,
+      );
+    }
+
+    this._browserMod = browserMod;
+    const browserCfg = typeof config.browser === "object" ? config.browser : {};
+    const agentId = this.parsedAgent?.frontmatter.id ?? this.parsedAgent?.frontmatter.name ?? "default";
+    const session = new browserMod.BrowserSession(`poncho-${agentId}`, browserCfg);
+    this._browserSession = session;
+
+    const tools = browserMod.createBrowserTools(
+      () => session,
+      () => this._currentRunConversationId ?? "__default__",
+    );
+    for (const tool of tools) {
+      if (this.isToolEnabled(tool.name)) {
+        this.registerIfMissing(tool);
+      }
+    }
+  }
+
+  /** Conversation ID of the currently executing run (set during run, cleared after). */
+  private _currentRunConversationId?: string;
+
+  get browserSession(): unknown {
+    return this._browserSession;
+  }
+
   async shutdown(): Promise<void> {
+    if (this._browserSession) {
+      try { await (this._browserSession as { close(): Promise<void> }).close(); } catch { /* best-effort */ }
+      this._browserSession = undefined;
+    }
+
     await this.mcpBridge?.stopLocalServers();
     if (this.latitudeTelemetry) {
       await this.latitudeTelemetry.shutdown().catch((err) => {
@@ -966,6 +1085,9 @@ export class AgentHarness {
     }
     await this.refreshSkillsIfChanged();
 
+    // Track which conversation this run belongs to so browser tools resolve the right session
+    this._currentRunConversationId = input.conversationId;
+
     const agent = this.parsedAgent as ParsedAgent;
     const runId = `run_${randomUUID()}`;
     const start = now();
@@ -1041,6 +1163,34 @@ ${boundedMainMemory.trim()}`
       contextWindow,
     });
 
+    // Subscribe to browser frame/status events for this conversation's tab.
+    const browserEventQueue: AgentEvent[] = [];
+    const browserCleanups: Array<() => void> = [];
+    const browserSession = this._browserSession as
+      | { onFrame: (cid: string, cb: (f: { data: string; width: number; height: number }) => void) => () => void;
+          onStatus: (cid: string, cb: (s: { active: boolean; url?: string; interactionAllowed: boolean }) => void) => () => void;
+          saveState: (path: string) => Promise<void>;
+          close: () => Promise<void>;
+          profileDir: string;
+          isLaunched: boolean }
+      | undefined;
+    const conversationId = input.conversationId ?? "__default__";
+    if (browserSession) {
+      browserCleanups.push(
+        browserSession.onFrame(conversationId, (frame) => {
+          browserEventQueue.push({ type: "browser:frame", data: frame.data, width: frame.width, height: frame.height });
+        }),
+        browserSession.onStatus(conversationId, (status) => {
+          browserEventQueue.push({ type: "browser:status", ...status });
+        }),
+      );
+    }
+    const drainBrowserEvents = function* (): Generator<AgentEvent> {
+      while (browserEventQueue.length > 0) {
+        yield browserEventQueue.shift()!;
+      }
+    };
+
     if (input.task != null) {
       if (input.files && input.files.length > 0) {
         const parts: ContentPart[] = [
@@ -1088,6 +1238,7 @@ ${boundedMainMemory.trim()}`
 
     for (let step = 1; step <= maxSteps; step += 1) {
       try {
+        yield* drainBrowserEvents();
         if (isCancelled()) {
           yield emitCancellation();
           return;
@@ -1145,8 +1296,22 @@ ${boundedMainMemory.trim()}`
         // Convert messages to ModelMessage format
         const convertMessage = async (msg: Message): Promise<ModelMessage[]> => {
           if (msg.role === "tool") {
-            // Tool messages are provider-sensitive; skip malformed historical records
-            // instead of failing the entire run continuation.
+            // When rich (multi-part) tool results are attached from the
+            // current run, use them directly — they include proper image
+            // content blocks instead of base64 text.
+            const meta = msg.metadata as Record<string, unknown> | undefined;
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const rich = (meta as any)?._richToolResults as unknown[] | undefined;
+            if (rich && rich.length > 0) {
+              // The rich array already conforms to the AI SDK ToolContent shape
+              // (tool-result parts with multi-part content outputs).  Cast
+              // through `any` because the exact generic types are internal.
+              // eslint-disable-next-line @typescript-eslint/no-explicit-any
+              return [{ role: "tool" as const, content: rich as any }];
+            }
+
+            // Fallback for historical messages loaded from storage (base64
+            // already stripped, so this is always token-safe).
             const textContent = typeof msg.content === "string" ? msg.content : getTextContent(msg);
             try {
               const parsed: unknown = JSON.parse(textContent);
@@ -1171,7 +1336,6 @@ ${boundedMainMemory.trim()}`
               return [{
                 role: "tool" as const,
                 content: toolResults.map((tr) => {
-                  // Parse JSON content for successful results, keep error messages as strings.
                   if (tr.content.startsWith("Tool error:")) {
                     return {
                       type: "tool-result" as const,
@@ -1589,6 +1753,17 @@ ${boundedMainMemory.trim()}`
         content: string;
       }> = [];
 
+      // Rich tool results that use multi-part content for images (proper
+      // vision tokens instead of base64 text).  Used for the *current* step
+      // model call; the `toolResultsForModel` array holds the storage-safe
+      // version with base64 stripped.
+      const richToolResults: Array<{
+        type: "tool-result";
+        toolCallId: string;
+        toolName: string;
+        output: { type: "json"; value: unknown } | { type: "content"; value: Array<{ type: "text"; text: string } | { type: "media"; data: string; mediaType: string }> };
+      }> = [];
+
       const approvedCalls: Array<{
         id: string;
         name: string;
@@ -1697,6 +1872,12 @@ ${boundedMainMemory.trim()}`
             tool_name: result.tool,
             content: `Tool error: ${result.error}`,
           });
+          richToolResults.push({
+            type: "tool-result",
+            toolCallId: result.callId,
+            toolName: result.tool,
+            output: { type: "json", value: { error: result.error } },
+          });
         } else {
           span?.end({ result: { value: result.output ?? null, isError: false } });
           yield pushEvent({
@@ -1705,12 +1886,36 @@ ${boundedMainMemory.trim()}`
             output: result.output,
             duration: now() - batchStart,
           });
+
+          const { mediaItems, strippedOutput } = extractMediaFromToolOutput(result.output);
           toolResultsForModel.push({
             type: "tool_result",
             tool_use_id: result.callId,
             tool_name: result.tool,
-            content: JSON.stringify(result.output ?? null),
+            content: JSON.stringify(strippedOutput ?? null),
           });
+
+          if (mediaItems.length > 0) {
+            richToolResults.push({
+              type: "tool-result",
+              toolCallId: result.callId,
+              toolName: result.tool,
+              output: {
+                type: "content",
+                value: [
+                  { type: "text", text: JSON.stringify(strippedOutput ?? null) },
+                  ...mediaItems,
+                ],
+              },
+            });
+          } else {
+            richToolResults.push({
+              type: "tool-result",
+              toolCallId: result.callId,
+              toolName: result.tool,
+              output: { type: "json", value: result.output ?? null },
+            });
+          }
         }
       }
 
@@ -1731,10 +1936,11 @@ ${boundedMainMemory.trim()}`
         content: assistantContent,
         metadata: { timestamp: now(), id: randomUUID(), step },
       });
+      const toolMsgMeta: Record<string, unknown> = { timestamp: now(), id: randomUUID(), step, _richToolResults: richToolResults };
       messages.push({
         role: "tool",
         content: JSON.stringify(toolResultsForModel),
-        metadata: { timestamp: now(), id: randomUUID(), step },
+        metadata: toolMsgMeta as Message["metadata"],
       });
 
         yield pushEvent({
@@ -1793,6 +1999,10 @@ ${boundedMainMemory.trim()}`
         },
       });
     }
+
+    // Drain any remaining browser events and clean up subscriptions
+    yield* drainBrowserEvents();
+    for (const cleanup of browserCleanups) cleanup();
   }
 
   async executeTools(

package/src/telemetry.ts

@@ -1,5 +1,16 @@
 import type { AgentEvent } from "@poncho-ai/sdk";
 
+const MAX_FIELD_LENGTH = 200;
+
+function sanitizeEventForLog(event: AgentEvent): string {
+  return JSON.stringify(event, (_key, value) => {
+    if (typeof value === "string" && value.length > MAX_FIELD_LENGTH) {
+      return `${value.slice(0, 80)}...[${value.length} chars]`;
+    }
+    return value;
+  });
+}
+
 export interface TelemetryConfig {
   enabled?: boolean;
   otlp?: string;
@@ -33,7 +44,8 @@ export class TelemetryEmitter {
     // Latitude telemetry is handled by LatitudeTelemetry (from
     // @latitude-data/telemetry) via harness.runWithTelemetry().
     // Default behavior in local dev: print concise structured logs.
-    process.stdout.write(`[event] ${event.type} ${JSON.stringify(event)}\n`);
+    // Strip large binary payloads (e.g. base64 images) to keep logs readable.
+    process.stdout.write(`[event] ${event.type} ${sanitizeEventForLog(event)}\n`);
   }
 
   private async sendOtlp(event: AgentEvent): Promise<void> {