@poncho-ai/harness 0.15.1 → 0.16.1

package/.turbo/turbo-build.log

@@ -1,5 +1,5 @@
 
-> @poncho-ai/harness@0.15.1 build /Users/cesar/Dev/latitude/poncho-ai/packages/harness
+> @poncho-ai/harness@0.16.1 build /Users/cesar/Dev/latitude/poncho-ai/packages/harness
 > tsup src/index.ts --format esm --dts
 
 CLI Building entry: src/index.ts
@@ -7,8 +7,8 @@
 CLI tsup v8.5.1
 CLI Target: es2022
 ESM Build start
-ESM dist/index.js 182.55 KB
-ESM ⚡️ Build success in 131ms
+ESM dist/index.js 187.36 KB
+ESM ⚡️ Build success in 91ms
 DTS Build start
-DTS ⚡️ Build success in 6605ms
-DTS dist/index.d.ts 22.34 KB
+DTS ⚡️ Build success in 3621ms
+DTS dist/index.d.ts 22.38 KB

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @poncho-ai/harness
 
+## 0.16.1
+
+### Patch Changes
+
+- [`7475da5`](https://github.com/cesr/poncho-ai/commit/7475da5c0c2399e79064a2622137c0eb2fb16871) Thanks [@cesr](https://github.com/cesr)! - Inject browser usage context into agent system prompt (auth flow, session persistence, tool selection guidance).
+
+## 0.16.0
+
+### Minor Changes
+
+- [`12f2845`](https://github.com/cesr/poncho-ai/commit/12f28457c20e650640ff2a1c1dbece2a6e4a9ddd) Thanks [@cesr](https://github.com/cesr)! - Add browser storage persistence (cookies/localStorage survive restarts via configured storage provider) and new `browser_content` tool for fast text extraction from pages.
+
 ## 0.15.1
 
 ### Patch Changes

package/dist/index.d.ts

@@ -486,6 +486,7 @@ declare class AgentHarness {
     private registerSkillTools;
     private refreshSkillsIfChanged;
     initialize(): Promise<void>;
+    private buildBrowserStoragePersistence;
     private initBrowserTools;
     /** Conversation ID of the currently executing run (set during run, cleared after). */
     private _currentRunConversationId?;

package/dist/index.js

@@ -3290,6 +3290,85 @@ var AgentHarness = class {
       );
     }
   }
+  async buildBrowserStoragePersistence(config, sessionId) {
+    const provider = config.storage?.provider ?? config.state?.provider ?? "local";
+    const stateKey = `poncho:browser:state:${sessionId}`;
+    if (provider === "memory") return void 0;
+    if (provider === "local") {
+      const { resolve: pathResolve } = await import("path");
+      const { homedir: home } = await import("os");
+      const stateDir = pathResolve(home(), ".poncho", "browser-state");
+      const filePath = pathResolve(stateDir, `${sessionId}.json`);
+      return {
+        async save(json) {
+          const { mkdir: mkdir5, writeFile: writeFile6 } = await import("fs/promises");
+          await mkdir5(stateDir, { recursive: true });
+          await writeFile6(filePath, json, "utf8");
+        },
+        async load() {
+          const { readFile: readFile8 } = await import("fs/promises");
+          try {
+            return await readFile8(filePath, "utf8");
+          } catch {
+            return void 0;
+          }
+        }
+      };
+    }
+    if (provider === "upstash") {
+      const urlEnv = config.storage?.urlEnv ?? (process.env.UPSTASH_REDIS_REST_URL ? "UPSTASH_REDIS_REST_URL" : "KV_REST_API_URL");
+      const tokenEnv = config.storage?.tokenEnv ?? (process.env.UPSTASH_REDIS_REST_TOKEN ? "UPSTASH_REDIS_REST_TOKEN" : "KV_REST_API_TOKEN");
+      const baseUrl = (process.env[urlEnv] ?? "").replace(/\/+$/, "");
+      const token = process.env[tokenEnv] ?? "";
+      if (!baseUrl || !token) return void 0;
+      const headers = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
+      return {
+        async save(json) {
+          await fetch(`${baseUrl}/set/${encodeURIComponent(stateKey)}/${encodeURIComponent(json)}`, { method: "POST", headers });
+        },
+        async load() {
+          const res = await fetch(`${baseUrl}/get/${encodeURIComponent(stateKey)}`, { headers });
+          if (!res.ok) return void 0;
+          const body = await res.json();
+          return body.result ?? void 0;
+        }
+      };
+    }
+    if (provider === "redis") {
+      const urlEnv = config.storage?.urlEnv ?? "REDIS_URL";
+      const url = process.env[urlEnv] ?? "";
+      if (!url) return void 0;
+      let clientPromise;
+      const getClient = () => {
+        if (!clientPromise) {
+          clientPromise = (async () => {
+            try {
+              const mod = await import("redis");
+              const c = mod.createClient({ url });
+              await c.connect();
+              return c;
+            } catch {
+              return void 0;
+            }
+          })();
+        }
+        return clientPromise;
+      };
+      return {
+        async save(json) {
+          const c = await getClient();
+          if (c) await c.set(stateKey, json);
+        },
+        async load() {
+          const c = await getClient();
+          if (!c) return void 0;
+          const val = await c.get(stateKey);
+          return val ?? void 0;
+        }
+      };
+    }
+    return void 0;
+  }
   async initBrowserTools(config) {
     const spec = ["@poncho-ai", "browser"].join("/");
     let browserMod;
@@ -3318,9 +3397,14 @@ var AgentHarness = class {
       );
     }
     this._browserMod = browserMod;
-    const browserCfg = typeof config.browser === "object" ? config.browser : {};
+    const browserCfg = typeof config.browser === "object" ? { ...config.browser } : {};
     const agentId = this.parsedAgent?.frontmatter.id ?? this.parsedAgent?.frontmatter.name ?? "default";
-    const session = new browserMod.BrowserSession(`poncho-${agentId}`, browserCfg);
+    const sessionId = `poncho-${agentId}`;
+    const storagePersistence = await this.buildBrowserStoragePersistence(config, sessionId);
+    if (storagePersistence) {
+      browserCfg.storagePersistence = storagePersistence;
+    }
+    const session = new browserMod.BrowserSession(sessionId, browserCfg);
     this._browserSession = session;
     const tools = browserMod.createBrowserTools(
       () => session,
@@ -3454,9 +3538,29 @@ var AgentHarness = class {
     const developmentContext = this.environment === "development" ? `
 
 ${DEVELOPMENT_MODE_CONTEXT}` : "";
+    const browserContext = this._browserSession ? `
+
+## Browser Tools
+
+The user has a live browser viewport displayed alongside the conversation. They can see everything the browser shows in real time and interact with it directly (click, type, scroll, paste).
+
+### Authentication
+When a website requires authentication or credentials, do NOT ask the user to send them in the chat. Instead, navigate to the login page and let the user enter their credentials directly in the browser viewport. Wait for them to confirm they have logged in before continuing.
+
+### Session persistence
+Browser sessions (cookies, localStorage, login state) are automatically saved and restored across conversations. If the user logged into a website in a previous conversation, that session is likely still active. Try navigating directly to the authenticated page before asking the user to log in again.
+
+### Reading page content
+- Use \`browser_content\` to read the visible text on a page. This is fast and token-efficient.
+- Use \`browser_snapshot\` to get the accessibility tree with interactive element refs for clicking and typing.
+- Use \`browser_screenshot\` only when you need to see visual layout or images. Screenshots consume significantly more tokens.
+- The accessibility tree may be sparse on some pages. If \`browser_snapshot\` returns little or no content, fall back to \`browser_content\` or \`browser_screenshot\`.
+
+### Tabs and resources
+Each conversation gets its own browser tab sharing a single browser instance. Call \`browser_close\` when done to free the tab. If you don't close it, the tab stays open and the user can continue interacting with it.` : "";
     const promptWithSkills = this.skillContextWindow ? `${systemPrompt}${developmentContext}
 
-${this.skillContextWindow}` : `${systemPrompt}${developmentContext}`;
+${this.skillContextWindow}${browserContext}` : `${systemPrompt}${developmentContext}${browserContext}`;
     const mainMemory = this.memoryStore ? await this.memoryStore.getMainMemory() : void 0;
     const boundedMainMemory = mainMemory && mainMemory.content.length > 4e3 ? `${mainMemory.content.slice(0, 4e3)}
 ...[truncated]` : mainMemory?.content;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@poncho-ai/harness",
-  "version": "0.15.1",
+  "version": "0.16.1",
   "description": "Agent execution runtime - conversation loop, tool dispatch, streaming",
   "repository": {
     "type": "git",

package/src/harness.ts

@@ -914,6 +914,94 @@ export class AgentHarness {
     }
   }
 
+  private async buildBrowserStoragePersistence(
+    config: PonchoConfig,
+    sessionId: string,
+  ): Promise<{ save(json: string): Promise<void>; load(): Promise<string | undefined> } | undefined> {
+    const provider = config.storage?.provider ?? (config.state as Record<string, unknown> | undefined)?.provider as string | undefined ?? "local";
+    const stateKey = `poncho:browser:state:${sessionId}`;
+
+    if (provider === "memory") return undefined;
+
+    if (provider === "local") {
+      const { resolve: pathResolve } = await import("node:path");
+      const { homedir: home } = await import("node:os");
+      const stateDir = pathResolve(home(), ".poncho", "browser-state");
+      const filePath = pathResolve(stateDir, `${sessionId}.json`);
+      return {
+        async save(json: string) {
+          const { mkdir, writeFile } = await import("node:fs/promises");
+          await mkdir(stateDir, { recursive: true });
+          await writeFile(filePath, json, "utf8");
+        },
+        async load() {
+          const { readFile } = await import("node:fs/promises");
+          try { return await readFile(filePath, "utf8"); } catch { return undefined; }
+        },
+      };
+    }
+
+    if (provider === "upstash") {
+      const urlEnv = config.storage?.urlEnv ?? (process.env.UPSTASH_REDIS_REST_URL ? "UPSTASH_REDIS_REST_URL" : "KV_REST_API_URL");
+      const tokenEnv = config.storage?.tokenEnv ?? (process.env.UPSTASH_REDIS_REST_TOKEN ? "UPSTASH_REDIS_REST_TOKEN" : "KV_REST_API_TOKEN");
+      const baseUrl = (process.env[urlEnv] ?? "").replace(/\/+$/, "");
+      const token = process.env[tokenEnv] ?? "";
+      if (!baseUrl || !token) return undefined;
+      const headers = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
+      return {
+        async save(json: string) {
+          await fetch(`${baseUrl}/set/${encodeURIComponent(stateKey)}/${encodeURIComponent(json)}`, { method: "POST", headers });
+        },
+        async load() {
+          const res = await fetch(`${baseUrl}/get/${encodeURIComponent(stateKey)}`, { headers });
+          if (!res.ok) return undefined;
+          const body = await res.json() as { result?: string | null };
+          return body.result ?? undefined;
+        },
+      };
+    }
+
+    if (provider === "redis") {
+      const urlEnv = config.storage?.urlEnv ?? "REDIS_URL";
+      const url = process.env[urlEnv] ?? "";
+      if (!url) return undefined;
+      let clientPromise: Promise<{ get(k: string): Promise<string | null>; set(k: string, v: string): Promise<unknown> } | undefined> | undefined;
+      const getClient = () => {
+        if (!clientPromise) {
+          clientPromise = (async () => {
+            try {
+              const mod = (await import("redis")) as unknown as {
+                createClient: (opts: { url: string }) => {
+                  connect(): Promise<unknown>;
+                  get(k: string): Promise<string | null>;
+                  set(k: string, v: string): Promise<unknown>;
+                };
+              };
+              const c = mod.createClient({ url });
+              await c.connect();
+              return c;
+            } catch { return undefined; }
+          })();
+        }
+        return clientPromise;
+      };
+      return {
+        async save(json: string) {
+          const c = await getClient();
+          if (c) await c.set(stateKey, json);
+        },
+        async load() {
+          const c = await getClient();
+          if (!c) return undefined;
+          const val = await c.get(stateKey);
+          return val ?? undefined;
+        },
+      };
+    }
+
+    return undefined;
+  }
+
   private async initBrowserTools(config: PonchoConfig): Promise<void> {
     const spec = ["@poncho-ai", "browser"].join("/");
     let browserMod: {
@@ -947,9 +1035,16 @@ export class AgentHarness {
     }
 
     this._browserMod = browserMod;
-    const browserCfg = typeof config.browser === "object" ? config.browser : {};
+    const browserCfg: Record<string, unknown> = typeof config.browser === "object" ? { ...config.browser } : {};
     const agentId = this.parsedAgent?.frontmatter.id ?? this.parsedAgent?.frontmatter.name ?? "default";
-    const session = new browserMod.BrowserSession(`poncho-${agentId}`, browserCfg);
+    const sessionId = `poncho-${agentId}`;
+
+    const storagePersistence = await this.buildBrowserStoragePersistence(config, sessionId);
+    if (storagePersistence) {
+      browserCfg.storagePersistence = storagePersistence;
+    }
+
+    const session = new browserMod.BrowserSession(sessionId, browserCfg);
     this._browserSession = session;
 
     const tools = browserMod.createBrowserTools(
@@ -1115,9 +1210,29 @@ export class AgentHarness {
     });
     const developmentContext =
       this.environment === "development" ? `\n\n${DEVELOPMENT_MODE_CONTEXT}` : "";
+    const browserContext = this._browserSession
+      ? `\n\n## Browser Tools
+
+The user has a live browser viewport displayed alongside the conversation. They can see everything the browser shows in real time and interact with it directly (click, type, scroll, paste).
+
+### Authentication
+When a website requires authentication or credentials, do NOT ask the user to send them in the chat. Instead, navigate to the login page and let the user enter their credentials directly in the browser viewport. Wait for them to confirm they have logged in before continuing.
+
+### Session persistence
+Browser sessions (cookies, localStorage, login state) are automatically saved and restored across conversations. If the user logged into a website in a previous conversation, that session is likely still active. Try navigating directly to the authenticated page before asking the user to log in again.
+
+### Reading page content
+- Use \`browser_content\` to read the visible text on a page. This is fast and token-efficient.
+- Use \`browser_snapshot\` to get the accessibility tree with interactive element refs for clicking and typing.
+- Use \`browser_screenshot\` only when you need to see visual layout or images. Screenshots consume significantly more tokens.
+- The accessibility tree may be sparse on some pages. If \`browser_snapshot\` returns little or no content, fall back to \`browser_content\` or \`browser_screenshot\`.
+
+### Tabs and resources
+Each conversation gets its own browser tab sharing a single browser instance. Call \`browser_close\` when done to free the tab. If you don't close it, the tab stays open and the user can continue interacting with it.`
+      : "";
     const promptWithSkills = this.skillContextWindow
-      ? `${systemPrompt}${developmentContext}\n\n${this.skillContextWindow}`
-      : `${systemPrompt}${developmentContext}`;
+      ? `${systemPrompt}${developmentContext}\n\n${this.skillContextWindow}${browserContext}`
+      : `${systemPrompt}${developmentContext}${browserContext}`;
     const mainMemory = this.memoryStore
       ? await this.memoryStore.getMainMemory()
       : undefined;