@elizaos/plugin-browser 2.0.0-alpha.9 → 2.0.3-beta.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shaw Walters and elizaOS Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,116 +1,131 @@
-# Browser Automation Plugin for ElizaOS (TypeScript)
+# @elizaos/plugin-browser
 
-Browser automation plugin enabling AI agents to browse websites, interact with elements, and extract data.
+Browser automation and companion bridge plugin for elizaOS. Adds the `BROWSER` action and `MANAGE_BROWSER_BRIDGE` action to any Eliza agent, owns the Eliza browser workspace (electrobun-embedded `BrowserView` on desktop, JSDOM fallback on web/mobile), and manages the Chrome/Safari Agent Browser Bridge companion extension.
 
-## Features
+## What this plugin provides
 
-- **Navigation**: Navigate to URLs, go back/forward, refresh pages
-- **AI-Powered Interactions**: Click, type, and select elements using natural language
-- **Data Extraction**: Extract structured data from web pages
-- **Screenshots**: Capture page screenshots
-- **CAPTCHA Solving**: Automatic CAPTCHA solving (Turnstile, reCAPTCHA, hCaptcha)
-- **Session Management**: Handle multiple browser sessions
-- **Security**: URL validation, domain filtering, rate limiting
+### Actions
 
-## Installation
+**BROWSER** — Controls a registered browser target. The agent picks the best available backend automatically, or you can pin a specific target with the `target` parameter. Supported operations:
 
-```bash
-npm install @elizaos/plugin-browser
-```
-
-## Configuration
+| `action` value | What it does |
+|---|---|
+| `open` | Open a URL in a new tab |
+| `navigate` | Navigate an existing tab to a URL |
+| `click` | Click a DOM element by CSS selector |
+| `type` | Type text into a selector |
+| `press` | Press a keyboard key |
+| `get` | Get a DOM value |
+| `state` | Return current tab state (URL, title) |
+| `snapshot` | Capture a DOM snapshot |
+| `screenshot` | Capture a screenshot |
+| `reload` | Reload the current tab |
+| `back` / `forward` | Browser history navigation |
+| `close` | Close a tab |
+| `show` / `hide` | Show or hide the browser window |
+| `wait` | Wait for a selector to appear |
+| `tab` | Tab management (list/new/close/switch) |
+| `realistic_click` | Animated cursor click (visible to user) |
+| `realistic_fill` | Animated fill with per-character delay |
+| `realistic_type` | Animated typing |
+| `realistic_press` | Animated key press |
+| `cursor_move` | Animate cursor to a position |
+| `cursor_hide` | Hide the cursor overlay |
+| `autofill_login` | Fill saved credentials into a browser tab (vault-gated; requires `domain`) |
 
-```bash
-# Optional - for cloud browser
-BROWSERBASE_API_KEY=your_api_key
-BROWSERBASE_PROJECT_ID=your_project_id
+**MANAGE_BROWSER_BRIDGE** — Manages the Chrome/Safari companion extension. Subactions: `install` (build + reveal + open manager), `reveal_folder` (open the build folder in Finder/Explorer), `open_manager` (`chrome://extensions`), `refresh` (report paired companions and settings). Owner-only.
 
-# Optional - for AI-powered interactions
-OPENAI_API_KEY=your_openai_key
-ANTHROPIC_API_KEY=your_anthropic_key
+### Browser targets
 
-# Optional - for CAPTCHA solving
-CAPSOLVER_API_KEY=your_capsolver_key
+The plugin uses a pluggable target registry in `BrowserService`. Targets are selected automatically by availability and score:
 
-# Browser settings
-BROWSER_HEADLESS=true
-BROWSER_ENABLED=true
-```
+| Target ID | Backend | When available |
+|---|---|---|
+| `workspace` | Electrobun `BrowserView` (desktop) or JSDOM (web) | Always |
+| `bridge` | Paired Chrome/Safari via companion extension | At least one companion paired |
+| `stagehand` | Playwright/Stagehand via HTTP endpoint | `ELIZA_BROWSER_STAGEHAND_COMMAND_URL` or `STAGEHAND_SERVER_URL` set |
 
-## Usage
+External plugins can register additional targets by calling `BrowserService.registerTarget(target)`.
 
-```typescript
-import { browserPlugin } from '@elizaos/plugin-browser';
+### Provider
 
-const agent = {
-  name: 'BrowserAgent',
-  plugins: [browserPlugin],
-};
-```
+`browser_workspace` — Injects the current dispatch mode (`desktop` / `web`) and a capped list of open tabs into agent context. Active when the `browser` or `web` context is selected.
 
-## Actions
+### Routes
 
-| Action               | Description        | Examples                            |
-| -------------------- | ------------------ | ----------------------------------- |
-| `BROWSER_NAVIGATE`   | Navigate to URL    | "Go to google.com"                  |
-| `BROWSER_BACK`       | Go back in history | "Go back"                           |
-| `BROWSER_FORWARD`    | Go forward         | "Go forward"                        |
-| `BROWSER_REFRESH`    | Refresh page       | "Refresh the page"                  |
-| `BROWSER_CLICK`      | Click element      | "Click the search button"           |
-| `BROWSER_TYPE`       | Type text          | "Type 'hello' in the search box"    |
-| `BROWSER_SELECT`     | Select option      | "Select 'US' from country dropdown" |
-| `BROWSER_EXTRACT`    | Extract data       | "Extract the main heading"          |
-| `BROWSER_SCREENSHOT` | Take screenshot    | "Take a screenshot"                 |
+`/api/browser-bridge/*` — HTTP surface for the companion extension: pairing, settings, tab sync, page-context ingest, session progress, and extension package build/download.
 
-## Providers
+## Requirements
 
-| Provider        | Description                   |
-| --------------- | ----------------------------- |
-| `BROWSER_STATE` | Current browser session state |
+### Auto-enable
 
-## API
+The plugin is opt-in. It activates when `config.features.browser` is truthy in the elizaOS agent config:
 
-### BrowserService
-
-```typescript
-class BrowserService extends Service {
-  async createSession(sessionId: string): Promise<BrowserSession>;
-  async getSession(sessionId: string): Promise<BrowserSession | undefined>;
-  async getCurrentSession(): Promise<BrowserSession | undefined>;
-  async destroySession(sessionId: string): Promise<void>;
+```json
+{
+  "features": {
+    "browser": true
+  }
 }
 ```
 
-### BrowserSession
+### Environment variables
 
-```typescript
-interface BrowserSession {
-  id: string;
-  createdAt: Date;
-  url?: string;
-  title?: string;
-}
-```
+| Variable | Purpose |
+|---|---|
+| `ELIZA_BROWSER_STAGEHAND_COMMAND_URL` | Full URL for the Stagehand command endpoint |
+| `STAGEHAND_SERVER_URL` | Stagehand base URL (commands go to `<url>/api/browser-command`) |
+| `ELIZA_BROWSER_STAGEHAND_URL` | Alias for `STAGEHAND_SERVER_URL` |
+| `ELIZA_BROWSER_STAGEHAND_AUTO_SETUP` | Set `false` to disable automatic stagehand-server install/build |
+| `ELIZA_BROWSER_ALLOW_STAGEHAND_ON_MOBILE` | Set `true` to allow stagehand target on mobile |
+| `ELIZA_MOBILE_PLATFORM` / `ELIZA_PLATFORM` / `CAPACITOR_PLATFORM` | Platform hint for target scoring (`ios`/`android`/`mobile`) |
+
+### Vault keys (set by the user, not env vars)
 
-## Development
+`autofill_login` only fires when the user has pre-authorized it per domain:
 
-```bash
-# Install dependencies
-npm install
+- `creds.<domain>.:autoallow = "1"` — set via Settings → Vault → Logins.
 
-# Build
-npm run build
+Without this flag, the action returns an error rather than prompting interactively.
 
-# Run tests
-npm test
+## Companion extension authentication
 
-# Type check
-npm run typecheck
+Companion-scoped endpoints require two headers:
+
+```
+X-Browser-Bridge-Companion-Id: <companion uuid>
+Authorization: Bearer <pairing token>
 ```
 
-## License
+Legacy header aliases (`X-LifeOps-Browser-Companion-Id`, `x-eliza-browser-companion-id`) are not accepted.
+
+## Database
+
+Drizzle tables in the `browser` PostgreSQL schema (applied by elizaOS `plugin-sql` migrator):
 
-MIT
+- `browser_bridge_companions`
+- `browser_bridge_settings`
+- `browser_bridge_tabs`
+- `browser_bridge_page_contexts`
 
+## Registering a custom browser target
 
+Any plugin can extend the browser dispatch surface at runtime:
 
+```ts
+import { BrowserService, BROWSER_SERVICE_TYPE } from "@elizaos/plugin-browser";
+import type { BrowserTarget } from "@elizaos/plugin-browser";
+
+const myTarget: BrowserTarget = {
+  id: "my-target",
+  name: "My Browser",
+  description: "Custom browser backend.",
+  kind: "external",
+  priority: 50,
+  available: async () => true,
+  execute: async (command) => { /* ... */ },
+};
+
+const browserService = runtime.getService<BrowserService>(BROWSER_SERVICE_TYPE);
+browserService?.registerTarget(myTarget);
+```

package/auto-enable.ts

@@ -0,0 +1,24 @@
+// Auto-enable check for @elizaos/plugin-browser.
+//
+// Plugin manifest entry-point — referenced by package.json's
+// `elizaos.plugin.autoEnableModule`. Keep this module light: env reads only,
+// no service init, no transitive imports of the full plugin runtime. The
+// auto-enable engine loads dozens of these per boot.
+import type { PluginAutoEnableContext } from "@elizaos/core";
+
+function isFeatureEnabled(
+  config: PluginAutoEnableContext["config"],
+  key: string,
+): boolean {
+  const f = (config?.features as Record<string, unknown> | undefined)?.[key];
+  if (f === true) return true;
+  if (f && typeof f === "object" && f !== null) {
+    return (f as Record<string, unknown>).enabled !== false;
+  }
+  return false;
+}
+
+/** Enable when `config.features.browser` is truthy / not explicitly disabled. */
+export function shouldEnable(ctx: PluginAutoEnableContext): boolean {
+  return isFeatureEnabled(ctx.config, "browser");
+}

package/dist/actions/browser-autofill-login.d.ts

@@ -0,0 +1,43 @@
+/**
+ * BROWSER autofill-login subaction — agent-driven browser login autofill.
+ *
+ * Invoked via BROWSER with `subaction: "autofill-login"` (canonical). The legacy
+ * planner action name `BROWSER_AUTOFILL_LOGIN` normalizes to BROWSER with this
+ * subaction in the planner dispatch pipeline.
+ *
+ * Lets the agent say "log into github.com for me" and have the saved
+ * credentials filled into an open Eliza browser tab without a per-call
+ * consent prompt.
+ *
+ * Authorization model (mirrors the user-driven autofill flow):
+ *   - The user must have set `creds.<domain>.:autoallow = "1"` on the
+ *     domain. This is the same vault key the React-side consent flow
+ *     uses; toggling it from Settings -> Vault -> Logins is the
+ *     SOLE way to let the agent autofill silently.
+ *   - Without that flag, this action returns
+ *     `{ ok: false, reason: "user has not pre-authorized agent autofill for <domain>" }`.
+ *     The agent should NOT fall back to the user-driven flow on its own
+ *     because the user-driven flow is gated by an interactive React
+ *     modal that an autonomous agent cannot consent to.
+ *
+ * Tab selection:
+ *   - Lists the live browser-workspace tabs and picks the first one
+ *     whose URL hostname matches `domain` (registrable hostname,
+ *     case-insensitive). Returns a clean error when no such tab exists
+ *     so the agent can decide whether to open one first via
+ *     BROWSER (open/navigate).
+ *
+ * Fill mechanism:
+ *   - Injects a small JS snippet that mirrors the same form-detection
+ *     and `setNativeInputValue` helpers the in-tab preload uses, so
+ *     React-controlled inputs see the change.
+ *   - When `submit: true`, the snippet also calls `form.submit()` (or
+ *     clicks a likely submit button) after filling. Off by default —
+ *     the safer behaviour is fill-only and let the user click submit.
+ */
+import type { ActionResult, HandlerOptions, IAgentRuntime, Memory } from "@elizaos/core";
+/**
+ * Executes the vault-gated workspace autofill flow for {@link AUTOFILL_SUBACTION}.
+ */
+export declare function executeBrowserAutofillLogin(_runtime: IAgentRuntime, _message: Memory | undefined, options: HandlerOptions | undefined): Promise<ActionResult>;
+//# sourceMappingURL=browser-autofill-login.d.ts.map

package/dist/actions/browser-autofill-login.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-autofill-login.d.ts","sourceRoot":"","sources":["../../src/actions/browser-autofill-login.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,KAAK,EACV,YAAY,EACZ,cAAc,EACd,aAAa,EACb,MAAM,EACP,MAAM,eAAe,CAAC;AAyIvB;;GAEG;AACH,wBAAsB,2BAA2B,CAC/C,QAAQ,EAAE,aAAa,EACvB,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,OAAO,EAAE,cAAc,GAAG,SAAS,GAClC,OAAO,CAAC,YAAY,CAAC,CAkLvB"}

package/dist/actions/browser-autofill-login.js

@@ -0,0 +1,278 @@
+import { logger } from "@elizaos/core";
+import {
+  createManager,
+  getAutofillAllowed,
+  getSavedLogin,
+  listSavedLogins
+} from "@elizaos/vault";
+import {
+  evaluateBrowserWorkspaceTab,
+  isBrowserWorkspaceBridgeConfigured,
+  listBrowserWorkspaceTabs
+} from "../workspace/browser-workspace.js";
+const AUTOFILL_SUBACTION = "autofill-login";
+const MAX_BROWSER_TAB_SCAN = 100;
+const MAX_FILL_REASON_CHARS = 240;
+let cachedVault = null;
+function sharedAutofillVault() {
+  cachedVault ??= createManager().vault;
+  return cachedVault;
+}
+function tabUrlMatchesDomain(tabUrl, domain) {
+  if (!tabUrl) return false;
+  let hostname;
+  try {
+    hostname = new URL(tabUrl).hostname;
+  } catch {
+    return false;
+  }
+  return hostname.toLowerCase() === domain.toLowerCase();
+}
+function buildAutofillScript(args) {
+  return `
+(() => {
+  const USERNAME = ${JSON.stringify(args.username)};
+  const PASSWORD = ${JSON.stringify(args.password)};
+  const SUBMIT = ${args.submit ? "true" : "false"};
+
+  function setNativeInputValue(input, value) {
+    const proto = Object.getPrototypeOf(input);
+    const desc = Object.getOwnPropertyDescriptor(proto, "value");
+    if (desc && typeof desc.set === "function") {
+      desc.set.call(input, value);
+    } else {
+      input.value = value;
+    }
+    input.dispatchEvent(new Event("input", { bubbles: true }));
+    input.dispatchEvent(new Event("change", { bubbles: true }));
+  }
+
+  function findPrecedingTextInput(passwordInput) {
+    const root = passwordInput.form || document.body;
+    const candidates = root.querySelectorAll(
+      'input[type="text"], input[type="email"], input:not([type])'
+    );
+    let lastBefore = null;
+    for (const el of candidates) {
+      if (el.compareDocumentPosition(passwordInput) & Node.DOCUMENT_POSITION_FOLLOWING) {
+        lastBefore = el;
+      }
+    }
+    return lastBefore;
+  }
+
+  const password = document.querySelector('input[type="password"]');
+  if (!password) {
+    return { ok: false, reason: "no_password_input" };
+  }
+  const form = password.form;
+  const username =
+    (form && form.querySelector(
+      'input[type="email"], input[name*="user" i], input[name*="email" i], input[name*="login" i]'
+    )) || findPrecedingTextInput(password);
+
+  if (username) setNativeInputValue(username, USERNAME);
+  setNativeInputValue(password, PASSWORD);
+
+  if (SUBMIT) {
+    if (form && typeof form.requestSubmit === "function") {
+      form.requestSubmit();
+    } else if (form && typeof form.submit === "function") {
+      form.submit();
+    } else {
+      const button =
+        (form && form.querySelector('button[type="submit"], input[type="submit"]')) ||
+        document.querySelector('button[type="submit"], input[type="submit"]');
+      if (button) (button).click();
+    }
+  }
+
+  return {
+    ok: true,
+    filled: { username: !!username, password: true },
+    submitted: SUBMIT,
+  };
+})();
+`;
+}
+function narrowSnippetResult(raw) {
+  if (!raw || typeof raw !== "object") {
+    return { filled: false, fillReason: null };
+  }
+  const obj = raw;
+  const hasFilledProp = "filled" in obj && Boolean(obj.filled);
+  let fillReason = null;
+  const reasonVal = "reason" in obj ? obj.reason : void 0;
+  if (typeof reasonVal === "string") {
+    fillReason = reasonVal.slice(0, MAX_FILL_REASON_CHARS);
+  }
+  return { filled: hasFilledProp, fillReason };
+}
+async function executeBrowserAutofillLogin(_runtime, _message, options) {
+  const params = options?.parameters;
+  const domain = params?.domain?.trim().toLowerCase() ?? "";
+  const requestedUsername = params?.username?.trim();
+  const submit = params?.submit === true;
+  if (!domain) {
+    return {
+      text: `BROWSER requires subaction "${AUTOFILL_SUBACTION}" and a \`domain\` parameter.`,
+      success: false,
+      values: {
+        success: false,
+        error: "BROWSER_AUTOFILL_BAD_PARAMS",
+        subaction: AUTOFILL_SUBACTION
+      },
+      data: { actionName: "BROWSER", subaction: AUTOFILL_SUBACTION }
+    };
+  }
+  if (!isBrowserWorkspaceBridgeConfigured(process.env)) {
+    return {
+      text: `BROWSER ${AUTOFILL_SUBACTION} requires the desktop browser workspace bridge.`,
+      success: false,
+      values: {
+        success: false,
+        error: "BROWSER_BRIDGE_UNAVAILABLE",
+        subaction: AUTOFILL_SUBACTION
+      },
+      data: { actionName: "BROWSER", subaction: AUTOFILL_SUBACTION }
+    };
+  }
+  const vault = sharedAutofillVault();
+  const allowed = await getAutofillAllowed(vault, domain);
+  if (!allowed) {
+    const text = `User has not pre-authorized agent autofill for ${domain}. Toggle "Allow agent to autofill" for this domain under Settings -> Vault -> Logins.`;
+    return {
+      text,
+      success: false,
+      values: {
+        success: false,
+        error: "AGENT_AUTOFILL_NOT_AUTHORIZED",
+        domain,
+        subaction: AUTOFILL_SUBACTION
+      },
+      data: {
+        actionName: "BROWSER",
+        subaction: AUTOFILL_SUBACTION,
+        domain,
+        reason: text
+      }
+    };
+  }
+  let savedLogin = null;
+  if (requestedUsername) {
+    savedLogin = await getSavedLogin(vault, domain, requestedUsername);
+    if (!savedLogin) {
+      return {
+        text: `No saved login for ${requestedUsername} on ${domain}.`,
+        success: false,
+        values: {
+          success: false,
+          error: "AGENT_AUTOFILL_NO_LOGIN",
+          domain,
+          username: requestedUsername,
+          subaction: AUTOFILL_SUBACTION
+        },
+        data: { actionName: "BROWSER", subaction: AUTOFILL_SUBACTION }
+      };
+    }
+  } else {
+    const summaries = await listSavedLogins(vault, domain);
+    if (summaries.length === 0) {
+      return {
+        text: `No saved logins for ${domain}.`,
+        success: false,
+        values: {
+          success: false,
+          error: "AGENT_AUTOFILL_NO_LOGIN",
+          domain,
+          subaction: AUTOFILL_SUBACTION
+        },
+        data: { actionName: "BROWSER", subaction: AUTOFILL_SUBACTION }
+      };
+    }
+    const sorted = [...summaries].sort(
+      (a, b) => b.lastModified - a.lastModified
+    );
+    const chosen = sorted[0];
+    if (!chosen) {
+      return {
+        text: `No saved logins for ${domain}.`,
+        success: false,
+        values: {
+          success: false,
+          error: "AGENT_AUTOFILL_NO_LOGIN",
+          domain,
+          subaction: AUTOFILL_SUBACTION
+        },
+        data: { actionName: "BROWSER", subaction: AUTOFILL_SUBACTION }
+      };
+    }
+    savedLogin = await getSavedLogin(vault, domain, chosen.username);
+    if (!savedLogin) {
+      return {
+        text: `Saved login ${chosen.username} on ${domain} disappeared between list and reveal.`,
+        success: false,
+        values: {
+          success: false,
+          error: "AGENT_AUTOFILL_RACE",
+          domain,
+          subaction: AUTOFILL_SUBACTION
+        },
+        data: { actionName: "BROWSER", subaction: AUTOFILL_SUBACTION }
+      };
+    }
+  }
+  const tabs = await listBrowserWorkspaceTabs();
+  const matchingTab = tabs.slice(0, MAX_BROWSER_TAB_SCAN).find((t) => tabUrlMatchesDomain(t.url, domain));
+  if (!matchingTab) {
+    return {
+      text: `No open browser tab on ${domain}. Open one with BROWSER (open/navigate) first.`,
+      success: false,
+      values: {
+        success: false,
+        error: "AGENT_AUTOFILL_NO_TAB",
+        domain,
+        subaction: AUTOFILL_SUBACTION
+      },
+      data: { actionName: "BROWSER", subaction: AUTOFILL_SUBACTION }
+    };
+  }
+  const script = buildAutofillScript({
+    username: savedLogin.username,
+    password: savedLogin.password,
+    submit
+  });
+  const rawResult = await evaluateBrowserWorkspaceTab({
+    id: matchingTab.id,
+    script
+  });
+  const { filled, fillReason } = narrowSnippetResult(rawResult);
+  logger.info(
+    `[browser-autofill-login] domain=${domain} tabId=${matchingTab.id} submit=${submit} filled=${filled}`
+  );
+  return {
+    text: submit ? `Filled and submitted login on ${domain} (tab ${matchingTab.id}).` : `Filled login on ${domain} (tab ${matchingTab.id}). User must click submit.`,
+    success: true,
+    values: {
+      success: true,
+      domain,
+      tabId: matchingTab.id,
+      submitted: submit,
+      filled,
+      subaction: AUTOFILL_SUBACTION,
+      ...fillReason ? { fillReason } : {}
+    },
+    data: {
+      actionName: "BROWSER",
+      subaction: AUTOFILL_SUBACTION,
+      domain,
+      tabId: matchingTab.id,
+      filled,
+      ...fillReason ? { fillReason } : {}
+    }
+  };
+}
+export {
+  executeBrowserAutofillLogin
+};
+//# sourceMappingURL=browser-autofill-login.js.map

package/dist/actions/browser-autofill-login.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/actions/browser-autofill-login.ts"],"sourcesContent":["/**\n * BROWSER autofill-login subaction — agent-driven browser login autofill.\n *\n * Invoked via BROWSER with `subaction: \"autofill-login\"` (canonical). The legacy\n * planner action name `BROWSER_AUTOFILL_LOGIN` normalizes to BROWSER with this\n * subaction in the planner dispatch pipeline.\n *\n * Lets the agent say \"log into github.com for me\" and have the saved\n * credentials filled into an open Eliza browser tab without a per-call\n * consent prompt.\n *\n * Authorization model (mirrors the user-driven autofill flow):\n *   - The user must have set `creds.<domain>.:autoallow = \"1\"` on the\n *     domain. This is the same vault key the React-side consent flow\n *     uses; toggling it from Settings -> Vault -> Logins is the\n *     SOLE way to let the agent autofill silently.\n *   - Without that flag, this action returns\n *     `{ ok: false, reason: \"user has not pre-authorized agent autofill for <domain>\" }`.\n *     The agent should NOT fall back to the user-driven flow on its own\n *     because the user-driven flow is gated by an interactive React\n *     modal that an autonomous agent cannot consent to.\n *\n * Tab selection:\n *   - Lists the live browser-workspace tabs and picks the first one\n *     whose URL hostname matches `domain` (registrable hostname,\n *     case-insensitive). Returns a clean error when no such tab exists\n *     so the agent can decide whether to open one first via\n *     BROWSER (open/navigate).\n *\n * Fill mechanism:\n *   - Injects a small JS snippet that mirrors the same form-detection\n *     and `setNativeInputValue` helpers the in-tab preload uses, so\n *     React-controlled inputs see the change.\n *   - When `submit: true`, the snippet also calls `form.submit()` (or\n *     clicks a likely submit button) after filling. Off by default —\n *     the safer behaviour is fill-only and let the user click submit.\n */\n\nimport type {\n  ActionResult,\n  HandlerOptions,\n  IAgentRuntime,\n  Memory,\n} from \"@elizaos/core\";\nimport { logger } from \"@elizaos/core\";\nimport {\n  createManager,\n  getAutofillAllowed,\n  getSavedLogin,\n  listSavedLogins,\n  type Vault,\n} from \"@elizaos/vault\";\nimport {\n  evaluateBrowserWorkspaceTab,\n  isBrowserWorkspaceBridgeConfigured,\n  listBrowserWorkspaceTabs,\n} from \"../workspace/browser-workspace.js\";\n\ninterface BrowserAutofillLoginParameters {\n  domain?: string;\n  username?: string;\n  /** When true, attempt to submit the form after filling. Default: false. */\n  submit?: boolean;\n}\n\nconst AUTOFILL_SUBACTION = \"autofill-login\";\n\nconst MAX_BROWSER_TAB_SCAN = 100;\nconst MAX_FILL_REASON_CHARS = 240;\n\nlet cachedVault: Vault | null = null;\n\nfunction sharedAutofillVault(): Vault {\n  cachedVault ??= createManager().vault;\n  return cachedVault;\n}\n\nfunction tabUrlMatchesDomain(tabUrl: string, domain: string): boolean {\n  if (!tabUrl) return false;\n  let hostname: string;\n  try {\n    hostname = new URL(tabUrl).hostname;\n  } catch {\n    return false;\n  }\n  return hostname.toLowerCase() === domain.toLowerCase();\n}\n\nfunction buildAutofillScript(args: {\n  username: string;\n  password: string;\n  submit: boolean;\n}): string {\n  // Inline snippet — runs in the OOPIF (the page's content world) via\n  // electrobun tab eval. Uses setNativeInputValue to bypass React's\n  // value-setter override.\n  return `\n(() => {\n  const USERNAME = ${JSON.stringify(args.username)};\n  const PASSWORD = ${JSON.stringify(args.password)};\n  const SUBMIT = ${args.submit ? \"true\" : \"false\"};\n\n  function setNativeInputValue(input, value) {\n    const proto = Object.getPrototypeOf(input);\n    const desc = Object.getOwnPropertyDescriptor(proto, \"value\");\n    if (desc && typeof desc.set === \"function\") {\n      desc.set.call(input, value);\n    } else {\n      input.value = value;\n    }\n    input.dispatchEvent(new Event(\"input\", { bubbles: true }));\n    input.dispatchEvent(new Event(\"change\", { bubbles: true }));\n  }\n\n  function findPrecedingTextInput(passwordInput) {\n    const root = passwordInput.form || document.body;\n    const candidates = root.querySelectorAll(\n      'input[type=\"text\"], input[type=\"email\"], input:not([type])'\n    );\n    let lastBefore = null;\n    for (const el of candidates) {\n      if (el.compareDocumentPosition(passwordInput) & Node.DOCUMENT_POSITION_FOLLOWING) {\n        lastBefore = el;\n      }\n    }\n    return lastBefore;\n  }\n\n  const password = document.querySelector('input[type=\"password\"]');\n  if (!password) {\n    return { ok: false, reason: \"no_password_input\" };\n  }\n  const form = password.form;\n  const username =\n    (form && form.querySelector(\n      'input[type=\"email\"], input[name*=\"user\" i], input[name*=\"email\" i], input[name*=\"login\" i]'\n    )) || findPrecedingTextInput(password);\n\n  if (username) setNativeInputValue(username, USERNAME);\n  setNativeInputValue(password, PASSWORD);\n\n  if (SUBMIT) {\n    if (form && typeof form.requestSubmit === \"function\") {\n      form.requestSubmit();\n    } else if (form && typeof form.submit === \"function\") {\n      form.submit();\n    } else {\n      const button =\n        (form && form.querySelector('button[type=\"submit\"], input[type=\"submit\"]')) ||\n        document.querySelector('button[type=\"submit\"], input[type=\"submit\"]');\n      if (button) (button).click();\n    }\n  }\n\n  return {\n    ok: true,\n    filled: { username: !!username, password: true },\n    submitted: SUBMIT,\n  };\n})();\n`;\n}\n\nfunction narrowSnippetResult(raw: unknown): {\n  filled: boolean;\n  fillReason: string | null;\n} {\n  if (!raw || typeof raw !== \"object\") {\n    return { filled: false, fillReason: null };\n  }\n  const obj = raw as { filled?: { username?: boolean; password?: boolean } };\n  const hasFilledProp = \"filled\" in obj && Boolean(obj.filled);\n  let fillReason: string | null = null;\n  const reasonVal = \"reason\" in obj ? obj.reason : undefined;\n  if (typeof reasonVal === \"string\") {\n    fillReason = reasonVal.slice(0, MAX_FILL_REASON_CHARS);\n  }\n  return { filled: hasFilledProp, fillReason };\n}\n\n/**\n * Executes the vault-gated workspace autofill flow for {@link AUTOFILL_SUBACTION}.\n */\nexport async function executeBrowserAutofillLogin(\n  _runtime: IAgentRuntime,\n  _message: Memory | undefined,\n  options: HandlerOptions | undefined,\n): Promise<ActionResult> {\n  const params = options?.parameters as\n    | BrowserAutofillLoginParameters\n    | undefined;\n  const domain = params?.domain?.trim().toLowerCase() ?? \"\";\n  const requestedUsername = params?.username?.trim();\n  const submit = params?.submit === true;\n\n  if (!domain) {\n    return {\n      text: `BROWSER requires subaction \"${AUTOFILL_SUBACTION}\" and a \\`domain\\` parameter.`,\n      success: false,\n      values: {\n        success: false,\n        error: \"BROWSER_AUTOFILL_BAD_PARAMS\",\n        subaction: AUTOFILL_SUBACTION,\n      },\n      data: { actionName: \"BROWSER\", subaction: AUTOFILL_SUBACTION },\n    };\n  }\n\n  if (!isBrowserWorkspaceBridgeConfigured(process.env)) {\n    return {\n      text: `BROWSER ${AUTOFILL_SUBACTION} requires the desktop browser workspace bridge.`,\n      success: false,\n      values: {\n        success: false,\n        error: \"BROWSER_BRIDGE_UNAVAILABLE\",\n        subaction: AUTOFILL_SUBACTION,\n      },\n      data: { actionName: \"BROWSER\", subaction: AUTOFILL_SUBACTION },\n    };\n  }\n\n  const vault = sharedAutofillVault();\n\n  const allowed = await getAutofillAllowed(vault, domain);\n  if (!allowed) {\n    const text = `User has not pre-authorized agent autofill for ${domain}. Toggle \"Allow agent to autofill\" for this domain under Settings -> Vault -> Logins.`;\n    return {\n      text,\n      success: false,\n      values: {\n        success: false,\n        error: \"AGENT_AUTOFILL_NOT_AUTHORIZED\",\n        domain,\n        subaction: AUTOFILL_SUBACTION,\n      },\n      data: {\n        actionName: \"BROWSER\",\n        subaction: AUTOFILL_SUBACTION,\n        domain,\n        reason: text,\n      },\n    };\n  }\n\n  let savedLogin: Awaited<ReturnType<typeof getSavedLogin>> = null;\n  if (requestedUsername) {\n    savedLogin = await getSavedLogin(vault, domain, requestedUsername);\n    if (!savedLogin) {\n      return {\n        text: `No saved login for ${requestedUsername} on ${domain}.`,\n        success: false,\n        values: {\n          success: false,\n          error: \"AGENT_AUTOFILL_NO_LOGIN\",\n          domain,\n          username: requestedUsername,\n          subaction: AUTOFILL_SUBACTION,\n        },\n        data: { actionName: \"BROWSER\", subaction: AUTOFILL_SUBACTION },\n      };\n    }\n  } else {\n    const summaries = await listSavedLogins(vault, domain);\n    if (summaries.length === 0) {\n      return {\n        text: `No saved logins for ${domain}.`,\n        success: false,\n        values: {\n          success: false,\n          error: \"AGENT_AUTOFILL_NO_LOGIN\",\n          domain,\n          subaction: AUTOFILL_SUBACTION,\n        },\n        data: { actionName: \"BROWSER\", subaction: AUTOFILL_SUBACTION },\n      };\n    }\n    const sorted = [...summaries].sort(\n      (a, b) => b.lastModified - a.lastModified,\n    );\n    const chosen = sorted[0];\n    if (!chosen) {\n      return {\n        text: `No saved logins for ${domain}.`,\n        success: false,\n        values: {\n          success: false,\n          error: \"AGENT_AUTOFILL_NO_LOGIN\",\n          domain,\n          subaction: AUTOFILL_SUBACTION,\n        },\n        data: { actionName: \"BROWSER\", subaction: AUTOFILL_SUBACTION },\n      };\n    }\n    savedLogin = await getSavedLogin(vault, domain, chosen.username);\n    if (!savedLogin) {\n      return {\n        text: `Saved login ${chosen.username} on ${domain} disappeared between list and reveal.`,\n        success: false,\n        values: {\n          success: false,\n          error: \"AGENT_AUTOFILL_RACE\",\n          domain,\n          subaction: AUTOFILL_SUBACTION,\n        },\n        data: { actionName: \"BROWSER\", subaction: AUTOFILL_SUBACTION },\n      };\n    }\n  }\n\n  const tabs = await listBrowserWorkspaceTabs();\n  const matchingTab = tabs\n    .slice(0, MAX_BROWSER_TAB_SCAN)\n    .find((t) => tabUrlMatchesDomain(t.url, domain));\n  if (!matchingTab) {\n    return {\n      text: `No open browser tab on ${domain}. Open one with BROWSER (open/navigate) first.`,\n      success: false,\n      values: {\n        success: false,\n        error: \"AGENT_AUTOFILL_NO_TAB\",\n        domain,\n        subaction: AUTOFILL_SUBACTION,\n      },\n      data: { actionName: \"BROWSER\", subaction: AUTOFILL_SUBACTION },\n    };\n  }\n\n  const script = buildAutofillScript({\n    username: savedLogin.username,\n    password: savedLogin.password,\n    submit,\n  });\n  const rawResult = await evaluateBrowserWorkspaceTab({\n    id: matchingTab.id,\n    script,\n  });\n  const { filled, fillReason } = narrowSnippetResult(rawResult);\n\n  logger.info(\n    `[browser-autofill-login] domain=${domain} tabId=${matchingTab.id} submit=${submit} filled=${filled}`,\n  );\n\n  return {\n    text: submit\n      ? `Filled and submitted login on ${domain} (tab ${matchingTab.id}).`\n      : `Filled login on ${domain} (tab ${matchingTab.id}). User must click submit.`,\n    success: true,\n    values: {\n      success: true,\n      domain,\n      tabId: matchingTab.id,\n      submitted: submit,\n      filled,\n      subaction: AUTOFILL_SUBACTION,\n      ...(fillReason ? { fillReason } : {}),\n    },\n    data: {\n      actionName: \"BROWSER\",\n      subaction: AUTOFILL_SUBACTION,\n      domain,\n      tabId: matchingTab.id,\n      filled,\n      ...(fillReason ? { fillReason } : {}),\n    },\n  };\n}\n"],"mappings":"AA4CA,SAAS,cAAc;AACvB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AASP,MAAM,qBAAqB;AAE3B,MAAM,uBAAuB;AAC7B,MAAM,wBAAwB;AAE9B,IAAI,cAA4B;AAEhC,SAAS,sBAA6B;AACpC,kBAAgB,cAAc,EAAE;AAChC,SAAO;AACT;AAEA,SAAS,oBAAoB,QAAgB,QAAyB;AACpE,MAAI,CAAC,OAAQ,QAAO;AACpB,MAAI;AACJ,MAAI;AACF,eAAW,IAAI,IAAI,MAAM,EAAE;AAAA,EAC7B,QAAQ;AACN,WAAO;AAAA,EACT;AACA,SAAO,SAAS,YAAY,MAAM,OAAO,YAAY;AACvD;AAEA,SAAS,oBAAoB,MAIlB;AAIT,SAAO;AAAA;AAAA,qBAEY,KAAK,UAAU,KAAK,QAAQ,CAAC;AAAA,qBAC7B,KAAK,UAAU,KAAK,QAAQ,CAAC;AAAA,mBAC/B,KAAK,SAAS,SAAS,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA6DjD;AAEA,SAAS,oBAAoB,KAG3B;AACA,MAAI,CAAC,OAAO,OAAO,QAAQ,UAAU;AACnC,WAAO,EAAE,QAAQ,OAAO,YAAY,KAAK;AAAA,EAC3C;AACA,QAAM,MAAM;AACZ,QAAM,gBAAgB,YAAY,OAAO,QAAQ,IAAI,MAAM;AAC3D,MAAI,aAA4B;AAChC,QAAM,YAAY,YAAY,MAAM,IAAI,SAAS;AACjD,MAAI,OAAO,cAAc,UAAU;AACjC,iBAAa,UAAU,MAAM,GAAG,qBAAqB;AAAA,EACvD;AACA,SAAO,EAAE,QAAQ,eAAe,WAAW;AAC7C;AAKA,eAAsB,4BACpB,UACA,UACA,SACuB;AACvB,QAAM,SAAS,SAAS;AAGxB,QAAM,SAAS,QAAQ,QAAQ,KAAK,EAAE,YAAY,KAAK;AACvD,QAAM,oBAAoB,QAAQ,UAAU,KAAK;AACjD,QAAM,SAAS,QAAQ,WAAW;AAElC,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,MACL,MAAM,+BAA+B,kBAAkB;AAAA,MACvD,SAAS;AAAA,MACT,QAAQ;AAAA,QACN,SAAS;AAAA,QACT,OAAO;AAAA,QACP,WAAW;AAAA,MACb;AAAA,MACA,MAAM,EAAE,YAAY,WAAW,WAAW,mBAAmB;AAAA,IAC/D;AAAA,EACF;AAEA,MAAI,CAAC,mCAAmC,QAAQ,GAAG,GAAG;AACpD,WAAO;AAAA,MACL,MAAM,WAAW,kBAAkB;AAAA,MACnC,SAAS;AAAA,MACT,QAAQ;AAAA,QACN,SAAS;AAAA,QACT,OAAO;AAAA,QACP,WAAW;AAAA,MACb;AAAA,MACA,MAAM,EAAE,YAAY,WAAW,WAAW,mBAAmB;AAAA,IAC/D;AAAA,EACF;AAEA,QAAM,QAAQ,oBAAoB;AAElC,QAAM,UAAU,MAAM,mBAAmB,OAAO,MAAM;AACtD,MAAI,CAAC,SAAS;AACZ,UAAM,OAAO,kDAAkD,MAAM;AACrE,WAAO;AAAA,MACL;AAAA,MACA,SAAS;AAAA,MACT,QAAQ;AAAA,QACN,SAAS;AAAA,QACT,OAAO;AAAA,QACP;AAAA,QACA,WAAW;AAAA,MACb;AAAA,MACA,MAAM;AAAA,QACJ,YAAY;AAAA,QACZ,WAAW;AAAA,QACX;AAAA,QACA,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF;AAEA,MAAI,aAAwD;AAC5D,MAAI,mBAAmB;AACrB,iBAAa,MAAM,cAAc,OAAO,QAAQ,iBAAiB;AACjE,QAAI,CAAC,YAAY;AACf,aAAO;AAAA,QACL,MAAM,sBAAsB,iBAAiB,OAAO,MAAM;AAAA,QAC1D,SAAS;AAAA,QACT,QAAQ;AAAA,UACN,SAAS;AAAA,UACT,OAAO;AAAA,UACP;AAAA,UACA,UAAU;AAAA,UACV,WAAW;AAAA,QACb;AAAA,QACA,MAAM,EAAE,YAAY,WAAW,WAAW,mBAAmB;AAAA,MAC/D;AAAA,IACF;AAAA,EACF,OAAO;AACL,UAAM,YAAY,MAAM,gBAAgB,OAAO,MAAM;AACrD,QAAI,UAAU,WAAW,GAAG;AAC1B,aAAO;AAAA,QACL,MAAM,uBAAuB,MAAM;AAAA,QACnC,SAAS;AAAA,QACT,QAAQ;AAAA,UACN,SAAS;AAAA,UACT,OAAO;AAAA,UACP;AAAA,UACA,WAAW;AAAA,QACb;AAAA,QACA,MAAM,EAAE,YAAY,WAAW,WAAW,mBAAmB;AAAA,MAC/D;AAAA,IACF;AACA,UAAM,SAAS,CAAC,GAAG,SAAS,EAAE;AAAA,MAC5B,CAAC,GAAG,MAAM,EAAE,eAAe,EAAE;AAAA,IAC/B;AACA,UAAM,SAAS,OAAO,CAAC;AACvB,QAAI,CAAC,QAAQ;AACX,aAAO;AAAA,QACL,MAAM,uBAAuB,MAAM;AAAA,QACnC,SAAS;AAAA,QACT,QAAQ;AAAA,UACN,SAAS;AAAA,UACT,OAAO;AAAA,UACP;AAAA,UACA,WAAW;AAAA,QACb;AAAA,QACA,MAAM,EAAE,YAAY,WAAW,WAAW,mBAAmB;AAAA,MAC/D;AAAA,IACF;AACA,iBAAa,MAAM,cAAc,OAAO,QAAQ,OAAO,QAAQ;AAC/D,QAAI,CAAC,YAAY;AACf,aAAO;AAAA,QACL,MAAM,eAAe,OAAO,QAAQ,OAAO,MAAM;AAAA,QACjD,SAAS;AAAA,QACT,QAAQ;AAAA,UACN,SAAS;AAAA,UACT,OAAO;AAAA,UACP;AAAA,UACA,WAAW;AAAA,QACb;AAAA,QACA,MAAM,EAAE,YAAY,WAAW,WAAW,mBAAmB;AAAA,MAC/D;AAAA,IACF;AAAA,EACF;AAEA,QAAM,OAAO,MAAM,yBAAyB;AAC5C,QAAM,cAAc,KACjB,MAAM,GAAG,oBAAoB,EAC7B,KAAK,CAAC,MAAM,oBAAoB,EAAE,KAAK,MAAM,CAAC;AACjD,MAAI,CAAC,aAAa;AAChB,WAAO;AAAA,MACL,MAAM,0BAA0B,MAAM;AAAA,MACtC,SAAS;AAAA,MACT,QAAQ;AAAA,QACN,SAAS;AAAA,QACT,OAAO;AAAA,QACP;AAAA,QACA,WAAW;AAAA,MACb;AAAA,MACA,MAAM,EAAE,YAAY,WAAW,WAAW,mBAAmB;AAAA,IAC/D;AAAA,EACF;AAEA,QAAM,SAAS,oBAAoB;AAAA,IACjC,UAAU,WAAW;AAAA,IACrB,UAAU,WAAW;AAAA,IACrB;AAAA,EACF,CAAC;AACD,QAAM,YAAY,MAAM,4BAA4B;AAAA,IAClD,IAAI,YAAY;AAAA,IAChB;AAAA,EACF,CAAC;AACD,QAAM,EAAE,QAAQ,WAAW,IAAI,oBAAoB,SAAS;AAE5D,SAAO;AAAA,IACL,mCAAmC,MAAM,UAAU,YAAY,EAAE,WAAW,MAAM,WAAW,MAAM;AAAA,EACrG;AAEA,SAAO;AAAA,IACL,MAAM,SACF,iCAAiC,MAAM,SAAS,YAAY,EAAE,OAC9D,mBAAmB,MAAM,SAAS,YAAY,EAAE;AAAA,IACpD,SAAS;AAAA,IACT,QAAQ;AAAA,MACN,SAAS;AAAA,MACT;AAAA,MACA,OAAO,YAAY;AAAA,MACnB,WAAW;AAAA,MACX;AAAA,MACA,WAAW;AAAA,MACX,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC;AAAA,IACA,MAAM;AAAA,MACJ,YAAY;AAAA,MACZ,WAAW;AAAA,MACX;AAAA,MACA,OAAO,YAAY;AAAA,MACnB;AAAA,MACA,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC;AAAA,EACF;AACF;","names":[]}

package/dist/actions/browser.d.ts

@@ -0,0 +1,11 @@
+import type { Action } from "@elizaos/core";
+/**
+ * Targets are the registered browser backends. The agent uses what is
+ * available; specifying a target overrides automatic routing. `workspace`
+ * is the app-owned browser surface, `bridge` is the paired Chrome/Safari
+ * companion, `stagehand` is the Playwright/Stagehand fallback, and other
+ * plugins may register additional target ids.
+ */
+export type BrowserTarget = string;
+export declare const browserAction: Action;
+//# sourceMappingURL=browser.d.ts.map

package/dist/actions/browser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/actions/browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,MAAM,EAIP,MAAM,eAAe,CAAC;AAYvB;;;;;;GAMG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC;AA2SnC,eAAO,MAAM,aAAa,EAAE,MAyS3B,CAAC"}