@elizaos/plugin-browser 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/actions/browser.js

@@ -2,11 +2,15 @@ import { logger } from "@elizaos/core";
 import {
   BROWSER_SERVICE_TYPE
 } from "../browser-service.js";
-import { executeBrowserAutofillLogin } from "./browser-autofill-login.js";
 import {
   executeBrowserWorkspaceCommand,
   getBrowserWorkspaceMode
 } from "../workspace/browser-workspace.js";
+import {
+  WAIT_FOR_URL_DEFAULT_POLL_INTERVAL_MS,
+  WAIT_FOR_URL_DEFAULT_TIMEOUT_MS,
+  waitForUrl
+} from "./wait-for-url.js";
 function getMessageText(message) {
   const content = message?.content;
   if (typeof content === "string") {
@@ -19,10 +23,14 @@ function extractFirstUrl(value) {
   return match?.[0] ?? null;
 }
 function inferBrowserSubaction(params, messageText) {
-  if (params?.action === "autofill-login" || params?.subaction === "autofill-login") {
+  const normalizedAction = normalizeBrowserAction(params?.action);
+  if (normalizedAction === "autofill-login" || params?.subaction === "autofill-login") {
     return "autofill-login";
   }
-  const legacySubaction = normalizeLegacyBrowserAction(params?.action);
+  if (normalizedAction === "wait-for-url" || params?.subaction === "wait-for-url") {
+    return "wait-for-url";
+  }
+  const legacySubaction = normalizeLegacyBrowserAction(normalizedAction);
   if (legacySubaction) {
     return legacySubaction;
   }
@@ -44,8 +52,32 @@ function inferBrowserSubaction(params, messageText) {
   }
   return "state";
 }
-function normalizeLegacyBrowserAction(action) {
+function normalizeBrowserAction(action) {
   switch (action) {
+    case "realistic_click":
+      return "realistic-click";
+    case "realistic_fill":
+      return "realistic-fill";
+    case "realistic_type":
+      return "realistic-type";
+    case "realistic_press":
+      return "realistic-press";
+    case "cursor_move":
+      return "cursor-move";
+    case "cursor_hide":
+      return "cursor-hide";
+    case "autofill_login":
+      return "autofill-login";
+    case "wait_for_url":
+    case "wait-for-url":
+      return "wait-for-url";
+    default:
+      return action;
+  }
+}
+function normalizeLegacyBrowserAction(action) {
+  const normalizedAction = normalizeBrowserAction(action);
+  switch (normalizedAction) {
     case "info":
     case "context":
     case "get_context":
@@ -56,15 +88,19 @@ function normalizeLegacyBrowserAction(action) {
     case "switch_tab":
       return "tab";
     case "autofill-login":
+    case "wait-for-url":
       return void 0;
     case void 0:
       return void 0;
     default:
-      return action;
+      return isWorkspaceSubaction(normalizedAction) ? normalizedAction : void 0;
   }
 }
+function isWorkspaceSubaction(action) {
+  return action === "back" || action === "click" || action === "close" || action === "forward" || action === "get" || action === "hide" || action === "navigate" || action === "open" || action === "press" || action === "reload" || action === "screenshot" || action === "show" || action === "snapshot" || action === "state" || action === "tab" || action === "type" || action === "wait" || action === "realistic-click" || action === "realistic-fill" || action === "realistic-type" || action === "realistic-press" || action === "cursor-move" || action === "cursor-hide";
+}
 function normalizeLegacyTabAction(action) {
-  switch (action) {
+  switch (normalizeBrowserAction(action)) {
     case "list_tabs":
       return "list";
     case "open_tab":
@@ -105,6 +141,193 @@ ${serialized}`;
   }
   return `Browser ${command.subaction} completed in ${result.mode} mode.`;
 }
+function browserProgressRationale(command, params, messageText) {
+  const explicit = params?.rationale?.trim();
+  if (explicit) return explicit;
+  switch (command.subaction) {
+    case "open":
+    case "navigate":
+      return command.url ? `open ${command.url}` : "open requested page";
+    case "click":
+    case "realistic-click":
+      return command.selector ? `click ${command.selector}` : "click requested target";
+    case "type":
+    case "realistic-fill":
+    case "realistic-type":
+      return command.selector ? `fill ${command.selector}` : "type requested text";
+    case "press":
+    case "realistic-press":
+      return command.key ? `press ${command.key}` : "press requested key";
+    case "tab":
+      return command.tabAction ? `${command.tabAction} browser tab` : "manage browser tabs";
+    case "wait":
+      return command.selector ? `wait for ${command.selector}` : "wait for browser state";
+    case "state":
+      return messageText.trim() || "read browser state";
+    default:
+      return `run browser ${command.subaction}`;
+  }
+}
+function buildBrowserStepProgressContent(command, params, messageText, success, error) {
+  const rationale = error ? `failed: ${error}` : browserProgressRationale(command, params, messageText);
+  return {
+    text: `Step 1: ${command.subaction} \u2014 ${rationale}`,
+    source: "action_progress",
+    merge: "replace",
+    metadata: {
+      transient: true,
+      compactProgress: true,
+      progress: {
+        source: "browser",
+        actionName: "BROWSER",
+        step: 1,
+        kind: command.subaction,
+        rationale,
+        success,
+        error
+      }
+    }
+  };
+}
+async function emitBrowserStepProgress(callback, command, params, messageText, success, error) {
+  if (params?.streamProgress !== true || !callback) return;
+  try {
+    await callback(
+      buildBrowserStepProgressContent(
+        command,
+        params,
+        messageText,
+        success,
+        error
+      ),
+      "BROWSER"
+    );
+  } catch (callbackError) {
+    logger.warn(
+      {
+        src: "plugin:browser",
+        action: command.subaction,
+        error: callbackError instanceof Error ? callbackError.message : String(callbackError)
+      },
+      "Failed to emit browser progress callback"
+    );
+  }
+}
+function currentUrlFromResult(result) {
+  if (result.tab?.url) {
+    return result.tab.url;
+  }
+  if (typeof result.value === "string" && result.value.trim()) {
+    return result.value.trim();
+  }
+  if (Array.isArray(result.tabs)) {
+    const visible = result.tabs.find((tab) => tab.visible) ?? result.tabs[0];
+    if (visible?.url) {
+      return visible.url;
+    }
+  }
+  return null;
+}
+async function readCurrentBrowserUrl(browserService, target, tabId) {
+  const run = async (command) => browserService ? browserService.execute(command, target) : executeBrowserWorkspaceCommand(command);
+  try {
+    const got = await run({
+      subaction: "get",
+      getMode: "url",
+      id: tabId
+    });
+    const url = currentUrlFromResult(got);
+    if (url) {
+      return url;
+    }
+  } catch (error) {
+    logger.debug(
+      `[BROWSER] wait_for_url get-url poll failed, falling back to state: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+  try {
+    const state = await run({ subaction: "state", id: tabId });
+    return currentUrlFromResult(state);
+  } catch (error) {
+    logger.debug(
+      `[BROWSER] wait_for_url state poll could not read URL: ${error instanceof Error ? error.message : String(error)}`
+    );
+    return null;
+  }
+}
+async function executeBrowserWaitForUrl(runtime, params, messageText, callback) {
+  const pattern = (params?.pattern ?? "").trim();
+  if (!pattern) {
+    const text = "wait_for_url needs a `pattern` (substring or regex) to watch for.";
+    logger.warn(`[BROWSER] ${text}`);
+    return {
+      text,
+      success: false,
+      values: { success: false, error: "BROWSER_WAIT_FOR_URL_NO_PATTERN" },
+      data: { actionName: "BROWSER", subaction: "wait_for_url" }
+    };
+  }
+  const browserService = runtime.getService(BROWSER_SERVICE_TYPE) ?? null;
+  const target = params?.target;
+  const startUrl = params?.url?.trim() || extractFirstUrl(messageText) || void 0;
+  const timeoutMs = params?.timeoutMs ?? WAIT_FOR_URL_DEFAULT_TIMEOUT_MS;
+  const pollIntervalMs = params?.pollIntervalMs ?? WAIT_FOR_URL_DEFAULT_POLL_INTERVAL_MS;
+  let tabId = params?.id?.trim() || void 0;
+  if (startUrl) {
+    const openCommand = {
+      subaction: tabId ? "navigate" : "open",
+      url: startUrl,
+      id: tabId
+    };
+    try {
+      const opened = browserService ? await browserService.execute(openCommand, target) : await executeBrowserWorkspaceCommand(openCommand);
+      tabId = opened.tab?.id ?? tabId;
+    } catch (error) {
+      const reason = error instanceof Error ? error.message : String(error);
+      logger.warn(
+        `[BROWSER] wait_for_url could not open ${startUrl}: ${reason}`
+      );
+      return {
+        text: `Couldn't open ${startUrl} to watch for "${pattern}": ${reason}`,
+        success: false,
+        values: { success: false, error: "BROWSER_WAIT_FOR_URL_OPEN_FAILED" },
+        data: { actionName: "BROWSER", subaction: "wait_for_url" }
+      };
+    }
+  }
+  await callback?.({
+    text: startUrl ? `I opened ${startUrl} \u2014 watching for "${pattern}" and I'll resume when it's reached.` : `Watching the current tab for "${pattern}" \u2014 I'll resume when it's reached.`
+  });
+  logger.info(
+    `[BROWSER] wait_for_url pattern="${pattern}" timeoutMs=${timeoutMs} pollIntervalMs=${pollIntervalMs} target=${target ?? "auto"}`
+  );
+  const outcome = await waitForUrl(
+    { pattern, timeoutMs, pollIntervalMs },
+    {
+      getCurrentUrl: () => readCurrentBrowserUrl(browserService, target, tabId),
+      emitStatus: async (text) => {
+        await callback?.({ text });
+      }
+    }
+  );
+  return {
+    text: outcome.message,
+    success: outcome.matched,
+    userFacingText: outcome.message,
+    values: {
+      success: outcome.matched,
+      subaction: "wait_for_url",
+      status: outcome.status,
+      matched: outcome.matched,
+      polls: outcome.polls
+    },
+    data: {
+      actionName: "BROWSER",
+      subaction: "wait_for_url",
+      outcome
+    }
+  };
+}
 const browserAction = {
   name: "BROWSER",
   contexts: ["browser", "web", "automation", "secrets"],
@@ -115,7 +338,6 @@ const browserAction = {
     "CONTROL_BROWSER",
     "CONTROL_BROWSER_SESSION",
     "MANAGE_ELIZA_BROWSER_WORKSPACE",
-    "MANAGE_LIFEOPS_BROWSER",
     "NAVIGATE_SITE",
     "OPEN_SITE",
     "USE_BROWSER",
@@ -128,16 +350,20 @@ const browserAction = {
     "LOG_INTO_SITE",
     "SIGN_IN_TO_SITE"
   ],
-  description: "Single BROWSER action \u2014 control whichever browser target is registered. Targets are pluggable: `workspace` (electrobun-embedded BrowserView, the default; falls back to a JSDOM web mode when the desktop bridge isn't configured), `bridge` (the user's real Chrome/Safari via the Agent Browser Bridge companion extension), and `computeruse` (a local puppeteer-driven Chromium via plugin-computeruse). The agent uses what is available \u2014 the BrowserService picks the active target when none is specified. Use `subaction: \"autofill-login\"` with `domain` (and optional `username`, `submit`) to vault-gated autofill into an open workspace tab.",
-  descriptionCompressed: "Browser tab/page control: open/navigate/click/type/screenshot/state; subaction autofill-login + domain autofill vault-gated credential into workspace tab pre-authorized in Settings Vault Logins. Bridge settings/status use MANAGE_BROWSER_BRIDGE.",
+  description: "BROWSER action. Control registered browser target: app workspace, bridge Chrome/Safari companion, computeruse Chromium, or Stagehand fallback. BrowserService picks target if omitted. action=autofill_login + domain vault-gated autofills open workspace tab. action=wait_for_url + pattern opens an optional url then watches the tab and resumes when its URL matches (OAuth callback, deploy/CI done), streaming progress.",
+  descriptionCompressed: "Browser open|navigate|click|type|screenshot|state|autofill_login|wait_for_url; bridge status elsewhere",
   validate: async () => true,
-  handler: async (runtime, message, _state, options) => {
+  handler: async (runtime, message, _state, options, callback) => {
     const params = options?.parameters;
     const messageText = getMessageText(message);
     const subaction = inferBrowserSubaction(params, messageText);
     if (subaction === "autofill-login") {
+      const { executeBrowserAutofillLogin } = await import("./browser-autofill-login.js");
       return executeBrowserAutofillLogin(runtime, message, options);
     }
+    if (subaction === "wait-for-url") {
+      return executeBrowserWaitForUrl(runtime, params, messageText, callback);
+    }
     const url = params?.url?.trim() || extractFirstUrl(messageText) || void 0;
     const command = {
       id: params?.id?.trim(),
@@ -148,6 +374,7 @@ const browserAction = {
       subaction,
       tabAction: params?.tabAction ?? normalizeLegacyTabAction(params?.action),
       text: params?.text,
+      value: params?.text,
       timeoutMs: params?.timeoutMs,
       url,
       cursorDurationMs: params?.cursorDurationMs,
@@ -156,14 +383,19 @@ const browserAction = {
       x: params?.x,
       y: params?.y
     };
-    const browserService = runtime.getService(
-      BROWSER_SERVICE_TYPE
-    );
+    const browserService = runtime.getService(BROWSER_SERVICE_TYPE);
     try {
       logger.info(
         `[BROWSER] ${command.subaction} via target=${params?.target ?? "auto"} (workspace mode=${getBrowserWorkspaceMode(process.env)})`
       );
       const result = browserService ? await browserService.execute(command, params?.target) : await executeBrowserWorkspaceCommand(command);
+      await emitBrowserStepProgress(
+        callback,
+        command,
+        params,
+        messageText,
+        true
+      );
       return {
         text: formatBrowserSessionResult(command, result),
         success: true,
@@ -179,10 +411,18 @@ const browserAction = {
         }
       };
     } catch (error) {
-      const messageText2 = error instanceof Error ? error.message : "Browser action failed";
-      logger.warn(`[BROWSER] Failed: ${messageText2}`);
+      const errorText = error instanceof Error ? error.message : "Browser action failed";
+      logger.warn(`[BROWSER] Failed: ${errorText}`);
+      await emitBrowserStepProgress(
+        callback,
+        command,
+        params,
+        messageText,
+        false,
+        errorText
+      );
       return {
-        text: `Browser action failed: ${messageText2}`,
+        text: `Browser action failed: ${errorText}`,
         success: false,
         values: { success: false, error: "BROWSER_FAILED" },
         data: {
@@ -193,9 +433,27 @@ const browserAction = {
     }
   },
   parameters: [
+    {
+      name: "target",
+      description: "Optional browser target id. Common values: workspace, bridge, computeruse, stagehand.",
+      required: false,
+      schema: { type: "string" }
+    },
+    {
+      name: "streamProgress",
+      description: "When true, emit a compact Step 1 progress callback after the browser command dispatches.",
+      required: false,
+      schema: { type: "boolean", default: false }
+    },
+    {
+      name: "rationale",
+      description: "Optional rationale shown in streamProgress callback text.",
+      required: false,
+      schema: { type: "string" }
+    },
     {
       name: "action",
-      description: "Browser action to perform. Legacy subaction is also accepted.",
+      description: "Browser action. Snake_case canonical; legacy kebab-case and subaction accepted.",
       required: false,
       schema: {
         type: "string",
@@ -224,49 +482,28 @@ const browserAction = {
           "wait",
           "close_tab",
           "switch_tab",
-          "realistic-click",
-          "realistic-fill",
-          "realistic-type",
-          "realistic-press",
-          "cursor-move",
-          "cursor-hide",
-          "autofill-login"
+          "realistic_click",
+          "realistic_fill",
+          "realistic_type",
+          "realistic_press",
+          "cursor_move",
+          "cursor_hide",
+          "autofill_login",
+          "wait_for_url"
         ]
       }
     },
     {
-      name: "subaction",
-      description: "Legacy alias for action.",
+      name: "pattern",
+      description: "For action=wait_for_url: substring or /regex/ to match the tab URL (e.g. callback?code=, or /\\/done$/).",
       required: false,
-      schema: {
-        type: "string",
-        enum: [
-          "back",
-          "click",
-          "close",
-          "forward",
-          "get",
-          "hide",
-          "navigate",
-          "open",
-          "press",
-          "reload",
-          "screenshot",
-          "show",
-          "snapshot",
-          "state",
-          "tab",
-          "type",
-          "wait",
-          "realistic-click",
-          "realistic-fill",
-          "realistic-type",
-          "realistic-press",
-          "cursor-move",
-          "cursor-hide",
-          "autofill-login"
-        ]
-      }
+      schema: { type: "string" }
+    },
+    {
+      name: "pollIntervalMs",
+      description: "For action=wait_for_url: poll cadence in ms. Default 2000.",
+      required: false,
+      schema: { type: "number" }
     },
     {
       name: "tabAction",
@@ -279,19 +516,19 @@ const browserAction = {
     },
     {
       name: "domain",
-      description: "Required when subaction is autofill-login: registrable hostname (e.g. `github.com`).",
+      description: "Required for action=autofill_login: registrable hostname, e.g. github.com.",
       required: false,
       schema: { type: "string" }
     },
     {
       name: "username",
-      description: "When using autofill-login: specific saved login; omit for most recently modified.",
+      description: "For autofill-login: saved login username; omit for latest.",
       required: false,
       schema: { type: "string" }
     },
     {
       name: "submit",
-      description: "When using autofill-login: submit the form after filling (default false).",
+      description: "For autofill-login: submit after filling. Default false.",
       required: false,
       schema: { type: "boolean" }
     },
@@ -345,7 +582,7 @@ const browserAction = {
     },
     {
       name: "watchMode",
-      description: "Hint that the user is watching; prefers realistic-* subactions for click/fill so the cursor moves visibly and pointer events fire faithfully.",
+      description: "User watching hint; prefer realistic-* click/fill, visible cursor, pointer events.",
       required: false,
       schema: { type: "boolean" }
     },
@@ -363,7 +600,7 @@ const browserAction = {
     },
     {
       name: "replace",
-      description: "Replace existing input value when filling (vs append) \u2014 applies to realistic-fill",
+      description: "For realistic-fill: replace existing input, not append.",
       required: false,
       schema: { type: "boolean" }
     },
@@ -408,6 +645,21 @@ const browserAction = {
           text: "click completed in desktop mode."
         }
       }
+    ],
+    [
+      {
+        name: "{{name1}}",
+        content: {
+          text: "Open the GitHub OAuth page and let me know when it redirects back to our callback."
+        }
+      },
+      {
+        name: "{{agentName}}",
+        content: {
+          text: `I opened https://github.com/login/oauth/authorize \u2014 watching for "callback?code=" and I'll resume when it's reached.`,
+          actions: ["BROWSER"]
+        }
+      }
     ]
   ]
 };