@vellumai/assistant 0.6.4 → 0.6.6

package/src/tools/browser/browser-execution.ts

@@ -4,6 +4,7 @@ import { getLogger } from "../../util/logger.js";
 import { truncate } from "../../util/truncate.js";
 import { safeStringSlice } from "../../util/unicode.js";
 import { credentialBroker } from "../credentials/broker.js";
+import { BROWSER_FILL_CAPABILITY } from "../credentials/tool-policy.js";
 import {
   isPrivateOrLocalHost,
   parseUrl,
@@ -86,19 +87,22 @@ type StatusCheckMode = BrowserStatusMode;
 
 const MODE_TRADEOFFS: Record<StatusCheckMode, string[]> = {
   [BROWSER_STATUS_MODE.EXTENSION]: [
-    "Controls the user's existing Chrome profile and tabs.",
-    "Requires the Vellum extension to be paired and actively connected.",
-    "Best when the user wants the assistant to operate in their real browser session.",
+    "This is the preferred approach for all things browser-use.",
+    "On macOS, the host browser proxy is provisioned automatically via the desktop client's SSE bridge — no extension install required.",
+    "When the Chrome extension is also installed, it takes priority for direct WebSocket routing to the user's Chrome session.",
+    "More secure than relying on Chrome's native remote debugging functionality.",
   ],
   [BROWSER_STATUS_MODE.CDP_INSPECT]: [
-    "Controls an existing Chrome instance launched with --remote-debugging-port.",
-    "Does not require the extension to be connected.",
-    "Requires remote debugging to stay enabled on localhost, which is more manual to maintain.",
+    "This is the second-best approach for all things browser-use, after the native Vellum Assistant Chrome Extension.",
+    "It requires Chrome version 146 or greater",
+    "It requires toggling on remote debugging in Chrome Settings",
+    "It's prone to phishing attacks from other local processes that may try to do their own remote debugging.",
   ],
   [BROWSER_STATUS_MODE.LOCAL]: [
-    "Runs a dedicated Playwright-managed Chromium profile.",
-    "Most isolated and reliable fallback when extension/CDP inspect are unavailable.",
+    "The least-preferred approach for all things browser-use.",
+    "Considered a last-resort fallback if the user has not installed the Chrome Extension or enabled remote debugging in Chrome, and has indicated that they do not want to.",
     "Does not use the user's existing browser profile, so sessions/cookies may differ.",
+    "Requires that Playwright and Chromium are installed on the host machine,",
   ],
 };
 
@@ -186,15 +190,16 @@ export function parseBrowserMode(
 const REMEDIATION_HINTS: Record<string, string[]> = {
   // Extension backend
   "extension:transport_error": [
-    "Ensure the Vellum browser extension is installed and enabled.",
-    "Check that the extension WebSocket connection is active (extension popup → status).",
-    "Try reconnecting the extension or reloading the extension service worker.",
+    "Ensure the Vellum browser extension is installed and enabled, or that the macOS desktop client is running for host browser proxy mode.",
+    "For extension mode: check that the extension WebSocket connection is active (extension popup → status).",
+    "For macOS host browser proxy: verify the desktop client is running and has an active SSE connection to the assistant.",
+    "Try reconnecting the extension or restarting the desktop client.",
   ],
   // cdp-inspect backend — discovery-level failures
   "cdp-inspect:unreachable": [
-    "Ensure Chrome/Chromium is running with --remote-debugging-port=9222.",
+    "Ensure that Chrome is on version 146 or higher by going to chrome://settings/help.",
+    'Ensure that you have toggled on "Allow remote debugging for this browser instance" by going to chrome://inspect/#remote-debugging',
     "Verify no firewall or antivirus is blocking localhost:9222.",
-    "Try: /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=9222",
   ],
   "cdp-inspect:non_chrome": [
     "The process listening on the configured port is not Chrome/Chromium.",
@@ -322,6 +327,14 @@ function collectRemediationHints(
   return hints;
 }
 
+/**
+ * Detect the common extension CDP failure where the active tab is a
+ * restricted Chrome internal page (e.g. `chrome://newtab`).
+ */
+function isRestrictedChromePageProbeError(error: CdpError): boolean {
+  return error.message.toLowerCase().includes("chrome://");
+}
+
 /**
  * Parse browser_mode from input and acquire a CdpClient. Returns
  * either a `{ cdp, browserMode }` pair on success or a pre-formatted
@@ -530,7 +543,7 @@ export function resolveElement(
     }
     return {
       resolved: null,
-      error: `Error: element_id "${elementId}" not found. Run browser_snapshot first to get current element IDs.`,
+      error: `Error: element_id "${elementId}" not found. Run a snapshot first to get current element IDs.`,
     };
   }
 
@@ -740,8 +753,8 @@ export async function executeBrowserNavigate(
           ).blockedAddress)
       ) {
         // Navigate the page away from the private target to prevent
-        // follow-up tool calls (e.g. browser_snapshot) from reading
-        // the already-loaded private content.
+        // follow-up operations (e.g. snapshot) from reading the
+        // already-loaded private content.
         try {
           await navigateAndWait(
             cdp,
@@ -847,7 +860,7 @@ export async function executeBrowserNavigate(
 
     if (navigationTimedOut) {
       lines.push(
-        `Note: Page is still loading (document.readyState timed out). The page should still be interactive - use browser_snapshot to check.`,
+        `Note: Page is still loading (document.readyState timed out). The page should still be interactive - take a snapshot to check.`,
       );
     }
 
@@ -917,14 +930,12 @@ export async function executeBrowserNavigate(
               lines.push("");
               lines.push(formatAuthChallenge(postCaptchaAuth));
               lines.push("");
+              lines.push("Handle this by interacting with the login form:");
               lines.push(
-                "Handle this by using browser tools to interact with the login form:",
-              );
-              lines.push(
-                "1. Use browser_snapshot to find the sign-in form elements",
+                "1. Take a snapshot to find the sign-in form elements",
               );
               lines.push(
-                "2. Use browser_fill_credential to fill email/password from credential_store",
+                "2. Use credential fill to enter email/password from credential_store",
               );
               lines.push(
                 "3. For email verification codes, use ui_show with a form to ask the user for the code mid-turn",
@@ -944,18 +955,14 @@ export async function executeBrowserNavigate(
           }
         } else {
           // Login / 2FA / OAuth - the agent should handle these itself
-          // using browser tools + credential_store. Don't hand off.
+          // using browser operations + credential_store. Don't hand off.
           lines.push("");
           lines.push(formatAuthChallenge(challenge));
           lines.push("");
+          lines.push("Handle this by interacting with the login form:");
+          lines.push("1. Take a snapshot to find the sign-in form elements");
           lines.push(
-            "Handle this by using browser tools to interact with the login form:",
-          );
-          lines.push(
-            "1. Use browser_snapshot to find the sign-in form elements",
-          );
-          lines.push(
-            "2. Use browser_fill_credential to fill email/password from credential_store",
+            "2. Use credential fill to enter email/password from credential_store",
           );
           lines.push(
             "3. For email verification codes, use ui_show with a form to ask the user for the code mid-turn",
@@ -1006,7 +1013,7 @@ export async function executeBrowserNavigate(
   }
 }
 
-// ── browser_snapshot ─────────────────────────────────────────────────
+// ── snapshot ─────────────────────────────────────────────────────────
 
 export async function executeBrowserSnapshot(
   _input: Record<string, unknown>,
@@ -1938,7 +1945,7 @@ export async function executeBrowserExtract(
   }
 }
 
-// ── browser_fill_credential ──────────────────────────────────────────
+// ── browser credential fill ──────────────────────────────────────────
 
 export async function executeBrowserFillCredential(
   input: Record<string, unknown>,
@@ -1996,7 +2003,7 @@ export async function executeBrowserFillCredential(
     const result = await credentialBroker.browserFill({
       service,
       field,
-      toolName: "browser_fill_credential",
+      toolName: BROWSER_FILL_CAPABILITY,
       domain: pageDomain,
       fill: async (value) => {
         // Clear-then-focus-then-insert via the shared helper. We
@@ -2087,24 +2094,59 @@ function modeTradeoffs(mode: StatusCheckMode): string[] {
 
 function extensionSetupActions(): string[] {
   return [
-    "Install and enable the Vellum Relay Chrome extension.",
-    "Open the extension popup and click Pair with local assistant.",
-    "Keep the extension connected to the assistant relay.",
+    "Install the Vellum Assistant Chrome extension from the Chrome Web Store: https://chromewebstore.google.com/detail/vellum-assistant-browser/hphbdmpffeigpcdjkckleobjmhhokpne",
+    "Open the extension and pair with your assistant.",
   ];
 }
 
+function macOSHostBrowserSetupActions(): string[] {
+  return [
+    "Ensure the Vellum desktop app is running and connected to the assistant.",
+    "Open Google Chrome on the desktop machine so the host browser proxy can attach.",
+    "If the desktop client is not running, launch it and wait for the SSE connection to establish.",
+  ];
+}
+
+function macOSHostBrowserReconnectActions(): string[] {
+  return [
+    "Verify the Vellum desktop app is still running and has an active network connection.",
+    "If the desktop client was recently restarted, send a new message to re-establish the SSE bridge.",
+    "Ensure Chrome is open on the desktop machine for the host browser proxy to target.",
+  ];
+}
+
+function macOSHostBrowserProbeFailureActions(error: CdpError): string[] {
+  const actions: string[] = [
+    "Ensure Google Chrome is running on the desktop machine.",
+  ];
+  const message = error.message.toLowerCase();
+  if (message.includes("timeout") || message.includes("timed out")) {
+    actions.push(
+      "The desktop client may be unresponsive — try restarting the Vellum desktop app.",
+    );
+  }
+  if (message.includes("transport") || message.includes("disconnected")) {
+    actions.push(
+      "The SSE bridge between the assistant and the desktop client appears broken. Send a new message to re-establish the connection.",
+    );
+  }
+  actions.push(
+    "Switch Chrome to a regular http(s) tab (not chrome://...) and retry.",
+  );
+  return dedupeStrings(actions);
+}
+
 function cdpInspectSetupActions(): string[] {
   return [
-    "Launch Chrome with --remote-debugging-port=9222 (or your configured port).",
-    "Keep Chrome running while browser tools are in use.",
-    "Ensure the configured host is loopback (localhost / 127.0.0.1 / ::1).",
+    "Update Chrome to the latest version by going to chrome://settings/help",
+    "Navigate directly to chrome://inspect/#remote-debugging",
+    'Check the box next to "Allow remote debugging for this browser instance"',
   ];
 }
 
 function localSetupActions(): string[] {
   return [
-    "Install assistant dependencies with bun install in assistant/.",
-    "Install Chromium for Playwright: bunx playwright install chromium.",
+    "Ask your assistant to install playwright and chromium on your host machine.",
   ];
 }
 
@@ -2243,6 +2285,7 @@ async function checkExtensionModeStatus(
 ): Promise<BrowserStatusModeResult> {
   const proxyBound = Boolean(context.hostBrowserProxy);
   const proxyConnected = context.hostBrowserProxy?.isAvailable() ?? false;
+  const isMacOS = context.transportInterface === "macos";
 
   if (!proxyBound) {
     return {
@@ -2250,13 +2293,20 @@ async function checkExtensionModeStatus(
       available: false,
       verified: "preflight",
       autoCandidate,
-      summary:
-        "Extension mode is unavailable: no host browser proxy is bound to this conversation.",
-      userActions: extensionSetupActions(),
+      summary: isMacOS
+        ? "Extension mode is unavailable: the macOS host browser proxy is not bound to this conversation. Ensure the desktop client is connected."
+        : "Extension mode is unavailable: no host browser proxy is bound to this conversation.",
+      userActions: isMacOS
+        ? macOSHostBrowserSetupActions()
+        : extensionSetupActions(),
       tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
       details: {
         proxyBound,
         proxyConnected,
+        transport:
+          isMacOS && !context.hostBrowserRegistryRouted
+            ? "macos-sse"
+            : "extension-ws",
       },
     };
   }
@@ -2267,13 +2317,20 @@ async function checkExtensionModeStatus(
       available: false,
       verified: "preflight",
       autoCandidate,
-      summary:
-        "Extension mode is unavailable: the extension transport is currently disconnected.",
-      userActions: extensionSetupActions(),
+      summary: isMacOS
+        ? "Extension mode is unavailable: the macOS host browser proxy is bound but the SSE transport is currently disconnected. Verify the desktop client is running and connected."
+        : "Extension mode is unavailable: the extension transport is currently disconnected.",
+      userActions: isMacOS
+        ? macOSHostBrowserReconnectActions()
+        : extensionSetupActions(),
       tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
       details: {
         proxyBound,
         proxyConnected,
+        transport:
+          isMacOS && !context.hostBrowserRegistryRouted
+            ? "macos-sse"
+            : "extension-ws",
       },
     };
   }
@@ -2288,13 +2345,47 @@ async function checkExtensionModeStatus(
       available: true,
       verified: "active_probe",
       autoCandidate,
-      summary: "Extension mode is ready and responded to an active CDP probe.",
+      summary: isMacOS
+        ? "Extension mode is ready via macOS host browser proxy and responded to an active CDP probe."
+        : "Extension mode is ready and responded to an active CDP probe.",
       userActions: [],
       tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
       details: {
         proxyBound,
         proxyConnected,
         backendKind: probe.backendKind,
+        transport:
+          isMacOS && !context.hostBrowserRegistryRouted
+            ? "macos-sse"
+            : "extension-ws",
+      },
+    };
+  }
+
+  if (isRestrictedChromePageProbeError(probe.error)) {
+    return {
+      mode: BROWSER_STATUS_MODE.EXTENSION,
+      available: true,
+      verified: "active_probe",
+      autoCandidate,
+      summary: isMacOS
+        ? "Extension mode transport is connected via macOS host browser proxy, but the active Chrome tab is a restricted chrome:// page. Switch to a regular website tab if browser actions fail."
+        : "Extension mode transport is connected, but the active Chrome tab is a restricted chrome:// page. Switch to a regular website tab if browser actions fail.",
+      userActions: [
+        "Switch Chrome to a regular http(s) tab (not chrome://...) and retry.",
+      ],
+      tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
+      details: {
+        proxyBound,
+        proxyConnected,
+        restrictedActiveTab: true,
+        errorCode: probe.error.code,
+        diagnostic: probe.diagnostic,
+        attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
+        transport:
+          isMacOS && !context.hostBrowserRegistryRouted
+            ? "macos-sse"
+            : "extension-ws",
       },
     };
   }
@@ -2304,11 +2395,12 @@ async function checkExtensionModeStatus(
     available: false,
     verified: "active_probe",
     autoCandidate,
-    summary: `Extension mode probe failed: ${probe.error.message}`,
-    userActions: probeFailureActions(
-      BROWSER_STATUS_MODE.EXTENSION,
-      probe.error,
-    ),
+    summary: isMacOS
+      ? `Extension mode probe failed via macOS host browser proxy: ${probe.error.message}`
+      : `Extension mode probe failed: ${probe.error.message}`,
+    userActions: isMacOS
+      ? macOSHostBrowserProbeFailureActions(probe.error)
+      : probeFailureActions(BROWSER_STATUS_MODE.EXTENSION, probe.error),
     tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
     details: {
       proxyBound,
@@ -2316,6 +2408,10 @@ async function checkExtensionModeStatus(
       errorCode: probe.error.code,
       diagnostic: probe.diagnostic,
       attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
+      transport:
+        isMacOS && !context.hostBrowserRegistryRouted
+          ? "macos-sse"
+          : "extension-ws",
     },
   };
 }
@@ -2547,7 +2643,7 @@ export async function executeBrowserStatus(
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
     return {
-      content: `Error: browser_status failed: ${msg}`,
+      content: `Error: browser status check failed: ${msg}`,
       isError: true,
     };
   }

package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts

@@ -356,4 +356,234 @@ describe("ExtensionCdpClient", () => {
     expect(err.message).toBe("CDP call aborted");
     expect(err.cdpMethod).toBe("Browser.getVersion");
   });
+
+  // ── Structured transport error classification ────────────────────────
+
+  test("structured error with code 'transport_error' is classified as transport_error", async () => {
+    const errorBody = {
+      code: "transport_error",
+      message: "Extension WebSocket disconnected",
+    };
+    const { proxy } = fakeProxy(async () => ({
+      content: JSON.stringify(errorBody),
+      isError: true,
+    }));
+
+    const client = createExtensionCdpClient(proxy, "conv-transport-1");
+
+    let caught: unknown;
+    try {
+      await client.send("Page.navigate", { url: "https://example.com" });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const err = caught as CdpError;
+    expect(err.code).toBe("transport_error");
+    expect(err.message).toBe("Extension WebSocket disconnected");
+    expect(err.cdpMethod).toBe("Page.navigate");
+    expect(err.cdpParams).toEqual({ url: "https://example.com" });
+    expect(err.underlying).toEqual(errorBody);
+  });
+
+  test("structured error with code 'unreachable' is classified as transport_error", async () => {
+    const errorBody = {
+      code: "unreachable",
+      message: "Host browser not reachable",
+    };
+    const { proxy } = fakeProxy(async () => ({
+      content: JSON.stringify(errorBody),
+      isError: true,
+    }));
+
+    const client = createExtensionCdpClient(proxy, "conv-transport-2");
+
+    let caught: unknown;
+    try {
+      await client.send("Runtime.evaluate", { expression: "1+1" });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const err = caught as CdpError;
+    expect(err.code).toBe("transport_error");
+    expect(err.message).toBe("Host browser not reachable");
+    expect(err.underlying).toEqual(errorBody);
+  });
+
+  test("structured error with code 'timeout' is classified as transport_error", async () => {
+    const errorBody = {
+      code: "timeout",
+      message: "CDP call timed out",
+    };
+    const { proxy } = fakeProxy(async () => ({
+      content: JSON.stringify(errorBody),
+      isError: true,
+    }));
+
+    const client = createExtensionCdpClient(proxy, "conv-transport-3");
+
+    let caught: unknown;
+    try {
+      await client.send("Page.captureScreenshot");
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const err = caught as CdpError;
+    expect(err.code).toBe("transport_error");
+    expect(err.message).toBe("CDP call timed out");
+    expect(err.underlying).toEqual(errorBody);
+  });
+
+  test("structured error with code 'non_loopback' is classified as transport_error", async () => {
+    const errorBody = {
+      code: "non_loopback",
+      message: "Only loopback addresses are allowed",
+    };
+    const { proxy } = fakeProxy(async () => ({
+      content: JSON.stringify(errorBody),
+      isError: true,
+    }));
+
+    const client = createExtensionCdpClient(proxy, "conv-transport-4");
+
+    let caught: unknown;
+    try {
+      await client.send("Browser.getVersion");
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const err = caught as CdpError;
+    expect(err.code).toBe("transport_error");
+    expect(err.message).toBe("Only loopback addresses are allowed");
+    expect(err.underlying).toEqual(errorBody);
+  });
+
+  // ── Command-level CDP errors remain cdp_error ────────────────────────
+
+  test("structured error with numeric code (CDP JSON-RPC) remains cdp_error", async () => {
+    const errorBody = {
+      code: -32000,
+      message: "Cannot find context with specified id",
+    };
+    const { proxy } = fakeProxy(async () => ({
+      content: JSON.stringify(errorBody),
+      isError: true,
+    }));
+
+    const client = createExtensionCdpClient(proxy, "conv-cdp-1");
+
+    let caught: unknown;
+    try {
+      await client.send("Runtime.evaluate", { expression: "boom" });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const err = caught as CdpError;
+    expect(err.code).toBe("cdp_error");
+    expect(err.message).toBe("Cannot find context with specified id");
+  });
+
+  test("structured error with unknown string code remains cdp_error", async () => {
+    const errorBody = {
+      code: "some_unknown_code",
+      message: "Unrecognized error",
+    };
+    const { proxy } = fakeProxy(async () => ({
+      content: JSON.stringify(errorBody),
+      isError: true,
+    }));
+
+    const client = createExtensionCdpClient(proxy, "conv-cdp-2");
+
+    let caught: unknown;
+    try {
+      await client.send("Page.navigate", { url: "https://example.com" });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const err = caught as CdpError;
+    expect(err.code).toBe("cdp_error");
+    expect(err.message).toBe("Unrecognized error");
+  });
+
+  test("structured error without code field remains cdp_error", async () => {
+    const errorBody = {
+      message: "Something went wrong",
+    };
+    const { proxy } = fakeProxy(async () => ({
+      content: JSON.stringify(errorBody),
+      isError: true,
+    }));
+
+    const client = createExtensionCdpClient(proxy, "conv-cdp-3");
+
+    let caught: unknown;
+    try {
+      await client.send("DOM.getDocument");
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const err = caught as CdpError;
+    expect(err.code).toBe("cdp_error");
+    expect(err.message).toBe("Something went wrong");
+  });
+
+  test("structured error with code: null remains cdp_error", async () => {
+    const errorBody = {
+      code: null,
+      message: "Null code error",
+    };
+    const { proxy } = fakeProxy(async () => ({
+      content: JSON.stringify(errorBody),
+      isError: true,
+    }));
+
+    const client = createExtensionCdpClient(proxy, "conv-cdp-4");
+
+    let caught: unknown;
+    try {
+      await client.send("Browser.getVersion");
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const err = caught as CdpError;
+    expect(err.code).toBe("cdp_error");
+  });
+
+  test("plain-text isError content still throws cdp_error (no failover)", async () => {
+    const { proxy } = fakeProxy(async () => ({
+      content: "Connection lost unexpectedly",
+      isError: true,
+    }));
+
+    const client = createExtensionCdpClient(proxy, "conv-cdp-5");
+
+    let caught: unknown;
+    try {
+      await client.send("Page.navigate", { url: "https://example.com" });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const err = caught as CdpError;
+    expect(err.code).toBe("cdp_error");
+    expect(err.message).toBe("Connection lost unexpectedly");
+    expect(err.underlying).toBe("Connection lost unexpectedly");
+  });
 });