@vellumai/assistant 0.6.3 → 0.6.4

package/src/tools/browser/browser-manager.ts

@@ -4,6 +4,7 @@ import { isAbsolute, join, relative, resolve } from "node:path";
 import { getLogger } from "../../util/logger.js";
 import { getDataDir } from "../../util/platform.js";
 import { authSessionCache } from "./auth-cache.js";
+import type { CdpClientKind } from "./cdp-client/types.js";
 import type { ExtractedCredential } from "./network-recording-types.js";
 import { importPlaywright } from "./runtime-check.js";
 
@@ -210,6 +211,16 @@ class BrowserManager {
    * resulting backendNodeId.
    */
   private snapshotBackendNodeMaps = new Map<string, Map<string, number>>();
+  /**
+   * The CDP backend kind that a conversation resolved to on an earlier
+   * tool call. When a subsequent browser_* call arrives with
+   * `browser_mode: "auto"`, the factory is pinned to this kind so
+   * navigate and screenshot cannot land on different Chromes within
+   * the same conversation. Explicit non-auto modes overwrite the entry.
+   * Cleared when the conversation's page is closed or all pages are
+   * closed.
+   */
+  private preferredBackendKinds = new Map<string, CdpClientKind>();
   private browserCdpSession: CDPSession | null = null;
   private browserWindowId: number | null = null;
   private interactiveModeSessions = new Set<string>();
@@ -444,6 +455,7 @@ class BrowserManager {
     this.pages.delete(conversationId);
     this.rawPages.delete(conversationId);
     this.snapshotBackendNodeMaps.delete(conversationId);
+    this.preferredBackendKinds.delete(conversationId);
     this.downloads.delete(conversationId);
     // Reject any pending download waiters
     const pending = this.pendingDownloads.get(conversationId);
@@ -477,6 +489,7 @@ class BrowserManager {
     this.pages.clear();
     this.rawPages.clear();
     this.snapshotBackendNodeMaps.clear();
+    this.preferredBackendKinds.clear();
     this.downloads.clear();
     for (const pending of this.pendingDownloads.values()) {
       for (const waiter of pending) waiter.reject(new Error("Browser closed"));
@@ -532,6 +545,38 @@ class BrowserManager {
     }
   }
 
+  /**
+   * Look up the backend kind this conversation has been using. Returns
+   * `null` before the first successful CDP send or after an explicit
+   * teardown. Callers should pass the result to the factory as a
+   * pinned mode so `auto`-mode calls stay consistent across tool
+   * invocations.
+   */
+  getPreferredBackendKind(conversationId: string): CdpClientKind | null {
+    return this.preferredBackendKinds.get(conversationId) ?? null;
+  }
+
+  /**
+   * Record the backend kind that a tool call resolved to. Overwrites
+   * any prior value so an explicit `browser_mode` override on one call
+   * becomes the default for subsequent `auto`-mode calls.
+   */
+  setPreferredBackendKind(
+    conversationId: string,
+    kind: CdpClientKind,
+  ): void {
+    this.preferredBackendKinds.set(conversationId, kind);
+  }
+
+  /**
+   * Drop the remembered backend kind for a conversation. Called on
+   * explicit teardown (browser_close, closeAllPages) so the next call
+   * re-runs the auto priority list.
+   */
+  clearPreferredBackendKind(conversationId: string): void {
+    this.preferredBackendKinds.delete(conversationId);
+  }
+
   /**
    * Store the backendNodeId lookup produced by the AX-tree-based
    * `executeBrowserSnapshot`. Callers overwrite any prior map for the

package/src/tools/browser/browser-mode-constants.ts

@@ -0,0 +1,12 @@
+import type { BrowserMode } from "./cdp-client/types.js";
+
+/**
+ * Canonical browser-mode identifiers shared across parsing and status
+ * reporting paths.
+ */
+export const BROWSER_MODE = {
+  AUTO: "auto",
+  EXTENSION: "extension",
+  CDP_INSPECT: "cdp-inspect",
+  LOCAL: "local",
+} as const satisfies Record<string, BrowserMode>;

package/src/tools/browser/browser-mode.ts

@@ -0,0 +1,92 @@
+/**
+ * Normalization helper for the `browser_mode` tool input parameter.
+ *
+ * Canonical values map directly to {@link CdpClientKind}:
+ *   - `auto`        -- let the factory pick the best backend (default)
+ *   - `extension`   -- force the Chrome extension transport
+ *   - `cdp-inspect` -- force the CDP inspect/debugger transport
+ *   - `local`       -- force the Playwright-managed local browser
+ *
+ * Aliases are accepted and normalized to their canonical form:
+ *   - `cdp-debugger` -> `cdp-inspect`
+ *   - `playwright`   -> `local`
+ */
+
+import { BROWSER_MODE } from "./browser-mode-constants.js";
+import type { BrowserMode } from "./cdp-client/types.js";
+
+/** Canonical browser mode values. Re-exported from cdp-client/types. */
+export type { BrowserMode } from "./cdp-client/types.js";
+
+/** All accepted values (canonical + aliases). */
+const ALIAS_MAP: Record<string, BrowserMode> = {
+  [BROWSER_MODE.AUTO]: BROWSER_MODE.AUTO,
+  [BROWSER_MODE.EXTENSION]: BROWSER_MODE.EXTENSION,
+  [BROWSER_MODE.CDP_INSPECT]: BROWSER_MODE.CDP_INSPECT,
+  "cdp-debugger": BROWSER_MODE.CDP_INSPECT,
+  [BROWSER_MODE.LOCAL]: BROWSER_MODE.LOCAL,
+  playwright: BROWSER_MODE.LOCAL,
+};
+
+/** Ordered list of accepted values for error messages. */
+const ACCEPTED_VALUES = Object.keys(ALIAS_MAP);
+
+/**
+ * Human-readable alias mapping for error messages.
+ * Only includes entries where the alias differs from the canonical value.
+ */
+const ALIAS_DISPLAY: Record<string, string> = {
+  "cdp-debugger": "cdp-inspect",
+  playwright: "local",
+};
+
+export interface NormalizeBrowserModeResult {
+  /** The normalized canonical mode. */
+  mode: BrowserMode;
+}
+
+export interface NormalizeBrowserModeError {
+  /** Deterministic error message describing the invalid value. */
+  error: string;
+}
+
+/**
+ * Normalize a raw `browser_mode` input value to a canonical {@link BrowserMode}.
+ *
+ * - `undefined` / `null` / empty string -> `{ mode: "auto" }`
+ * - Valid canonical value or alias       -> `{ mode: <canonical> }`
+ * - Invalid value                        -> `{ error: "..." }` with accepted list and alias mapping
+ */
+export function normalizeBrowserMode(
+  raw: unknown,
+): NormalizeBrowserModeResult | NormalizeBrowserModeError {
+  if (raw === undefined || raw === null || raw === "") {
+    return { mode: BROWSER_MODE.AUTO };
+  }
+
+  if (typeof raw !== "string") {
+    return buildError(String(raw));
+  }
+
+  const lower = raw.toLowerCase().trim();
+  const canonical = ALIAS_MAP[lower];
+
+  if (canonical !== undefined) {
+    return { mode: canonical };
+  }
+
+  return buildError(raw);
+}
+
+function buildError(value: string): NormalizeBrowserModeError {
+  const aliasHints = Object.entries(ALIAS_DISPLAY)
+    .map(([alias, canonical]) => `${alias}->${canonical}`)
+    .join(", ");
+
+  return {
+    error:
+      `Invalid browser_mode "${value}". ` +
+      `Accepted values: ${ACCEPTED_VALUES.join(", ")}. ` +
+      `Aliases: ${aliasHints}.`,
+  };
+}

package/src/tools/browser/browser-status-constants.ts

@@ -0,0 +1,33 @@
+import { BROWSER_MODE } from "./browser-mode-constants.js";
+import { DEVTOOLS_DISCOVERY_CODE } from "./cdp-client/cdp-inspect/discovery.js";
+
+export const BROWSER_STATUS_MODE = {
+  EXTENSION: BROWSER_MODE.EXTENSION,
+  CDP_INSPECT: BROWSER_MODE.CDP_INSPECT,
+  LOCAL: BROWSER_MODE.LOCAL,
+} as const;
+
+export type BrowserStatusMode =
+  (typeof BROWSER_STATUS_MODE)[keyof typeof BROWSER_STATUS_MODE];
+
+export const BROWSER_STATUS_MODES: readonly BrowserStatusMode[] = [
+  BROWSER_STATUS_MODE.EXTENSION,
+  BROWSER_STATUS_MODE.CDP_INSPECT,
+  BROWSER_STATUS_MODE.LOCAL,
+] as const;
+
+export const BROWSER_STATUS_INPUT_FIELD = {
+  CHECK_LOCAL_LAUNCH: "check_local_launch",
+} as const;
+
+export const EXTENSION_STATUS_ERROR_MARKER = {
+  UNAUTHORIZED_ORIGIN: "unauthorized_origin",
+  NATIVE_MESSAGING_HOST: "native messaging host",
+  HTTP_401: "401",
+} as const;
+
+export const CDP_INSPECT_STATUS_DISCOVERY_CODE = {
+  NO_TARGETS: DEVTOOLS_DISCOVERY_CODE.NO_TARGETS,
+  INVALID_RESPONSE: DEVTOOLS_DISCOVERY_CODE.INVALID_RESPONSE,
+  WS_FALLBACK_FAILED: DEVTOOLS_DISCOVERY_CODE.WS_FALLBACK_FAILED,
+} as const;

package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts

@@ -16,6 +16,7 @@ const { CdpInspectClient, createCdpInspectClient } =
   await import("../cdp-inspect-client.js");
 const { CdpError } = await import("../errors.js");
 const { CdpWsTransportError } = await import("../cdp-inspect/ws-transport.js");
+const { DevToolsDiscoveryError } = await import("../cdp-inspect/discovery.js");
 
 type CdpInspectClientInstance = InstanceType<typeof CdpInspectClient>;
 
@@ -868,3 +869,395 @@ describe("CdpInspectClient", () => {
     expect(harness.attachCallCount()).toBe(1);
   });
 });
+
+// ---------------------------------------------------------------------------
+// WS-only fallback — HTTP discovery absent, WS fallback succeeds.
+// ---------------------------------------------------------------------------
+
+describe("CdpInspectClient — WS-only fallback", () => {
+  test("falls back to WS when HTTP discovery returns invalid_response", async () => {
+    const sends: Array<{
+      method: string;
+      params?: Record<string, unknown>;
+      sessionId?: string;
+    }> = [];
+    let connectCount = 0;
+    let discoverViaWsCalls = 0;
+
+    const client = createCdpInspectClient("conv-ws-fallback", {
+      host: "127.0.0.1",
+      port: 9222,
+      discoveryTimeoutMs: 100,
+      wsConnectTimeoutMs: 100,
+      helpers: {
+        probeDevToolsJsonVersion: async () => {
+          throw new DevToolsDiscoveryError(
+            "invalid_response",
+            "DevTools /json/version returned HTTP 404.",
+          );
+        },
+        listDevToolsTargets: async () => {
+          throw new Error("listDevToolsTargets should not be called");
+        },
+        connectCdpWsTransport: async () => {
+          connectCount += 1;
+          return createFakeTransport({
+            onSend: async (method) => {
+              if (method === "Target.attachToTarget") {
+                return { sessionId: "ws-session" };
+              }
+              return { ok: true };
+            },
+            trackSends: sends,
+          });
+        },
+        discoverTargetsViaWs: async () => {
+          discoverViaWsCalls += 1;
+          return [
+            {
+              id: "ws-target-1",
+              type: "page",
+              title: "WS Page",
+              url: "https://example.com/",
+              webSocketDebuggerUrl:
+                "ws://127.0.0.1:9222/devtools/page/ws-target-1",
+            },
+          ];
+        },
+        buildBrowserWsUrl: (host: string, port: number) =>
+          `ws://${host}:${port}/devtools/browser`,
+      },
+    });
+
+    const result = await client.send<{ ok: boolean }>("Browser.getVersion");
+    expect(result).toEqual({ ok: true });
+    expect(connectCount).toBe(1);
+    expect(discoverViaWsCalls).toBe(1);
+    // attach + forwarded call
+    expect(sends).toEqual([
+      {
+        method: "Target.attachToTarget",
+        params: { targetId: "ws-target-1", flatten: true },
+        sessionId: undefined,
+      },
+      {
+        method: "Browser.getVersion",
+        params: undefined,
+        sessionId: "ws-session",
+      },
+    ]);
+  });
+
+  test("falls back to WS when HTTP discovery is unreachable", async () => {
+    let discoverViaWsCalls = 0;
+
+    const client = createCdpInspectClient("conv-ws-unreachable", {
+      host: "127.0.0.1",
+      port: 9222,
+      discoveryTimeoutMs: 100,
+      helpers: {
+        probeDevToolsJsonVersion: async () => {
+          throw new DevToolsDiscoveryError(
+            "unreachable",
+            "Failed to reach DevTools endpoint: ECONNREFUSED",
+          );
+        },
+        listDevToolsTargets: async () => {
+          throw new Error("should not be called");
+        },
+        connectCdpWsTransport: async () =>
+          createFakeTransport({
+            onSend: async (method) => {
+              if (method === "Target.attachToTarget") {
+                return { sessionId: "ws-session" };
+              }
+              return { ok: true };
+            },
+          }),
+        discoverTargetsViaWs: async () => {
+          discoverViaWsCalls += 1;
+          return [
+            {
+              id: "ws-target-1",
+              type: "page",
+              title: "Page",
+              url: "https://example.com/",
+              webSocketDebuggerUrl:
+                "ws://127.0.0.1:9222/devtools/page/ws-target-1",
+            },
+          ];
+        },
+        buildBrowserWsUrl: (host: string, port: number) =>
+          `ws://${host}:${port}/devtools/browser`,
+      },
+    });
+
+    const result = await client.send<{ ok: boolean }>("Runtime.enable");
+    expect(result).toEqual({ ok: true });
+    expect(discoverViaWsCalls).toBe(1);
+  });
+
+  test("does NOT fall back to WS on non_loopback error (safety constraint)", async () => {
+    let connectCalled = false;
+
+    const client = createCdpInspectClient("conv-no-fallback-loopback", {
+      host: "127.0.0.1",
+      port: 9222,
+      discoveryTimeoutMs: 100,
+      helpers: {
+        probeDevToolsJsonVersion: async () => {
+          throw new DevToolsDiscoveryError(
+            "non_loopback",
+            "Refusing to probe non-loopback host",
+          );
+        },
+        listDevToolsTargets: async () => {
+          throw new Error("should not be called");
+        },
+        connectCdpWsTransport: async () => {
+          connectCalled = true;
+          return createFakeTransport({});
+        },
+        discoverTargetsViaWs: async () => {
+          throw new Error("should not be called");
+        },
+        buildBrowserWsUrl: () => "ws://127.0.0.1:9222/devtools/browser",
+      },
+    });
+
+    let caught: unknown;
+    try {
+      await client.send("Browser.getVersion");
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const cdpErr = caught as InstanceType<typeof CdpError>;
+    expect(cdpErr.code).toBe("transport_error");
+    expect(cdpErr.message).toContain("non-loopback");
+    expect(connectCalled).toBe(false);
+  });
+
+  test("does NOT fall back to WS on non_chrome error", async () => {
+    let connectCalled = false;
+
+    const client = createCdpInspectClient("conv-no-fallback-chrome", {
+      host: "127.0.0.1",
+      port: 9222,
+      discoveryTimeoutMs: 100,
+      helpers: {
+        probeDevToolsJsonVersion: async () => {
+          throw new DevToolsDiscoveryError(
+            "non_chrome",
+            "Not Chrome: Firefox/115.0",
+          );
+        },
+        listDevToolsTargets: async () => {
+          throw new Error("should not be called");
+        },
+        connectCdpWsTransport: async () => {
+          connectCalled = true;
+          return createFakeTransport({});
+        },
+        discoverTargetsViaWs: async () => {
+          throw new Error("should not be called");
+        },
+        buildBrowserWsUrl: () => "ws://127.0.0.1:9222/devtools/browser",
+      },
+    });
+
+    let caught: unknown;
+    try {
+      await client.send("Browser.getVersion");
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const cdpErr = caught as InstanceType<typeof CdpError>;
+    expect(cdpErr.code).toBe("transport_error");
+    expect(connectCalled).toBe(false);
+  });
+
+  test("does NOT fall back to WS on timeout error", async () => {
+    let connectCalled = false;
+
+    const client = createCdpInspectClient("conv-no-fallback-timeout", {
+      host: "127.0.0.1",
+      port: 9222,
+      discoveryTimeoutMs: 100,
+      helpers: {
+        probeDevToolsJsonVersion: async () => {
+          throw new DevToolsDiscoveryError(
+            "timeout",
+            "Timed out waiting for DevTools HTTP response.",
+          );
+        },
+        listDevToolsTargets: async () => {
+          throw new Error("should not be called");
+        },
+        connectCdpWsTransport: async () => {
+          connectCalled = true;
+          return createFakeTransport({});
+        },
+        discoverTargetsViaWs: async () => {
+          throw new Error("should not be called");
+        },
+        buildBrowserWsUrl: () => "ws://127.0.0.1:9222/devtools/browser",
+      },
+    });
+
+    let caught: unknown;
+    try {
+      await client.send("Browser.getVersion");
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const cdpErr = caught as InstanceType<typeof CdpError>;
+    expect(cdpErr.code).toBe("transport_error");
+    expect(connectCalled).toBe(false);
+  });
+
+  test("full fallback failure: HTTP invalid_response + WS connect fails", async () => {
+    const client = createCdpInspectClient("conv-full-fail", {
+      host: "127.0.0.1",
+      port: 9222,
+      discoveryTimeoutMs: 100,
+      helpers: {
+        probeDevToolsJsonVersion: async () => {
+          throw new DevToolsDiscoveryError(
+            "invalid_response",
+            "DevTools /json/version returned HTTP 404.",
+          );
+        },
+        listDevToolsTargets: async () => {
+          throw new Error("should not be called");
+        },
+        connectCdpWsTransport: async () => {
+          throw new CdpWsTransportError(
+            "transport_error",
+            "websocket closed before open",
+          );
+        },
+        discoverTargetsViaWs: async () => {
+          throw new Error("should not be called");
+        },
+        buildBrowserWsUrl: (host: string, port: number) =>
+          `ws://${host}:${port}/devtools/browser`,
+      },
+    });
+
+    let caught: unknown;
+    try {
+      await client.send("Browser.getVersion");
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const cdpErr = caught as InstanceType<typeof CdpError>;
+    expect(cdpErr.code).toBe("transport_error");
+    // Error message should explain both HTTP and WS failures
+    expect(cdpErr.message).toContain("HTTP discovery failed");
+    expect(cdpErr.message).toContain("WS-only fallback also failed");
+    expect(cdpErr.message).toContain("127.0.0.1:9222");
+  });
+
+  test("WS fallback: no page targets yields stable error", async () => {
+    const client = createCdpInspectClient("conv-ws-no-targets", {
+      host: "127.0.0.1",
+      port: 9222,
+      discoveryTimeoutMs: 100,
+      helpers: {
+        probeDevToolsJsonVersion: async () => {
+          throw new DevToolsDiscoveryError("unreachable", "ECONNREFUSED");
+        },
+        listDevToolsTargets: async () => {
+          throw new Error("should not be called");
+        },
+        connectCdpWsTransport: async () =>
+          createFakeTransport({
+            onSend: async () => ({ ok: true }),
+          }),
+        discoverTargetsViaWs: async () => {
+          throw new DevToolsDiscoveryError(
+            "no_targets",
+            "No usable page targets returned by CDP Target.getTargets.",
+          );
+        },
+        buildBrowserWsUrl: (host: string, port: number) =>
+          `ws://${host}:${port}/devtools/browser`,
+      },
+    });
+
+    let caught: unknown;
+    try {
+      await client.send("Browser.getVersion");
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const cdpErr = caught as InstanceType<typeof CdpError>;
+    expect(cdpErr.code).toBe("transport_error");
+    expect(cdpErr.message).toContain("No usable page targets");
+  });
+
+  test("WS fallback caches session for subsequent calls", async () => {
+    let connectCount = 0;
+    let discoverViaWsCalls = 0;
+
+    const client = createCdpInspectClient("conv-ws-cache", {
+      host: "127.0.0.1",
+      port: 9222,
+      discoveryTimeoutMs: 100,
+      helpers: {
+        probeDevToolsJsonVersion: async () => {
+          throw new DevToolsDiscoveryError(
+            "invalid_response",
+            "no HTTP discovery",
+          );
+        },
+        listDevToolsTargets: async () => {
+          throw new Error("should not be called");
+        },
+        connectCdpWsTransport: async () => {
+          connectCount += 1;
+          return createFakeTransport({
+            onSend: async (method) => {
+              if (method === "Target.attachToTarget") {
+                return { sessionId: "ws-session-cached" };
+              }
+              return { ok: true };
+            },
+          });
+        },
+        discoverTargetsViaWs: async () => {
+          discoverViaWsCalls += 1;
+          return [
+            {
+              id: "t1",
+              type: "page",
+              title: "Page",
+              url: "https://example.com/",
+              webSocketDebuggerUrl: "ws://127.0.0.1:9222/devtools/page/t1",
+            },
+          ];
+        },
+        buildBrowserWsUrl: (host: string, port: number) =>
+          `ws://${host}:${port}/devtools/browser`,
+      },
+    });
+
+    await client.send("Runtime.enable");
+    await client.send("Page.enable");
+    await client.send("DOM.enable");
+
+    // Only one WS fallback discovery + one connect + one attach
+    expect(connectCount).toBe(1);
+    expect(discoverViaWsCalls).toBe(1);
+  });
+});

package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts

@@ -185,6 +185,35 @@ describe("ExtensionCdpClient", () => {
     expect(err.underlying).toBeInstanceOf(Error);
   });
 
+  test("result.isError with non-JSON content throws CdpError('cdp_error')", async () => {
+    const { proxy } = fakeProxy(async () => ({
+      content:
+        "Host browser proxy timed out waiting for extension response (check callback failures).",
+      isError: true,
+    }));
+
+    const client = createExtensionCdpClient(proxy, "conv-6b");
+
+    let caught: unknown;
+    try {
+      await client.send("Runtime.evaluate", { expression: "document.title" });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(CdpError);
+    const err = caught as CdpError;
+    expect(err.code).toBe("cdp_error");
+    expect(err.message).toContain(
+      "Host browser proxy timed out waiting for extension response",
+    );
+    expect(err.cdpMethod).toBe("Runtime.evaluate");
+    expect(err.cdpParams).toEqual({ expression: "document.title" });
+    expect(err.underlying).toBe(
+      "Host browser proxy timed out waiting for extension response (check callback failures).",
+    );
+  });
+
   test("result.content === 'Aborted' throws CdpError('aborted')", async () => {
     const { proxy, request } = fakeProxy(async () => ({
       content: "Aborted",