@vellumai/assistant 0.6.2 → 0.6.4

package/src/tools/browser/browser-manager.ts

@@ -4,6 +4,7 @@ import { isAbsolute, join, relative, resolve } from "node:path";
 import { getLogger } from "../../util/logger.js";
 import { getDataDir } from "../../util/platform.js";
 import { authSessionCache } from "./auth-cache.js";
+import type { CdpClientKind } from "./cdp-client/types.js";
 import type { ExtractedCredential } from "./network-recording-types.js";
 import { importPlaywright } from "./runtime-check.js";
 
@@ -203,7 +204,23 @@ class BrowserManager {
   private pages = new Map<string, Page>();
   private rawPages = new Map<string, unknown>();
   private cdpSessions = new Map<string, CDPSession>();
-  private snapshotMaps = new Map<string, Map<string, string>>();
+  /**
+   * CDP backendNodeId lookup per conversation, populated by the
+   * AX-tree-based `executeBrowserSnapshot`. Click/type/hover/etc. tools
+   * resolve an element_id against this map and talk to CDP with the
+   * resulting backendNodeId.
+   */
+  private snapshotBackendNodeMaps = new Map<string, Map<string, number>>();
+  /**
+   * The CDP backend kind that a conversation resolved to on an earlier
+   * tool call. When a subsequent browser_* call arrives with
+   * `browser_mode: "auto"`, the factory is pinned to this kind so
+   * navigate and screenshot cannot land on different Chromes within
+   * the same conversation. Explicit non-auto modes overwrite the entry.
+   * Cleared when the conversation's page is closed or all pages are
+   * closed.
+   */
+  private preferredBackendKinds = new Map<string, CdpClientKind>();
   private browserCdpSession: CDPSession | null = null;
   private browserWindowId: number | null = null;
   private interactiveModeSessions = new Set<string>();
@@ -358,7 +375,7 @@ class BrowserManager {
           this.rawPages.clear();
           this.cdpSessions.clear();
 
-          this.snapshotMaps.clear();
+          this.snapshotBackendNodeMaps.clear();
           this.downloads.clear();
           for (const pending of this.pendingDownloads.values()) {
             for (const waiter of pending)
@@ -384,7 +401,7 @@ class BrowserManager {
     }
 
     // Clear stale snapshot mappings and CDP state when replacing a closed page
-    this.snapshotMaps.delete(conversationId);
+    this.snapshotBackendNodeMaps.delete(conversationId);
     await this.stopScreencast(conversationId);
 
     const page = await context.newPage();
@@ -437,7 +454,8 @@ class BrowserManager {
     }
     this.pages.delete(conversationId);
     this.rawPages.delete(conversationId);
-    this.snapshotMaps.delete(conversationId);
+    this.snapshotBackendNodeMaps.delete(conversationId);
+    this.preferredBackendKinds.delete(conversationId);
     this.downloads.delete(conversationId);
     // Reject any pending download waiters
     const pending = this.pendingDownloads.get(conversationId);
@@ -470,7 +488,8 @@ class BrowserManager {
     }
     this.pages.clear();
     this.rawPages.clear();
-    this.snapshotMaps.clear();
+    this.snapshotBackendNodeMaps.clear();
+    this.preferredBackendKinds.clear();
     this.downloads.clear();
     for (const pending of this.pendingDownloads.values()) {
       for (const waiter of pending) waiter.reject(new Error("Browser closed"));
@@ -526,19 +545,69 @@ class BrowserManager {
     }
   }
 
-  storeSnapshotMap(conversationId: string, map: Map<string, string>): void {
-    this.snapshotMaps.set(conversationId, map);
+  /**
+   * Look up the backend kind this conversation has been using. Returns
+   * `null` before the first successful CDP send or after an explicit
+   * teardown. Callers should pass the result to the factory as a
+   * pinned mode so `auto`-mode calls stay consistent across tool
+   * invocations.
+   */
+  getPreferredBackendKind(conversationId: string): CdpClientKind | null {
+    return this.preferredBackendKinds.get(conversationId) ?? null;
+  }
+
+  /**
+   * Record the backend kind that a tool call resolved to. Overwrites
+   * any prior value so an explicit `browser_mode` override on one call
+   * becomes the default for subsequent `auto`-mode calls.
+   */
+  setPreferredBackendKind(
+    conversationId: string,
+    kind: CdpClientKind,
+  ): void {
+    this.preferredBackendKinds.set(conversationId, kind);
+  }
+
+  /**
+   * Drop the remembered backend kind for a conversation. Called on
+   * explicit teardown (browser_close, closeAllPages) so the next call
+   * re-runs the auto priority list.
+   */
+  clearPreferredBackendKind(conversationId: string): void {
+    this.preferredBackendKinds.delete(conversationId);
+  }
+
+  /**
+   * Store the backendNodeId lookup produced by the AX-tree-based
+   * `executeBrowserSnapshot`. Callers overwrite any prior map for the
+   * conversation so each snapshot fully supersedes the previous one.
+   */
+  storeSnapshotBackendNodeMap(
+    conversationId: string,
+    map: Map<string, number>,
+  ): void {
+    this.snapshotBackendNodeMaps.set(conversationId, map);
   }
 
-  clearSnapshotMap(conversationId: string): void {
-    this.snapshotMaps.delete(conversationId);
+  /**
+   * Drop the backendNodeId lookup for a conversation so stale element
+   * IDs from a previous snapshot cannot resolve after a navigation or
+   * page close.
+   */
+  clearSnapshotBackendNodeMap(conversationId: string): void {
+    this.snapshotBackendNodeMaps.delete(conversationId);
   }
 
-  resolveSnapshotSelector(
+  /**
+   * Look up the CDP backendNodeId for an element id produced by the
+   * most recent AX-tree snapshot. Returns `null` if no snapshot has
+   * been taken or the id is unknown.
+   */
+  resolveSnapshotBackendNodeId(
     conversationId: string,
     elementId: string,
-  ): string | null {
-    const map = this.snapshotMaps.get(conversationId);
+  ): number | null {
+    const map = this.snapshotBackendNodeMaps.get(conversationId);
     if (!map) return null;
     return map.get(elementId) ?? null;
   }

package/src/tools/browser/browser-mode-constants.ts

@@ -0,0 +1,12 @@
+import type { BrowserMode } from "./cdp-client/types.js";
+
+/**
+ * Canonical browser-mode identifiers shared across parsing and status
+ * reporting paths.
+ */
+export const BROWSER_MODE = {
+  AUTO: "auto",
+  EXTENSION: "extension",
+  CDP_INSPECT: "cdp-inspect",
+  LOCAL: "local",
+} as const satisfies Record<string, BrowserMode>;

package/src/tools/browser/browser-mode.ts

@@ -0,0 +1,92 @@
+/**
+ * Normalization helper for the `browser_mode` tool input parameter.
+ *
+ * Canonical values map directly to {@link CdpClientKind}:
+ *   - `auto`        -- let the factory pick the best backend (default)
+ *   - `extension`   -- force the Chrome extension transport
+ *   - `cdp-inspect` -- force the CDP inspect/debugger transport
+ *   - `local`       -- force the Playwright-managed local browser
+ *
+ * Aliases are accepted and normalized to their canonical form:
+ *   - `cdp-debugger` -> `cdp-inspect`
+ *   - `playwright`   -> `local`
+ */
+
+import { BROWSER_MODE } from "./browser-mode-constants.js";
+import type { BrowserMode } from "./cdp-client/types.js";
+
+/** Canonical browser mode values. Re-exported from cdp-client/types. */
+export type { BrowserMode } from "./cdp-client/types.js";
+
+/** All accepted values (canonical + aliases). */
+const ALIAS_MAP: Record<string, BrowserMode> = {
+  [BROWSER_MODE.AUTO]: BROWSER_MODE.AUTO,
+  [BROWSER_MODE.EXTENSION]: BROWSER_MODE.EXTENSION,
+  [BROWSER_MODE.CDP_INSPECT]: BROWSER_MODE.CDP_INSPECT,
+  "cdp-debugger": BROWSER_MODE.CDP_INSPECT,
+  [BROWSER_MODE.LOCAL]: BROWSER_MODE.LOCAL,
+  playwright: BROWSER_MODE.LOCAL,
+};
+
+/** Ordered list of accepted values for error messages. */
+const ACCEPTED_VALUES = Object.keys(ALIAS_MAP);
+
+/**
+ * Human-readable alias mapping for error messages.
+ * Only includes entries where the alias differs from the canonical value.
+ */
+const ALIAS_DISPLAY: Record<string, string> = {
+  "cdp-debugger": "cdp-inspect",
+  playwright: "local",
+};
+
+export interface NormalizeBrowserModeResult {
+  /** The normalized canonical mode. */
+  mode: BrowserMode;
+}
+
+export interface NormalizeBrowserModeError {
+  /** Deterministic error message describing the invalid value. */
+  error: string;
+}
+
+/**
+ * Normalize a raw `browser_mode` input value to a canonical {@link BrowserMode}.
+ *
+ * - `undefined` / `null` / empty string -> `{ mode: "auto" }`
+ * - Valid canonical value or alias       -> `{ mode: <canonical> }`
+ * - Invalid value                        -> `{ error: "..." }` with accepted list and alias mapping
+ */
+export function normalizeBrowserMode(
+  raw: unknown,
+): NormalizeBrowserModeResult | NormalizeBrowserModeError {
+  if (raw === undefined || raw === null || raw === "") {
+    return { mode: BROWSER_MODE.AUTO };
+  }
+
+  if (typeof raw !== "string") {
+    return buildError(String(raw));
+  }
+
+  const lower = raw.toLowerCase().trim();
+  const canonical = ALIAS_MAP[lower];
+
+  if (canonical !== undefined) {
+    return { mode: canonical };
+  }
+
+  return buildError(raw);
+}
+
+function buildError(value: string): NormalizeBrowserModeError {
+  const aliasHints = Object.entries(ALIAS_DISPLAY)
+    .map(([alias, canonical]) => `${alias}->${canonical}`)
+    .join(", ");
+
+  return {
+    error:
+      `Invalid browser_mode "${value}". ` +
+      `Accepted values: ${ACCEPTED_VALUES.join(", ")}. ` +
+      `Aliases: ${aliasHints}.`,
+  };
+}

package/src/tools/browser/browser-status-constants.ts

@@ -0,0 +1,33 @@
+import { BROWSER_MODE } from "./browser-mode-constants.js";
+import { DEVTOOLS_DISCOVERY_CODE } from "./cdp-client/cdp-inspect/discovery.js";
+
+export const BROWSER_STATUS_MODE = {
+  EXTENSION: BROWSER_MODE.EXTENSION,
+  CDP_INSPECT: BROWSER_MODE.CDP_INSPECT,
+  LOCAL: BROWSER_MODE.LOCAL,
+} as const;
+
+export type BrowserStatusMode =
+  (typeof BROWSER_STATUS_MODE)[keyof typeof BROWSER_STATUS_MODE];
+
+export const BROWSER_STATUS_MODES: readonly BrowserStatusMode[] = [
+  BROWSER_STATUS_MODE.EXTENSION,
+  BROWSER_STATUS_MODE.CDP_INSPECT,
+  BROWSER_STATUS_MODE.LOCAL,
+] as const;
+
+export const BROWSER_STATUS_INPUT_FIELD = {
+  CHECK_LOCAL_LAUNCH: "check_local_launch",
+} as const;
+
+export const EXTENSION_STATUS_ERROR_MARKER = {
+  UNAUTHORIZED_ORIGIN: "unauthorized_origin",
+  NATIVE_MESSAGING_HOST: "native messaging host",
+  HTTP_401: "401",
+} as const;
+
+export const CDP_INSPECT_STATUS_DISCOVERY_CODE = {
+  NO_TARGETS: DEVTOOLS_DISCOVERY_CODE.NO_TARGETS,
+  INVALID_RESPONSE: DEVTOOLS_DISCOVERY_CODE.INVALID_RESPONSE,
+  WS_FALLBACK_FAILED: DEVTOOLS_DISCOVERY_CODE.WS_FALLBACK_FAILED,
+} as const;

package/src/tools/browser/cdp-client/__tests__/accessibility-snapshot.test.ts

@@ -0,0 +1,318 @@
+import { describe, expect, it } from "bun:test";
+
+import {
+  type AxSnapshotElement,
+  type AxSnapshotResult,
+  formatAxSnapshot,
+  transformAxTree,
+} from "../accessibility-snapshot.js";
+import nestedFramesFixture from "./fixtures/ax-tree-nested-frames.json" with { type: "json" };
+import simpleFixture from "./fixtures/ax-tree-simple.json" with { type: "json" };
+
+// ── transformAxTree ──────────────────────────────────────────────────
+
+describe("transformAxTree", () => {
+  it("happy path: simple fixture returns exactly 3 elements with stable eids", () => {
+    const result = transformAxTree(simpleFixture);
+
+    expect(result.elements).toHaveLength(3);
+
+    const [e1, e2, e3] = result.elements as [
+      AxSnapshotElement,
+      AxSnapshotElement,
+      AxSnapshotElement,
+    ];
+
+    // e1: the button (trimmed + focusable prop not surfaced as attr).
+    expect(e1.eid).toBe("e1");
+    expect(e1.role).toBe("button");
+    expect(e1.name).toBe("Submit Form");
+    expect(e1.backendNodeId).toBe(101);
+    expect(e1.attrs).toEqual({ disabled: "false" });
+    expect(e1.value).toBeUndefined();
+
+    // e2: the link carries its url property in attrs.
+    expect(e2.eid).toBe("e2");
+    expect(e2.role).toBe("link");
+    expect(e2.name).toBe("About Us");
+    expect(e2.backendNodeId).toBe(102);
+    expect(e2.attrs).toEqual({ url: "https://example.com/about" });
+
+    // e3: the textbox carries placeholder + required + a `value` field.
+    expect(e3.eid).toBe("e3");
+    expect(e3.role).toBe("textbox");
+    expect(e3.name).toBe("Email address");
+    expect(e3.backendNodeId).toBe(103);
+    expect(e3.value).toBe("user@example.com");
+    expect(e3.attrs).toEqual({
+      placeholder: "you@example.com",
+      required: "true",
+    });
+  });
+
+  it("filters out ignored nodes", () => {
+    const result = transformAxTree(simpleFixture);
+    for (const el of result.elements) {
+      expect(el.name).not.toBe("Hidden Action");
+    }
+  });
+
+  it("filters out nodes without backendDOMNodeId", () => {
+    const result = transformAxTree(simpleFixture);
+    for (const el of result.elements) {
+      expect(el.name).not.toBe("Orphan Button");
+    }
+  });
+
+  it("truncates to opts.maxElements", () => {
+    const result = transformAxTree(simpleFixture, { maxElements: 2 });
+    expect(result.elements).toHaveLength(2);
+    expect(result.elements[0]?.eid).toBe("e1");
+    expect(result.elements[1]?.eid).toBe("e2");
+    // selectorMap only contains the kept elements.
+    expect(result.selectorMap.size).toBe(2);
+    expect(result.selectorMap.get("e1")).toBe(101);
+    expect(result.selectorMap.get("e2")).toBe(102);
+    expect(result.selectorMap.has("e3")).toBe(false);
+  });
+
+  it("selectorMap.get() returns the backendNodeId for each eid", () => {
+    const result = transformAxTree(simpleFixture);
+    expect(result.selectorMap.size).toBe(3);
+    expect(result.selectorMap.get("e1")).toBe(101);
+    expect(result.selectorMap.get("e2")).toBe(102);
+    expect(result.selectorMap.get("e3")).toBe(103);
+  });
+
+  it("filters out non-interactive roles (StaticText, generic, RootWebArea)", () => {
+    const fixture = {
+      nodes: [
+        {
+          nodeId: "1",
+          role: { value: "RootWebArea" },
+          name: { value: "Root" },
+          backendDOMNodeId: 1,
+          childIds: ["2", "3", "4"],
+          ignored: false,
+        },
+        {
+          nodeId: "2",
+          role: { value: "StaticText" },
+          name: { value: "Just some text" },
+          backendDOMNodeId: 2,
+          childIds: [],
+          ignored: false,
+        },
+        {
+          nodeId: "3",
+          role: { value: "generic" },
+          name: { value: "A div" },
+          backendDOMNodeId: 3,
+          childIds: [],
+          ignored: false,
+        },
+        {
+          nodeId: "4",
+          role: { value: "button" },
+          name: { value: "Real Button" },
+          backendDOMNodeId: 4,
+          childIds: [],
+          ignored: false,
+        },
+      ],
+    };
+
+    const result = transformAxTree(fixture);
+    expect(result.elements).toHaveLength(1);
+    expect(result.elements[0]?.name).toBe("Real Button");
+    expect(result.elements[0]?.role).toBe("button");
+  });
+
+  it("includes focusable nodes even when the role is not interactive", () => {
+    // Regression test for contenteditable divs and custom widgets: a
+    // generic-role node with `focusable: true` must still surface.
+    const fixture = {
+      nodes: [
+        {
+          nodeId: "1",
+          role: { value: "RootWebArea" },
+          name: { value: "Root" },
+          backendDOMNodeId: 1,
+          childIds: ["2"],
+          ignored: false,
+        },
+        {
+          nodeId: "2",
+          role: { value: "generic" },
+          name: { value: "Contenteditable field" },
+          properties: [
+            { name: "focusable", value: { type: "boolean", value: true } },
+          ],
+          backendDOMNodeId: 42,
+          childIds: [],
+          ignored: false,
+        },
+      ],
+    };
+
+    const result = transformAxTree(fixture);
+    expect(result.elements).toHaveLength(1);
+    const [el] = result.elements;
+    expect(el?.role).toBe("generic");
+    expect(el?.backendNodeId).toBe(42);
+    expect(el?.name).toBe("Contenteditable field");
+  });
+
+  it("traverses nested RootWebArea (iframe) children in document order", () => {
+    const result = transformAxTree(nestedFramesFixture);
+
+    // Expect 4 kept elements across parent + inner frame, in doc order.
+    expect(result.elements).toHaveLength(4);
+    expect(result.elements.map((el) => el.name)).toEqual([
+      "Parent Button",
+      "Inner Link",
+      "Inner Search",
+      "Subscribe",
+    ]);
+    expect(result.elements.map((el) => el.backendNodeId)).toEqual([
+      201, 203, 204, 205,
+    ]);
+    expect(result.elements.map((el) => el.eid)).toEqual([
+      "e1",
+      "e2",
+      "e3",
+      "e4",
+    ]);
+  });
+
+  it("truncates names longer than 80 chars", () => {
+    const longName = "x".repeat(200);
+    const fixture = {
+      nodes: [
+        {
+          nodeId: "1",
+          role: { value: "button" },
+          name: { value: longName },
+          backendDOMNodeId: 1,
+          childIds: [],
+          ignored: false,
+        },
+      ],
+    };
+    const result = transformAxTree(fixture);
+    expect(result.elements).toHaveLength(1);
+    expect(result.elements[0]?.name.length).toBe(80);
+  });
+
+  it("returns an empty result for malformed input", () => {
+    expect(transformAxTree(null)).toEqual({
+      elements: [],
+      selectorMap: new Map(),
+    });
+    expect(transformAxTree(undefined)).toEqual({
+      elements: [],
+      selectorMap: new Map(),
+    });
+    expect(transformAxTree({})).toEqual({
+      elements: [],
+      selectorMap: new Map(),
+    });
+    expect(transformAxTree({ nodes: [] })).toEqual({
+      elements: [],
+      selectorMap: new Map(),
+    });
+  });
+});
+
+// ── formatAxSnapshot ──────────────────────────────────────────────────
+
+describe("formatAxSnapshot", () => {
+  it("byte-matches the legacy executeBrowserSnapshot format for the simple fixture", () => {
+    const result = transformAxTree(simpleFixture);
+    const formatted = formatAxSnapshot(result, {
+      url: "https://example.com/",
+      title: "Example",
+    });
+
+    const expected = [
+      "URL: https://example.com/",
+      "Title: Example",
+      "",
+      `[e1] <button disabled="false"> Submit Form`,
+      `[e2] <link url="https://example.com/about"> About Us`,
+      `[e3] <textbox placeholder="you@example.com" required="true" value="user@example.com"> Email address`,
+      "",
+      "3 interactive elements found.",
+    ].join("\n");
+
+    expect(formatted).toBe(expected);
+  });
+
+  it("uses '(none)' when title is empty", () => {
+    const result: AxSnapshotResult = {
+      elements: [],
+      selectorMap: new Map(),
+    };
+    const formatted = formatAxSnapshot(result, {
+      url: "about:blank",
+      title: "",
+    });
+    expect(formatted).toBe(
+      [
+        "URL: about:blank",
+        "Title: (none)",
+        "",
+        "(no interactive elements found)",
+      ].join("\n"),
+    );
+  });
+
+  it("renders 'interactive element' (singular) for a single-element result", () => {
+    const result: AxSnapshotResult = {
+      elements: [
+        {
+          eid: "e1",
+          role: "button",
+          name: "Click",
+          attrs: {},
+          backendNodeId: 1,
+        },
+      ],
+      selectorMap: new Map([["e1", 1]]),
+    };
+    const formatted = formatAxSnapshot(result, {
+      url: "https://example.com/",
+      title: "T",
+    });
+    expect(formatted).toBe(
+      [
+        "URL: https://example.com/",
+        "Title: T",
+        "",
+        "[e1] <button> Click",
+        "",
+        "1 interactive element found.",
+      ].join("\n"),
+    );
+  });
+
+  it("omits the trailing ' name' segment when name is empty", () => {
+    const result: AxSnapshotResult = {
+      elements: [
+        {
+          eid: "e1",
+          role: "button",
+          name: "",
+          attrs: {},
+          backendNodeId: 1,
+        },
+      ],
+      selectorMap: new Map([["e1", 1]]),
+    };
+    const formatted = formatAxSnapshot(result, {
+      url: "https://example.com/",
+      title: "T",
+    });
+    expect(formatted.split("\n")[3]).toBe("[e1] <button>");
+  });
+});