@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.4

package/src/desktop/desktop-browser-client.ts

@@ -0,0 +1,430 @@
+import { browserManager } from "../tools/browser/browser-manager.js";
+import {
+  type CdpWsTransport,
+  CdpWsTransportError,
+} from "../tools/browser/cdp-client/cdp-inspect/ws-transport.js";
+import { CdpError } from "../tools/browser/cdp-client/errors.js";
+import type {
+  ScopedCdpClient,
+  TabInfo,
+} from "../tools/browser/cdp-client/types.js";
+import {
+  desktopCursorExpression,
+  REMOVE_DESKTOP_CURSOR,
+} from "./desktop-browser-cursor.js";
+
+type Target = { targetId: string; type: string; url: string; title: string };
+type HeldInput = {
+  targetId: string;
+  method: string;
+  params: Record<string, unknown>;
+};
+
+export class DesktopBrowserClient {
+  private transport?: CdpWsTransport;
+  private targets = new Map<number, string>();
+  private sessions = new Map<string, string>();
+  private selected?: number;
+  private nextTab = 1;
+  private held = new Map<string, HeldInput>();
+  private conversation?: string;
+  private cursorPositions = new Map<string, { x: number; y: number }>();
+  private cursorRestored: Promise<void> = Promise.resolve();
+  generation = 0;
+
+  constructor(
+    private readonly connect: (signal: AbortSignal) => Promise<CdpWsTransport>,
+  ) {}
+
+  invalidateSnapshot(): void {
+    this.generation++;
+    if (this.conversation) {
+      browserManager.clearSnapshotBackendNodeMap(this.conversation);
+    }
+  }
+
+  async client(
+    conversationId: string,
+    signal: AbortSignal,
+  ): Promise<ScopedCdpClient> {
+    signal = AbortSignal.any([signal, AbortSignal.timeout(120_000)]);
+    signal.throwIfAborted();
+    if (this.transport?.closed) {
+      await this.release();
+      signal.throwIfAborted();
+    }
+    if (!this.transport) {
+      const transport = await this.connect(signal);
+      if (signal.aborted) {
+        transport.dispose();
+        signal.throwIfAborted();
+      }
+      this.transport = transport;
+      transport.addEventListener((event) => {
+        if (event.method === "Page.domContentEventFired" && event.sessionId) {
+          const sessionId = event.sessionId;
+          this.cursorRestored = this.cursorRestored.then(async () => {
+            const position = this.cursorPositions.get(sessionId);
+            if (this.transport === transport && position) {
+              await this.drawCursor(
+                transport,
+                sessionId,
+                position.x,
+                position.y,
+                AbortSignal.timeout(3_000),
+              );
+            }
+          });
+        }
+        if (
+          [
+            "Page.frameNavigated",
+            "Page.navigatedWithinDocument",
+            "DOM.documentUpdated",
+            "Target.detachedFromTarget",
+            "Target.targetDestroyed",
+          ].includes(event.method)
+        ) {
+          this.invalidateSnapshot();
+        }
+        if (event.method === "Target.detachedFromTarget") {
+          const detached = (event.params as { sessionId?: string })?.sessionId;
+          if (detached) {
+            this.cursorPositions.delete(detached);
+          }
+          for (const [target, session] of this.sessions) {
+            if (session === detached) {
+              this.sessions.delete(target);
+            }
+          }
+        }
+      });
+    }
+    this.conversation = `desktop-browser:${conversationId}`;
+    const transport = this.transport;
+    await this.session(transport, signal);
+    const generation = this.generation;
+    const activeSignal = (child?: AbortSignal) =>
+      child ? AbortSignal.any([signal, child]) : signal;
+    return {
+      kind: "cdp-inspect",
+      conversationId: this.conversation,
+      send: <T>(
+        method: string,
+        params?: Record<string, unknown>,
+        child?: AbortSignal,
+      ) =>
+        this.send<T>(
+          transport,
+          method,
+          params ?? {},
+          activeSignal(child),
+          generation,
+        ),
+      listTabs: () => this.listTabs(transport, signal),
+      selectTab: async (tabId) => {
+        const tabs = await this.listTabs(transport, signal);
+        const tab = tabs.find((item) => item.tabId === tabId);
+        const targetId = this.targets.get(tabId);
+        if (!tab || !targetId) {
+          throw new Error("Desktop browser tab is unavailable");
+        }
+        await transport.send("Target.activateTarget", { targetId }, { signal });
+        this.selected = tabId;
+        this.invalidateSnapshot();
+        return { ...tab, clientId: "managed-desktop" };
+      },
+      closeTab: async (tabId) => {
+        const targetId = this.targets.get(tabId);
+        if (!targetId) {
+          throw new Error("Desktop browser tab is unavailable");
+        }
+        await transport.send("Target.closeTarget", { targetId }, { signal });
+        if (this.selected === tabId) {
+          this.selected = undefined;
+        }
+        this.invalidateSnapshot();
+        return { closed: true, tabId, clientId: "managed-desktop" };
+      },
+      setCdpSessionId: (tabId) => {
+        signal.throwIfAborted();
+        if (this.transport !== transport) {
+          throw new Error("Desktop browser session expired");
+        }
+        this.selected = tabId === undefined ? undefined : Number(tabId);
+        this.invalidateSnapshot();
+      },
+      dispose: () => {},
+    };
+  }
+
+  private async listTabs(
+    transport: CdpWsTransport,
+    signal: AbortSignal,
+  ): Promise<TabInfo[]> {
+    signal.throwIfAborted();
+    if (this.transport !== transport) {
+      throw new Error("Desktop browser session expired");
+    }
+    const { targetInfos } = await transport.send<{ targetInfos: Target[] }>(
+      "Target.getTargets",
+      {},
+      { signal },
+    );
+    const pages = targetInfos.filter(
+      (target) => target.type === "page" && !target.url.startsWith("devtools:"),
+    );
+    const liveTargets = new Set(pages.map((page) => page.targetId));
+    for (const [id, target] of this.targets) {
+      if (!liveTargets.has(target)) {
+        this.targets.delete(id);
+      }
+    }
+    const aliases = new Map(
+      [...this.targets].map(([id, target]) => [target, id]),
+    );
+    return pages.map((target) => {
+      let id = aliases.get(target.targetId);
+      if (id === undefined) {
+        id = this.nextTab++;
+        this.targets.set(id, target.targetId);
+      }
+      return {
+        tabId: id,
+        url: target.url,
+        title: target.title,
+        active: id === this.selected,
+        pinned: false,
+      };
+    });
+  }
+
+  private async createTab(
+    transport: CdpWsTransport,
+    signal: AbortSignal,
+  ): Promise<{ tabId: number }> {
+    const { targetId } = await transport.send<{ targetId: string }>(
+      "Target.createTarget",
+      { url: "about:blank" },
+      { signal },
+    );
+    const tabId = this.nextTab++;
+    this.targets.set(tabId, targetId);
+    this.selected = tabId;
+    this.invalidateSnapshot();
+    return { tabId };
+  }
+
+  private async session(
+    transport: CdpWsTransport,
+    signal: AbortSignal,
+  ): Promise<string> {
+    const tabs = await this.listTabs(transport, signal);
+    if (this.selected !== undefined && !this.targets.has(this.selected)) {
+      this.selected = undefined;
+      this.invalidateSnapshot();
+      throw new Error(
+        "Desktop browser tab closed. Take a fresh snapshot before continuing.",
+      );
+    }
+    if (this.selected === undefined) {
+      this.selected = tabs.find((tab) => /^https?:/.test(tab.url ?? ""))?.tabId;
+      if (this.selected === undefined) {
+        await this.createTab(transport, signal);
+      }
+    }
+    const targetId = this.targets.get(this.selected!)!;
+    let sessionId = this.sessions.get(targetId);
+    if (!sessionId) {
+      const attached = await transport.send<{ sessionId: string }>(
+        "Target.attachToTarget",
+        { targetId, flatten: true },
+        { signal },
+      );
+      sessionId = attached.sessionId;
+      this.sessions.set(targetId, sessionId);
+      await transport.send("Page.enable", {}, { sessionId, signal });
+      await transport.send("DOM.enable", {}, { sessionId, signal });
+    }
+    await transport.send("Target.activateTarget", { targetId }, { signal });
+    return sessionId;
+  }
+
+  private async send<T>(
+    transport: CdpWsTransport,
+    method: string,
+    params: Record<string, unknown>,
+    signal: AbortSignal,
+    generation: number,
+  ): Promise<T> {
+    signal.throwIfAborted();
+    if (this.transport !== transport) {
+      throw new Error("Desktop browser session expired");
+    }
+    if (method === "Vellum.createTab") {
+      return (await this.createTab(transport, signal)) as T;
+    }
+    const targetId =
+      this.selected === undefined ? undefined : this.targets.get(this.selected);
+    const sessionId =
+      (targetId && this.sessions.get(targetId)) ||
+      (await this.session(transport, signal));
+    const pointer =
+      method === "Input.dispatchMouseEvent" &&
+      typeof params.x === "number" &&
+      typeof params.y === "number"
+        ? { x: params.x, y: params.y }
+        : undefined;
+    if (pointer) {
+      await this.drawCursor(transport, sessionId, pointer.x, pointer.y, signal);
+      this.cursorPositions.set(sessionId, pointer);
+      signal.throwIfAborted();
+    }
+    if (method.startsWith("Input.") && generation !== this.generation) {
+      throw new Error(
+        "The page changed before input. Take a fresh snapshot before continuing.",
+      );
+    }
+    const key = JSON.stringify([
+      sessionId,
+      method,
+      params.code ?? params.key ?? params.button,
+    ]);
+    const pressed =
+      params.type === "keyDown" ||
+      params.type === "rawKeyDown" ||
+      params.type === "mousePressed";
+    const previous = this.held.get(key);
+    const heldInput = {
+      targetId: this.targets.get(this.selected!)!,
+      method,
+      params: {
+        ...params,
+        type: method === "Input.dispatchKeyEvent" ? "keyUp" : "mouseReleased",
+      },
+    };
+    if (pressed) {
+      this.held.set(key, heldInput);
+    }
+    try {
+      const result = await transport.send<T>(method, params, {
+        sessionId,
+        signal,
+      });
+      if (params.type === "keyUp" || params.type === "mouseReleased") {
+        this.held.delete(key);
+      }
+      const cursor = this.cursorPositions.get(sessionId);
+      if (
+        cursor &&
+        (params.type === "mouseReleased" || params.type === "keyUp") &&
+        generation === this.generation
+      ) {
+        await this.drawCursor(transport, sessionId, cursor.x, cursor.y, signal);
+      }
+      return result;
+    } catch (error) {
+      if (
+        pressed &&
+        error instanceof CdpWsTransportError &&
+        error.code === "cdp_error" &&
+        this.held.get(key) === heldInput
+      ) {
+        if (previous) {
+          this.held.set(key, previous);
+        } else {
+          this.held.delete(key);
+        }
+      }
+      throw new CdpError(
+        error instanceof CdpWsTransportError && error.code === "aborted"
+          ? "aborted"
+          : "cdp_error",
+        "Desktop browser command failed. A dispatched action may have completed; do not retry it automatically.",
+        { underlying: error, cdpMethod: method },
+      );
+    }
+  }
+
+  private async drawCursor(
+    transport: CdpWsTransport,
+    sessionId: string,
+    x: number,
+    y: number,
+    signal: AbortSignal,
+  ): Promise<void> {
+    await transport
+      .send(
+        "Runtime.evaluate",
+        {
+          expression: desktopCursorExpression(x, y),
+          awaitPromise: true,
+        },
+        { sessionId, signal },
+      )
+      .catch(() => {});
+  }
+
+  async release(): Promise<void> {
+    this.cursorPositions.clear();
+    await this.cursorRestored;
+    if (!this.transport) {
+      return;
+    }
+    const signal = AbortSignal.timeout(3000);
+    const cleanup = await this.connect(signal);
+    try {
+      const { targetInfos } = await cleanup.send<{ targetInfos: Target[] }>(
+        "Target.getTargets",
+        {},
+        { signal },
+      );
+      const targets = new Set([
+        ...this.sessions.keys(),
+        ...[...this.held.values()].map((input) => input.targetId),
+      ]);
+      for (const targetId of targets) {
+        if (!targetInfos.some((target) => target.targetId === targetId)) {
+          for (const [key, input] of this.held) {
+            if (input.targetId === targetId) {
+              this.held.delete(key);
+            }
+          }
+          continue;
+        }
+        const { sessionId } = await cleanup.send<{ sessionId: string }>(
+          "Target.attachToTarget",
+          { targetId, flatten: true },
+          { signal },
+        );
+        for (const [key, input] of this.held) {
+          if (input.targetId !== targetId) {
+            continue;
+          }
+          await cleanup.send(input.method, input.params, { sessionId, signal });
+          this.held.delete(key);
+        }
+        await cleanup.send(
+          "Runtime.evaluate",
+          { expression: REMOVE_DESKTOP_CURSOR },
+          { sessionId, signal },
+        );
+      }
+    } finally {
+      cleanup.dispose();
+    }
+    this.dispose();
+  }
+
+  dispose(): void {
+    this.invalidateSnapshot();
+    this.transport?.dispose();
+    this.transport = undefined;
+    this.sessions.clear();
+    this.targets.clear();
+    this.selected = undefined;
+    this.held.clear();
+    this.conversation = undefined;
+    this.cursorPositions.clear();
+  }
+}

package/src/desktop/desktop-browser-cursor.ts

@@ -0,0 +1,40 @@
+export const REMOVE_DESKTOP_CURSOR = `document.querySelector('[data-vellum-desktop-cursor]')?.remove()`;
+
+export function desktopCursorExpression(x: number, y: number): string {
+  if (!Number.isFinite(x) || !Number.isFinite(y)) {
+    throw new Error("Invalid desktop cursor coordinates");
+  }
+  return `(async () => {
+    let cursor = document.querySelector('[data-vellum-desktop-cursor]');
+    if (!cursor) {
+      cursor = document.createElement('div');
+      cursor.setAttribute('data-vellum-desktop-cursor', '');
+      cursor.setAttribute('aria-hidden', 'true');
+      cursor.setAttribute('popover', 'manual');
+      cursor.style.cssText = 'all:initial;position:fixed;inset:auto;left:0;top:0;margin:0;padding:0;border:0;width:24px;height:30px;pointer-events:none;z-index:2147483647;';
+      const shadow = cursor.attachShadow({mode:'closed'});
+      const style = document.createElement('style');
+      style.textContent = ':host::backdrop{pointer-events:none;background:transparent}';
+      shadow.append(style);
+      const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
+      svg.setAttribute('width', '24');
+      svg.setAttribute('height', '30');
+      svg.setAttribute('viewBox', '0 0 24 30');
+      const arrow = document.createElementNS('http://www.w3.org/2000/svg', 'path');
+      arrow.setAttribute('d', 'M2 2 L2 23 L8 18 L13 28 L17 26 L12 16 L21 16 Z');
+      arrow.setAttribute('fill', '#7057ff');
+      arrow.setAttribute('stroke', 'white');
+      arrow.setAttribute('stroke-width', '2');
+      svg.append(arrow);
+      shadow.append(svg);
+      document.documentElement.append(cursor);
+    }
+    if (cursor.matches(':popover-open')) { cursor.hidePopover(); }
+    cursor.showPopover();
+    const destination = 'translate(${x}px, ${y}px)';
+    if (cursor.style.transform === destination) { return; }
+    const animation = cursor.animate([{transform:cursor.style.transform || destination},{transform:destination}], {duration:100,easing:'ease-out'});
+    cursor.style.transform = destination;
+    await animation.finished;
+  })()`;
+}

package/src/desktop/desktop-browser-endpoint.test.ts

@@ -0,0 +1,78 @@
+import {
+  mkdir,
+  mkdtemp,
+  realpath,
+  rm,
+  symlink,
+  writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { expect, test } from "bun:test";
+
+import {
+  findDesktopBrowserPid,
+  validateDesktopWebSocket,
+} from "./desktop-browser-endpoint.js";
+
+test("discovery only accepts the allocated loopback browser endpoint", () => {
+  expect(
+    validateDesktopWebSocket(
+      "ws://127.0.0.1:9222/devtools/browser/abc-123",
+      9222,
+    ),
+  ).toBe("ws://127.0.0.1:9222/devtools/browser/abc-123");
+  for (const endpoint of [
+    "ws://example.com:9222/devtools/browser/abc",
+    "ws://127.0.0.1:9223/devtools/browser/abc",
+    "ws://user:password@127.0.0.1:9222/devtools/browser/abc",
+    "ws://127.0.0.1:9222/devtools/page/abc",
+    "ws://127.0.0.1:9222/devtools/browser/abc?redirect=1",
+    "ws://127.0.0.1:9222/devtools/browser/abc#fragment",
+    undefined,
+  ]) {
+    expect(() => validateDesktopWebSocket(endpoint, 9222)).toThrow();
+  }
+});
+
+test("dock browser discovery validates the profile lock against the live executable and arguments", async () => {
+  const directory = await mkdtemp(join(tmpdir(), "desktop-pid-"));
+  const executable = join(directory, "chrome");
+  const profile = join(directory, "profile");
+  const proc = join(directory, "proc");
+  const processDir = join(proc, "1234");
+  try {
+    await mkdir(profile);
+    await mkdir(processDir, { recursive: true });
+    await writeFile(executable, "test executable");
+    await symlink("desktop-1234", join(profile, "SingletonLock"));
+    await symlink(await realpath(executable), join(processDir, "exe"));
+    const args = [
+      executable,
+      `--user-data-dir=${profile}`,
+      "--remote-debugging-port=9222",
+      "--remote-debugging-address=127.0.0.1",
+    ];
+    const discover = () =>
+      findDesktopBrowserPid(executable, profile, 9222, proc);
+    await writeFile(join(processDir, "cmdline"), args.join("\0"));
+    expect(await discover()).toBe(1234);
+    for (const changed of [
+      args.map((arg) => arg.replace("port=9222", "port=9223")),
+      args.map((arg) => arg.replace(profile, join(directory, "other-profile"))),
+      args.map((arg) => arg.replace("address=127.0.0.1", "address=0.0.0.0")),
+      [...args, "--type=renderer"],
+    ]) {
+      await writeFile(join(processDir, "cmdline"), changed.join("\0"));
+      expect(await discover()).toBeUndefined();
+    }
+    await writeFile(join(processDir, "cmdline"), args.join("\0"));
+    await rm(join(processDir, "exe"));
+    await symlink(join(directory, "other-executable"), join(processDir, "exe"));
+    expect(await discover()).toBeUndefined();
+    await rm(processDir, { recursive: true });
+    expect(await discover()).toBeUndefined();
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+});

package/src/desktop/desktop-browser-endpoint.ts

@@ -0,0 +1,145 @@
+import { readdir, readFile, readlink, realpath } from "node:fs/promises";
+import { createServer } from "node:net";
+import { join } from "node:path";
+
+import { shouldRestoreDesktopChromeSession } from "./desktop-chrome-session.js";
+
+export async function allocateDesktopDebugPort(): Promise<number> {
+  const server = createServer();
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", resolve);
+  });
+  const address = server.address();
+  await new Promise<void>((resolve, reject) =>
+    server.close((error) => (error ? reject(error) : resolve())),
+  );
+  if (!address || typeof address === "string") {
+    throw new Error("Cannot allocate desktop browser port");
+  }
+  return address.port;
+}
+
+export function validateDesktopWebSocket(value: unknown, port: number): string {
+  if (typeof value !== "string") {
+    throw new Error("Desktop browser discovery returned no endpoint");
+  }
+  const url = new URL(value);
+  if (
+    url.protocol !== "ws:" ||
+    url.hostname !== "127.0.0.1" ||
+    url.port !== String(port) ||
+    url.username ||
+    url.password ||
+    url.search ||
+    url.hash ||
+    !/^\/devtools\/browser\/[a-zA-Z0-9-]+$/.test(url.pathname)
+  ) {
+    throw new Error("Desktop browser discovery returned an unsafe endpoint");
+  }
+  return url.href;
+}
+
+// Match the listener to the managed Chrome process, including occupied-port races.
+export async function assertDesktopListener(
+  pid: number,
+  port: number,
+): Promise<void> {
+  const table = await readFile(`/proc/${pid}/net/tcp`, "utf8");
+  const address = `0100007F:${port.toString(16).toUpperCase().padStart(4, "0")}`;
+  const inodes = new Set(
+    table.split("\n").flatMap((line) => {
+      const fields = line.trim().split(/\s+/);
+      return fields[1] === address && fields[3] === "0A" ? [fields[9]] : [];
+    }),
+  );
+  const fds = await readdir(`/proc/${pid}/fd`);
+  for (const fd of fds) {
+    const link = await readlink(`/proc/${pid}/fd/${fd}`).catch(() => "");
+    if (inodes.has(/^socket:\[(\d+)\]$/.exec(link)?.[1] ?? "")) {
+      return;
+    }
+  }
+  throw new Error(
+    "Debug listener does not belong to the managed desktop Chrome",
+  );
+}
+
+export async function findDesktopBrowserPid(
+  executable: string,
+  profileDir: string,
+  port: number,
+  procDir = "/proc",
+): Promise<number | undefined> {
+  try {
+    const lock = await readlink(join(profileDir, "SingletonLock"));
+    const pid = Number(/-(\d+)$/.exec(lock)?.[1]);
+    if (!Number.isSafeInteger(pid) || pid <= 0) {
+      return undefined;
+    }
+    const [expectedExecutable, actualExecutable, commandLine] =
+      await Promise.all([
+        realpath(executable),
+        readlink(join(procDir, String(pid), "exe")),
+        readFile(join(procDir, String(pid), "cmdline"), "utf8"),
+      ]);
+    const args = commandLine.split("\0");
+    if (
+      actualExecutable !== expectedExecutable ||
+      !args.includes(`--user-data-dir=${profileDir}`) ||
+      !args.includes(`--remote-debugging-port=${port}`) ||
+      !args.includes("--remote-debugging-address=127.0.0.1") ||
+      args.some((arg) => arg.startsWith("--type="))
+    ) {
+      return undefined;
+    }
+    return pid;
+  } catch {
+    return undefined;
+  }
+}
+
+export async function discoverDesktopBrowser(
+  pid: number,
+  port: number,
+  signal: AbortSignal,
+): Promise<string> {
+  await assertDesktopListener(pid, port);
+  signal.throwIfAborted();
+  const response = await fetch(`http://127.0.0.1:${port}/json/version`, {
+    signal,
+    redirect: "error",
+  });
+  if (!response.ok) {
+    throw new Error("Desktop browser discovery failed");
+  }
+  const body = await response.text();
+  if (body.length > 16_384) {
+    throw new Error("Desktop browser discovery exceeded its size limit");
+  }
+  const data = JSON.parse(body) as { webSocketDebuggerUrl?: unknown };
+  await assertDesktopListener(pid, port);
+  return validateDesktopWebSocket(data.webSocketDebuggerUrl, port);
+}
+
+export function desktopChromeArguments(
+  profileDir: string,
+  debugPort?: number,
+): string[] {
+  return [
+    "--no-sandbox",
+    "--no-first-run",
+    "--disable-dev-shm-usage",
+    ...(shouldRestoreDesktopChromeSession(profileDir)
+      ? ["--restore-last-session", "--hide-crash-restore-bubble"]
+      : []),
+    "--start-maximized",
+    `--user-data-dir=${profileDir}`,
+    ...(debugPort
+      ? [
+          "--remote-debugging-address=127.0.0.1",
+          `--remote-debugging-port=${debugPort}`,
+        ]
+      : []),
+  ];
+}