@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.4

package/src/desktop/desktop-browser-operations.ts

@@ -0,0 +1,130 @@
+import { executeBrowserOperation } from "../browser/operations.js";
+import type { BrowserOperation } from "../browser/types.js";
+import type { ToolContext, ToolExecutionResult } from "../tools/types.js";
+import { desktopAutomationLease } from "./desktop-automation-lease.js";
+import { getDesktopSessionManager } from "./desktop-session-manager.js";
+
+export function executeDesktopBrowserOperation(
+  operation: BrowserOperation,
+  input: Record<string, unknown>,
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  if (
+    input.target_client_id ||
+    (input.browser_mode &&
+      input.browser_mode !== "auto" &&
+      input.browser_mode !== "cdp-inspect")
+  ) {
+    throw new Error(
+      "--virtual-desktop targets the streamed Chrome directly. Omit other browser targets and modes.",
+    );
+  }
+  if (input.use_active_tab) {
+    throw new Error(
+      "Use assistant browser --virtual-desktop tabs list and tabs select to choose a streamed browser tab.",
+    );
+  }
+  if (operation === "wait_for_download") {
+    throw new Error(
+      "Download waiting is unavailable for the streamed desktop browser",
+    );
+  }
+  return desktopAutomationLease.runBrowser(
+    context,
+    async (signal) => {
+      const cdpClient = await getDesktopSessionManager().browser.client(
+        context.conversationId,
+        signal,
+      );
+      if (operation === "status") {
+        await cdpClient.listTabs();
+        return {
+          isError: false,
+          content: JSON.stringify({
+            requestedMode: "desktop",
+            recommendedMode: "desktop",
+            modes: [
+              {
+                mode: "desktop",
+                available: true,
+                summary:
+                  "Virtual desktop Chrome is ready. Use assistant browser --virtual-desktop commands.",
+                userActions: [],
+              },
+            ],
+          }),
+        };
+      }
+      const browser = getDesktopSessionManager().browser;
+      const generation = browser.generation;
+      const result = await executeBrowserOperation(operation, input, {
+        ...context,
+        signal,
+        cdpClient,
+      });
+      if (operation === "snapshot" && generation !== browser.generation) {
+        browser.invalidateSnapshot();
+        return {
+          isError: true,
+          content:
+            "The page changed during the snapshot. Take a fresh snapshot before acting.",
+        };
+      }
+      return result;
+    },
+    operation === "detach" || operation === "close",
+  );
+}
+
+export function executeDesktopBrowserTabs(
+  params: {
+    command: "list" | "select" | "new" | "close";
+    tabId?: number;
+    url?: string;
+  },
+  context: ToolContext,
+): Promise<ToolExecutionResult> {
+  return desktopAutomationLease.runBrowser(context, async (signal) => {
+    const cdpClient = await getDesktopSessionManager().browser.client(
+      context.conversationId,
+      signal,
+    );
+    let result: unknown;
+    if (params.command === "list") {
+      const tabs = await cdpClient.listTabs();
+      result = {
+        ok: true,
+        tabs: [
+          ...tabs.filter((tab) => tab.active),
+          ...tabs.filter((tab) => !tab.active),
+        ].slice(0, 100),
+      };
+    } else if (params.command === "new") {
+      const created = await cdpClient.send<{ tabId: number }>(
+        "Vellum.createTab",
+        {},
+        signal,
+      );
+      if (params.url) {
+        const navigated = await executeBrowserOperation(
+          "navigate",
+          { url: params.url },
+          { ...context, signal, cdpClient },
+        );
+        if (navigated.isError) {
+          return navigated;
+        }
+      }
+      result = { ok: true, ...created };
+    } else {
+      if (params.tabId === undefined) {
+        throw new Error("A tab ID is required");
+      }
+      result =
+        params.command === "select"
+          ? { ok: true, tab: await cdpClient.selectTab(params.tabId) }
+          : { ok: true, ...(await cdpClient.closeTab(params.tabId)) };
+    }
+    return { isError: false, content: JSON.stringify(result) };
+  });
+}

package/src/desktop/desktop-chrome-session.test.ts

@@ -0,0 +1,48 @@
+import {
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, expect, test } from "bun:test";
+
+import { configureDesktopChromeFrame } from "./desktop-chrome-session.js";
+
+const directories: string[] = [];
+
+afterEach(() => {
+  for (const directory of directories.splice(0)) {
+    rmSync(directory, { recursive: true, force: true });
+  }
+});
+
+test.each(["{truncated", "[]", '{"browser":[]}'])(
+  "leaves unreadable or malformed preferences untouched: %s",
+  (contents) => {
+    const directory = mkdtempSync(join(tmpdir(), "desktop-frame-"));
+    directories.push(directory);
+    mkdirSync(join(directory, "Default"));
+    const path = join(directory, "Default", "Preferences");
+    writeFileSync(path, contents);
+
+    expect(() => configureDesktopChromeFrame(directory)).toThrow();
+    expect(readFileSync(path, "utf8")).toBe(contents);
+  },
+);
+
+test("does not rewrite an already configured profile", () => {
+  const directory = mkdtempSync(join(tmpdir(), "desktop-frame-"));
+  directories.push(directory);
+  mkdirSync(join(directory, "Default"));
+  const path = join(directory, "Default", "Preferences");
+  const contents =
+    '{ "browser": { "custom_chrome_frame": true }, "profile": { "exit_type": "Crashed" } }';
+  writeFileSync(path, contents);
+
+  configureDesktopChromeFrame(directory);
+
+  expect(readFileSync(path, "utf8")).toBe(contents);
+});

package/src/desktop/desktop-chrome-session.ts

@@ -1,7 +1,60 @@
-import { readFileSync } from "node:fs";
-import { basename, join } from "node:path";
+import {
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { basename, dirname, join } from "node:path";
+
+import { isPlainObject } from "../util/object.js";
 
 export function shouldRestoreDesktopChromeSession(profileDir: string): boolean {
+  return (
+    readProfileMetadata(chromePreferencesPath(profileDir))?.exit_type ===
+    "Crashed"
+  );
+}
+
+/** Configure Chrome's own title bar before the browser or dock starts. */
+export function configureDesktopChromeFrame(profileDir: string): void {
+  const path = chromePreferencesPath(profileDir);
+  let preferences: Record<string, unknown> = {};
+  try {
+    const value: unknown = JSON.parse(readFileSync(path, "utf8"));
+    if (!isPlainObject(value)) {
+      throw new Error("Desktop Chrome preferences must be a JSON object");
+    }
+    preferences = value;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+      throw err;
+    }
+  }
+  const browser = preferences.browser ?? {};
+  if (!isPlainObject(browser)) {
+    throw new Error("Desktop Chrome browser preferences must be a JSON object");
+  }
+  if (browser.custom_chrome_frame === true) {
+    return;
+  }
+  preferences.browser = { ...browser, custom_chrome_frame: true };
+  const directory = dirname(path);
+  mkdirSync(directory, { recursive: true });
+  const staging = mkdtempSync(join(directory, ".vellum-frame-"));
+  try {
+    const temporaryPath = join(staging, "Preferences");
+    writeFileSync(temporaryPath, JSON.stringify(preferences) + "\n", {
+      mode: 0o600,
+    });
+    renameSync(temporaryPath, path);
+  } finally {
+    rmSync(staging, { recursive: true, force: true });
+  }
+}
+
+function chromePreferencesPath(profileDir: string): string {
   const lastUsed = readProfileMetadata(
     join(profileDir, "Local State"),
   )?.last_used;
@@ -13,10 +66,7 @@ export function shouldRestoreDesktopChromeSession(profileDir: string): boolean {
     basename(lastUsed) === lastUsed
       ? lastUsed
       : "Default";
-  return (
-    readProfileMetadata(join(profileDir, profileName, "Preferences"))
-      ?.exit_type === "Crashed"
-  );
+  return join(profileDir, profileName, "Preferences");
 }
 
 function readProfileMetadata(
@@ -24,9 +74,9 @@ function readProfileMetadata(
 ): { last_used?: unknown; exit_type?: unknown } | undefined {
   try {
     const value: unknown = JSON.parse(readFileSync(path, "utf8"));
-    if (value && typeof value === "object" && "profile" in value) {
+    if (isPlainObject(value)) {
       const profile = value.profile;
-      if (profile && typeof profile === "object" && !Array.isArray(profile)) {
+      if (isPlainObject(profile)) {
         return profile;
       }
     }

package/src/desktop/desktop-dependencies.test.ts

@@ -100,3 +100,32 @@ describe("desktop dependency installation", () => {
     expect(f.install).not.toHaveBeenCalled();
   });
 });
+
+test("browser and viewer share installation while a cancelled waiter stops immediately", async () => {
+  const f = setup();
+  const abort = new AbortController();
+  const browser = f.installer
+    .ensureReady(abort.signal)
+    .catch((error: unknown) => error);
+  const viewer = f.installer.ensureReady();
+  await flush();
+  expect(f.install).toHaveBeenCalledTimes(1);
+  abort.abort();
+  expect(await browser).toBeInstanceOf(Error);
+  expect(f.installer.getStatus().state).toBe("installing");
+  f.finish();
+  await viewer;
+  expect(f.installer.getStatus().state).toBe("ready");
+});
+
+test("failed setup reports the stage without running an automatic retry", async () => {
+  const f = setup();
+  const result = f.installer.ensureReady().catch((error: Error) => error);
+  await flush();
+  f.progress();
+  f.fail();
+  const error = await result;
+  expect(error).toBeInstanceOf(Error);
+  expect((error as Error).message).toContain("failed during chrome");
+  expect(f.install).toHaveBeenCalledTimes(1);
+});

package/src/desktop/desktop-dependencies.ts

@@ -156,7 +156,7 @@ export class DesktopDependencyInstaller {
       })
       .catch((err: unknown) => {
         log.warn({ err }, "Desktop installation failed");
-        this.status = { state: "failed" };
+        this.status = { state: "failed", stage: this.status?.stage };
       })
       .finally(() => {
         this.installing = null;
@@ -166,11 +166,32 @@ export class DesktopDependencyInstaller {
     return this.status;
   }
 
-  async ensureReady(): Promise<void> {
+  async ensureReady(signal?: AbortSignal): Promise<void> {
+    signal?.throwIfAborted();
     this.start();
-    await this.installing;
-    if (this.getStatus().state !== "ready") {
-      throw new Error("Desktop setup did not complete");
+    let onAbort: (() => void) | undefined;
+    try {
+      await new Promise<void>((resolve, reject) => {
+        onAbort = () => reject(signal?.reason);
+        signal?.addEventListener("abort", onAbort, { once: true });
+        void Promise.resolve(this.installing).then(() => resolve(), reject);
+      });
+      signal?.throwIfAborted();
+      const status = this.getStatus();
+      if (status.state === "unsupported") {
+        throw new Error(
+          "Virtual desktop installation is unsupported on this assistant.",
+        );
+      }
+      if (status.state !== "ready") {
+        throw new Error(
+          `Virtual desktop setup failed during ${status.stage ?? "installation"}. Open the Virtual desktop panel to retry. Do not launch Chrome manually.`,
+        );
+      }
+    } finally {
+      if (onAbort) {
+        signal?.removeEventListener("abort", onAbort);
+      }
     }
   }
 

package/src/desktop/desktop-display.ts

@@ -0,0 +1,9 @@
+export const DESKTOP_DISPLAY = ":99";
+export const DESKTOP_WIDTH = 1600;
+export const DESKTOP_HEIGHT = 900;
+export const DESKTOP_OVERRIDABLE_PARAMETERS = [
+  "desktop",
+  "SendCutText",
+  "SendPrimary",
+  "SetPrimary",
+] as const;

package/src/desktop/desktop-panel-config.ts

@@ -9,6 +9,8 @@ import {
 import { dirname, join } from "node:path";
 import { pathToFileURL } from "node:url";
 
+import { desktopChromeArguments } from "./desktop-browser-endpoint.js";
+
 // Absolute icon paths work without an installed icon theme.
 const TERMINAL_ICON_BASE64 = [
   "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAACBklEQVR42u2bP2vCQBjG/T",
@@ -32,6 +34,7 @@ export interface DesktopPanelConfigRequest {
   /** Profile the launcher shares with that browser, so it reuses the window. */
   readonly chromiumProfileDir: string;
   readonly terminalPath: string;
+  readonly debugPort?: number;
 }
 
 /** Generate managed launchers and seed the desktop dock on first use. */
@@ -60,7 +63,17 @@ export function writeDesktopPanelConfig(
       // Chrome includes its profile path in WM_CLASS.
       windowClass: `google-chrome (${request.chromiumProfileDir})`,
       icon: browserIcon,
-      exec: `"${request.chromiumPath}" --no-sandbox --no-first-run --disable-dev-shm-usage "--user-data-dir=${request.chromiumProfileDir}"`,
+      exec: [
+        request.chromiumPath,
+        ...desktopChromeArguments(
+          request.chromiumProfileDir,
+          request.debugPort,
+        ),
+      ]
+        .map(
+          (arg) => `"${arg.replace(/[\\"`$]/g, "\\$&").replace(/%/g, "%%")}"`,
+        )
+        .join(" "),
     }),
   );
   writeFileSync(

package/src/desktop/desktop-session-manager.test.ts

@@ -7,7 +7,7 @@ import {
 } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { afterAll, describe, expect, test } from "bun:test";
+import { afterAll, describe, expect, spyOn, test } from "bun:test";
 
 import { waitFor } from "../__tests__/helpers/wait-for.js";
 import { sleep } from "../util/retry.js";
@@ -19,6 +19,7 @@ import {
   newViewer,
   settle,
 } from "./__tests__/fake-desktop.js";
+import * as browserEndpoint from "./desktop-browser-endpoint.js";
 import {
   DESKTOP_VNC_PORT,
   type DesktopChildRole,
@@ -543,7 +544,12 @@ describe("DesktopSessionManager process tree", () => {
           JSON.stringify({ profile: { last_used: profileName } }),
         );
         const preferences = join(dir, profileName, "Preferences");
-        const crashed = JSON.stringify({ profile: { exit_type: "Crashed" } });
+        const original = {
+          profile: { exit_type: "Crashed" },
+          browser: { custom_chrome_frame: false, show_home_button: true },
+          session: { restore_on_startup: 1 },
+        };
+        const crashed = JSON.stringify(original);
         writeFileSync(preferences, crashed);
         h.manager.acquireViewerSlot(newViewer().viewer);
         await h.manager.ensureDesktopRunning();
@@ -553,7 +559,10 @@ describe("DesktopSessionManager process tree", () => {
         expect(h.child("browser").request.cmd).toContain(
           "--hide-crash-restore-bubble",
         );
-        expect(readFileSync(preferences, "utf8")).toBe(crashed);
+        expect(JSON.parse(readFileSync(preferences, "utf8"))).toEqual({
+          ...original,
+          browser: { ...original.browser, custom_chrome_frame: true },
+        });
 
         h.child("browser").exit(1);
         await settle();
@@ -728,3 +737,77 @@ describe("DesktopSessionManager viewer slot", () => {
     expect(h.count("x-server")).toBe(1);
   });
 });
+
+describe("desktop automation lifecycle", () => {
+  test("an automation session outlives viewer disconnect and idles after both release", async () => {
+    const f = newManager({ exitOnTerm: true });
+    const viewer = newViewer();
+    const automation = newViewer();
+    expect(f.manager.acquireViewerSlot(viewer.viewer)).toEqual({ ok: true });
+    expect(f.manager.acquireAutomationSlot(automation.viewer)).toEqual({
+      ok: true,
+    });
+    await f.manager.ensureDesktopRunning();
+    f.manager.releaseViewerSlot(viewer.viewer);
+    await sleep(LINGER_MS + 20);
+    expect(f.terminated()).toHaveLength(0);
+    f.manager.releaseAutomationSlot(automation.viewer);
+    await waitFor(() => f.terminated().length > 0);
+    await f.manager.destroy();
+  });
+
+  test("shutdown notifies both the viewer and automation owner", async () => {
+    const f = newManager({ exitOnTerm: true });
+    const viewer = newViewer();
+    const automation = newViewer();
+    f.manager.acquireViewerSlot(viewer.viewer);
+    f.manager.acquireAutomationSlot(automation.viewer);
+    await f.manager.ensureDesktopRunning();
+    await f.manager.destroy();
+    expect(viewer.lost[0]?.code).toBe(1001);
+    expect(automation.lost[0]?.code).toBe(1001);
+    expect(f.manager.acquireAutomationSlot(newViewer().viewer)).toEqual(
+      SHUTTING_DOWN,
+    );
+  });
+});
+
+test.each([undefined, 4321])(
+  "browser control uses the reopened dock PID %s or launches Chrome when absent",
+  async (dockPid) => {
+    const h = newManager({ exitOnTerm: true });
+    const owner = newViewer().viewer;
+    h.manager.acquireViewerSlot(owner);
+    await h.manager.ensureDesktopRunning();
+    await settle();
+    h.child("browser").exit(0);
+    await settle();
+    h.manager.acquireAutomationSlot(owner);
+    const abort = new AbortController();
+    const find = spyOn(
+      browserEndpoint,
+      "findDesktopBrowserPid",
+    ).mockResolvedValue(dockPid);
+    const discover = spyOn(
+      browserEndpoint,
+      "discoverDesktopBrowser",
+    ).mockImplementation(async () => {
+      abort.abort();
+      throw new Error("Reached verified discovery");
+    });
+    try {
+      await expect(
+        h.manager.browser.client("conv-123", abort.signal),
+      ).rejects.toThrow("Reached verified discovery");
+      expect(h.count("browser")).toBe(dockPid ? 1 : 2);
+      expect(discover.mock.calls[0]?.[0]).toBe(
+        dockPid ?? h.child("browser").pid,
+      );
+      expect(find).toHaveBeenCalledTimes(1);
+    } finally {
+      find.mockRestore();
+      discover.mockRestore();
+      await h.manager.destroy();
+    }
+  },
+);