@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.4

package/src/desktop/desktop-session-manager.ts

@@ -12,16 +12,30 @@ import { join } from "node:path";
 import { readAvatarState } from "../avatar/avatar-manifest.js";
 import { resolveNotificationAccentHex } from "../avatar/notification-avatar.js";
 import { getIsContainerized } from "../config/env-registry.js";
+import { connectCdpWsTransport } from "../tools/browser/cdp-client/cdp-inspect/ws-transport.js";
 import { terminateProcessTree } from "../util/host-process.js";
 import { getLogger } from "../util/logger.js";
 import { getDataDir } from "../util/platform.js";
 import { sleep } from "../util/retry.js";
+import { DesktopBrowserClient } from "./desktop-browser-client.js";
+import {
+  allocateDesktopDebugPort,
+  desktopChromeArguments,
+  discoverDesktopBrowser,
+  findDesktopBrowserPid,
+} from "./desktop-browser-endpoint.js";
 import { writeDesktopChromePolicy } from "./desktop-chrome-policy.js";
-import { shouldRestoreDesktopChromeSession } from "./desktop-chrome-session.js";
+import { configureDesktopChromeFrame } from "./desktop-chrome-session.js";
 import {
   desktopChromePath,
   resolveDesktopBinaries,
 } from "./desktop-dependencies.js";
+import {
+  DESKTOP_DISPLAY,
+  DESKTOP_HEIGHT,
+  DESKTOP_OVERRIDABLE_PARAMETERS,
+  DESKTOP_WIDTH,
+} from "./desktop-display.js";
 import { writeDesktopPanelConfig } from "./desktop-panel-config.js";
 import { renderCurrentDesktopWallpaper } from "./desktop-wallpaper.js";
 import { DESKTOP_WINDOW_DRAG_SCRIPT } from "./desktop-window-drag.js";
@@ -30,10 +44,7 @@ import { writeDesktopWindowTheme } from "./desktop-window-theme.js";
 
 const log = getLogger("desktop-session");
 
-const DESKTOP_DISPLAY = ":99";
 export const DESKTOP_VNC_PORT = 5999;
-const DESKTOP_WIDTH = 1600;
-const DESKTOP_HEIGHT = 900;
 const DESKTOP_GEOMETRY = `${DESKTOP_WIDTH}x${DESKTOP_HEIGHT}`;
 const DESKTOP_LINGER_MS = 5 * 60_000;
 const VNC_READY_DEADLINE_MS = 10_000;
@@ -153,6 +164,7 @@ type ViewerSlotResult =
   | { readonly ok: false; readonly loss: DesktopLoss };
 
 interface DesktopSessionManagerOptions {
+  readonly allocateDebugPort?: () => Promise<number>;
   readonly spawn?: (
     role: DesktopChildRole,
     request: DesktopSpawnRequest,
@@ -194,6 +206,7 @@ export class DesktopSessionManager {
   /** Bumped on every teardown so an in-flight start notices it lost its tree. */
   private generation = 0;
   private viewer: DesktopViewer | null = null;
+  private automation: DesktopViewer | null = null;
   private lingerTimer: ReturnType<typeof setTimeout> | null = null;
   private ingressClosed = false;
   /** Resolved for the current tree, and read again when the dock comes up. */
@@ -211,6 +224,70 @@ export class DesktopSessionManager {
     refreshQueued: boolean;
   } | null = null;
 
+  private debugPort?: number;
+  private browserStarting: Promise<void> | null = null;
+  private readonly allocateDebugPort: () => Promise<number>;
+  readonly browser = new DesktopBrowserClient(async (signal) => {
+    if (!this.automation) {
+      throw new Error("Desktop browser requires an automation slot");
+    }
+    const generation = this.generation;
+    const owner = this.automation;
+    let child = this.children.get("browser");
+    let pid = child?.pid;
+    if (!pid && this.debugPort) {
+      pid = await findDesktopBrowserPid(
+        await this.resolveChromePath(),
+        this.profileDir,
+        this.debugPort,
+      );
+    }
+    if (this.generation !== generation || this.automation !== owner) {
+      throw new Error("Desktop Chrome session changed. Observe again.");
+    }
+    signal.throwIfAborted();
+    if (!pid) {
+      await this.ensureBrowser(this.childEnv(), generation);
+      child = this.children.get("browser");
+      pid = child?.pid;
+    }
+    const port = this.debugPort;
+    const isCurrent = () =>
+      this.generation === generation &&
+      this.automation === owner &&
+      (!child || this.children.get("browser") === child);
+    if (!pid || !port) {
+      throw new Error(
+        "Desktop Chrome is unavailable. Open the Desktop modal to inspect it, then retry the browser command.",
+      );
+    }
+    signal = AbortSignal.any([signal, AbortSignal.timeout(5_000)]);
+    const deadline = Date.now() + 5_000;
+    while (true) {
+      signal.throwIfAborted();
+      if (!isCurrent()) {
+        throw new Error("Desktop Chrome session changed. Observe again.");
+      }
+      try {
+        const url = await discoverDesktopBrowser(pid, port, signal);
+        const transport = await connectCdpWsTransport(url, {
+          signal,
+          connectTimeoutMs: 3000,
+        });
+        if (!isCurrent() || signal.aborted) {
+          transport.dispose();
+          throw new Error("Desktop Chrome session changed. Observe again.");
+        }
+        return transport;
+      } catch (error) {
+        if (Date.now() >= deadline || signal.aborted) {
+          throw error;
+        }
+        await sleep(100);
+      }
+    }
+  });
+
   private readonly spawn: NonNullable<DesktopSessionManagerOptions["spawn"]>;
   private readonly which: NonNullable<DesktopSessionManagerOptions["which"]>;
   private readonly probeVncPort: NonNullable<
@@ -235,6 +312,8 @@ export class DesktopSessionManager {
   private readonly writeWindowManagerConfig: (configDir: string) => string;
 
   constructor(options: DesktopSessionManagerOptions = {}) {
+    this.allocateDebugPort =
+      options.allocateDebugPort ?? allocateDesktopDebugPort;
     this.spawn = options.spawn ?? spawnDetached;
     this.which = options.which ?? Bun.which;
     this.probeVncPort = options.probeVncPort ?? probeLoopbackPort;
@@ -308,7 +387,29 @@ export class DesktopSessionManager {
       return;
     }
     this.viewer = null;
-    if (this.running || this.starting) {
+    if (!this.automation && (this.running || this.starting)) {
+      this.armLinger();
+    }
+  }
+
+  acquireAutomationSlot(owner: DesktopViewer): ViewerSlotResult {
+    if (this.ingressClosed) {
+      return { ok: false, loss: SHUTTING_DOWN_LOSS };
+    }
+    if (this.automation) {
+      return { ok: false, loss: BUSY_LOSS };
+    }
+    this.automation = owner;
+    this.clearLinger();
+    return { ok: true };
+  }
+
+  releaseAutomationSlot(owner: DesktopViewer): void {
+    if (this.automation !== owner) {
+      return;
+    }
+    this.automation = null;
+    if (!this.viewer && (this.running || this.starting)) {
       this.armLinger();
     }
   }
@@ -393,7 +494,7 @@ export class DesktopSessionManager {
     this.running = true;
     log.info({ display: DESKTOP_DISPLAY }, "Desktop started");
     void this.refreshWallpaper(env, generation);
-    void this.startBrowser(env, generation);
+    void this.ensureBrowser(env, generation);
   }
 
   private async refreshWallpaper(
@@ -455,18 +556,42 @@ export class DesktopSessionManager {
   }
 
   /** Launch Chrome and its dock once the X server is ready. */
+  private ensureBrowser(
+    env: Record<string, string>,
+    generation: number,
+  ): Promise<void> {
+    if (this.children.has("browser")) {
+      return Promise.resolve();
+    }
+    this.browserStarting ??= this.startBrowser(env, generation).finally(() => {
+      this.browserStarting = null;
+    });
+    return this.browserStarting;
+  }
+
   private async startBrowser(
     env: Record<string, string>,
     generation: number,
   ): Promise<void> {
     try {
       const executable = await this.resolveChromePath();
-      if (this.generation !== generation) {
+      const debugPort = this.debugPort ?? (await this.allocateDebugPort());
+      if (this.generation !== generation || this.children.has("browser")) {
         return;
       }
       mkdirSync(this.profileDir, { recursive: true });
+      this.debugPort = debugPort;
+      try {
+        configureDesktopChromeFrame(this.profileDir);
+      } catch (err) {
+        log.warn({ err }, "Desktop Chrome frame could not be configured");
+      }
       this.startPanel(executable, env);
-      this.launch("browser", browserCommand(executable, this.profileDir), env);
+      this.launch(
+        "browser",
+        [executable, ...desktopChromeArguments(this.profileDir, debugPort)],
+        env,
+      );
     } catch (err) {
       log.warn({ err }, "Desktop browser failed to launch");
       if (this.generation === generation) {
@@ -491,6 +616,7 @@ export class DesktopSessionManager {
         configDir: this.panelConfigDir,
         chromiumPath,
         chromiumProfileDir: this.profileDir,
+        debugPort: this.debugPort,
         terminalPath: binaries.terminal,
       });
       this.launch("panel", [binaries.panelSession, "--", binaries.panel], {
@@ -577,6 +703,7 @@ export class DesktopSessionManager {
       return;
     }
     if (role === "browser") {
+      this.browser.dispose();
       log.info({ outcome }, "Desktop browser exited");
       return;
     }
@@ -599,6 +726,8 @@ export class DesktopSessionManager {
   private teardown(loss?: DesktopLoss): Promise<void> {
     this.clearLinger();
     this.generation += 1;
+    this.browser.dispose();
+    this.debugPort = undefined;
     this.running = false;
     this.binaries = null;
     this.panelStarted = false;
@@ -613,7 +742,12 @@ export class DesktopSessionManager {
     const children = new Map(this.children);
     this.children.clear();
     const viewer = this.viewer;
+    const automation = this.automation;
     this.viewer = null;
+    this.automation = null;
+    if (automation && loss) {
+      automation.onDesktopLost(loss);
+    }
     if (viewer && loss) {
       viewer.onDesktopLost(loss);
     }
@@ -733,6 +867,8 @@ function xServerCommand(executable: string): string[] {
     "None",
     "-rfbport",
     String(DESKTOP_VNC_PORT),
+    "-AllowOverride",
+    DESKTOP_OVERRIDABLE_PARAMETERS.join(","),
     "-geometry",
     DESKTOP_GEOMETRY,
     // Keep the dock and wallpaper anchored to a stable display.
@@ -744,22 +880,6 @@ function xServerCommand(executable: string): string[] {
   ];
 }
 
-function browserCommand(executable: string, profileDir: string): string[] {
-  // Explicit window bounds override Chrome's maximized startup state.
-  return [
-    executable,
-    "--no-sandbox",
-    "--no-first-run",
-    "--disable-dev-shm-usage",
-    // Crash recovery requires both flags; clean exits use normal startup.
-    ...(shouldRestoreDesktopChromeSession(profileDir)
-      ? ["--restore-last-session", "--hide-crash-restore-bubble"]
-      : []),
-    "--start-maximized",
-    `--user-data-dir=${profileDir}`,
-  ];
-}
-
 // ---------------------------------------------------------------------------
 // Default collaborators
 // ---------------------------------------------------------------------------

package/src/desktop/desktop-stream-bridge.ts

@@ -178,7 +178,7 @@ export class DesktopStreamBridge {
     if (!this.isEnabled()) {
       this.fail(
         DESKTOP_CLOSE.unavailable,
-        "Desktop is not available on this assistant",
+        "Virtual desktop is available only on enabled platform-hosted assistants",
       );
       return false;
     }

package/src/desktop/virtual-desktop-feature.ts

@@ -0,0 +1,18 @@
+import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
+import { getIsContainerized, getIsPlatform } from "../config/env-registry.js";
+import type { AssistantConfig } from "../config/schema.js";
+
+const VIRTUAL_DESKTOP_FLAG = "assistant-desktop" as const;
+
+/** Gates desktop streaming and control to enabled, platform-hosted containers. */
+export function isVirtualDesktopEnabled(
+  config: AssistantConfig,
+  containerized: boolean = getIsContainerized(),
+  platformHosted: boolean = getIsPlatform(),
+): boolean {
+  return (
+    platformHosted &&
+    containerized &&
+    isAssistantFeatureFlagEnabled(VIRTUAL_DESKTOP_FLAG, config)
+  );
+}

package/src/desktop/virtual-desktop-platform.test.ts

@@ -0,0 +1,61 @@
+import { afterEach, beforeEach, expect, spyOn, test } from "bun:test";
+
+import { setOverridesForTesting } from "../__tests__/feature-flag-test-helpers.js";
+import { shouldUseVirtualDesktopBrowser } from "../browser/virtual-desktop-target.js";
+import { executeDesktopBrowserOperation } from "./desktop-browser-operations.js";
+import { desktopDependencyInstaller } from "./desktop-dependencies.js";
+
+const originalPlatform = process.env.IS_PLATFORM;
+const originalContainerized = process.env.IS_CONTAINERIZED;
+const context = {
+  workingDir: "/tmp",
+  conversationId: "conv-platform-gate",
+  sourceActorPrincipalId: "user-123",
+  trustClass: "guardian" as const,
+  transportInterface: "web" as const,
+  clientOs: "web" as const,
+};
+let status: ReturnType<
+  typeof spyOn<typeof desktopDependencyInstaller, "getStatus">
+>;
+
+beforeEach(() => {
+  process.env.IS_CONTAINERIZED = "true";
+  process.env.IS_PLATFORM = "false";
+  setOverridesForTesting({ "assistant-desktop": true });
+  status = spyOn(desktopDependencyInstaller, "getStatus").mockReturnValue({
+    state: "ready",
+  });
+});
+
+afterEach(() => {
+  status.mockRestore();
+  setOverridesForTesting({});
+  if (originalPlatform === undefined) {
+    delete process.env.IS_PLATFORM;
+  } else {
+    process.env.IS_PLATFORM = originalPlatform;
+  }
+  if (originalContainerized === undefined) {
+    delete process.env.IS_CONTAINERIZED;
+  } else {
+    process.env.IS_CONTAINERIZED = originalContainerized;
+  }
+});
+
+test("self-hosted Docker cannot automatically select or explicitly control virtual Chrome", async () => {
+  expect(shouldUseVirtualDesktopBrowser(undefined, {}, context)).toBe(false);
+  await expect(
+    executeDesktopBrowserOperation("snapshot", {}, context),
+  ).rejects.toThrow("only on enabled platform-hosted assistants");
+  expect(status).not.toHaveBeenCalled();
+});
+
+test("platform web conversations select virtual Chrome before and during installation", () => {
+  process.env.IS_PLATFORM = "true";
+  expect(shouldUseVirtualDesktopBrowser(undefined, {}, context)).toBe(true);
+  status.mockReturnValue({ state: "installing" });
+  expect(shouldUseVirtualDesktopBrowser(undefined, {}, context)).toBe(true);
+  status.mockReturnValue({ state: "required" });
+  expect(shouldUseVirtualDesktopBrowser(undefined, {}, context)).toBe(true);
+});

package/src/i18n/__tests__/i18n.test.ts

@@ -35,9 +35,7 @@ describe("negotiateLocale", () => {
 
 describe("localeFromAcceptLanguage", () => {
   test("honors quality values", () => {
-    expect(localeFromAcceptLanguage("fr;q=0.8, es;q=0.9, en;q=0.5")).toBe(
-      "es",
-    );
+    expect(localeFromAcceptLanguage("fr;q=0.8, es;q=0.9, en;q=0.5")).toBe("es");
   });
 
   test("defaults on missing or empty", () => {
@@ -58,7 +56,7 @@ describe("catalog completeness", () => {
 });
 
 describe("t", () => {
-  test("resolves generating and untitled per locale", () => {
+  test("resolves message keys per locale", () => {
     expect(t(MESSAGE_KEYS.CONVERSATION_TITLE_GENERATING, "en")).toBe(
       "Generating title...",
     );
@@ -68,13 +66,19 @@ describe("t", () => {
     expect(t(MESSAGE_KEYS.CONVERSATION_TITLE_UNTITLED, "es")).toBe(
       "Sin título",
     );
+    expect(
+      t(MESSAGE_KEYS.PLUGIN_MCP_OAUTH_CREDENTIALS_UNCHECKED, "es"),
+    ).toContain("almacenamiento de credenciales");
   });
 });
 
 describe("resolveConversationTitle", () => {
   test("resolves a stored key and an empty title", () => {
     expect(
-      resolveConversationTitle(MESSAGE_KEYS.CONVERSATION_TITLE_GENERATING, "zh"),
+      resolveConversationTitle(
+        MESSAGE_KEYS.CONVERSATION_TITLE_GENERATING,
+        "zh",
+      ),
     ).toBe("标题生成中...");
     expect(resolveConversationTitle(null, "es")).toBe("Sin título");
     expect(resolveConversationTitle("")).toBe("Untitled Conversation");

package/src/i18n/messages.ts

@@ -11,14 +11,13 @@
  * a user title may be in any language.
  */
 
-import {
-  DEFAULT_LOCALE,
-  type SupportedLocale,
-} from "./locales.js";
+import { DEFAULT_LOCALE, type SupportedLocale } from "./locales.js";
 
 export const MESSAGE_KEYS = {
   CONVERSATION_TITLE_GENERATING: "conversation.title.generating",
   CONVERSATION_TITLE_UNTITLED: "conversation.title.untitled",
+  PLUGIN_MCP_OAUTH_CREDENTIALS_UNCHECKED:
+    "plugin.uninstall.mcp_oauth_credentials_unchecked",
 } as const;
 
 export type MessageKey = (typeof MESSAGE_KEYS)[keyof typeof MESSAGE_KEYS];
@@ -30,22 +29,32 @@ export const MESSAGE_CATALOGS: Record<
   en: {
     "conversation.title.generating": "Generating title...",
     "conversation.title.untitled": "Untitled Conversation",
+    "plugin.uninstall.mcp_oauth_credentials_unchecked":
+      "Credential storage is unavailable, so historical plugin MCP OAuth credentials could not be checked.",
   },
   es: {
     "conversation.title.generating": "Generando título...",
     "conversation.title.untitled": "Sin título",
+    "plugin.uninstall.mcp_oauth_credentials_unchecked":
+      "El almacenamiento de credenciales no está disponible, por lo que no se pudieron comprobar las credenciales históricas de OAuth de MCP del plugin.",
   },
   ru: {
     "conversation.title.generating": "Создание названия...",
     "conversation.title.untitled": "Без названия",
+    "plugin.uninstall.mcp_oauth_credentials_unchecked":
+      "Хранилище учетных данных недоступно, поэтому не удалось проверить сохраненные учетные данные OAuth MCP плагина.",
   },
   zh: {
     "conversation.title.generating": "标题生成中...",
     "conversation.title.untitled": "无标题",
+    "plugin.uninstall.mcp_oauth_credentials_unchecked":
+      "凭据存储不可用，因此无法检查该插件之前的 MCP OAuth 凭据。",
   },
   "zh-TW": {
     "conversation.title.generating": "標題產生中...",
     "conversation.title.untitled": "未命名",
+    "plugin.uninstall.mcp_oauth_credentials_unchecked":
+      "憑證儲存空間無法使用，因此無法檢查此外掛程式先前的 MCP OAuth 憑證。",
   },
 };