@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/desktop/desktop-automation-lease.ts

@@ -1,7 +1,7 @@
-import { getConfig } from "../config/loader.js";
+import { broadcastMessage } from "../runtime/assistant-event-hub.js";
 import type { ToolContext, ToolExecutionResult } from "../tools/types.js";
 import { getLogger } from "../util/logger.js";
-import { desktopDependencyInstaller } from "./desktop-dependencies.js";
+import { desktopDependencies } from "./desktop-dependencies.js";
 import {
   type DesktopSessionManager,
   type DesktopViewer,
@@ -21,6 +21,8 @@ type Owner = {
   actions: number;
   lastActivity: number;
   desktopLost: boolean;
+  humanHelp?: symbol;
+  observationId?: string;
 };
 
 export class DesktopAutomationLease {
@@ -33,16 +35,33 @@ export class DesktopAutomationLease {
     private readonly deps: {
       enabled: () => boolean;
       ready: () => boolean;
-      ensureReady: (signal: AbortSignal) => Promise<void>;
       manager: () => DesktopSessionManager;
+      notify: () => Promise<unknown>;
     } = {
-      enabled: () => isVirtualDesktopEnabled(getConfig()),
-      ready: () => desktopDependencyInstaller.getStatus().state === "ready",
-      ensureReady: (signal) => desktopDependencyInstaller.ensureReady(signal),
+      enabled: isVirtualDesktopEnabled,
+      ready: () => desktopDependencies.getStatus().state === "ready",
       manager: getDesktopSessionManager,
+      notify: async () =>
+        broadcastMessage({ type: "desktop_activity_changed" }),
     },
   ) {}
 
+  get isActive(): boolean {
+    return (
+      this.owner !== null &&
+      !this.owner.humanHelp &&
+      !this.owner.abort.signal.aborted
+    );
+  }
+
+  private notify(): void {
+    void this.deps
+      .notify()
+      .catch((err) =>
+        log.warn({ err }, "Desktop automation activity notification failed"),
+      );
+  }
+
   private exclusive<T>(run: () => Promise<T>): Promise<T> {
     const next = this.tail.then(run, run);
     this.tail = next.catch(() => {});
@@ -52,12 +71,12 @@ export class DesktopAutomationLease {
   private assertAvailable(): void {
     if (!this.deps.enabled()) {
       throw new Error(
-        "Virtual desktop browser automation is available only on enabled platform-hosted assistants",
+        "Virtual desktop automation is available only on enabled platform-hosted assistants",
       );
     }
     if (!this.deps.ready()) {
       throw new Error(
-        "Open the Virtual desktop panel and wait for automatic installation to finish before using desktop browser automation",
+        "Desktop components are missing from the assistant image. Update the assistant image. Do not install packages manually.",
       );
     }
   }
@@ -84,9 +103,25 @@ export class DesktopAutomationLease {
     }
     this.generation += 1;
     owner.abort.abort();
-    void this.exclusive(() => this.release()).catch((err) =>
-      log.warn({ err }, "Desktop browser session cleanup failed"),
-    );
+    this.releaseCancelledOwner(owner);
+  }
+
+  private releaseCancelledOwner(owner: Owner): void {
+    void this.exclusive(async () => {
+      if (this.owner === owner) {
+        await this.release();
+      }
+    }).catch((err) => {
+      log.warn({ err }, "Desktop automation session cleanup failed");
+      const retry = setTimeout(() => this.releaseCancelledOwner(owner), 1_000);
+      retry.unref?.();
+    });
+  }
+
+  releaseForConversation(conversationId: string): void {
+    if (this.owner?.conversationId === conversationId) {
+      this.cancel(this.owner);
+    }
   }
 
   private bindCancellation(owner: Owner, signal?: AbortSignal): void {
@@ -122,19 +157,27 @@ export class DesktopAutomationLease {
       throw new Error("The desktop is busy or shutting down");
     }
     this.owner = owner;
+    owner.abort.signal.addEventListener("abort", () => this.notify(), {
+      once: true,
+    });
+    this.notify();
     this.bindCancellation(owner, context.signal);
     const cancel = () => this.cancel(owner);
     this.watchdog = setInterval(() => {
       try {
         if (
-          Date.now() - owner.lastActivity > IDLE_TIMEOUT_MS ||
+          (!owner.humanHelp &&
+            Date.now() - owner.lastActivity > IDLE_TIMEOUT_MS) ||
           !this.deps.enabled() ||
           !this.deps.ready()
         ) {
           cancel();
         }
       } catch (err) {
-        log.warn({ err }, "Desktop browser session availability check failed");
+        log.warn(
+          { err },
+          "Desktop automation session availability check failed",
+        );
         cancel();
       }
     }, 1_000);
@@ -149,17 +192,65 @@ export class DesktopAutomationLease {
       await this.release().catch((cleanupError) =>
         log.warn(
           { err: cleanupError },
-          "Desktop browser session cleanup failed",
+          "Desktop automation session cleanup failed",
         ),
       );
       throw err;
     }
   }
 
+  async reserveForHuman(
+    context: ToolContext,
+  ): Promise<(resume: boolean) => Promise<void>> {
+    let reservedOwner: Owner | undefined;
+    const reservation = Symbol("human-help");
+    const result = await this.runBrowser(context, async () => {
+      await this.deps.manager().browser.release();
+      reservedOwner = this.owner!;
+      reservedOwner.humanHelp = reservation;
+      this.notify();
+      return { content: "", isError: false };
+    });
+    if (result.isError || !reservedOwner) {
+      throw new Error(
+        "Desktop help was interrupted before control could be handed over.",
+      );
+    }
+    const owner = reservedOwner;
+    return (resume) =>
+      this.exclusive(async () => {
+        if (this.owner !== owner || owner.humanHelp !== reservation) {
+          return;
+        }
+        if (resume && !owner.abort.signal.aborted) {
+          owner.humanHelp = undefined;
+          owner.lastActivity = Date.now();
+          this.notify();
+        } else {
+          try {
+            await this.release();
+          } catch (error) {
+            this.releaseCancelledOwner(owner);
+            throw error;
+          }
+        }
+      });
+  }
+
+  recordObservation(): string {
+    if (!this.owner || this.owner.abort.signal.aborted) {
+      throw new Error("Desktop observation was interrupted. Observe again.");
+    }
+    const id = crypto.randomUUID();
+    this.owner.observationId = id;
+    return id;
+  }
+
   runBrowser(
     context: ToolContext,
     operation: (signal: AbortSignal) => Promise<ToolExecutionResult>,
     done = false,
+    observation?: { id: unknown },
   ): Promise<ToolExecutionResult> {
     const generation = this.generation;
     return this.exclusive(async () => {
@@ -169,7 +260,7 @@ export class DesktopAutomationLease {
         !context.conversationId
       ) {
         throw new Error(
-          "Desktop browser session requires an identified guardian conversation",
+          "Desktop automation session requires an identified guardian conversation",
         );
       }
       if (
@@ -179,47 +270,30 @@ export class DesktopAutomationLease {
       ) {
         throw new Error("Another conversation is controlling the desktop");
       }
+      if (this.owner?.humanHelp) {
+        throw new Error(
+          "The desktop is reserved while the user is helping. Wait for Done or Skip.",
+        );
+      }
       if (done) {
         await this.release();
-        return { content: "Desktop browser session released.", isError: false };
+        return {
+          content: "Desktop automation session released.",
+          isError: false,
+        };
       }
       context.signal?.throwIfAborted();
+      if (
+        observation &&
+        (typeof observation.id !== "string" ||
+          !this.owner?.observationId ||
+          observation.id !== this.owner.observationId)
+      ) {
+        throw new Error(
+          "Desktop observation is missing or stale. Call computer_use_observe with target assistant-desktop before acting.",
+        );
+      }
       try {
-        if (
-          !this.owner &&
-          generation === this.generation &&
-          this.deps.enabled() &&
-          !this.deps.ready()
-        ) {
-          const abort = new AbortController();
-          const signal = AbortSignal.any([
-            abort.signal,
-            ...(context.signal ? [context.signal] : []),
-          ]);
-          const timeout = setTimeout(
-            () =>
-              abort.abort(
-                new Error(
-                  "Virtual desktop setup is taking too long. Check installation progress in the Virtual desktop panel. No browser action was performed.",
-                ),
-              ),
-            8 * 60_000,
-          );
-          const watchdog = setInterval(() => {
-            if (!this.deps.enabled()) {
-              abort.abort(
-                new Error("Virtual desktop was disabled during setup"),
-              );
-            }
-          }, 1_000);
-          try {
-            await this.deps.ensureReady(signal);
-            signal.throwIfAborted();
-          } finally {
-            clearTimeout(timeout);
-            clearInterval(watchdog);
-          }
-        }
         this.assertAvailable();
       } catch (error) {
         await this.release();
@@ -228,7 +302,7 @@ export class DesktopAutomationLease {
       if (generation !== this.generation) {
         return {
           content:
-            "Desktop browser session was interrupted. Take a fresh snapshot before continuing.",
+            "Desktop automation session was interrupted. Take a fresh snapshot before continuing.",
           isError: true,
         };
       }
@@ -239,8 +313,9 @@ export class DesktopAutomationLease {
       const signal = context.signal
         ? AbortSignal.any([context.signal, owner.abort.signal])
         : owner.abort.signal;
-      owner.lastActivity = Date.now();
       signal.throwIfAborted();
+      owner.observationId = undefined;
+      owner.lastActivity = Date.now();
       try {
         this.assertAvailable();
         if (++owner.actions > MAX_ACTIONS) {
@@ -258,7 +333,7 @@ export class DesktopAutomationLease {
         await this.release().catch((cleanupError) =>
           log.warn(
             { err: cleanupError },
-            "Desktop browser session cleanup failed",
+            "Desktop automation session cleanup failed",
           ),
         );
         throw err;

package/src/desktop/desktop-browser-endpoint.ts

@@ -4,6 +4,8 @@ import { join } from "node:path";
 
 import { shouldRestoreDesktopChromeSession } from "./desktop-chrome-session.js";
 
+export const DESKTOP_CHROME_WINDOW_CLASS = "vellum-desktop-chrome";
+
 export async function allocateDesktopDebugPort(): Promise<number> {
   const server = createServer();
   await new Promise<void>((resolve, reject) => {
@@ -129,6 +131,8 @@ export function desktopChromeArguments(
   return [
     "--no-sandbox",
     "--no-first-run",
+    `--class=${DESKTOP_CHROME_WINDOW_CLASS}`,
+    "--force-renderer-accessibility",
     "--disable-dev-shm-usage",
     ...(shouldRestoreDesktopChromeSession(profileDir)
       ? ["--restore-last-session", "--hide-crash-restore-bubble"]

package/src/desktop/desktop-computer-use-routing.test.ts

@@ -0,0 +1,330 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { afterEach, beforeEach, expect, spyOn, test } from "bun:test";
+
+import { setOverridesForTesting } from "../__tests__/feature-flag-test-helpers.js";
+import { asConversation } from "../__tests__/helpers/mock-conversation.js";
+import {
+  getBundledSkillsDir,
+  type SkillToolManifest,
+} from "../config/skills.js";
+import { surfaceProxyResolver } from "../daemon/conversation-surfaces.js";
+import { HostCuProxy } from "../daemon/host-cu-proxy.js";
+import type { TrustContext } from "../daemon/trust-context-types.js";
+import { supportsClientOsForSkillTool } from "../tools/client-os.js";
+import { resolveExecutionTarget } from "../tools/execution-target.js";
+import {
+  createSkillTool,
+  createSkillToolsFromManifest,
+} from "../tools/skills/skill-tool-factory.js";
+import { sensitiveToolReach } from "../tools/tool-approval-handler.js";
+import * as computerUse from "./desktop-computer-use.js";
+import { shouldUseVirtualDesktop } from "./virtual-desktop-feature.js";
+
+const originalPlatform = process.env.IS_PLATFORM;
+const originalContainerized = process.env.IS_CONTAINERIZED;
+const context = {
+  trustClass: "guardian" as const,
+  sourceActorPrincipalId: "user-123",
+  transportInterface: "web" as const,
+  clientOs: "web" as const,
+};
+const spies: Array<{ mockRestore(): void }> = [];
+
+beforeEach(() => {
+  process.env.IS_PLATFORM = "true";
+  process.env.IS_CONTAINERIZED = "true";
+  setOverridesForTesting({ "assistant-desktop": true });
+});
+
+afterEach(() => {
+  for (const spy of spies.splice(0)) {
+    spy.mockRestore();
+  }
+  setOverridesForTesting({});
+  if (originalPlatform === undefined) {
+    delete process.env.IS_PLATFORM;
+  } else {
+    process.env.IS_PLATFORM = originalPlatform;
+  }
+  if (originalContainerized === undefined) {
+    delete process.env.IS_CONTAINERIZED;
+  } else {
+    process.env.IS_CONTAINERIZED = originalContainerized;
+  }
+});
+
+function conversation() {
+  const proxy = new HostCuProxy(10);
+  const ctx = asConversation({
+    workingDir: "/tmp",
+    conversationId: "conv-123",
+    transportInterface: "web",
+    currentTurnClientOs: "web",
+    currentTurnSourceActorPrincipalId: "user-123",
+    getTurnOrRestingTrust: (): TrustContext => ({
+      sourceChannel: "vellum",
+      trustClass: "guardian",
+    }),
+    hostCuProxy: proxy,
+  });
+  return { ctx, proxy };
+}
+
+test("explicit computer use runs on the virtual desktop without a host client", async () => {
+  const { ctx, proxy } = conversation();
+  const execute = spyOn(
+    computerUse,
+    "executeDesktopComputerUse",
+  ).mockResolvedValue({ content: "desktop", isError: false });
+  spies.push(execute);
+  expect(
+    await surfaceProxyResolver(ctx, "computer_use_observe", {
+      target: "assistant-desktop",
+    }),
+  ).toEqual({
+    content: "desktop",
+    isError: false,
+  });
+  expect(execute.mock.calls[0][2]).toMatchObject({
+    conversationId: "conv-123",
+    ...context,
+  });
+  expect(execute.mock.calls[0][3]).toBe(proxy);
+});
+
+test("explicit host targets never fall through to the virtual desktop", async () => {
+  const { ctx, proxy } = conversation();
+  const execute = spyOn(
+    computerUse,
+    "executeDesktopComputerUse",
+  ).mockResolvedValue({ content: "desktop", isError: false });
+  spies.push(execute, spyOn(proxy, "isAvailable").mockReturnValue(false));
+  expect(
+    (
+      await surfaceProxyResolver(ctx, "computer_use_click", {
+        target_client_id: "missing-client",
+        x: 1,
+        y: 2,
+      })
+    ).isError,
+  ).toBe(true);
+  expect(execute).not.toHaveBeenCalled();
+});
+
+test("screen annotations stay on the host path", async () => {
+  const { ctx, proxy } = conversation();
+  const execute = spyOn(
+    computerUse,
+    "executeDesktopComputerUse",
+  ).mockResolvedValue({ content: "desktop", isError: false });
+  spies.push(execute, spyOn(proxy, "isAvailable").mockReturnValue(false));
+  expect(
+    (await surfaceProxyResolver(ctx, "computer_use_point_at", {})).isError,
+  ).toBe(true);
+  expect(execute).not.toHaveBeenCalled();
+});
+
+test("native, unidentified and non-guardian turns do not select the virtual desktop", () => {
+  for (const clientOs of ["macos", "windows", "linux"] as const) {
+    expect(shouldUseVirtualDesktop({ ...context, clientOs })).toBe(false);
+  }
+  expect(
+    shouldUseVirtualDesktop({ ...context, sourceActorPrincipalId: undefined }),
+  ).toBe(false);
+  expect(shouldUseVirtualDesktop({ ...context, trustClass: "unknown" })).toBe(
+    false,
+  );
+  process.env.IS_PLATFORM = "false";
+  expect(shouldUseVirtualDesktop(context)).toBe(false);
+});
+
+test("web skill tools include supported desktop actions without a native client", () => {
+  expect(
+    supportsClientOsForSkillTool(["macos"], "computer_use_sequence", context),
+  ).toBe(true);
+  expect(
+    supportsClientOsForSkillTool(
+      ["macos", "windows"],
+      "computer_use_drag",
+      context,
+    ),
+  ).toBe(true);
+  expect(
+    supportsClientOsForSkillTool(
+      ["macos"],
+      "computer_use_run_applescript",
+      context,
+    ),
+  ).toBe(false);
+});
+
+test("routing uses frozen turn trust even when the resting conversation is guardian", async () => {
+  const { ctx, proxy } = conversation();
+  ctx.trustContext = { sourceChannel: "vellum", trustClass: "guardian" };
+  ctx.getTurnOrRestingTrust = () => ({
+    sourceChannel: "vellum",
+    trustClass: "unknown",
+  });
+  const execute = spyOn(
+    computerUse,
+    "executeDesktopComputerUse",
+  ).mockResolvedValue({ content: "desktop", isError: false });
+  spies.push(execute, spyOn(proxy, "isAvailable").mockReturnValue(false));
+  expect(
+    (
+      await surfaceProxyResolver(ctx, "computer_use_observe", {
+        target: "assistant-desktop",
+      })
+    ).isError,
+  ).toBe(true);
+  expect(execute).not.toHaveBeenCalled();
+});
+
+for (const input of [
+  { target: "assistant-desktop", target_client_id: "client-123" },
+  { target: "connected-computer", observation_id: "stale" },
+  { target: "unknown" },
+]) {
+  test(`invalid target never dispatches: ${JSON.stringify(input)}`, async () => {
+    const { ctx, proxy } = conversation();
+    const execute = spyOn(computerUse, "executeDesktopComputerUse");
+    const host = spyOn(proxy, "request");
+    spies.push(execute, host);
+    await expect(
+      surfaceProxyResolver(ctx, "computer_use_observe", input),
+    ).rejects.toThrow();
+    expect(execute).not.toHaveBeenCalled();
+    expect(host).not.toHaveBeenCalled();
+    expect(proxy.stepCount).toBe(0);
+  });
+}
+
+for (const platform of [true, false]) {
+  for (const clientOs of ["web", "macos", "windows", "linux"] as const) {
+    test(`bundled tool policy and dispatch agree for ${platform ? "platform" : "local"} ${clientOs}`, async () => {
+      process.env.IS_PLATFORM = String(platform);
+      const skillDir = join(getBundledSkillsDir(), "computer-use");
+      const manifest = JSON.parse(
+        readFileSync(join(skillDir, "TOOLS.json"), "utf8"),
+      ) as SkillToolManifest;
+      const tools = createSkillToolsFromManifest(
+        manifest.tools,
+        skillDir,
+        "",
+        true,
+      );
+      const click = tools.find((tool) => tool.name === "computer_use_click")!;
+      const { ctx, proxy } = conversation();
+      ctx.currentTurnClientOs = clientOs;
+      const local = spyOn(
+        computerUse,
+        "executeDesktopComputerUse",
+      ).mockResolvedValue({ content: "desktop", isError: false });
+      spies.push(local, spyOn(proxy, "isAvailable").mockReturnValue(false));
+      const toolContext = {
+        ...context,
+        clientOs,
+        workingDir: "/tmp",
+        conversationId: "conv-123",
+        proxyToolResolver: (name: string, args: Record<string, unknown>) =>
+          surfaceProxyResolver(ctx, name, args),
+      };
+      for (const target of [
+        undefined,
+        "connected-computer",
+        "assistant-desktop",
+      ]) {
+        const virtual =
+          target === "assistant-desktop" ||
+          (target === undefined && platform && clientOs === "web");
+        const input = {
+          target,
+          x: 30,
+          y: 40,
+          reasoning: "Focus field",
+          ...(virtual ? { observation_id: "observation-123" } : {}),
+        };
+        const boundary = resolveExecutionTarget(click, input, toolContext);
+        expect(boundary).toBe(virtual ? "sandbox" : "host");
+        if (virtual) {
+          expect(sensitiveToolReach(click.name, boundary, input)).not.toBe(
+            "host",
+          );
+        } else {
+          expect(sensitiveToolReach(click.name, boundary, input)).toBe("host");
+        }
+        const calls = local.mock.calls.length;
+        const result = await click.execute(input, toolContext);
+        expect(result.isError).toBe(!(virtual && platform));
+        expect(local).toHaveBeenCalledTimes(
+          calls + (virtual && platform ? 1 : 0),
+        );
+      }
+      const calls = local.mock.calls.length;
+      const rejected = await click.execute(
+        { target: "invalid", reasoning: "Focus field" },
+        toolContext,
+      );
+      expect(rejected.isError).toBe(true);
+      expect(local).toHaveBeenCalledTimes(calls);
+      const entry = manifest.tools.find((tool) => tool.name === click.name)!;
+      for (const tool of [
+        createSkillTool(entry, "/tmp/skills/computer-use", "", true),
+        createSkillTool(entry, skillDir, "", false),
+        createSkillTool(entry, skillDir, "", true, "plugin-123"),
+      ]) {
+        expect(
+          resolveExecutionTarget(
+            tool,
+            { target: "assistant-desktop" },
+            toolContext,
+          ),
+        ).toBe("host");
+      }
+    });
+  }
+}
+
+test("desktop revocation between policy and dispatch never falls back to the host", async () => {
+  const { ctx, proxy } = conversation();
+  const skillDir = join(getBundledSkillsDir(), "computer-use");
+  const manifest = JSON.parse(
+    readFileSync(join(skillDir, "TOOLS.json"), "utf8"),
+  ) as SkillToolManifest;
+  const observe = createSkillToolsFromManifest(
+    manifest.tools,
+    skillDir,
+    "",
+    true,
+  ).find((tool) => tool.name === "computer_use_observe")!;
+  const toolContext = {
+    ...context,
+    workingDir: "/tmp",
+    conversationId: "conv-123",
+  };
+  expect(resolveExecutionTarget(observe, {}, toolContext)).toBe("sandbox");
+  setOverridesForTesting({ "assistant-desktop": false });
+  const local = spyOn(computerUse, "executeDesktopComputerUse");
+  const host = spyOn(proxy, "request");
+  spies.push(local, host, spyOn(proxy, "isAvailable").mockReturnValue(true));
+  expect(
+    (await surfaceProxyResolver(ctx, "computer_use_observe", {})).isError,
+  ).toBe(true);
+  expect(host).not.toHaveBeenCalled();
+  expect(local).not.toHaveBeenCalled();
+});
+
+test("a disabled assistant desktop never falls back to a connected computer", async () => {
+  setOverridesForTesting({ "assistant-desktop": false });
+  const { ctx, proxy } = conversation();
+  const local = spyOn(computerUse, "executeDesktopComputerUse");
+  const host = spyOn(proxy, "request");
+  spies.push(local, host, spyOn(proxy, "isAvailable").mockReturnValue(true));
+  const result = await surfaceProxyResolver(ctx, "computer_use_observe", {
+    target: "assistant-desktop",
+  });
+  expect(result.isError).toBe(true);
+  expect(local).not.toHaveBeenCalled();
+  expect(host).not.toHaveBeenCalled();
+});