@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/desktop/desktop-automation-lease.test.ts

@@ -1,7 +1,6 @@
 import { afterEach, expect, mock, test } from "bun:test";
 
 import { DesktopAutomationLease } from "./desktop-automation-lease.js";
-import { DesktopDependencyInstaller } from "./desktop-dependencies.js";
 import type { DesktopSessionManager } from "./desktop-session-manager.js";
 
 const context = {
@@ -21,17 +20,6 @@ afterEach(async () => {
 function fixture() {
   let enabled = true;
   let ready = true;
-  const setup = Promise.withResolvers<void>();
-  const install = mock(() => setup.promise);
-  const installer = new DesktopDependencyInstaller({
-    supported: () => true,
-    ready: () => ready,
-    install,
-    notify: async () => {},
-  });
-  const ensureReady = mock((signal: AbortSignal) =>
-    installer.ensureReady(signal),
-  );
   const started = mock(async () => {});
   const released = mock(() => {});
   const releaseBrowser = mock(async () => {});
@@ -40,7 +28,6 @@ function fixture() {
     notify,
     enabled: () => enabled,
     ready: () => ready,
-    ensureReady,
     manager: () =>
       ({
         browser: { release: releaseBrowser },
@@ -53,12 +40,8 @@ function fixture() {
   return {
     lease,
     notify,
-    ensureReady,
-    install,
-    failSetup: () => setup.reject(new Error("download failed")),
-    completeSetup: () => {
+    restoreComponents: () => {
       ready = true;
-      setup.resolve();
     },
     started,
     released,
@@ -66,7 +49,7 @@ function fixture() {
     disable: () => {
       enabled = false;
     },
-    uninstall: () => {
+    removeComponents: () => {
       ready = false;
     },
   };
@@ -86,13 +69,10 @@ test("browser session reuses one automation slot and isolates conversations", as
   expect(f.released).toHaveBeenCalledTimes(1);
 });
 
-for (const lose of ["disable", "uninstall"] as const) {
+for (const lose of ["disable", "removeComponents"] as const) {
   test(`${lose} blocks startup, reuse and startup races`, async () => {
     const unavailable = fixture();
     unavailable[lose]();
-    if (lose === "uninstall") {
-      unavailable.ensureReady.mockRejectedValueOnce(new Error("setup failed"));
-    }
     await expect(
       unavailable.lease.runBrowser(context, operation),
     ).rejects.toThrow();
@@ -151,51 +131,9 @@ test("failed browser cleanup retains ownership until release succeeds", async ()
   expect(f.released).toHaveBeenCalledTimes(1);
 });
 
-test("first browser call waits for one shared install then executes exactly once", async () => {
-  const f = fixture();
-  f.uninstall();
-  const callback = mock(operation);
-  const first = f.lease.runBrowser(context, callback);
-  await Bun.sleep(0);
-  expect(f.install).toHaveBeenCalledTimes(1);
-  expect(f.started).not.toHaveBeenCalled();
-  expect(callback).not.toHaveBeenCalled();
-  f.completeSetup();
-  expect((await first).isError).toBe(false);
-  expect(callback).toHaveBeenCalledTimes(1);
-  await f.lease.runBrowser(context, callback);
-  expect(f.install).toHaveBeenCalledTimes(1);
-  expect(f.started).toHaveBeenCalledTimes(1);
-});
-
-for (const interrupt of ["cancel", "disable", "failure"] as const) {
-  test(`${interrupt} during setup prevents a delayed browser action`, async () => {
-    const f = fixture();
-    f.uninstall();
-    const abort = new AbortController();
-    const callback = mock(operation);
-    const result = f.lease
-      .runBrowser({ ...context, signal: abort.signal }, callback)
-      .catch((error: unknown) => error);
-    await Bun.sleep(0);
-    if (interrupt === "cancel") {
-      abort.abort();
-    } else if (interrupt === "disable") {
-      f.disable();
-    } else {
-      f.failSetup();
-    }
-    expect(await result).toBeInstanceOf(Error);
-    f.completeSetup();
-    await Bun.sleep(0);
-    expect(f.started).not.toHaveBeenCalled();
-    expect(callback).not.toHaveBeenCalled();
-  });
-}
-
-test("disabled, unidentified, cancelled and released browser calls cannot install", async () => {
+test("disabled, unidentified, cancelled and released browser calls cannot start the desktop", async () => {
   const f = fixture();
-  f.uninstall();
+  f.removeComponents();
   const cancelled = new AbortController();
   cancelled.abort();
   for (const caller of [
@@ -208,7 +146,7 @@ test("disabled, unidentified, cancelled and released browser calls cannot instal
   await f.lease.runBrowser(context, operation, true);
   f.disable();
   await expect(f.lease.runBrowser(context, operation)).rejects.toThrow();
-  expect(f.ensureReady).not.toHaveBeenCalled();
+  expect(f.started).not.toHaveBeenCalled();
 });
 
 test("activity follows the browser lease and clears before failed cleanup", async () => {
@@ -436,3 +374,73 @@ test("ending human help retries failed input cleanup without another turn", asyn
   await f.lease.runBrowser(context, operation);
   expect(f.lease.isActive).toBe(true);
 });
+
+for (const interrupt of ["browser", "handoff", "release", "cancel"] as const) {
+  test(`${interrupt} invalidates native observations`, async () => {
+    const f = fixture();
+    const abort = new AbortController();
+    const ownerContext = { ...context, signal: abort.signal };
+    let id = "";
+    await f.lease.runBrowser(ownerContext, async () => {
+      id = f.lease.recordObservation();
+      return operation();
+    });
+    if (interrupt === "browser") {
+      await f.lease.runBrowser(ownerContext, operation);
+    } else if (interrupt === "handoff") {
+      const resume = await f.lease.reserveForHuman(ownerContext);
+      await resume(true);
+    } else if (interrupt === "release") {
+      await f.lease.runBrowser(ownerContext, operation, true);
+    } else {
+      abort.abort();
+    }
+    const input = mock(operation);
+    await expect(
+      f.lease.runBrowser(context, input, false, { id }),
+    ).rejects.toThrow("stale");
+    expect(input).not.toHaveBeenCalled();
+  });
+}
+
+test("queued actions cannot consume the same observation twice", async () => {
+  const f = fixture();
+  let id = "";
+  await f.lease.runBrowser(context, async () => {
+    id = f.lease.recordObservation();
+    return operation();
+  });
+  const input = mock(operation);
+  const actions = await Promise.allSettled([
+    f.lease.runBrowser(context, input, false, { id }),
+    f.lease.runBrowser(context, input, false, { id }),
+  ]);
+  expect(actions.map((action) => action.status)).toEqual([
+    "fulfilled",
+    "rejected",
+  ]);
+  expect(input).toHaveBeenCalledTimes(1);
+});
+
+test("a stale observation cannot start or reserve an unowned desktop", async () => {
+  const f = fixture();
+  const input = mock(operation);
+  f.removeComponents();
+  await expect(
+    f.lease.runBrowser(context, input, false, { id: "stale" }),
+  ).rejects.toThrow("stale");
+  expect(f.started).not.toHaveBeenCalled();
+  expect(f.lease.isActive).toBe(false);
+  expect(input).not.toHaveBeenCalled();
+  f.restoreComponents();
+  await f.lease.runBrowser(
+    { ...context, conversationId: "conv-456" },
+    operation,
+  );
+  expect(f.started).toHaveBeenCalledTimes(1);
+  await f.lease.runBrowser(
+    { ...context, conversationId: "conv-456" },
+    operation,
+    true,
+  );
+});

package/src/desktop/desktop-automation-lease.ts

@@ -1,7 +1,7 @@
 import { broadcastMessage } from "../runtime/assistant-event-hub.js";
 import type { ToolContext, ToolExecutionResult } from "../tools/types.js";
 import { getLogger } from "../util/logger.js";
-import { desktopDependencyInstaller } from "./desktop-dependencies.js";
+import { desktopDependencies } from "./desktop-dependencies.js";
 import {
   type DesktopSessionManager,
   type DesktopViewer,
@@ -22,6 +22,7 @@ type Owner = {
   lastActivity: number;
   desktopLost: boolean;
   humanHelp?: symbol;
+  observationId?: string;
 };
 
 export class DesktopAutomationLease {
@@ -34,13 +35,11 @@ export class DesktopAutomationLease {
     private readonly deps: {
       enabled: () => boolean;
       ready: () => boolean;
-      ensureReady: (signal: AbortSignal) => Promise<void>;
       manager: () => DesktopSessionManager;
       notify: () => Promise<unknown>;
     } = {
       enabled: isVirtualDesktopEnabled,
-      ready: () => desktopDependencyInstaller.getStatus().state === "ready",
-      ensureReady: (signal) => desktopDependencyInstaller.ensureReady(signal),
+      ready: () => desktopDependencies.getStatus().state === "ready",
       manager: getDesktopSessionManager,
       notify: async () =>
         broadcastMessage({ type: "desktop_activity_changed" }),
@@ -59,7 +58,7 @@ export class DesktopAutomationLease {
     void this.deps
       .notify()
       .catch((err) =>
-        log.warn({ err }, "Desktop browser activity notification failed"),
+        log.warn({ err }, "Desktop automation activity notification failed"),
       );
   }
 
@@ -72,12 +71,12 @@ export class DesktopAutomationLease {
   private assertAvailable(): void {
     if (!this.deps.enabled()) {
       throw new Error(
-        "Virtual desktop browser automation is available only on enabled platform-hosted assistants",
+        "Virtual desktop automation is available only on enabled platform-hosted assistants",
       );
     }
     if (!this.deps.ready()) {
       throw new Error(
-        "Open the Virtual desktop panel and wait for automatic installation to finish before using desktop browser automation",
+        "Desktop components are missing from the assistant image. Update the assistant image. Do not install packages manually.",
       );
     }
   }
@@ -113,7 +112,7 @@ export class DesktopAutomationLease {
         await this.release();
       }
     }).catch((err) => {
-      log.warn({ err }, "Desktop browser session cleanup failed");
+      log.warn({ err }, "Desktop automation session cleanup failed");
       const retry = setTimeout(() => this.releaseCancelledOwner(owner), 1_000);
       retry.unref?.();
     });
@@ -175,7 +174,10 @@ export class DesktopAutomationLease {
           cancel();
         }
       } catch (err) {
-        log.warn({ err }, "Desktop browser session availability check failed");
+        log.warn(
+          { err },
+          "Desktop automation session availability check failed",
+        );
         cancel();
       }
     }, 1_000);
@@ -190,7 +192,7 @@ export class DesktopAutomationLease {
       await this.release().catch((cleanupError) =>
         log.warn(
           { err: cleanupError },
-          "Desktop browser session cleanup failed",
+          "Desktop automation session cleanup failed",
         ),
       );
       throw err;
@@ -235,10 +237,20 @@ export class DesktopAutomationLease {
       });
   }
 
+  recordObservation(): string {
+    if (!this.owner || this.owner.abort.signal.aborted) {
+      throw new Error("Desktop observation was interrupted. Observe again.");
+    }
+    const id = crypto.randomUUID();
+    this.owner.observationId = id;
+    return id;
+  }
+
   runBrowser(
     context: ToolContext,
     operation: (signal: AbortSignal) => Promise<ToolExecutionResult>,
     done = false,
+    observation?: { id: unknown },
   ): Promise<ToolExecutionResult> {
     const generation = this.generation;
     return this.exclusive(async () => {
@@ -248,7 +260,7 @@ export class DesktopAutomationLease {
         !context.conversationId
       ) {
         throw new Error(
-          "Desktop browser session requires an identified guardian conversation",
+          "Desktop automation session requires an identified guardian conversation",
         );
       }
       if (
@@ -265,45 +277,23 @@ export class DesktopAutomationLease {
       }
       if (done) {
         await this.release();
-        return { content: "Desktop browser session released.", isError: false };
+        return {
+          content: "Desktop automation session released.",
+          isError: false,
+        };
       }
       context.signal?.throwIfAborted();
+      if (
+        observation &&
+        (typeof observation.id !== "string" ||
+          !this.owner?.observationId ||
+          observation.id !== this.owner.observationId)
+      ) {
+        throw new Error(
+          "Desktop observation is missing or stale. Call computer_use_observe with target assistant-desktop before acting.",
+        );
+      }
       try {
-        if (
-          !this.owner &&
-          generation === this.generation &&
-          this.deps.enabled() &&
-          !this.deps.ready()
-        ) {
-          const abort = new AbortController();
-          const signal = AbortSignal.any([
-            abort.signal,
-            ...(context.signal ? [context.signal] : []),
-          ]);
-          const timeout = setTimeout(
-            () =>
-              abort.abort(
-                new Error(
-                  "Virtual desktop setup is taking too long. Check installation progress in the Virtual desktop panel. No browser action was performed.",
-                ),
-              ),
-            8 * 60_000,
-          );
-          const watchdog = setInterval(() => {
-            if (!this.deps.enabled()) {
-              abort.abort(
-                new Error("Virtual desktop was disabled during setup"),
-              );
-            }
-          }, 1_000);
-          try {
-            await this.deps.ensureReady(signal);
-            signal.throwIfAborted();
-          } finally {
-            clearTimeout(timeout);
-            clearInterval(watchdog);
-          }
-        }
         this.assertAvailable();
       } catch (error) {
         await this.release();
@@ -312,7 +302,7 @@ export class DesktopAutomationLease {
       if (generation !== this.generation) {
         return {
           content:
-            "Desktop browser session was interrupted. Take a fresh snapshot before continuing.",
+            "Desktop automation session was interrupted. Take a fresh snapshot before continuing.",
           isError: true,
         };
       }
@@ -323,8 +313,9 @@ export class DesktopAutomationLease {
       const signal = context.signal
         ? AbortSignal.any([context.signal, owner.abort.signal])
         : owner.abort.signal;
-      owner.lastActivity = Date.now();
       signal.throwIfAborted();
+      owner.observationId = undefined;
+      owner.lastActivity = Date.now();
       try {
         this.assertAvailable();
         if (++owner.actions > MAX_ACTIONS) {
@@ -342,7 +333,7 @@ export class DesktopAutomationLease {
         await this.release().catch((cleanupError) =>
           log.warn(
             { err: cleanupError },
-            "Desktop browser session cleanup failed",
+            "Desktop automation session cleanup failed",
           ),
         );
         throw err;

package/src/desktop/desktop-browser-endpoint.ts

@@ -4,6 +4,8 @@ import { join } from "node:path";
 
 import { shouldRestoreDesktopChromeSession } from "./desktop-chrome-session.js";
 
+export const DESKTOP_CHROME_WINDOW_CLASS = "vellum-desktop-chrome";
+
 export async function allocateDesktopDebugPort(): Promise<number> {
   const server = createServer();
   await new Promise<void>((resolve, reject) => {
@@ -129,6 +131,8 @@ export function desktopChromeArguments(
   return [
     "--no-sandbox",
     "--no-first-run",
+    `--class=${DESKTOP_CHROME_WINDOW_CLASS}`,
+    "--force-renderer-accessibility",
     "--disable-dev-shm-usage",
     ...(shouldRestoreDesktopChromeSession(profileDir)
       ? ["--restore-last-session", "--hide-crash-restore-bubble"]