@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.4

package/src/daemon/conversation-tool-setup.ts

@@ -7,10 +7,9 @@
  */
 
 import type { AssistantEvent } from "../api/index.js";
+import { shouldUseVirtualDesktopBrowser } from "../browser/virtual-desktop-target.js";
 import {
-  type ClientOs,
   type HostProxyCapability,
-  parseClientOs,
   supportsHostProxy,
 } from "../channels/types.js";
 import { getIsPlatform } from "../config/env-registry.js";
@@ -74,6 +73,7 @@ import {
   conversationSupportsGuardianQuestionCards,
 } from "./channel-ui-capability.js";
 import type { Conversation } from "./conversation.js";
+import { resolveTurnClientOs } from "./conversation-client-surface.js";
 import { projectSkillTools } from "./conversation-skill-tools.js";
 import {
   restoreSurfaceStateEntry,
@@ -742,30 +742,6 @@ export const ALLOWLIST_ONLY_TOOL_NAMES = new Set<string>([
   "delete_memory_page",
 ]);
 
-/**
- * Host OS of the client driving this turn. The Electron renderer reports
- * `interface: "web"` and carries the real OS in `clientOs`, so this prefers
- * the frozen per-turn value and only falls back to a desktop transport.
- */
-function resolveTurnClientOs(ctx: Conversation): {
-  clientOs: ClientOs | undefined;
-  transportInterface: Conversation["transportInterface"];
-} {
-  const pin = ctx.toolContextPin;
-  const transportInterface = pin
-    ? pin.transportInterface
-    : ctx.transportInterface;
-  const clientOs = pin
-    ? pin.clientOs
-    : (parseClientOs(ctx.currentTurnClientOs ?? ctx.clientOs) ??
-      (transportInterface === "macos" ||
-      transportInterface === "windows" ||
-      transportInterface === "linux"
-        ? transportInterface
-        : undefined));
-  return { clientOs, transportInterface };
-}
-
 /**
  * Windows parity gate: skill tools may declare `supported_client_os`; drop
  * them when the turn's client OS (or pinned OS for wakes) is not listed.
@@ -1149,9 +1125,35 @@ export function createResolveToolsCallback(
     // while background calls persist the full surface content for the next
     // capable client that opens the conversation. Skill tools stay off this
     // list (`skill_execute` dispatch) and are a separate disclosure path.
-    let allBaseDefs = [...scopedCoreDefs, ...scopedWorkspaceDefs, ...scopedMcpDefs].filter(
-      (d) => !excluded.has(d.name),
-    );
+    let allBaseDefs = [
+      ...scopedCoreDefs,
+      ...scopedWorkspaceDefs,
+      ...scopedMcpDefs,
+    ].filter((d) => !excluded.has(d.name));
+    if (
+      ctx.transportInterface === "web" &&
+      shouldUseVirtualDesktopBrowser(
+        undefined,
+        {},
+        {
+          workingDir: ctx.workingDir,
+          conversationId: ctx.conversationId,
+          trustClass: ctx.trustContext?.trustClass ?? "unknown",
+          transportInterface: ctx.transportInterface,
+          clientOs: resolveTurnClientOs(ctx).clientOs,
+          sourceActorPrincipalId: ctx.getTurnActorPrincipalId?.(),
+        },
+      )
+    ) {
+      allBaseDefs = allBaseDefs.map((definition) =>
+        definition.name === "bash"
+          ? {
+              ...definition,
+              description: `${definition.description} For browser tasks, use assistant browser navigate --url <url> directly. It installs the virtual desktop if needed, starts Chrome, and completes the action in one call. Use timeout_seconds: ${getConfig().timeouts.shellMaxTimeoutSec} for first use; setup progress is visible in the Virtual desktop panel. Use assistant browser --help for other browser actions. Use this managed path even if saved notes describe manual setup. Do not install packages or launch Chrome, X servers, or screenshot scripts yourself.`,
+            }
+          : definition,
+      );
+    }
     // Activation-rail conversations carry the optional `activation_moment`
     // telemetry param on ui_show. The marker is written before the first
     // tool resolution (see `applyBootstrapTemplate` in system-prompt.ts), so

package/src/daemon/host-cu-proxy.ts

@@ -48,8 +48,8 @@ const SCREENSHOT_OMITTED_MESSAGE =
   "Screenshot omitted: the accessibility tree above is current. Pass include_screenshot: true on computer_use_observe to see the screen.";
 
 // computer_use_key combos that change only selection/cursor/clipboard state.
-// The AX tree models none of these, so they always produce an empty diff —
-// exempt them from the "NO VISIBLE EFFECT" signal (mirrors computer_use_wait).
+// The AX tree models none of these, so they always produce an empty diff and
+// are exempt from the unchanged-tree signal.
 // Stored in canonical form (see canonicalizeKeyCombo): modifier aliases
 // normalized and ordered, so `cmd + a`, `command+a`, `alt+tab`, `tab+shift`
 // all match.
@@ -129,12 +129,28 @@ export interface ActionRecord {
   reasoning?: string;
 }
 
+// Steps whose empty diff says nothing about whether they worked: waiting and
+// observing change nothing by design, and an AppleScript reports its own
+// outcome in the execution result, often for an app whose content the tree
+// cannot see. They neither warn nor move the unchanged streak.
+const NO_AX_DIFF_TOOLS = new Set([
+  "computer_use_wait",
+  "computer_use_observe",
+  "computer_use_run_applescript",
+]);
+
+type ActionIdentity = Pick<ActionRecord, "toolName" | "input">;
+
+function isNoDiffTool(action: ActionIdentity | undefined): boolean {
+  return action !== undefined && NO_AX_DIFF_TOOLS.has(action.toolName);
+}
+
 /**
  * True when `action` is a computer_use_key press whose key only mutates
- * selection/cursor/clipboard state — changes the AX tree cannot represent, so
+ * selection/cursor/clipboard state: changes the AX tree cannot represent, so
  * an empty diff is expected rather than a sign the action did nothing.
  */
-function isNoDiffKeyAction(action: ActionRecord | undefined): boolean {
+function isNoDiffKeyAction(action: ActionIdentity | undefined): boolean {
   if (action?.toolName !== "computer_use_key") {
     return false;
   }
@@ -245,6 +261,7 @@ export class HostCuProxy {
       resetGeneration: number;
       dispatchedAt: number;
       toolName: string;
+      input: Record<string, unknown>;
       step: number;
     }
   >();
@@ -455,6 +472,7 @@ export class HostCuProxy {
         resetGeneration: this._resetGeneration,
         dispatchedAt: Date.now(),
         toolName,
+        input,
         step: stepNumber,
       });
 
@@ -538,7 +556,7 @@ export class HostCuProxy {
     }
 
     // A targeted snapshot has no comparable action/diff baseline; neither it
-    // nor the first desktop observation after it can imply "no visible effect".
+    // nor the first desktop observation after it can imply an unchanged tree.
     // A response dispatched before the last reset belongs to a finished run,
     // so it must not change any state the new run has started building.
     const fromCurrentRun =
@@ -548,6 +566,11 @@ export class HostCuProxy {
       this._consecutiveUnchangedSteps = 0;
     }
     const prevAXTree = this._previousAXTree;
+    // Judge the empty diff against the action this request carried, for the
+    // same reason the timings line does.
+    const action: ActionIdentity | undefined = owned
+      ? { toolName: owned.toolName, input: owned.input }
+      : this.lastRecordedAction();
     const comparableObservation = scopedObservation
       ? { ...observation, axDiff: undefined, secondaryWindows: undefined }
       : observation;
@@ -557,7 +580,7 @@ export class HostCuProxy {
       !scopedObservation &&
       owned?.toolName !== POINT_AT_PROXY_TOOL
     ) {
-      this.updateStateFromObservation(comparableObservation);
+      this.updateStateFromObservation(comparableObservation, action);
       // A desktop has had its first look only once pixels from it arrived,
       // so a failed capture leaves the next request asking again.
       if (owned && observation.screenshot) {
@@ -568,6 +591,7 @@ export class HostCuProxy {
       comparableObservation,
       prevAXTree,
       owned?.screenshotSkipped ?? false,
+      action,
     );
     interaction.rpcResolve(result);
     return result;
@@ -672,6 +696,7 @@ export class HostCuProxy {
     obs: CuObservationResult,
     previousAXTree?: string,
     screenshotSkipped = false,
+    action: ActionIdentity | undefined = this.lastRecordedAction(),
   ): ToolExecutionResult {
     const prevTree = previousAXTree;
     const parts: string[] = [];
@@ -690,24 +715,17 @@ export class HostCuProxy {
       parts.push(obs.axDiff);
       parts.push("");
     } else if (prevTree != null && obs.axTree != null) {
-      const lastAction =
-        this._actionHistory.length > 0
-          ? this._actionHistory[this._actionHistory.length - 1]
-          : undefined;
-      const isWaitAction = lastAction?.toolName === "computer_use_wait";
-      const isNoDiffKey = isNoDiffKeyAction(lastAction);
-
-      if (!isWaitAction && !isNoDiffKey) {
+      if (!isNoDiffTool(action) && !isNoDiffKeyAction(action)) {
         if (
           this._consecutiveUnchangedSteps >=
           CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD
         ) {
           parts.push(
-            `WARNING: ${this._consecutiveUnchangedSteps} consecutive actions had NO VISIBLE EFFECT on the UI. You MUST try a completely different approach.`,
+            `WARNING: the accessibility tree did not change across ${this._consecutiveUnchangedSteps} consecutive actions. If the screenshot does not show them working either, try a different approach.`,
           );
         } else {
           parts.push(
-            "Your last action had NO VISIBLE EFFECT on the UI. Try something different.",
+            "The accessibility tree did not change after your last action. Apps that draw their own content (timelines, canvases, spreadsheet grids) change without it, so check the screenshot before deciding the action did nothing.",
           );
         }
         parts.push("");
@@ -822,19 +840,26 @@ export class HostCuProxy {
   // Private helpers
   // ---------------------------------------------------------------------------
 
-  private updateStateFromObservation(obs: CuObservationResult): void {
+  private lastRecordedAction(): ActionRecord | undefined {
+    return this._actionHistory.at(-1);
+  }
+
+  private updateStateFromObservation(
+    obs: CuObservationResult,
+    action: ActionIdentity | undefined,
+  ): void {
     if (this._stepCount > 0) {
-      const lastAction =
-        this._actionHistory.length > 0
-          ? this._actionHistory[this._actionHistory.length - 1]
-          : undefined;
-      if (obs.axDiff != null || isNoDiffKeyAction(lastAction)) {
+      if (obs.axDiff != null || isNoDiffKeyAction(action)) {
         // A real diff, or an exempt key whose effect is invisible by design,
-        // breaks the no-effect streak — clear it rather than preserving a
-        // stale count so an intervening cmd+a can't bridge two no-op actions
-        // into a false "consecutive" escalation.
+        // breaks the no-effect streak. Clearing it rather than preserving a
+        // stale count keeps an intervening cmd+a from bridging two no-op
+        // actions into a false "consecutive" escalation.
         this._consecutiveUnchangedSteps = 0;
-      } else if (this._previousAXTree != null && obs.axTree != null) {
+      } else if (
+        !isNoDiffTool(action) &&
+        this._previousAXTree != null &&
+        obs.axTree != null
+      ) {
         this._consecutiveUnchangedSteps++;
       }
     }

package/src/daemon/host-proxy-preactivation.ts

@@ -9,6 +9,8 @@
  *   - `daemon/process-message.ts` (create path, prepareConversationForMessage)
  *   - `daemon/conversation-process.ts` `drainSingleMessage` (re-add after dequeue)
  *   - `daemon/conversation-process.ts` `drainBatch` (re-add after dequeue)
+ *   - `calls/voice-session-bridge.ts` (tool-capable legs of a live-voice
+ *     session opened from the macOS desktop client)
  *
  * The create paths additionally instantiate the proxy itself; that
  * instantiation logic is per-proxy-class and stays inline at each create

package/src/daemon/mcp-reload-service.ts

@@ -14,6 +14,7 @@ import { getMcpServerManager } from "../mcp/manager.js";
 import { migrateLegacyMcpHeaders } from "../mcp/mcp-header-store.js";
 import { signalMcpReloaded } from "../mcp/reload-signal.js";
 import { loadWorkspaceMcpConfig } from "../mcp/workspace-mcp-config.js";
+import { publishMcpChanged } from "../runtime/sync/resource-sync-events.js";
 import { createMcpToolsFromServer } from "../tools/mcp/mcp-tool-factory.js";
 import { registerMcpTools, unregisterAllMcpTools } from "../tools/registry.js";
 import { getLogger } from "../util/logger.js";
@@ -80,6 +81,7 @@ export async function reconcilePluginMcpServers(): Promise<void> {
 }
 
 async function doReload(): Promise<McpReloadResult> {
+  let teardownStarted = false;
   try {
     const manager = getMcpServerManager();
 
@@ -98,6 +100,7 @@ async function doReload(): Promise<McpReloadResult> {
     invalidateConfigCache();
 
     // 2. Stop existing MCP servers + unregister their tools
+    teardownStarted = true;
     await manager.stop();
     unregisterAllMcpTools();
 
@@ -160,5 +163,9 @@ async function doReload(): Promise<McpReloadResult> {
     const error = err instanceof Error ? err.message : String(err);
     log.error({ err }, "MCP reload failed");
     return { success: false, error };
+  } finally {
+    if (teardownStarted) {
+      publishMcpChanged();
+    }
   }
 }

package/src/daemon/message-types/sync.ts

@@ -18,6 +18,7 @@ export const SYNC_TAGS = {
   appsList: "apps:list",
   documentsList: "documents:list",
   pluginsList: "plugins:list",
+  mcpList: "mcp:list",
   conversationsList: "conversations:list",
   /** Activation-checklist progress: task launches, step counts, completions. */
   activationProgress: "activation:progress",

package/src/daemon/providers-setup.ts

@@ -1,6 +1,7 @@
 import { maybeDefaultSpeechToManaged } from "../config/managed-speech-defaults.js";
 import { rehydratePlatformCredentials } from "../config/platform-rehydration.js";
 import type { AssistantConfig } from "../config/types.js";
+import { getMcpServerManager } from "../mcp/manager.js";
 import { startConfiguredMcpServers } from "../mcp/startup.js";
 import { gmailMessagingProvider } from "../messaging/providers/gmail/adapter.js";
 import { outlookMessagingProvider } from "../messaging/providers/outlook/adapter.js";
@@ -9,6 +10,7 @@ import { telegramBotMessagingProvider } from "../messaging/providers/telegram-bo
 import { whatsappMessagingProvider } from "../messaging/providers/whatsapp/adapter.js";
 import { registerMessagingProvider } from "../messaging/registry.js";
 import { initializeProviders } from "../providers/registry.js";
+import { publishMcpChanged } from "../runtime/sync/resource-sync-events.js";
 import { validateSubagentRoleAllowlists } from "../subagent/validate-allowlists.js";
 import { initializeTools } from "../tools/registry.js";
 import { getLogger } from "../util/logger.js";
@@ -63,7 +65,9 @@ export async function initializeProvidersAndTools(
   // Start MCP servers — workspace-configured and plugin-declared alike —
   // and register their tools. Shared with the schedule worker, which hosts
   // agent turns in its own process and so needs its own connections.
+  getMcpServerManager().setUnexpectedCloseHandler(() => publishMcpChanged());
   await startConfiguredMcpServers();
+  publishMcpChanged();
 
   log.info("Daemon startup: providers and tools initialized");
 }

package/src/desktop/__tests__/fake-desktop.ts

@@ -58,6 +58,7 @@ export function newFakeDesktop(options: FakeDesktopOptions) {
   let vncReady = true;
   let chromiumPath: () => Promise<string> = async () => "/fake/chromium";
   const manager = new DesktopSessionManager({
+    allocateDebugPort: async () => 9222,
     spawn: (role, request) => {
       if (options.failSpawn?.includes(role)) {
         throw new Error(`spawn ${role} failed`);

package/src/desktop/desktop-automation-lease.test.ts

@@ -0,0 +1,208 @@
+import { afterEach, expect, mock, test } from "bun:test";
+
+import { DesktopAutomationLease } from "./desktop-automation-lease.js";
+import { DesktopDependencyInstaller } from "./desktop-dependencies.js";
+import type { DesktopSessionManager } from "./desktop-session-manager.js";
+
+const context = {
+  conversationId: "conv-123",
+  sourceActorPrincipalId: "user-123",
+  trustClass: "guardian" as const,
+  workingDir: "/tmp",
+};
+const cleanups: DesktopAutomationLease[] = [];
+afterEach(async () => {
+  await Promise.all(
+    cleanups
+      .splice(0)
+      .map((lease) => lease.runBrowser(context, operation, true)),
+  );
+});
+function fixture() {
+  let enabled = true;
+  let ready = true;
+  const setup = Promise.withResolvers<void>();
+  const install = mock(() => setup.promise);
+  const installer = new DesktopDependencyInstaller({
+    supported: () => true,
+    ready: () => ready,
+    install,
+    notify: async () => {},
+  });
+  const ensureReady = mock((signal: AbortSignal) =>
+    installer.ensureReady(signal),
+  );
+  const started = mock(async () => {});
+  const released = mock(() => {});
+  const releaseBrowser = mock(async () => {});
+  const lease = new DesktopAutomationLease({
+    enabled: () => enabled,
+    ready: () => ready,
+    ensureReady,
+    manager: () =>
+      ({
+        browser: { release: releaseBrowser },
+        acquireAutomationSlot: () => ({ ok: true }),
+        releaseAutomationSlot: released,
+        ensureDesktopRunning: started,
+      }) as unknown as DesktopSessionManager,
+  });
+  cleanups.push(lease);
+  return {
+    lease,
+    ensureReady,
+    install,
+    failSetup: () => setup.reject(new Error("download failed")),
+    completeSetup: () => {
+      ready = true;
+      setup.resolve();
+    },
+    started,
+    released,
+    releaseBrowser,
+    disable: () => {
+      enabled = false;
+    },
+    uninstall: () => {
+      ready = false;
+    },
+  };
+}
+const operation = async () => ({ content: "ok", isError: false });
+
+test("browser session reuses one automation slot and isolates conversations", async () => {
+  const f = fixture();
+  await f.lease.runBrowser(context, operation);
+  await f.lease.runBrowser(context, operation);
+  expect(f.started).toHaveBeenCalledTimes(1);
+  await expect(
+    f.lease.runBrowser({ ...context, conversationId: "conv-456" }, operation),
+  ).rejects.toThrow("Another conversation");
+  expect(f.released).not.toHaveBeenCalled();
+  await f.lease.runBrowser(context, operation, true);
+  expect(f.released).toHaveBeenCalledTimes(1);
+});
+
+for (const lose of ["disable", "uninstall"] as const) {
+  test(`${lose} blocks startup, reuse and startup races`, async () => {
+    const unavailable = fixture();
+    unavailable[lose]();
+    if (lose === "uninstall") {
+      unavailable.ensureReady.mockRejectedValueOnce(new Error("setup failed"));
+    }
+    await expect(
+      unavailable.lease.runBrowser(context, operation),
+    ).rejects.toThrow();
+    expect(unavailable.started).not.toHaveBeenCalled();
+    const race = fixture();
+    race.started.mockImplementationOnce(async () => race[lose]());
+    const callback = mock(operation);
+    await expect(race.lease.runBrowser(context, callback)).rejects.toThrow();
+    expect(callback).not.toHaveBeenCalled();
+    expect(race.released).toHaveBeenCalledTimes(1);
+    const reused = fixture();
+    await reused.lease.runBrowser(context, operation);
+    reused[lose]();
+    await expect(reused.lease.runBrowser(context, callback)).rejects.toThrow();
+    expect(callback).not.toHaveBeenCalled();
+    expect(reused.released).toHaveBeenCalledTimes(1);
+  });
+}
+
+test("cancellation stops running and queued browser commands without blocking a new session", async () => {
+  const f = fixture();
+  const started = Promise.withResolvers<void>();
+  const abort = new AbortController();
+  const running = f.lease
+    .runBrowser({ ...context, signal: abort.signal }, async (signal) => {
+      started.resolve();
+      await new Promise<void>((_resolve, reject) => {
+        signal.addEventListener("abort", () => reject(new Error("aborted")), {
+          once: true,
+        });
+      });
+      return operation();
+    })
+    .catch((error: Error) => error);
+  await started.promise;
+  const callback = mock(operation);
+  const queued = f.lease.runBrowser(context, callback);
+  abort.abort();
+  expect(await running).toBeInstanceOf(Error);
+  expect((await queued).isError).toBe(true);
+  expect(callback).not.toHaveBeenCalled();
+  await f.lease.runBrowser(context, callback);
+  expect(callback).toHaveBeenCalledTimes(1);
+});
+
+test("failed browser cleanup retains ownership until release succeeds", async () => {
+  const f = fixture();
+  await f.lease.runBrowser(context, operation);
+  f.releaseBrowser.mockRejectedValueOnce(new Error("Chrome busy"));
+  await expect(f.lease.runBrowser(context, operation, true)).rejects.toThrow(
+    "Chrome busy",
+  );
+  expect(f.released).not.toHaveBeenCalled();
+  await f.lease.runBrowser(context, operation, true);
+  expect(f.released).toHaveBeenCalledTimes(1);
+});
+
+test("first browser call waits for one shared install then executes exactly once", async () => {
+  const f = fixture();
+  f.uninstall();
+  const callback = mock(operation);
+  const first = f.lease.runBrowser(context, callback);
+  await Bun.sleep(0);
+  expect(f.install).toHaveBeenCalledTimes(1);
+  expect(f.started).not.toHaveBeenCalled();
+  expect(callback).not.toHaveBeenCalled();
+  f.completeSetup();
+  expect((await first).isError).toBe(false);
+  expect(callback).toHaveBeenCalledTimes(1);
+  await f.lease.runBrowser(context, callback);
+  expect(f.install).toHaveBeenCalledTimes(1);
+  expect(f.started).toHaveBeenCalledTimes(1);
+});
+
+for (const interrupt of ["cancel", "disable", "failure"] as const) {
+  test(`${interrupt} during setup prevents a delayed browser action`, async () => {
+    const f = fixture();
+    f.uninstall();
+    const abort = new AbortController();
+    const callback = mock(operation);
+    const result = f.lease
+      .runBrowser({ ...context, signal: abort.signal }, callback)
+      .catch((error: unknown) => error);
+    await Bun.sleep(0);
+    if (interrupt === "cancel") {
+      abort.abort();
+    } else if (interrupt === "disable") {
+      f.disable();
+    } else {
+      f.failSetup();
+    }
+    expect(await result).toBeInstanceOf(Error);
+    f.completeSetup();
+    await Bun.sleep(0);
+    expect(f.started).not.toHaveBeenCalled();
+    expect(callback).not.toHaveBeenCalled();
+  });
+}
+
+test("disabled, unidentified, cancelled and released browser calls cannot install", async () => {
+  const f = fixture();
+  f.uninstall();
+  const cancelled = new AbortController();
+  cancelled.abort();
+  for (const caller of [
+    { ...context, trustClass: "unknown" as const },
+    { ...context, sourceActorPrincipalId: undefined },
+    { ...context, signal: cancelled.signal },
+  ]) {
+    await expect(f.lease.runBrowser(caller, operation)).rejects.toThrow();
+  }
+  await f.lease.runBrowser(context, operation, true);
+  f.disable();
+  await expect(f.lease.runBrowser(context, operation)).rejects.toThrow();
+  expect(f.ensureReady).not.toHaveBeenCalled();
+});