@vellumai/assistant 0.12.2-staging.1 → 0.12.2-staging.3

package/src/daemon/host-cu-proxy.ts

@@ -48,8 +48,8 @@ const SCREENSHOT_OMITTED_MESSAGE =
   "Screenshot omitted: the accessibility tree above is current. Pass include_screenshot: true on computer_use_observe to see the screen.";
 
 // computer_use_key combos that change only selection/cursor/clipboard state.
-// The AX tree models none of these, so they always produce an empty diff —
-// exempt them from the "NO VISIBLE EFFECT" signal (mirrors computer_use_wait).
+// The AX tree models none of these, so they always produce an empty diff and
+// are exempt from the unchanged-tree signal.
 // Stored in canonical form (see canonicalizeKeyCombo): modifier aliases
 // normalized and ordered, so `cmd + a`, `command+a`, `alt+tab`, `tab+shift`
 // all match.
@@ -129,12 +129,28 @@ export interface ActionRecord {
   reasoning?: string;
 }
 
+// Steps whose empty diff says nothing about whether they worked: waiting and
+// observing change nothing by design, and an AppleScript reports its own
+// outcome in the execution result, often for an app whose content the tree
+// cannot see. They neither warn nor move the unchanged streak.
+const NO_AX_DIFF_TOOLS = new Set([
+  "computer_use_wait",
+  "computer_use_observe",
+  "computer_use_run_applescript",
+]);
+
+type ActionIdentity = Pick<ActionRecord, "toolName" | "input">;
+
+function isNoDiffTool(action: ActionIdentity | undefined): boolean {
+  return action !== undefined && NO_AX_DIFF_TOOLS.has(action.toolName);
+}
+
 /**
  * True when `action` is a computer_use_key press whose key only mutates
- * selection/cursor/clipboard state — changes the AX tree cannot represent, so
+ * selection/cursor/clipboard state: changes the AX tree cannot represent, so
  * an empty diff is expected rather than a sign the action did nothing.
  */
-function isNoDiffKeyAction(action: ActionRecord | undefined): boolean {
+function isNoDiffKeyAction(action: ActionIdentity | undefined): boolean {
   if (action?.toolName !== "computer_use_key") {
     return false;
   }
@@ -225,6 +241,11 @@ export class HostCuProxy {
    * that reset cleared.
    */
   private _resetGeneration = 0;
+  /**
+   * Desktops a request has been sent to since the last reset, keyed by
+   * `observedTargetKey`, so the end of a task can reach each one.
+   */
+  private _dispatchedTargets = new Map<string, string | undefined>();
   /**
    * Owned request IDs mapped to whether their observation is scoped, whether
    * the helper was told to skip the screenshot, and when the request was
@@ -240,6 +261,7 @@ export class HostCuProxy {
       resetGeneration: number;
       dispatchedAt: number;
       toolName: string;
+      input: Record<string, unknown>;
       step: number;
     }
   >();
@@ -442,6 +464,7 @@ export class HostCuProxy {
         detachAbort = () => signal.removeEventListener("abort", onAbort);
       }
 
+      this._dispatchedTargets.set(targetKey, resolvedTargetClientId);
       this._ownedRequests.set(requestId, {
         scoped: scopedObservation,
         screenshotSkipped,
@@ -449,6 +472,7 @@ export class HostCuProxy {
         resetGeneration: this._resetGeneration,
         dispatchedAt: Date.now(),
         toolName,
+        input,
         step: stepNumber,
       });
 
@@ -532,7 +556,7 @@ export class HostCuProxy {
     }
 
     // A targeted snapshot has no comparable action/diff baseline; neither it
-    // nor the first desktop observation after it can imply "no visible effect".
+    // nor the first desktop observation after it can imply an unchanged tree.
     // A response dispatched before the last reset belongs to a finished run,
     // so it must not change any state the new run has started building.
     const fromCurrentRun =
@@ -542,6 +566,11 @@ export class HostCuProxy {
       this._consecutiveUnchangedSteps = 0;
     }
     const prevAXTree = this._previousAXTree;
+    // Judge the empty diff against the action this request carried, for the
+    // same reason the timings line does.
+    const action: ActionIdentity | undefined = owned
+      ? { toolName: owned.toolName, input: owned.input }
+      : this.lastRecordedAction();
     const comparableObservation = scopedObservation
       ? { ...observation, axDiff: undefined, secondaryWindows: undefined }
       : observation;
@@ -551,7 +580,7 @@ export class HostCuProxy {
       !scopedObservation &&
       owned?.toolName !== POINT_AT_PROXY_TOOL
     ) {
-      this.updateStateFromObservation(comparableObservation);
+      this.updateStateFromObservation(comparableObservation, action);
       // A desktop has had its first look only once pixels from it arrived,
       // so a failed capture leaves the next request asking again.
       if (owned && observation.screenshot) {
@@ -562,6 +591,7 @@ export class HostCuProxy {
       comparableObservation,
       prevAXTree,
       owned?.screenshotSkipped ?? false,
+      action,
     );
     interaction.rpcResolve(result);
     return result;
@@ -592,8 +622,36 @@ export class HostCuProxy {
     }
   }
 
+  /**
+   * Finish the task: tell every desktop this task drove that it is over, then
+   * reset. The notice rides `host_cu_cancel` with a fresh request ID, which
+   * matches nothing in flight; the helper treats any cancel as the run
+   * stopping and puts the pointer back where the user left it, instead of
+   * waiting out its idle fallback.
+   */
+  endTask(conversationId: string): void {
+    for (const targetClientId of this._dispatchedTargets.values()) {
+      try {
+        broadcastMessage(
+          {
+            type: "host_cu_cancel",
+            requestId: uuid(),
+            conversationId,
+            ...(targetClientId != null ? { targetClientId } : {}),
+          },
+          conversationId,
+          { targetClientId },
+        );
+      } catch {
+        // Best-effort: the helper still returns the pointer after its idle delay.
+      }
+    }
+    this.reset();
+  }
+
   /** Reset all CU state. Called on terminal tools (computer_use_done, etc.). */
   reset(): void {
+    this._dispatchedTargets.clear();
     this._stepCount = 0;
     this._previousAXTree = undefined;
     this._consecutiveUnchangedSteps = 0;
@@ -604,20 +662,24 @@ export class HostCuProxy {
 
   /**
    * Whether the request about to be dispatched should carry a screenshot. The
-   * accessibility tree comes back every step. Pixels come back on the first
-   * look since the last reset, for a window- or display-scoped capture, and
-   * when the model asks for them. Otherwise whether the tree is enough is the
-   * model's call.
+   * accessibility tree comes back every step. Pixels come back after every
+   * action, so the model sees what its action did, on the first look since
+   * the last reset, for a window- or display-scoped capture, and when the
+   * model asks for them. Only a plain observation leaves whether the tree is
+   * enough to the model.
    */
   private shouldAttachScreenshot(
     toolName: string,
     input: Record<string, unknown>,
     targetKey: string,
   ): boolean {
+    if (toolName !== "computer_use_observe") {
+      return true;
+    }
     return (
       !this._observedTargets.has(targetKey) ||
       hasCaptureTarget(input) ||
-      (toolName === "computer_use_observe" && input.include_screenshot === true)
+      input.include_screenshot === true
     );
   }
 
@@ -634,6 +696,7 @@ export class HostCuProxy {
     obs: CuObservationResult,
     previousAXTree?: string,
     screenshotSkipped = false,
+    action: ActionIdentity | undefined = this.lastRecordedAction(),
   ): ToolExecutionResult {
     const prevTree = previousAXTree;
     const parts: string[] = [];
@@ -652,24 +715,17 @@ export class HostCuProxy {
       parts.push(obs.axDiff);
       parts.push("");
     } else if (prevTree != null && obs.axTree != null) {
-      const lastAction =
-        this._actionHistory.length > 0
-          ? this._actionHistory[this._actionHistory.length - 1]
-          : undefined;
-      const isWaitAction = lastAction?.toolName === "computer_use_wait";
-      const isNoDiffKey = isNoDiffKeyAction(lastAction);
-
-      if (!isWaitAction && !isNoDiffKey) {
+      if (!isNoDiffTool(action) && !isNoDiffKeyAction(action)) {
         if (
           this._consecutiveUnchangedSteps >=
           CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD
         ) {
           parts.push(
-            `WARNING: ${this._consecutiveUnchangedSteps} consecutive actions had NO VISIBLE EFFECT on the UI. You MUST try a completely different approach.`,
+            `WARNING: the accessibility tree did not change across ${this._consecutiveUnchangedSteps} consecutive actions. If the screenshot does not show them working either, try a different approach.`,
           );
         } else {
           parts.push(
-            "Your last action had NO VISIBLE EFFECT on the UI. Try something different.",
+            "The accessibility tree did not change after your last action. Apps that draw their own content (timelines, canvases, spreadsheet grids) change without it, so check the screenshot before deciding the action did nothing.",
           );
         }
         parts.push("");
@@ -784,19 +840,26 @@ export class HostCuProxy {
   // Private helpers
   // ---------------------------------------------------------------------------
 
-  private updateStateFromObservation(obs: CuObservationResult): void {
+  private lastRecordedAction(): ActionRecord | undefined {
+    return this._actionHistory.at(-1);
+  }
+
+  private updateStateFromObservation(
+    obs: CuObservationResult,
+    action: ActionIdentity | undefined,
+  ): void {
     if (this._stepCount > 0) {
-      const lastAction =
-        this._actionHistory.length > 0
-          ? this._actionHistory[this._actionHistory.length - 1]
-          : undefined;
-      if (obs.axDiff != null || isNoDiffKeyAction(lastAction)) {
+      if (obs.axDiff != null || isNoDiffKeyAction(action)) {
         // A real diff, or an exempt key whose effect is invisible by design,
-        // breaks the no-effect streak — clear it rather than preserving a
-        // stale count so an intervening cmd+a can't bridge two no-op actions
-        // into a false "consecutive" escalation.
+        // breaks the no-effect streak. Clearing it rather than preserving a
+        // stale count keeps an intervening cmd+a from bridging two no-op
+        // actions into a false "consecutive" escalation.
         this._consecutiveUnchangedSteps = 0;
-      } else if (this._previousAXTree != null && obs.axTree != null) {
+      } else if (
+        !isNoDiffTool(action) &&
+        this._previousAXTree != null &&
+        obs.axTree != null
+      ) {
         this._consecutiveUnchangedSteps++;
       }
     }

package/src/daemon/host-proxy-preactivation.ts

@@ -9,6 +9,8 @@
  *   - `daemon/process-message.ts` (create path, prepareConversationForMessage)
  *   - `daemon/conversation-process.ts` `drainSingleMessage` (re-add after dequeue)
  *   - `daemon/conversation-process.ts` `drainBatch` (re-add after dequeue)
+ *   - `calls/voice-session-bridge.ts` (tool-capable legs of a live-voice
+ *     session opened from the macOS desktop client)
  *
  * The create paths additionally instantiate the proxy itself; that
  * instantiation logic is per-proxy-class and stays inline at each create

package/src/daemon/mcp-reload-service.ts

@@ -14,6 +14,7 @@ import { getMcpServerManager } from "../mcp/manager.js";
 import { migrateLegacyMcpHeaders } from "../mcp/mcp-header-store.js";
 import { signalMcpReloaded } from "../mcp/reload-signal.js";
 import { loadWorkspaceMcpConfig } from "../mcp/workspace-mcp-config.js";
+import { publishMcpChanged } from "../runtime/sync/resource-sync-events.js";
 import { createMcpToolsFromServer } from "../tools/mcp/mcp-tool-factory.js";
 import { registerMcpTools, unregisterAllMcpTools } from "../tools/registry.js";
 import { getLogger } from "../util/logger.js";
@@ -80,6 +81,7 @@ export async function reconcilePluginMcpServers(): Promise<void> {
 }
 
 async function doReload(): Promise<McpReloadResult> {
+  let teardownStarted = false;
   try {
     const manager = getMcpServerManager();
 
@@ -98,6 +100,7 @@ async function doReload(): Promise<McpReloadResult> {
     invalidateConfigCache();
 
     // 2. Stop existing MCP servers + unregister their tools
+    teardownStarted = true;
     await manager.stop();
     unregisterAllMcpTools();
 
@@ -160,5 +163,9 @@ async function doReload(): Promise<McpReloadResult> {
     const error = err instanceof Error ? err.message : String(err);
     log.error({ err }, "MCP reload failed");
     return { success: false, error };
+  } finally {
+    if (teardownStarted) {
+      publishMcpChanged();
+    }
   }
 }

package/src/daemon/message-types/sync.ts

@@ -18,6 +18,7 @@ export const SYNC_TAGS = {
   appsList: "apps:list",
   documentsList: "documents:list",
   pluginsList: "plugins:list",
+  mcpList: "mcp:list",
   conversationsList: "conversations:list",
   /** Activation-checklist progress: task launches, step counts, completions. */
   activationProgress: "activation:progress",

package/src/daemon/providers-setup.ts

@@ -1,6 +1,7 @@
 import { maybeDefaultSpeechToManaged } from "../config/managed-speech-defaults.js";
 import { rehydratePlatformCredentials } from "../config/platform-rehydration.js";
 import type { AssistantConfig } from "../config/types.js";
+import { getMcpServerManager } from "../mcp/manager.js";
 import { startConfiguredMcpServers } from "../mcp/startup.js";
 import { gmailMessagingProvider } from "../messaging/providers/gmail/adapter.js";
 import { outlookMessagingProvider } from "../messaging/providers/outlook/adapter.js";
@@ -9,6 +10,7 @@ import { telegramBotMessagingProvider } from "../messaging/providers/telegram-bo
 import { whatsappMessagingProvider } from "../messaging/providers/whatsapp/adapter.js";
 import { registerMessagingProvider } from "../messaging/registry.js";
 import { initializeProviders } from "../providers/registry.js";
+import { publishMcpChanged } from "../runtime/sync/resource-sync-events.js";
 import { validateSubagentRoleAllowlists } from "../subagent/validate-allowlists.js";
 import { initializeTools } from "../tools/registry.js";
 import { getLogger } from "../util/logger.js";
@@ -63,7 +65,9 @@ export async function initializeProvidersAndTools(
   // Start MCP servers — workspace-configured and plugin-declared alike —
   // and register their tools. Shared with the schedule worker, which hosts
   // agent turns in its own process and so needs its own connections.
+  getMcpServerManager().setUnexpectedCloseHandler(() => publishMcpChanged());
   await startConfiguredMcpServers();
+  publishMcpChanged();
 
   log.info("Daemon startup: providers and tools initialized");
 }

package/src/desktop/__tests__/fake-desktop.ts

@@ -58,6 +58,7 @@ export function newFakeDesktop(options: FakeDesktopOptions) {
   let vncReady = true;
   let chromiumPath: () => Promise<string> = async () => "/fake/chromium";
   const manager = new DesktopSessionManager({
+    allocateDebugPort: async () => 9222,
     spawn: (role, request) => {
       if (options.failSpawn?.includes(role)) {
         throw new Error(`spawn ${role} failed`);

package/src/desktop/desktop-automation-lease.test.ts

@@ -0,0 +1,208 @@
+import { afterEach, expect, mock, test } from "bun:test";
+
+import { DesktopAutomationLease } from "./desktop-automation-lease.js";
+import { DesktopDependencyInstaller } from "./desktop-dependencies.js";
+import type { DesktopSessionManager } from "./desktop-session-manager.js";
+
+const context = {
+  conversationId: "conv-123",
+  sourceActorPrincipalId: "user-123",
+  trustClass: "guardian" as const,
+  workingDir: "/tmp",
+};
+const cleanups: DesktopAutomationLease[] = [];
+afterEach(async () => {
+  await Promise.all(
+    cleanups
+      .splice(0)
+      .map((lease) => lease.runBrowser(context, operation, true)),
+  );
+});
+function fixture() {
+  let enabled = true;
+  let ready = true;
+  const setup = Promise.withResolvers<void>();
+  const install = mock(() => setup.promise);
+  const installer = new DesktopDependencyInstaller({
+    supported: () => true,
+    ready: () => ready,
+    install,
+    notify: async () => {},
+  });
+  const ensureReady = mock((signal: AbortSignal) =>
+    installer.ensureReady(signal),
+  );
+  const started = mock(async () => {});
+  const released = mock(() => {});
+  const releaseBrowser = mock(async () => {});
+  const lease = new DesktopAutomationLease({
+    enabled: () => enabled,
+    ready: () => ready,
+    ensureReady,
+    manager: () =>
+      ({
+        browser: { release: releaseBrowser },
+        acquireAutomationSlot: () => ({ ok: true }),
+        releaseAutomationSlot: released,
+        ensureDesktopRunning: started,
+      }) as unknown as DesktopSessionManager,
+  });
+  cleanups.push(lease);
+  return {
+    lease,
+    ensureReady,
+    install,
+    failSetup: () => setup.reject(new Error("download failed")),
+    completeSetup: () => {
+      ready = true;
+      setup.resolve();
+    },
+    started,
+    released,
+    releaseBrowser,
+    disable: () => {
+      enabled = false;
+    },
+    uninstall: () => {
+      ready = false;
+    },
+  };
+}
+const operation = async () => ({ content: "ok", isError: false });
+
+test("browser session reuses one automation slot and isolates conversations", async () => {
+  const f = fixture();
+  await f.lease.runBrowser(context, operation);
+  await f.lease.runBrowser(context, operation);
+  expect(f.started).toHaveBeenCalledTimes(1);
+  await expect(
+    f.lease.runBrowser({ ...context, conversationId: "conv-456" }, operation),
+  ).rejects.toThrow("Another conversation");
+  expect(f.released).not.toHaveBeenCalled();
+  await f.lease.runBrowser(context, operation, true);
+  expect(f.released).toHaveBeenCalledTimes(1);
+});
+
+for (const lose of ["disable", "uninstall"] as const) {
+  test(`${lose} blocks startup, reuse and startup races`, async () => {
+    const unavailable = fixture();
+    unavailable[lose]();
+    if (lose === "uninstall") {
+      unavailable.ensureReady.mockRejectedValueOnce(new Error("setup failed"));
+    }
+    await expect(
+      unavailable.lease.runBrowser(context, operation),
+    ).rejects.toThrow();
+    expect(unavailable.started).not.toHaveBeenCalled();
+    const race = fixture();
+    race.started.mockImplementationOnce(async () => race[lose]());
+    const callback = mock(operation);
+    await expect(race.lease.runBrowser(context, callback)).rejects.toThrow();
+    expect(callback).not.toHaveBeenCalled();
+    expect(race.released).toHaveBeenCalledTimes(1);
+    const reused = fixture();
+    await reused.lease.runBrowser(context, operation);
+    reused[lose]();
+    await expect(reused.lease.runBrowser(context, callback)).rejects.toThrow();
+    expect(callback).not.toHaveBeenCalled();
+    expect(reused.released).toHaveBeenCalledTimes(1);
+  });
+}
+
+test("cancellation stops running and queued browser commands without blocking a new session", async () => {
+  const f = fixture();
+  const started = Promise.withResolvers<void>();
+  const abort = new AbortController();
+  const running = f.lease
+    .runBrowser({ ...context, signal: abort.signal }, async (signal) => {
+      started.resolve();
+      await new Promise<void>((_resolve, reject) => {
+        signal.addEventListener("abort", () => reject(new Error("aborted")), {
+          once: true,
+        });
+      });
+      return operation();
+    })
+    .catch((error: Error) => error);
+  await started.promise;
+  const callback = mock(operation);
+  const queued = f.lease.runBrowser(context, callback);
+  abort.abort();
+  expect(await running).toBeInstanceOf(Error);
+  expect((await queued).isError).toBe(true);
+  expect(callback).not.toHaveBeenCalled();
+  await f.lease.runBrowser(context, callback);
+  expect(callback).toHaveBeenCalledTimes(1);
+});
+
+test("failed browser cleanup retains ownership until release succeeds", async () => {
+  const f = fixture();
+  await f.lease.runBrowser(context, operation);
+  f.releaseBrowser.mockRejectedValueOnce(new Error("Chrome busy"));
+  await expect(f.lease.runBrowser(context, operation, true)).rejects.toThrow(
+    "Chrome busy",
+  );
+  expect(f.released).not.toHaveBeenCalled();
+  await f.lease.runBrowser(context, operation, true);
+  expect(f.released).toHaveBeenCalledTimes(1);
+});
+
+test("first browser call waits for one shared install then executes exactly once", async () => {
+  const f = fixture();
+  f.uninstall();
+  const callback = mock(operation);
+  const first = f.lease.runBrowser(context, callback);
+  await Bun.sleep(0);
+  expect(f.install).toHaveBeenCalledTimes(1);
+  expect(f.started).not.toHaveBeenCalled();
+  expect(callback).not.toHaveBeenCalled();
+  f.completeSetup();
+  expect((await first).isError).toBe(false);
+  expect(callback).toHaveBeenCalledTimes(1);
+  await f.lease.runBrowser(context, callback);
+  expect(f.install).toHaveBeenCalledTimes(1);
+  expect(f.started).toHaveBeenCalledTimes(1);
+});
+
+for (const interrupt of ["cancel", "disable", "failure"] as const) {
+  test(`${interrupt} during setup prevents a delayed browser action`, async () => {
+    const f = fixture();
+    f.uninstall();
+    const abort = new AbortController();
+    const callback = mock(operation);
+    const result = f.lease
+      .runBrowser({ ...context, signal: abort.signal }, callback)
+      .catch((error: unknown) => error);
+    await Bun.sleep(0);
+    if (interrupt === "cancel") {
+      abort.abort();
+    } else if (interrupt === "disable") {
+      f.disable();
+    } else {
+      f.failSetup();
+    }
+    expect(await result).toBeInstanceOf(Error);
+    f.completeSetup();
+    await Bun.sleep(0);
+    expect(f.started).not.toHaveBeenCalled();
+    expect(callback).not.toHaveBeenCalled();
+  });
+}
+
+test("disabled, unidentified, cancelled and released browser calls cannot install", async () => {
+  const f = fixture();
+  f.uninstall();
+  const cancelled = new AbortController();
+  cancelled.abort();
+  for (const caller of [
+    { ...context, trustClass: "unknown" as const },
+    { ...context, sourceActorPrincipalId: undefined },
+    { ...context, signal: cancelled.signal },
+  ]) {
+    await expect(f.lease.runBrowser(caller, operation)).rejects.toThrow();
+  }
+  await f.lease.runBrowser(context, operation, true);
+  f.disable();
+  await expect(f.lease.runBrowser(context, operation)).rejects.toThrow();
+  expect(f.ensureReady).not.toHaveBeenCalled();
+});