@vellumai/assistant 0.12.1 → 0.12.2-staging.2

package/src/daemon/host-cu-proxy.ts

@@ -42,6 +42,11 @@ const MAX_HISTORY_ENTRIES = 10;
 const LOOP_DETECTION_WINDOW = 3;
 const CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD = 2;
 
+const SEQUENCE_TOOL = "computer_use_sequence";
+
+const SCREENSHOT_OMITTED_MESSAGE =
+  "Screenshot omitted: the accessibility tree above is current. Pass include_screenshot: true on computer_use_observe to see the screen.";
+
 // computer_use_key combos that change only selection/cursor/clipboard state.
 // The AX tree models none of these, so they always produce an empty diff —
 // exempt them from the "NO VISIBLE EFFECT" signal (mirrors computer_use_wait).
@@ -113,6 +118,8 @@ export interface CuObservationResult {
   executionResult?: string;
   executionError?: string;
   userGuidance?: string;
+  /** Per-phase helper timings in milliseconds, absent on older helpers. */
+  timings?: Record<string, number>;
 }
 
 export interface ActionRecord {
@@ -161,6 +168,40 @@ function actionSignature(record: ActionRecord): string {
   return `${record.toolName}:${JSON.stringify(record.input)}`;
 }
 
+/**
+ * The key a desktop's first look is tracked under. An untargeted request has
+ * no client id, so it gets a key no real id can take.
+ */
+function observedTargetKey(targetClientId: string | undefined): string {
+  return targetClientId === undefined
+    ? "\u0000untargeted"
+    : `client:${targetClientId}`;
+}
+
+/** Whether `input` scopes the observation to one window or display. */
+function hasCaptureTarget(input: Record<string, unknown>): boolean {
+  return (
+    Object.hasOwn(input, "capture_window_id") ||
+    Object.hasOwn(input, "captureWindowId") ||
+    Object.hasOwn(input, "captureDisplayId")
+  );
+}
+
+/**
+ * Whether the resolved client claimed `capability` on its connection. An
+ * untargeted request has no client to vouch for it, so it never qualifies.
+ */
+function clientAdvertises(
+  targetClientId: string | undefined,
+  capability: "host_cu_window_capture" | "host_cu_sequence",
+): boolean {
+  const client =
+    targetClientId == null
+      ? undefined
+      : assistantEventHub.getClientById(targetClientId);
+  return client?.capabilities.includes(capability) ?? false;
+}
+
 // ---------------------------------------------------------------------------
 // HostCuProxy
 // ---------------------------------------------------------------------------
@@ -172,8 +213,41 @@ export class HostCuProxy {
   private _previousAXTree: string | undefined;
   private _consecutiveUnchangedSteps = 0;
   private _actionHistory: ActionRecord[] = [];
-  /** Owned request IDs mapped to whether their observation is scoped. */
-  private _ownedRequests = new Map<string, boolean>();
+  /**
+   * Desktops an unscoped observation has come back from since the last reset,
+   * keyed by `observedTargetKey`. A first look is per desktop: having seen one
+   * machine says nothing about another the conversation switches to.
+   */
+  private _observedTargets = new Set<string>();
+  /**
+   * Bumped on every reset. A request carries the value it was dispatched
+   * under, so an observation that lands after a reset cannot restore state
+   * that reset cleared.
+   */
+  private _resetGeneration = 0;
+  /**
+   * Desktops a request has been sent to since the last reset, keyed by
+   * `observedTargetKey`, so the end of a task can reach each one.
+   */
+  private _dispatchedTargets = new Map<string, string | undefined>();
+  /**
+   * Owned request IDs mapped to whether their observation is scoped, whether
+   * the helper was told to skip the screenshot, and when the request was
+   * dispatched. The dispatch time gives the round trip, which is the part of
+   * a step the helper's own timings cannot see.
+   */
+  private _ownedRequests = new Map<
+    string,
+    {
+      scoped: boolean;
+      screenshotSkipped: boolean;
+      targetKey: string;
+      resetGeneration: number;
+      dispatchedAt: number;
+      toolName: string;
+      step: number;
+    }
+  >();
 
   constructor(maxSteps = loadConfig().maxStepsPerSession) {
     this._maxSteps = maxSteps;
@@ -288,11 +362,7 @@ export class HostCuProxy {
       }
       // Never dispatch this option to a legacy executor: it would forward the
       // unknown snake-case key and silently capture the entire desktop.
-      const client =
-        resolvedTargetClientId == null
-          ? undefined
-          : assistantEventHub.getClientById(resolvedTargetClientId);
-      if (!client?.capabilities.includes("host_cu_window_capture")) {
+      if (!clientAdvertises(resolvedTargetClientId, "host_cu_window_capture")) {
         return Promise.resolve({
           content:
             "Window-only observation requires a connected client advertising host_cu_window_capture; update the desktop app before retrying. No capture was requested.",
@@ -300,14 +370,40 @@ export class HostCuProxy {
         });
       }
     }
-    const scopedObservation =
-      hasWindowTarget ||
-      Object.hasOwn(input, "captureWindowId") ||
-      Object.hasOwn(input, "captureDisplayId");
+    // An older helper answers an unknown tool name as done, which would end
+    // the session with nothing run.
+    if (
+      toolName === SEQUENCE_TOOL &&
+      !clientAdvertises(resolvedTargetClientId, "host_cu_sequence")
+    ) {
+      return Promise.resolve({
+        content:
+          "Batched actions require a desktop app that supports computer_use_sequence; update the desktop app, or send the actions one at a time. Nothing was run.",
+        isError: true,
+      });
+    }
+    const scopedObservation = hasCaptureTarget(input);
     if (scopedObservation) {
       this._previousAXTree = undefined;
       this._consecutiveUnchangedSteps = 0;
     }
+    const targetKey = observedTargetKey(resolvedTargetClientId);
+    // Pointing never reaches the helper's capture path, so its input goes out
+    // as given. Every other request carries the screenshot decision made now,
+    // at dispatch, and never the model-facing snake_case key.
+    let dispatchInput = input;
+    let screenshotSkipped = false;
+    if (toolName !== POINT_AT_PROXY_TOOL) {
+      const { include_screenshot: _includeScreenshot, ...rest } = input;
+      screenshotSkipped = !this.shouldAttachScreenshot(
+        toolName,
+        input,
+        targetKey,
+      );
+      dispatchInput = screenshotSkipped
+        ? { ...rest, includeScreenshot: false }
+        : rest;
+    }
     const requestId = uuid();
 
     return new Promise<ToolExecutionResult>((resolve, reject) => {
@@ -351,7 +447,16 @@ export class HostCuProxy {
         detachAbort = () => signal.removeEventListener("abort", onAbort);
       }
 
-      this._ownedRequests.set(requestId, scopedObservation);
+      this._dispatchedTargets.set(targetKey, resolvedTargetClientId);
+      this._ownedRequests.set(requestId, {
+        scoped: scopedObservation,
+        screenshotSkipped,
+        targetKey,
+        resetGeneration: this._resetGeneration,
+        dispatchedAt: Date.now(),
+        toolName,
+        step: stepNumber,
+      });
 
       pendingInteractions.register(requestId, {
         conversationId,
@@ -375,7 +480,7 @@ export class HostCuProxy {
             requestId,
             conversationId,
             toolName,
-            input,
+            input: dispatchInput,
             stepNumber,
             reasoning,
             ...(resolvedTargetClientId != null
@@ -403,7 +508,8 @@ export class HostCuProxy {
     requestId: string,
     observation: CuObservationResult,
   ): ToolExecutionResult | undefined {
-    const scopedObservation = this._ownedRequests.get(requestId) ?? false;
+    const owned = this._ownedRequests.get(requestId);
+    const scopedObservation = owned?.scoped ?? false;
     this._ownedRequests.delete(requestId);
     const interaction = pendingInteractions.resolve(requestId, "answered");
     if (!interaction?.rpcResolve) {
@@ -411,9 +517,33 @@ export class HostCuProxy {
       return undefined;
     }
 
+    // Label the line from what this request was dispatched with, never from
+    // current proxy state. One model response can dispatch several CU tools,
+    // and the agent loop runs them concurrently, so a later call can advance
+    // the history and the step count before an earlier observation lands.
+    // Reading them here would file each measurement under whichever tool was
+    // dispatched last, and would mislabel computer_use_point_at, which never
+    // records an action at all.
+    if (owned) {
+      log.info(
+        {
+          requestId,
+          toolName: owned.toolName,
+          step: owned.step,
+          roundTripMs: Date.now() - owned.dispatchedAt,
+          ...(observation.timings ? { helper: observation.timings } : {}),
+        },
+        "Host CU step timings",
+      );
+    }
+
     // A targeted snapshot has no comparable action/diff baseline; neither it
     // nor the first desktop observation after it can imply "no visible effect".
-    if (scopedObservation) {
+    // A response dispatched before the last reset belongs to a finished run,
+    // so it must not change any state the new run has started building.
+    const fromCurrentRun =
+      owned === undefined || owned.resetGeneration === this._resetGeneration;
+    if (scopedObservation && fromCurrentRun) {
       this._previousAXTree = undefined;
       this._consecutiveUnchangedSteps = 0;
     }
@@ -421,10 +551,24 @@ export class HostCuProxy {
     const comparableObservation = scopedObservation
       ? { ...observation, axDiff: undefined, secondaryWindows: undefined }
       : observation;
-    if (!scopedObservation) {
+    // Pointing observes nothing, so its response leaves the screen state alone.
+    if (
+      fromCurrentRun &&
+      !scopedObservation &&
+      owned?.toolName !== POINT_AT_PROXY_TOOL
+    ) {
       this.updateStateFromObservation(comparableObservation);
+      // A desktop has had its first look only once pixels from it arrived,
+      // so a failed capture leaves the next request asking again.
+      if (owned && observation.screenshot) {
+        this._observedTargets.add(owned.targetKey);
+      }
     }
-    const result = this.formatObservation(comparableObservation, prevAXTree);
+    const result = this.formatObservation(
+      comparableObservation,
+      prevAXTree,
+      owned?.screenshotSkipped ?? false,
+    );
     interaction.rpcResolve(result);
     return result;
   }
@@ -454,12 +598,65 @@ export class HostCuProxy {
     }
   }
 
+  /**
+   * Finish the task: tell every desktop this task drove that it is over, then
+   * reset. The notice rides `host_cu_cancel` with a fresh request ID, which
+   * matches nothing in flight; the helper treats any cancel as the run
+   * stopping and puts the pointer back where the user left it, instead of
+   * waiting out its idle fallback.
+   */
+  endTask(conversationId: string): void {
+    for (const targetClientId of this._dispatchedTargets.values()) {
+      try {
+        broadcastMessage(
+          {
+            type: "host_cu_cancel",
+            requestId: uuid(),
+            conversationId,
+            ...(targetClientId != null ? { targetClientId } : {}),
+          },
+          conversationId,
+          { targetClientId },
+        );
+      } catch {
+        // Best-effort: the helper still returns the pointer after its idle delay.
+      }
+    }
+    this.reset();
+  }
+
   /** Reset all CU state. Called on terminal tools (computer_use_done, etc.). */
   reset(): void {
+    this._dispatchedTargets.clear();
     this._stepCount = 0;
     this._previousAXTree = undefined;
     this._consecutiveUnchangedSteps = 0;
     this._actionHistory = [];
+    this._observedTargets.clear();
+    this._resetGeneration++;
+  }
+
+  /**
+   * Whether the request about to be dispatched should carry a screenshot. The
+   * accessibility tree comes back every step. Pixels come back after every
+   * action, so the model sees what its action did, on the first look since
+   * the last reset, for a window- or display-scoped capture, and when the
+   * model asks for them. Only a plain observation leaves whether the tree is
+   * enough to the model.
+   */
+  private shouldAttachScreenshot(
+    toolName: string,
+    input: Record<string, unknown>,
+    targetKey: string,
+  ): boolean {
+    if (toolName !== "computer_use_observe") {
+      return true;
+    }
+    return (
+      !this._observedTargets.has(targetKey) ||
+      hasCaptureTarget(input) ||
+      input.include_screenshot === true
+    );
   }
 
   // ---------------------------------------------------------------------------
@@ -474,6 +671,7 @@ export class HostCuProxy {
   formatObservation(
     obs: CuObservationResult,
     previousAXTree?: string,
+    screenshotSkipped = false,
   ): ToolExecutionResult {
     const prevTree = previousAXTree;
     const parts: string[] = [];
@@ -552,6 +750,15 @@ export class HostCuProxy {
       parts.push(...screenshotMeta);
     }
 
+    // Only a deliberate omission is announced, and only beside a tree the line
+    // can point at. A step that asked for pixels and got none is a capture
+    // failure, which this line must not disguise. A refused or failed step
+    // still carries the line, since that is where asking for pixels helps most.
+    const isError = obs.executionError != null;
+    if (screenshotSkipped && !obs.screenshot && obs.axTree) {
+      parts.push("", SCREENSHOT_OMITTED_MESSAGE);
+    }
+
     const content = parts.join("\n").trim() || "Action executed";
 
     const contentBlocks: ContentBlock[] = [];
@@ -566,8 +773,6 @@ export class HostCuProxy {
       });
     }
 
-    const isError = obs.executionError != null;
-
     return {
       content: isError
         ? `Action failed: ${obs.executionError}\n\n${content}`

package/src/daemon/host-file-proxy.ts

@@ -6,10 +6,10 @@ import {
   broadcastMessage,
 } from "../runtime/assistant-event-hub.js";
 import {
-  ambiguousSameUserError,
   enforceSameActorOrErrorResult,
-  pickSameUserAutoResolve,
+  pickMostRecentSameUserClient,
   snapshotHostProxyActorPrincipalId,
+  unavailableSameUserClientError,
 } from "../runtime/auth/same-actor.js";
 import * as pendingInteractions from "../runtime/pending-interactions.js";
 import { readAudioBase64 } from "../tools/shared/filesystem/audio-read.js";
@@ -102,19 +102,17 @@ export class HostFileProxy {
         });
       }
     } else {
-      // Auto-resolve to the unique same-user client. Reject ambiguous
-      // (multi-machine) cases so a single targeted-style request cannot
-      // fan out across the user's machines.
-      const resolved = pickSameUserAutoResolve({
+      resolvedTargetClientId = pickMostRecentSameUserClient({
         hub: assistantEventHub,
         capability: "host_file",
         sourceActorPrincipalId,
       });
-      if (resolved.kind === "ambiguous") {
-        return Promise.resolve(ambiguousSameUserError("host_file"));
+      if (
+        resolvedTargetClientId === undefined &&
+        sourceActorPrincipalId !== undefined
+      ) {
+        return Promise.resolve(unavailableSameUserClientError("host_file"));
       }
-      resolvedTargetClientId =
-        resolved.kind === "match" ? resolved.clientId : undefined;
     }
 
     // Same-user check: targeted host_file requests must be bound to the same

package/src/daemon/host-proxy-preactivation.ts

@@ -105,12 +105,13 @@ export const HOST_PROXY_SKILL_PREACTIVATIONS: ReadonlyArray<{
  */
 /**
  * Capabilities a client asks for on its connection rather than getting from
- * what it is. Both ride the host_cu transport and are answered only by a
+ * what it is. Each rides the host_cu transport and is answered only by a
  * macOS build new enough to send the header (`events-routes.ts`), so the
  * interface alone cannot say whether one is really there.
  */
 const NEGOTIATED_CAPABILITIES: ReadonlySet<HostProxyCapability> = new Set([
   "host_cu_window_capture",
+  "host_cu_sequence",
   "host_cu_annotate",
 ]);
 

package/src/daemon/host-transfer-proxy.ts

@@ -10,10 +10,10 @@ import {
   broadcastMessage,
 } from "../runtime/assistant-event-hub.js";
 import {
-  ambiguousSameUserError,
   enforceSameActorOrErrorResult,
-  pickSameUserAutoResolve,
+  pickMostRecentSameUserClient,
   snapshotHostProxyActorPrincipalId,
+  unavailableSameUserClientError,
 } from "../runtime/auth/same-actor.js";
 import * as pendingInteractions from "../runtime/pending-interactions.js";
 import type { ToolExecutionResult } from "../tools/types.js";
@@ -162,19 +162,17 @@ export class HostTransferProxy {
         });
       }
     } else {
-      // Auto-resolve to the unique same-user client; reject ambiguous
-      // (multi-machine) cases so a single targeted-style transfer cannot
-      // fan out across the user's machines.
-      const resolved = pickSameUserAutoResolve({
+      resolvedTargetClientId = pickMostRecentSameUserClient({
         hub: assistantEventHub,
         capability: "host_file",
         sourceActorPrincipalId,
       });
-      if (resolved.kind === "ambiguous") {
-        return Promise.resolve(ambiguousSameUserError("host_file"));
+      if (
+        resolvedTargetClientId === undefined &&
+        sourceActorPrincipalId !== undefined
+      ) {
+        return Promise.resolve(unavailableSameUserClientError("host_file"));
       }
-      resolvedTargetClientId =
-        resolved.kind === "match" ? resolved.clientId : undefined;
     }
 
     if (resolvedTargetClientId != null) {
@@ -364,19 +362,17 @@ export class HostTransferProxy {
         });
       }
     } else {
-      // Auto-resolve to the unique same-user client; reject ambiguous
-      // (multi-machine) cases so a single targeted-style transfer cannot
-      // fan out across the user's machines.
-      const resolved = pickSameUserAutoResolve({
+      resolvedTargetClientId = pickMostRecentSameUserClient({
         hub: assistantEventHub,
         capability: "host_file",
         sourceActorPrincipalId,
       });
-      if (resolved.kind === "ambiguous") {
-        return Promise.resolve(ambiguousSameUserError("host_file"));
+      if (
+        resolvedTargetClientId === undefined &&
+        sourceActorPrincipalId !== undefined
+      ) {
+        return Promise.resolve(unavailableSameUserClientError("host_file"));
       }
-      resolvedTargetClientId =
-        resolved.kind === "match" ? resolved.clientId : undefined;
     }
 
     if (resolvedTargetClientId != null) {

package/src/daemon/lifecycle.ts

@@ -71,7 +71,6 @@ import { ensureCompleteCustomProfiles } from "../workspace/custom-profile-ensure
 import { ensureDefaultProvider } from "../workspace/default-provider-ensure.js";
 import { WORKSPACE_MIGRATIONS } from "../workspace/migrations/registry.js";
 import { runWorkspaceMigrations } from "../workspace/migrations/runner.js";
-import { startAppSourceWatcher } from "./app-source-watcher.js";
 import { startConfigWatcher } from "./config-watcher.js";
 import { startConversationEvictor } from "./conversation-evictor.js";
 import { writePid } from "./daemon-control.js";
@@ -706,10 +705,6 @@ export async function runDaemon(): Promise<void> {
   // config, and broadcast the relevant resource-changed events to clients.
   startConfigWatcher();
 
-  // Watch app source directories so edits recompile + refresh surfaces across
-  // all conversations.
-  startAppSourceWatcher();
-
   // Start the CLI IPC server. Throws on EADDRINUSE to abort startup when another
   // daemon already holds the socket, so this process never runs background jobs
   // against the shared database as an unmanageable duplicate.

package/src/daemon/mcp-reload-service.ts

@@ -1,11 +1,11 @@
 /**
  * Shared MCP reload business logic.
  *
- * Called by the ConfigWatcher when config.json changes or a reload signal
+ * Called by the ConfigWatcher when mcp.json changes or a reload signal
  * file is detected, so the daemon automatically reconnects MCP servers.
  */
 
-import { getConfig, invalidateConfigCache } from "../config/loader.js";
+import { invalidateConfigCache } from "../config/loader.js";
 import {
   buildEffectiveMcpConfig,
   pluginMcpServersChangedSinceLastBuild,
@@ -13,6 +13,7 @@ import {
 import { getMcpServerManager } from "../mcp/manager.js";
 import { migrateLegacyMcpHeaders } from "../mcp/mcp-header-store.js";
 import { signalMcpReloaded } from "../mcp/reload-signal.js";
+import { loadWorkspaceMcpConfig } from "../mcp/workspace-mcp-config.js";
 import { createMcpToolsFromServer } from "../tools/mcp/mcp-tool-factory.js";
 import { registerMcpTools, unregisterAllMcpTools } from "../tools/registry.js";
 import { getLogger } from "../util/logger.js";
@@ -95,16 +96,15 @@ async function doReload(): Promise<McpReloadResult> {
     //    If the config is broken we abort early, preserving the current
     //    working MCP setup instead of leaving zero servers.
     invalidateConfigCache();
-    const config = getConfig();
 
     // 2. Stop existing MCP servers + unregister their tools
     await manager.stop();
     unregisterAllMcpTools();
 
     // Plugins are re-read here too: installing or removing one changes the
-    // server set exactly like editing config.json does, and both arrive
+    // server set exactly like editing mcp.json does, and both arrive
     // through this same reload.
-    const mcpConfig = buildEffectiveMcpConfig(config.mcp);
+    const mcpConfig = buildEffectiveMcpConfig(loadWorkspaceMcpConfig());
     const serverIds = Object.keys(mcpConfig.servers);
 
     // 3. Restart MCP servers

package/src/daemon/message-types/sync.ts

@@ -13,6 +13,7 @@ export const SYNC_TAGS = {
   assistantConfig: "assistant:self:config",
   assistantSounds: "assistant:self:sounds",
   assistantSchedules: "assistant:self:schedules",
+  assistantDesktop: "assistant:self:desktop",
   assistantTheme: "assistant:self:theme",
   appsList: "apps:list",
   documentsList: "documents:list",

package/src/daemon/orphan-reaper.ts

@@ -3,7 +3,7 @@
  *
  * Tools run commands in their own process group (`detached: true`) and, on
  * timeout/abort, group-kill with `process.kill(-pgid, SIGKILL)` (the bash and
- * host_bash tools, the skill sandbox runner, and the debug-bash route). The
+ * host_bash tools, and the skill sandbox runner). The
  * immediate child is reaped by Bun/libuv, but its descendants — e.g. git's
  * transport helpers or a skill runner's `bun` process — were never spawned by
  * the daemon, so when the group dies they reparent to PID 1. When the daemon

package/src/daemon/providers-setup.ts

@@ -63,7 +63,7 @@ export async function initializeProvidersAndTools(
   // Start MCP servers — workspace-configured and plugin-declared alike —
   // and register their tools. Shared with the schedule worker, which hosts
   // agent turns in its own process and so needs its own connections.
-  await startConfiguredMcpServers(config.mcp);
+  await startConfiguredMcpServers();
 
   log.info("Daemon startup: providers and tools initialized");
 }

package/src/daemon/reaction-history-render.ts

@@ -28,6 +28,7 @@ import {
   unwrapExternalContentForDisplay,
   wrapUntrustedContent,
 } from "../security/untrusted-content.js";
+import { safeStringSlice } from "../util/unicode.js";
 
 /**
  * Slack-style emoji names render in colon form. Anything else (a unicode
@@ -53,7 +54,7 @@ function snippetOf(text: string): string {
   if (flat.length <= TARGET_SNIPPET_MAX_CHARS) {
     return flat;
   }
-  return `${flat.slice(0, TARGET_SNIPPET_MAX_CHARS)}...`;
+  return `${safeStringSlice(flat, 0, TARGET_SNIPPET_MAX_CHARS)}...`;
 }
 
 /**

package/src/daemon/shutdown-handlers.ts

@@ -24,11 +24,7 @@ import { browserManager } from "../tools/browser/browser-manager.js";
 import { getLogger } from "../util/logger.js";
 import { APP_VERSION } from "../version.js";
 import { getEnrichmentService } from "../workspace/commit-message-enrichment-service.js";
-import {
-  commitAllPendingWorkspaceChanges,
-  stopWorkspaceHeartbeatService,
-} from "../workspace/heartbeat-service.js";
-import { stopAppSourceWatcher } from "./app-source-watcher.js";
+import { commitAllPendingWorkspaceChanges } from "../workspace/heartbeat-service.js";
 import { stopConfigWatcher } from "./config-watcher.js";
 import { stopConversationEvictor } from "./conversation-evictor.js";
 import { stopConversations } from "./conversation-store.js";
@@ -98,9 +94,6 @@ async function shutdown(): Promise<void> {
   }, 30_000);
   forceTimer.unref();
 
-  // Workspace git heartbeat timer lives in the monitor process. This
-  // in-process stop is a no-op unless the singleton was started here.
-  await stopWorkspaceHeartbeatService();
   await stopHeartbeatService();
 
   // Stop the periodic consent-cache refresh (a daemon-owned interval).
@@ -134,7 +127,6 @@ async function shutdown(): Promise<void> {
   disposeAcpSessionManager();
   stopConversationEvictor();
   stopConfigWatcher();
-  stopAppSourceWatcher();
   stopCliIpcServer();
   stopConversations();
   await stopCes();