@vellumai/assistant 0.4.46 → 0.4.49

package/src/daemon/message-types/computer-use.ts

@@ -1,49 +1,9 @@
-// Computer use, task routing, ride shotgun, and watch observation types.
+// Computer use, task routing, and watch observation types.
 
 import type { CommandIntent, UserMessageAttachment } from "./shared.js";
 
 // === Client → Server ===
 
-export interface CuSessionCreate {
-  type: "cu_session_create";
-  sessionId: string;
-  task: string;
-  screenWidth: number;
-  screenHeight: number;
-  attachments?: UserMessageAttachment[];
-  interactionType?: "computer_use" | "text_qa";
-}
-
-export interface CuSessionAbort {
-  type: "cu_session_abort";
-  sessionId: string;
-}
-
-export interface CuObservation {
-  type: "cu_observation";
-  sessionId: string;
-  axTree?: string;
-  axDiff?: string;
-  secondaryWindows?: string;
-  screenshot?: string;
-  /** Screenshot image width in pixels (`Px`). */
-  screenshotWidthPx?: number;
-  /** Screenshot image height in pixels (`Px`). */
-  screenshotHeightPx?: number;
-  /** Screen width in macOS points (`Pt`) used by native execution. */
-  screenWidthPt?: number;
-  /** Screen height in macOS points (`Pt`) used by native execution. */
-  screenHeightPt?: number;
-  /** Coordinate origin convention used by the observation payload. */
-  coordinateOrigin?: "top_left";
-  /** Display ID used by screenshot capture for this observation. */
-  captureDisplayId?: number;
-  executionResult?: string;
-  executionError?: string;
-  /** Free-form guidance from the user, injected mid-turn to steer the agent. */
-  userGuidance?: string;
-}
-
 export interface TaskSubmit {
   type: "task_submit";
   task: string;
@@ -55,22 +15,6 @@ export interface TaskSubmit {
   commandIntent?: CommandIntent;
 }
 
-export interface RideShotgunStart {
-  type: "ride_shotgun_start";
-  durationSeconds: number;
-  intervalSeconds: number;
-  mode?: "observe" | "learn";
-  targetDomain?: string;
-  /** Domain to auto-navigate (may differ from targetDomain, e.g. open.spotify.com vs spotify.com). */
-  navigateDomain?: string;
-  autoNavigate?: boolean;
-}
-
-export interface RideShotgunStop {
-  type: "ride_shotgun_stop";
-  watchId: string;
-}
-
 export interface WatchObservation {
   type: "watch_observation";
   watchId: string;
@@ -145,58 +89,16 @@ export interface RecordingResume {
   recordingId: string;
 }
 
-export interface CuAction {
-  type: "cu_action";
-  sessionId: string;
-  toolName: string;
-  input: Record<string, unknown>;
-  reasoning?: string;
-  stepNumber: number;
-}
-
-export interface CuComplete {
-  type: "cu_complete";
-  sessionId: string;
-  summary: string;
-  stepCount: number;
-  isResponse?: boolean;
-}
-
-export interface CuError {
-  type: "cu_error";
-  sessionId: string;
-  message: string;
-}
-
 export interface TaskRouted {
   type: "task_routed";
   sessionId: string;
   interactionType: "computer_use" | "text_qa";
   /** The task text passed to the escalated session. */
   task?: string;
-  /** Set when a text_qa session escalates to computer_use via computer_use_request_control. */
+  /** Set when a text_qa session escalates to computer_use. */
   escalatedFrom?: string;
 }
 
-export interface RideShotgunProgress {
-  type: "ride_shotgun_progress";
-  watchId: string;
-  message: string;
-  networkEntryCount?: number;
-  statusMessage?: string;
-  idleHint?: boolean;
-}
-
-export interface RideShotgunResult {
-  type: "ride_shotgun_result";
-  sessionId: string;
-  watchId: string;
-  summary: string;
-  observationCount: number;
-  recordingId?: string;
-  recordingPath?: string;
-}
-
 export interface WatchStarted {
   type: "watch_started";
   sessionId: string;
@@ -211,34 +113,15 @@ export interface WatchCompleteRequest {
   watchId: string;
 }
 
-/** Server → Client: bootstrap failure during learn-mode recording setup. */
-export interface RideShotgunError {
-  type: "ride_shotgun_error";
-  watchId: string;
-  sessionId: string;
-  message: string;
-}
-
 // --- Domain-level union aliases (consumed by the barrel file) ---
 
 export type _ComputerUseClientMessages =
-  | CuSessionCreate
-  | CuSessionAbort
-  | CuObservation
   | TaskSubmit
-  | RideShotgunStart
-  | RideShotgunStop
   | WatchObservation
   | RecordingStatus;
 
 export type _ComputerUseServerMessages =
-  | CuAction
-  | CuComplete
-  | CuError
   | TaskRouted
-  | RideShotgunProgress
-  | RideShotgunResult
-  | RideShotgunError
   | WatchStarted
   | WatchCompleteRequest
   | RecordingStart

package/src/daemon/message-types/host-cu.ts

@@ -0,0 +1,19 @@
+// Host computer-use proxy types.
+// Enables proxying computer-use actions (click, type, screenshot, etc.)
+// to the desktop client when running as a managed assistant.
+
+// === Server → Client ===
+
+export interface HostCuRequest {
+  type: "host_cu_request";
+  requestId: string;
+  sessionId: string;
+  toolName: string; // "computer_use_click", "computer_use_type_text", etc.
+  input: Record<string, unknown>;
+  stepNumber: number;
+  reasoning?: string;
+}
+
+// --- Domain-level union aliases (consumed by the barrel file) ---
+
+export type _HostCuServerMessages = HostCuRequest;

package/src/daemon/message-types/integrations.ts

@@ -194,6 +194,7 @@ export interface IntegrationConnectResult {
 export interface OAuthConnectResultResponse {
   type: "oauth_connect_result";
   success: boolean;
+  service?: string;
   grantedScopes?: string[];
   accountInfo?: string;
   error?: string;

package/src/daemon/message-types/messages.ts

@@ -88,6 +88,7 @@ export interface ToolOutputChunk {
   type: "tool_output_chunk";
   chunk: string;
   sessionId?: string;
+  toolUseId?: string;
   subType?: "tool_start" | "tool_complete" | "status";
   subToolName?: string;
   subToolInput?: string;
@@ -155,6 +156,8 @@ export interface ConfirmationRequest {
   persistentDecisionsAllowed?: boolean;
   /** Which temporary approval options the client should render (e.g. "Allow for 10 minutes", "Allow for this thread"). */
   temporaryOptionsAvailable?: Array<"allow_10m" | "allow_thread">;
+  /** The tool_use block ID for client-side correlation with specific tool calls. */
+  toolUseId?: string;
 }
 
 export interface SecretRequest {

package/src/daemon/server.ts

@@ -48,7 +48,6 @@ import {
   getWorkspacePromptPath,
 } from "../util/platform.js";
 import { registerDaemonCallbacks } from "../work-items/work-item-runner.js";
-import { ComputerUseSession } from "./computer-use-session.js";
 import { ConfigWatcher } from "./config-watcher.js";
 import { parseIdentityFields } from "./handlers/identity.js";
 import type {
@@ -57,6 +56,7 @@ import type {
 } from "./handlers/shared.js";
 import type { SkillOperationContext } from "./handlers/skills.js";
 import { HostBashProxy } from "./host-bash-proxy.js";
+import { HostCuProxy } from "./host-cu-proxy.js";
 import { HostFileProxy } from "./host-file-proxy.js";
 import type { ServerMessage } from "./message-protocol.js";
 import {
@@ -214,14 +214,18 @@ function makePendingInteractionRegistrar(
         conversationId,
         kind: "host_file",
       });
+    } else if (msg.type === "host_cu_request") {
+      pendingInteractions.register(msg.requestId, {
+        session,
+        conversationId,
+        kind: "host_cu",
+      });
     }
   };
 }
 
 export class DaemonServer {
   private sessions = new Map<string, Session>();
-  private cuSessions = new Map<string, ComputerUseSession>();
-  private cuObservationParseSequence = new Map<string, number>();
   private sessionOptions = new Map<string, SessionCreateOptions>();
   private sessionCreating = new Map<string, Promise<Session>>();
   private sharedRequestTimestamps: number[] = [];
@@ -412,11 +416,6 @@ export class DaemonServer {
     }
     this.sessions.clear();
 
-    for (const cuSession of this.cuSessions.values()) {
-      cuSession.abort();
-    }
-    this.cuSessions.clear();
-
     log.info("Daemon server stopped");
   }
 
@@ -563,8 +562,6 @@ export class DaemonServer {
   private handlerContext(): HandlerContext {
     return {
       sessions: this.sessions,
-      cuSessions: this.cuSessions,
-      cuObservationParseSequence: this.cuObservationParseSequence,
       sharedRequestTimestamps: this.sharedRequestTimestamps,
       debounceTimers: this.configWatcher.timers,
       suppressConfigReload: this.configWatcher.suppressConfigReload,
@@ -665,9 +662,13 @@ export class DaemonServer {
           }),
         );
       }
+      if (!session.isProcessing() || !session.hostCuProxy) {
+        session.setHostCuProxy(new HostCuProxy(session.getCurrentSender()));
+      }
     } else if (!session.isProcessing()) {
       session.setHostBashProxy(undefined);
       session.setHostFileProxy(undefined);
+      session.setHostCuProxy(undefined);
     }
     session.setCommandIntent(options?.commandIntent ?? null);
     session.setTurnChannelContext({
@@ -907,23 +908,15 @@ export class DaemonServer {
 
   /**
    * Look up an active session by ID without creating one.
-   * Checks both normal sessions and computer-use sessions so the HTTP
-   * surface-action path is consistent with dispatch.
    */
-  findSession(sessionId: string): Session | ComputerUseSession | undefined {
-    return this.cuSessions.get(sessionId) ?? this.sessions.get(sessionId);
+  findSession(sessionId: string): Session | undefined {
+    return this.sessions.get(sessionId);
   }
 
   /**
    * Look up an active session that owns a given surfaceId.
-   * Falls back across both normal and computer-use sessions.
    */
-  findSessionBySurfaceId(
-    surfaceId: string,
-  ): Session | ComputerUseSession | undefined {
-    for (const s of this.cuSessions.values()) {
-      if (s.surfaceState.has(surfaceId)) return s;
-    }
+  findSessionBySurfaceId(surfaceId: string): Session | undefined {
     for (const s of this.sessions.values()) {
       if (s.surfaceState.has(surfaceId)) return s;
     }

package/src/daemon/session-agent-loop-handlers.ts

@@ -384,6 +384,7 @@ export function handleToolOutputChunk(
       type: "tool_output_chunk",
       chunk: event.chunk,
       sessionId: deps.ctx.conversationId,
+      toolUseId: event.toolUseId,
       subType: structured.subType,
       subToolName: structured.subToolName,
       subToolInput: structured.subToolInput,
@@ -395,6 +396,7 @@ export function handleToolOutputChunk(
       type: "tool_output_chunk",
       chunk: event.chunk,
       sessionId: deps.ctx.conversationId,
+      toolUseId: event.toolUseId,
     });
   }
 }

package/src/daemon/session-attachments.ts

@@ -175,8 +175,7 @@ export async function resolveAssistantAttachments(
       accumulatedToolContentBlocks,
       toolContentBlockToolNames,
     );
-    // Most recent tool outputs (e.g., final browser screenshot) should win
-    // the MAX_ASSISTANT_ATTACHMENTS cap over older intermediate screenshots.
+    // Most recent tool outputs first so deduplication keeps the latest version.
     toolDrafts.reverse();
     const merged = deduplicateDrafts([
       ...directiveDrafts.drafts,

package/src/daemon/session-messaging.ts

@@ -446,7 +446,9 @@ export function redirectToSecurePrompt(
           "Ingress redirect: transient credential injected",
         );
       } else {
-        const key = `credential:${target.service}:${target.field}`;
+        const { credentialKey: credKey } =
+          await import("../security/credential-key.js");
+        const key = credKey(target.service, target.field);
         const stored = await setSecureKeyAsync(key, result.value);
         if (stored) {
           try {

package/src/daemon/session-slash.ts

@@ -465,7 +465,7 @@ function resolvePairCommand(content: string): SlashResolution | null {
       kind: "unknown",
       message:
         "Cannot generate pairing info — no gateway URL is configured and no LAN address was detected.\n\n" +
-        "Set a public gateway URL with `config set ingress.publicBaseUrl <url>` or the `INGRESS_PUBLIC_BASE_URL` environment variable.",
+        "Set a public gateway URL with `config set ingress.publicBaseUrl <url>`.",
     };
   }
 

package/src/daemon/session-surfaces.ts

@@ -202,7 +202,8 @@ export interface SurfaceSessionContext {
     }>;
     display?: string;
   }>;
-  onEscalateToComputerUse?: (task: string, sourceSessionId: string) => boolean;
+  /** Optional proxy for delegating computer-use actions to a connected desktop client. */
+  hostCuProxy?: import("./host-cu-proxy.js").HostCuProxy;
   isProcessing(): boolean;
   enqueueMessage(
     content: string,
@@ -931,13 +932,50 @@ export function buildUserFacingLabel(
 
 /**
  * Resolve a proxy tool call that targets a UI surface.
- * Handles ui_show, ui_update, ui_dismiss, computer_use_request_control, and app_open.
+ * Handles ui_show, ui_update, ui_dismiss, computer_use_* proxy tools, and app_open.
  */
 export async function surfaceProxyResolver(
   ctx: SurfaceSessionContext,
   toolName: string,
   input: Record<string, unknown>,
 ): Promise<ToolExecutionResult> {
+  // Route CU proxy tools (all computer_use_* action tools)
+  if (toolName.startsWith("computer_use_")) {
+    if (!ctx.hostCuProxy) {
+      return {
+        content: "Computer use is not available — no desktop client connected.",
+        isError: true,
+      };
+    }
+
+    // Terminal tools resolve immediately without a client round-trip
+    if (
+      toolName === "computer_use_done" ||
+      toolName === "computer_use_respond"
+    ) {
+      const summary =
+        typeof input.summary === "string"
+          ? input.summary
+          : typeof input.answer === "string"
+            ? input.answer
+            : "Task complete";
+      ctx.hostCuProxy.reset();
+      return { content: summary, isError: false };
+    }
+
+    // Record the action and proxy to the connected desktop client
+    const reasoning =
+      typeof input.reasoning === "string" ? input.reasoning : undefined;
+    ctx.hostCuProxy.recordAction(toolName, input, reasoning);
+    return ctx.hostCuProxy.request(
+      toolName,
+      input,
+      ctx.conversationId,
+      ctx.hostCuProxy.stepCount,
+      reasoning,
+    );
+  }
+
   if (toolName === "ui_show" || toolName === "ui_update") {
     const caps = ctx.channelCapabilities;
     if (caps && !caps.supportsDynamicUi) {
@@ -1152,32 +1190,6 @@ export async function surfaceProxyResolver(
     };
   }
 
-  if (toolName === "computer_use_request_control") {
-    const task =
-      typeof input.task === "string"
-        ? input.task
-        : "Perform the requested task";
-    if (!ctx.onEscalateToComputerUse) {
-      return {
-        content:
-          "Computer control escalation is not available in this session.",
-        isError: true,
-      };
-    }
-    const success = ctx.onEscalateToComputerUse(task, ctx.conversationId);
-    if (!success) {
-      return {
-        content: "Computer control escalation failed — no active connection.",
-        isError: true,
-      };
-    }
-    return {
-      content:
-        "Computer control activated. The task has been handed off to foreground computer use.",
-      isError: false,
-    };
-  }
-
   if (toolName === "app_open") {
     const appId = input.app_id as string;
     const preview = input.preview as DynamicPageSurfaceData["preview"];

package/src/daemon/session-tool-setup.ts

@@ -25,12 +25,15 @@ import type { TrustClass } from "../runtime/actor-trust-resolver.js";
 import { getEffectiveMode } from "../runtime/session-approval-overrides.js";
 import { coreAppProxyTools } from "../tools/apps/definitions.js";
 import { registerSessionSender } from "../tools/browser/browser-screencast.js";
-import { requestComputerControlTool } from "../tools/computer-use/request-computer-control.js";
 import type { ToolExecutor } from "../tools/executor.js";
 import {
   getAllToolDefinitions,
   getMcpToolDefinitions,
 } from "../tools/registry.js";
+import {
+  injectReasonField,
+  REASON_SKIP_SET,
+} from "../tools/schema-transforms.js";
 import type {
   ProxyApprovalCallback,
   ProxyApprovalRequest,
@@ -111,16 +114,13 @@ export interface ToolSetupContext extends SurfaceSessionContext {
 
 /**
  * Collect all tool definitions for the agent loop: built-in tools,
- * UI surface proxy tools, app proxy tools, and the computer-use
- * escalation tool.
+ * UI surface proxy tools, and app proxy tools.
  */
 export function buildToolDefinitions(): ToolDefinition[] {
   return [
     ...getAllToolDefinitions(),
     ...allUiSurfaceTools.map((t) => t.getDefinition()),
     ...coreAppProxyTools.map((t) => t.getDefinition()),
-    // Escalation tool: allows text_qa sessions to hand off to computer use
-    requestComputerControlTool.getDefinition(),
   ];
 }
 
@@ -335,11 +335,23 @@ export function createToolExecutor(
     // with the real tool name.
     if (name === "skill_execute") {
       const toolName = typeof input.tool === "string" ? input.tool : "";
-      const toolInput =
+      const rawToolInput =
         input.input != null && typeof input.input === "object"
           ? (input.input as Record<string, unknown>)
           : {};
 
+      // Clone to avoid mutating shared input objects
+      const toolInput = { ...rawToolInput };
+
+      // Propagate outer reason when inner input lacks a valid one
+      if (
+        typeof input.reason === "string" &&
+        input.reason &&
+        (typeof toolInput.reason !== "string" || toolInput.reason.length === 0)
+      ) {
+        toolInput.reason = input.reason;
+      }
+
       if (!toolName) {
         return {
           content:
@@ -550,10 +562,7 @@ const HOST_TOOL_NAMES = new Set([
   "host_bash",
 ]);
 const ASSET_TOOL_NAMES = new Set(["asset_search", "asset_materialize"]);
-const CLIENT_CAPABILITY_TOOL_NAMES = new Set([
-  "app_open",
-  "computer_use_request_control",
-]);
+const CLIENT_CAPABILITY_TOOL_NAMES = new Set(["app_open"]);
 const PLATFORM_TOOL_NAMES = new Set(["request_system_permission"]);
 
 /**
@@ -663,6 +672,6 @@ export function createResolveToolsCallback(
       turnAllowed.add(name);
     }
     ctx.allowedToolNames = turnAllowed;
-    return allBaseDefs;
+    return injectReasonField(allBaseDefs, REASON_SKIP_SET);
   };
 }