@vellumai/assistant 0.4.48 → 0.4.50

package/src/daemon/message-types/computer-use.ts

@@ -1,49 +1,9 @@
-// Computer use, task routing, ride shotgun, and watch observation types.
+// Computer use and watch observation types.
 
 import type { CommandIntent, UserMessageAttachment } from "./shared.js";
 
 // === Client → Server ===
 
-export interface CuSessionCreate {
-  type: "cu_session_create";
-  sessionId: string;
-  task: string;
-  screenWidth: number;
-  screenHeight: number;
-  attachments?: UserMessageAttachment[];
-  interactionType?: "computer_use" | "text_qa";
-}
-
-export interface CuSessionAbort {
-  type: "cu_session_abort";
-  sessionId: string;
-}
-
-export interface CuObservation {
-  type: "cu_observation";
-  sessionId: string;
-  axTree?: string;
-  axDiff?: string;
-  secondaryWindows?: string;
-  screenshot?: string;
-  /** Screenshot image width in pixels (`Px`). */
-  screenshotWidthPx?: number;
-  /** Screenshot image height in pixels (`Px`). */
-  screenshotHeightPx?: number;
-  /** Screen width in macOS points (`Pt`) used by native execution. */
-  screenWidthPt?: number;
-  /** Screen height in macOS points (`Pt`) used by native execution. */
-  screenHeightPt?: number;
-  /** Coordinate origin convention used by the observation payload. */
-  coordinateOrigin?: "top_left";
-  /** Display ID used by screenshot capture for this observation. */
-  captureDisplayId?: number;
-  executionResult?: string;
-  executionError?: string;
-  /** Free-form guidance from the user, injected mid-turn to steer the agent. */
-  userGuidance?: string;
-}
-
 export interface TaskSubmit {
   type: "task_submit";
   task: string;
@@ -55,22 +15,6 @@ export interface TaskSubmit {
   commandIntent?: CommandIntent;
 }
 
-export interface RideShotgunStart {
-  type: "ride_shotgun_start";
-  durationSeconds: number;
-  intervalSeconds: number;
-  mode?: "observe" | "learn";
-  targetDomain?: string;
-  /** Domain to auto-navigate (may differ from targetDomain, e.g. open.spotify.com vs spotify.com). */
-  navigateDomain?: string;
-  autoNavigate?: boolean;
-}
-
-export interface RideShotgunStop {
-  type: "ride_shotgun_stop";
-  watchId: string;
-}
-
 export interface WatchObservation {
   type: "watch_observation";
   watchId: string;
@@ -145,58 +89,6 @@ export interface RecordingResume {
   recordingId: string;
 }
 
-export interface CuAction {
-  type: "cu_action";
-  sessionId: string;
-  toolName: string;
-  input: Record<string, unknown>;
-  reasoning?: string;
-  stepNumber: number;
-}
-
-export interface CuComplete {
-  type: "cu_complete";
-  sessionId: string;
-  summary: string;
-  stepCount: number;
-  isResponse?: boolean;
-}
-
-export interface CuError {
-  type: "cu_error";
-  sessionId: string;
-  message: string;
-}
-
-export interface TaskRouted {
-  type: "task_routed";
-  sessionId: string;
-  interactionType: "computer_use" | "text_qa";
-  /** The task text passed to the escalated session. */
-  task?: string;
-  /** Set when a text_qa session escalates to computer_use via computer_use_request_control. */
-  escalatedFrom?: string;
-}
-
-export interface RideShotgunProgress {
-  type: "ride_shotgun_progress";
-  watchId: string;
-  message: string;
-  networkEntryCount?: number;
-  statusMessage?: string;
-  idleHint?: boolean;
-}
-
-export interface RideShotgunResult {
-  type: "ride_shotgun_result";
-  sessionId: string;
-  watchId: string;
-  summary: string;
-  observationCount: number;
-  recordingId?: string;
-  recordingPath?: string;
-}
-
 export interface WatchStarted {
   type: "watch_started";
   sessionId: string;
@@ -211,34 +103,14 @@ export interface WatchCompleteRequest {
   watchId: string;
 }
 
-/** Server → Client: bootstrap failure during learn-mode recording setup. */
-export interface RideShotgunError {
-  type: "ride_shotgun_error";
-  watchId: string;
-  sessionId: string;
-  message: string;
-}
-
 // --- Domain-level union aliases (consumed by the barrel file) ---
 
 export type _ComputerUseClientMessages =
-  | CuSessionCreate
-  | CuSessionAbort
-  | CuObservation
   | TaskSubmit
-  | RideShotgunStart
-  | RideShotgunStop
   | WatchObservation
   | RecordingStatus;
 
 export type _ComputerUseServerMessages =
-  | CuAction
-  | CuComplete
-  | CuError
-  | TaskRouted
-  | RideShotgunProgress
-  | RideShotgunResult
-  | RideShotgunError
   | WatchStarted
   | WatchCompleteRequest
   | RecordingStart

package/src/daemon/message-types/host-cu.ts

@@ -0,0 +1,19 @@
+// Host computer-use proxy types.
+// Enables proxying computer-use actions (click, type, screenshot, etc.)
+// to the desktop client when running as a managed assistant.
+
+// === Server → Client ===
+
+export interface HostCuRequest {
+  type: "host_cu_request";
+  requestId: string;
+  sessionId: string;
+  toolName: string; // "computer_use_click", "computer_use_type_text", etc.
+  input: Record<string, unknown>;
+  stepNumber: number;
+  reasoning?: string;
+}
+
+// --- Domain-level union aliases (consumed by the barrel file) ---
+
+export type _HostCuServerMessages = HostCuRequest;

package/src/daemon/message-types/memory.ts

@@ -11,7 +11,6 @@ export interface MemoryRecalledCandidateDebug {
   type: string;
   kind: string;
   finalScore: number;
-  lexical: number;
   semantic: number;
   recency: number;
 }
@@ -21,18 +20,14 @@ export interface MemoryRecalled {
   provider: string;
   model: string;
   degradation?: MemoryRecalledDegradation;
-  lexicalHits: number;
   semanticHits: number;
   recencyHits: number;
-  entityHits: number;
-  relationSeedEntityCount?: number;
-  relationTraversedEdgeCount?: number;
-  relationNeighborEntityCount?: number;
-  relationExpandedItemCount?: number;
-  earlyTerminated?: boolean;
+  tier1Count: number;
+  tier2Count: number;
+  hybridSearchLatencyMs: number;
+  sparseVectorUsed: boolean;
   mergedCount: number;
   selectedCount: number;
-  rerankApplied: boolean;
   injectedTokens: number;
   latencyMs: number;
   topCandidates: MemoryRecalledCandidateDebug[];
@@ -46,13 +41,6 @@ export interface MemoryStatus {
   reason?: string;
   provider?: string;
   model?: string;
-  conflictsPending: number;
-  conflictsResolved: number;
-  oldestPendingConflictAgeMs: number | null;
-  cleanupResolvedJobsPending: number;
-  cleanupSupersededJobsPending: number;
-  cleanupResolvedJobsCompleted24h: number;
-  cleanupSupersededJobsCompleted24h: number;
 }
 
 // --- Domain-level union aliases (consumed by the barrel file) ---

package/src/daemon/message-types/messages.ts

@@ -88,6 +88,7 @@ export interface ToolOutputChunk {
   type: "tool_output_chunk";
   chunk: string;
   sessionId?: string;
+  toolUseId?: string;
   subType?: "tool_start" | "tool_complete" | "status";
   subToolName?: string;
   subToolInput?: string;
@@ -155,6 +156,8 @@ export interface ConfirmationRequest {
   persistentDecisionsAllowed?: boolean;
   /** Which temporary approval options the client should render (e.g. "Allow for 10 minutes", "Allow for this thread"). */
   temporaryOptionsAvailable?: Array<"allow_10m" | "allow_thread">;
+  /** The tool_use block ID for client-side correlation with specific tool calls. */
+  toolUseId?: string;
 }
 
 export interface SecretRequest {
@@ -287,6 +290,7 @@ export interface AssistantActivityState {
     | "tool_result_received"
     | "confirmation_requested"
     | "confirmation_resolved"
+    | "context_compacting"
     | "message_complete"
     | "generation_cancelled"
     | "error_terminal";

package/src/daemon/message-types/sessions.ts

@@ -394,6 +394,8 @@ export type SessionErrorCode =
   | "PROVIDER_RATE_LIMIT"
   | "PROVIDER_API"
   | "PROVIDER_BILLING"
+  | "PROVIDER_ORDERING"
+  | "PROVIDER_WEB_SEARCH"
   | "CONTEXT_TOO_LARGE"
   | "SESSION_ABORTED"
   | "SESSION_PROCESSING_FAILED"
@@ -407,6 +409,8 @@ export interface SessionErrorMessage {
   userMessage: string;
   retryable: boolean;
   debugDetails?: string;
+  /** Machine-readable error category for log report metadata and triage. */
+  errorCategory?: string;
 }
 
 /** Server push — broadcast when a schedule creates a conversation, so the client can show it as a chat thread. */

package/src/daemon/server.ts

@@ -48,7 +48,6 @@ import {
   getWorkspacePromptPath,
 } from "../util/platform.js";
 import { registerDaemonCallbacks } from "../work-items/work-item-runner.js";
-import { ComputerUseSession } from "./computer-use-session.js";
 import { ConfigWatcher } from "./config-watcher.js";
 import { parseIdentityFields } from "./handlers/identity.js";
 import type {
@@ -57,7 +56,9 @@ import type {
 } from "./handlers/shared.js";
 import type { SkillOperationContext } from "./handlers/skills.js";
 import { HostBashProxy } from "./host-bash-proxy.js";
+import { HostCuProxy } from "./host-cu-proxy.js";
 import { HostFileProxy } from "./host-file-proxy.js";
+import { reloadMcpServers } from "./mcp-reload-service.js";
 import type { ServerMessage } from "./message-protocol.js";
 import {
   DEFAULT_MEMORY_POLICY,
@@ -214,14 +215,18 @@ function makePendingInteractionRegistrar(
         conversationId,
         kind: "host_file",
       });
+    } else if (msg.type === "host_cu_request") {
+      pendingInteractions.register(msg.requestId, {
+        session,
+        conversationId,
+        kind: "host_cu",
+      });
     }
   };
 }
 
 export class DaemonServer {
   private sessions = new Map<string, Session>();
-  private cuSessions = new Map<string, ComputerUseSession>();
-  private cuObservationParseSequence = new Map<string, number>();
   private sessionOptions = new Map<string, SessionCreateOptions>();
   private sessionCreating = new Map<string, Promise<Session>>();
   private sharedRequestTimestamps: number[] = [];
@@ -388,6 +393,11 @@ export class DaemonServer {
     this.configWatcher.start(
       () => this.evictSessionsForReload(),
       () => this.broadcastIdentityChanged(),
+      () => {
+        reloadMcpServers().catch((err: unknown) => {
+          log.error({ err }, "MCP reload triggered by config change failed");
+        });
+      },
     );
 
     // Broadcast contacts_changed to all clients when any contact mutation occurs.
@@ -412,11 +422,6 @@ export class DaemonServer {
     }
     this.sessions.clear();
 
-    for (const cuSession of this.cuSessions.values()) {
-      cuSession.abort();
-    }
-    this.cuSessions.clear();
-
     log.info("Daemon server stopped");
   }
 
@@ -563,8 +568,6 @@ export class DaemonServer {
   private handlerContext(): HandlerContext {
     return {
       sessions: this.sessions,
-      cuSessions: this.cuSessions,
-      cuObservationParseSequence: this.cuObservationParseSequence,
       sharedRequestTimestamps: this.sharedRequestTimestamps,
       debounceTimers: this.configWatcher.timers,
       suppressConfigReload: this.configWatcher.suppressConfigReload,
@@ -665,9 +668,18 @@ export class DaemonServer {
           }),
         );
       }
+      if (!session.isProcessing() || !session.hostCuProxy) {
+        session.setHostCuProxy(
+          new HostCuProxy(session.getCurrentSender(), (requestId) => {
+            pendingInteractions.resolve(requestId);
+          }),
+        );
+      }
+      session.addPreactivatedSkillId("computer-use");
     } else if (!session.isProcessing()) {
       session.setHostBashProxy(undefined);
       session.setHostFileProxy(undefined);
+      session.setHostCuProxy(undefined);
     }
     session.setCommandIntent(options?.commandIntent ?? null);
     session.setTurnChannelContext({
@@ -907,23 +919,15 @@ export class DaemonServer {
 
   /**
    * Look up an active session by ID without creating one.
-   * Checks both normal sessions and computer-use sessions so the HTTP
-   * surface-action path is consistent with dispatch.
    */
-  findSession(sessionId: string): Session | ComputerUseSession | undefined {
-    return this.cuSessions.get(sessionId) ?? this.sessions.get(sessionId);
+  findSession(sessionId: string): Session | undefined {
+    return this.sessions.get(sessionId);
   }
 
   /**
    * Look up an active session that owns a given surfaceId.
-   * Falls back across both normal and computer-use sessions.
    */
-  findSessionBySurfaceId(
-    surfaceId: string,
-  ): Session | ComputerUseSession | undefined {
-    for (const s of this.cuSessions.values()) {
-      if (s.surfaceState.has(surfaceId)) return s;
-    }
+  findSessionBySurfaceId(surfaceId: string): Session | undefined {
     for (const s of this.sessions.values()) {
       if (s.surfaceState.has(surfaceId)) return s;
     }

package/src/daemon/session-agent-loop-handlers.ts

@@ -57,6 +57,8 @@ export interface EventHandlerState {
   orderingErrorDetected: boolean;
   deferredOrderingError: string | null;
   contextTooLargeDetected: boolean;
+  /** The raw error message from the provider when context_too_large is detected. */
+  contextTooLargeErrorMessage: string | null;
   providerErrorUserMessage: string | null;
   lastAssistantMessageId: string | undefined;
   readonly pendingToolResults: Map<string, PendingToolResult>;
@@ -121,6 +123,7 @@ export function createEventHandlerState(): EventHandlerState {
     orderingErrorDetected: false,
     deferredOrderingError: null,
     contextTooLargeDetected: false,
+    contextTooLargeErrorMessage: null,
     providerErrorUserMessage: null,
     lastAssistantMessageId: undefined,
     pendingToolResults: new Map(),
@@ -384,6 +387,7 @@ export function handleToolOutputChunk(
       type: "tool_output_chunk",
       chunk: event.chunk,
       sessionId: deps.ctx.conversationId,
+      toolUseId: event.toolUseId,
       subType: structured.subType,
       subToolName: structured.subToolName,
       subToolInput: structured.subToolInput,
@@ -395,6 +399,7 @@ export function handleToolOutputChunk(
       type: "tool_output_chunk",
       chunk: event.chunk,
       sessionId: deps.ctx.conversationId,
+      toolUseId: event.toolUseId,
     });
   }
 }
@@ -593,12 +598,22 @@ export function handleError(
     state.deferredOrderingError = event.error.message;
   } else if (isContextTooLarge(event.error.message)) {
     state.contextTooLargeDetected = true;
+    state.contextTooLargeErrorMessage = event.error.message;
   } else {
     const classified = classifySessionError(event.error, {
       phase: "agent_loop",
     });
     if (classified.code === "CONTEXT_TOO_LARGE") {
       state.contextTooLargeDetected = true;
+      state.contextTooLargeErrorMessage = event.error.message;
+    } else if (
+      classified.code === "PROVIDER_ORDERING" ||
+      classified.code === "PROVIDER_WEB_SEARCH"
+    ) {
+      // Ordering errors detected via classifySessionError (e.g. from ProviderError
+      // with statusCode 400 and ordering message) — trigger the retry path.
+      state.orderingErrorDetected = true;
+      state.deferredOrderingError = event.error.message;
     } else {
       deps.onEvent(
         buildSessionErrorMessage(deps.ctx.conversationId, classified),
@@ -829,6 +844,31 @@ export async function dispatchAgentEvent(
         deps.reqId,
         statusText,
       );
+      // Emit tool_use_start so the client renders a tool chip (like other tools)
+      deps.onEvent({
+        type: "tool_use_start",
+        toolName: event.name,
+        input: event.input,
+        sessionId: deps.ctx.conversationId,
+        toolUseId: event.toolUseId,
+      });
+      break;
+    }
+    case "server_tool_complete": {
+      deps.ctx.emitActivityState(
+        "streaming",
+        "tool_result_received",
+        "assistant_turn",
+        deps.reqId,
+      );
+      deps.onEvent({
+        type: "tool_result",
+        toolName: "",
+        result: "",
+        isError: false,
+        sessionId: deps.ctx.conversationId,
+        toolUseId: event.toolUseId,
+      });
       break;
     }
     case "error":