@vellumai/assistant 0.4.48 → 0.4.49

package/src/daemon/handlers/sessions.ts

@@ -35,6 +35,7 @@ import * as pendingInteractions from "../../runtime/pending-interactions.js";
 import { getSubagentManager } from "../../subagent/index.js";
 import { truncate } from "../../util/truncate.js";
 import { HostBashProxy } from "../host-bash-proxy.js";
+import { HostCuProxy } from "../host-cu-proxy.js";
 import { HostFileProxy } from "../host-file-proxy.js";
 import type {
   CancelRequest,
@@ -60,7 +61,6 @@ import {
   type HandlerContext,
   log,
   pendingStandaloneSecrets,
-  wireEscalationHandler,
 } from "./shared.js";
 
 /**
@@ -165,6 +165,12 @@ export function makeEventSender(params: {
         conversationId,
         kind: "host_file",
       });
+    } else if (event.type === "host_cu_request") {
+      pendingInteractions.register(event.requestId, {
+        session,
+        conversationId,
+        kind: "host_cu",
+      });
     }
 
     ctx.send(event);
@@ -195,21 +201,6 @@ export function handleConfirmationResponse(
     }
   }
 
-  // Also check computer-use sessions — they have their own PermissionPrompter
-  for (const [, cuSession] of ctx.cuSessions) {
-    if (cuSession.hasPendingConfirmation(msg.requestId)) {
-      cuSession.handleConfirmationResponse(
-        msg.requestId,
-        msg.decision,
-        msg.selectedPattern,
-        msg.selectedScope,
-      );
-      syncCanonicalStatusFromConfirmationDecision(msg.requestId, msg.decision);
-      pendingInteractions.resolve(msg.requestId);
-      return;
-    }
-  }
-
   log.warn(
     { requestId: msg.requestId },
     "No session found with pending confirmation for requestId",
@@ -362,7 +353,6 @@ export async function handleSessionCreate(
     maxResponseTokens: msg.maxResponseTokens,
     transport: msg.transport,
   });
-  wireEscalationHandler(session, ctx);
 
   // Pre-activate skills before sending session_info so they're available
   // for the initial message processing.
@@ -431,6 +421,8 @@ export async function handleSessionCreate(
         pendingInteractions.resolve(requestId);
       });
       session.setHostFileProxy(fileProxy);
+      const cuProxy = new HostCuProxy(sendEvent);
+      session.setHostCuProxy(cuProxy);
     }
     session.updateClient(sendEvent, false);
     session
@@ -492,13 +484,7 @@ export async function switchSession(
     // Load the session without rebinding the client — the session stays headless
     await ctx.getOrCreateSession(sessionId);
   } else {
-    const session = await ctx.getOrCreateSession(sessionId);
-    // Only wire the escalation handler if one isn't already set — handleTaskSubmit
-    // sets a handler with the client's actual screen dimensions, and overwriting it
-    // here would replace those dimensions with the daemon's defaults.
-    if (!session.hasEscalationHandler()) {
-      wireEscalationHandler(session, ctx);
-    }
+    await ctx.getOrCreateSession(sessionId);
   }
 
   return {

package/src/daemon/handlers/shared.ts

@@ -1,17 +1,12 @@
-import { execSync } from "node:child_process";
-
 import { v4 as uuid } from "uuid";
 
 import { getConfig } from "../../config/loader.js";
 import type { HeartbeatService } from "../../heartbeat/heartbeat-service.js";
 import type { SecretPromptResult } from "../../permissions/secret-prompter.js";
-import { RateLimitProvider } from "../../providers/ratelimit.js";
-import { getFailoverProvider } from "../../providers/registry.js";
 import type { AuthContext } from "../../runtime/auth/types.js";
 import type { DebouncerMap } from "../../util/debounce.js";
 import { getLogger } from "../../util/logger.js";
 import { estimateBase64Bytes } from "../assistant-attachments.js";
-import { ComputerUseSession } from "../computer-use-session.js";
 import type {
   ServerMessage,
   SessionTransportMetadata,
@@ -28,9 +23,6 @@ export const CONFIG_RELOAD_DEBOUNCE_MS = 300;
 
 const HISTORY_ATTACHMENT_TEXT_LIMIT = 500;
 
-export const FALLBACK_SCREEN = { width: 1920, height: 1080 };
-let cachedScreenDims: { width: number; height: number } | null = null;
-
 // Module-level map for non-session secret prompts (e.g. publish_page)
 export const pendingStandaloneSecrets = new Map<
   string,
@@ -150,8 +142,6 @@ export interface SessionCreateOptions {
  */
 export interface HandlerContext {
   sessions: Map<string, Session>;
-  cuSessions: Map<string, ComputerUseSession>;
-  cuObservationParseSequence: Map<string, number>;
   sharedRequestTimestamps: number[];
   debounceTimers: DebouncerMap;
   suppressConfigReload: boolean;
@@ -170,126 +160,6 @@ export interface HandlerContext {
   heartbeatService?: HeartbeatService;
 }
 
-/**
- * Query the main display dimensions via CoreGraphics.
- * Cached after the first successful call; falls back to 1920x1080.
- */
-export function getScreenDimensions(): { width: number; height: number } {
-  if (cachedScreenDims) return cachedScreenDims;
-  if (process.platform !== "darwin") return FALLBACK_SCREEN;
-  try {
-    // Use osascript (JXA) instead of `swift` to avoid the
-    // "Install Command Line Developer Tools" popup on fresh macOS installs.
-    const out = execSync(
-      `osascript -l JavaScript -e 'ObjC.import("AppKit"); var f = $.NSScreen.mainScreen.frame; Math.round(f.size.width) + "x" + Math.round(f.size.height)'`,
-      { timeout: 10_000, encoding: "utf-8" },
-    ).trim();
-    const [w, h] = out.split("x").map(Number);
-    if (w > 0 && h > 0) {
-      cachedScreenDims = { width: w, height: h };
-      return cachedScreenDims;
-    }
-  } catch (err) {
-    log.debug({ err }, "Failed to query screen dimensions, using fallback");
-  }
-  return FALLBACK_SCREEN;
-}
-
-/**
- * Wire the escalation handler on a text_qa session so that invoking
- * `computer_use_request_control` creates a CU session and notifies the client.
- *
- * In the HTTP-only world, the escalation handler broadcasts events via
- * `ctx.broadcast` instead of targeting a specific socket.
- */
-export function wireEscalationHandler(
-  session: Session,
-  ctx: HandlerContext,
-  explicitWidth?: number,
-  explicitHeight?: number,
-): void {
-  const dims =
-    explicitWidth && explicitHeight
-      ? { width: explicitWidth, height: explicitHeight }
-      : getScreenDimensions();
-  const screenWidth = dims.width;
-  const screenHeight = dims.height;
-  session.setEscalationHandler(
-    (task: string, sourceSessionId: string): boolean => {
-      const cuSessionId = uuid();
-
-      // Inline CU session creation (previously delegated to deleted handlers/computer-use.ts)
-      const existingSession = ctx.cuSessions.get(cuSessionId);
-      if (existingSession) {
-        existingSession.abort();
-        ctx.cuSessions.delete(cuSessionId);
-        ctx.cuObservationParseSequence.delete(cuSessionId);
-      }
-
-      const config = getConfig();
-      let provider = getFailoverProvider(config.provider, config.providerOrder);
-      const { rateLimit } = config;
-      if (
-        rateLimit.maxRequestsPerMinute > 0 ||
-        rateLimit.maxTokensPerSession > 0
-      ) {
-        provider = new RateLimitProvider(
-          provider,
-          rateLimit,
-          ctx.sharedRequestTimestamps,
-        );
-      }
-
-      const sendToClient = (serverMsg: ServerMessage) => {
-        ctx.send(serverMsg);
-      };
-
-      const sessionRef: { current?: ComputerUseSession } = {};
-      const onTerminal = (sid: string) => {
-        const current = ctx.cuSessions.get(sid);
-        if (sessionRef.current && current && current !== sessionRef.current) {
-          return;
-        }
-        ctx.cuSessions.delete(sid);
-        ctx.cuObservationParseSequence.delete(sid);
-        log.info(
-          { sessionId: sid },
-          "Computer-use session cleaned up after terminal state",
-        );
-      };
-
-      const cuSession = new ComputerUseSession(
-        cuSessionId,
-        task,
-        screenWidth,
-        screenHeight,
-        provider,
-        sendToClient,
-        "computer_use",
-        onTerminal,
-      );
-      sessionRef.current = cuSession;
-
-      ctx.cuSessions.set(cuSessionId, cuSession);
-
-      log.info(
-        { sessionId: cuSessionId, taskLength: task.length },
-        "Computer-use session created via escalation",
-      );
-
-      ctx.broadcast({
-        type: "task_routed",
-        sessionId: cuSessionId,
-        interactionType: "computer_use",
-        task,
-        escalatedFrom: sourceSessionId,
-      });
-
-      return true;
-    },
-  );
-}
-
 export function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value != null;
 }

package/src/daemon/host-cu-proxy.ts

@@ -0,0 +1,401 @@
+/**
+ * Host computer-use proxy.
+ *
+ * Proxies computer-use actions to the desktop client when running as a
+ * managed assistant, following the same request/resolve pattern as
+ * HostBashProxy. Also owns CU-specific state tracking (step counting,
+ * loop detection, observation formatting) for the unified agent loop.
+ */
+
+import { v4 as uuid } from "uuid";
+
+import type { ContentBlock } from "../providers/types.js";
+import type { ToolExecutionResult } from "../tools/types.js";
+import { AssistantError, ErrorCode } from "../util/errors.js";
+import { getLogger } from "../util/logger.js";
+import type { ServerMessage } from "./message-protocol.js";
+
+const log = getLogger("host-cu-proxy");
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const REQUEST_TIMEOUT_SEC = 60;
+const MAX_STEPS = 50;
+const MAX_HISTORY_ENTRIES = 10;
+const LOOP_DETECTION_WINDOW = 3;
+const CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD = 2;
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface CuObservationResult {
+  axTree?: string;
+  axDiff?: string;
+  secondaryWindows?: string;
+  screenshot?: string; // base64 JPEG
+  screenshotWidthPx?: number;
+  screenshotHeightPx?: number;
+  screenWidthPt?: number;
+  screenHeightPt?: number;
+  executionResult?: string;
+  executionError?: string;
+  userGuidance?: string;
+}
+
+export interface ActionRecord {
+  step: number;
+  toolName: string;
+  input: Record<string, unknown>;
+  reasoning?: string;
+}
+
+interface PendingRequest {
+  resolve: (result: ToolExecutionResult) => void;
+  reject: (err: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+}
+
+// ---------------------------------------------------------------------------
+// HostCuProxy
+// ---------------------------------------------------------------------------
+
+export class HostCuProxy {
+  private pending = new Map<string, PendingRequest>();
+  private sendToClient: (msg: ServerMessage) => void;
+  private clientConnected = false;
+
+  // CU state tracking (per-conversation)
+  private _stepCount = 0;
+  private _maxSteps: number;
+  private _previousAXTree: string | undefined;
+  private _consecutiveUnchangedSteps = 0;
+  private _actionHistory: ActionRecord[] = [];
+
+  constructor(
+    sendToClient: (msg: ServerMessage) => void,
+    maxSteps = MAX_STEPS,
+  ) {
+    this.sendToClient = sendToClient;
+    this._maxSteps = maxSteps;
+  }
+
+  // ---------------------------------------------------------------------------
+  // CU state accessors (for testing / external inspection)
+  // ---------------------------------------------------------------------------
+
+  get stepCount(): number {
+    return this._stepCount;
+  }
+
+  get maxSteps(): number {
+    return this._maxSteps;
+  }
+
+  get previousAXTree(): string | undefined {
+    return this._previousAXTree;
+  }
+
+  get consecutiveUnchangedSteps(): number {
+    return this._consecutiveUnchangedSteps;
+  }
+
+  get actionHistory(): readonly ActionRecord[] {
+    return this._actionHistory;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Sender management
+  // ---------------------------------------------------------------------------
+
+  updateSender(
+    sendToClient: (msg: ServerMessage) => void,
+    clientConnected: boolean,
+  ): void {
+    this.sendToClient = sendToClient;
+    this.clientConnected = clientConnected;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Request / resolve lifecycle
+  // ---------------------------------------------------------------------------
+
+  request(
+    toolName: string,
+    input: Record<string, unknown>,
+    sessionId: string,
+    stepNumber: number,
+    reasoning?: string,
+    signal?: AbortSignal,
+  ): Promise<ToolExecutionResult> {
+    if (signal?.aborted) {
+      return Promise.resolve({
+        content: "Aborted",
+        isError: true,
+      });
+    }
+
+    // Enforce step limit before sending to client
+    if (this._stepCount > this._maxSteps) {
+      return Promise.resolve({
+        content: `Step limit (${this._maxSteps}) exceeded. Call computer_use_done to finish.`,
+        isError: true,
+      });
+    }
+
+    const requestId = uuid();
+
+    return new Promise<ToolExecutionResult>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(requestId);
+        log.warn({ requestId, toolName }, "Host CU proxy request timed out");
+        resolve({
+          content: "Host CU proxy timed out waiting for client response",
+          isError: true,
+        });
+      }, REQUEST_TIMEOUT_SEC * 1000);
+
+      this.pending.set(requestId, { resolve, reject, timer });
+
+      if (signal) {
+        const onAbort = () => {
+          if (this.pending.has(requestId)) {
+            clearTimeout(timer);
+            this.pending.delete(requestId);
+            resolve({ content: "Aborted", isError: true });
+          }
+        };
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
+
+      this.sendToClient({
+        type: "host_cu_request",
+        requestId,
+        sessionId,
+        toolName,
+        input,
+        stepNumber,
+        reasoning,
+      } as ServerMessage);
+    });
+  }
+
+  resolve(requestId: string, observation: CuObservationResult): void {
+    const entry = this.pending.get(requestId);
+    if (!entry) {
+      log.warn({ requestId }, "No pending host CU request for response");
+      return;
+    }
+    clearTimeout(entry.timer);
+    this.pending.delete(requestId);
+
+    // Update CU state from observation
+    this.updateStateFromObservation(observation);
+
+    const result = this.formatObservation(observation);
+    entry.resolve(result);
+  }
+
+  hasPendingRequest(requestId: string): boolean {
+    return this.pending.has(requestId);
+  }
+
+  // ---------------------------------------------------------------------------
+  // CU state management
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Increment step count and record an action. Call this before sending
+   * each non-terminal tool request.
+   */
+  recordAction(
+    toolName: string,
+    input: Record<string, unknown>,
+    reasoning?: string,
+  ): void {
+    this._stepCount++;
+    this._actionHistory.push({
+      step: this._stepCount,
+      toolName,
+      input,
+      reasoning,
+    });
+    // Keep history bounded
+    if (this._actionHistory.length > MAX_HISTORY_ENTRIES) {
+      this._actionHistory = this._actionHistory.slice(-MAX_HISTORY_ENTRIES);
+    }
+  }
+
+  /** Reset all CU state. Called on terminal tools (computer_use_done, etc.). */
+  reset(): void {
+    this._stepCount = 0;
+    this._previousAXTree = undefined;
+    this._consecutiveUnchangedSteps = 0;
+    this._actionHistory = [];
+  }
+
+  // ---------------------------------------------------------------------------
+  // Observation formatting
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Formats a CU observation into a ToolExecutionResult with text content
+   * (AX tree wrapped in markers, diff, warnings) and optional screenshot
+   * as an image content block.
+   */
+  formatObservation(obs: CuObservationResult): ToolExecutionResult {
+    const parts: string[] = [];
+
+    // Surface user guidance prominently so the model sees it first
+    if (obs.userGuidance) {
+      parts.push(`USER GUIDANCE: ${obs.userGuidance}`);
+      parts.push("");
+    }
+
+    if (obs.executionResult) {
+      parts.push(obs.executionResult);
+      parts.push("");
+    }
+
+    // AX tree diff / unchanged warning
+    if (obs.axDiff) {
+      parts.push(obs.axDiff);
+      parts.push("");
+    } else if (this._previousAXTree != null && obs.axTree != null) {
+      // No diff means the screen didn't change
+      if (
+        this._consecutiveUnchangedSteps >=
+        CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD
+      ) {
+        parts.push(
+          `WARNING: ${this._consecutiveUnchangedSteps} consecutive actions had NO VISIBLE EFFECT on the UI. You MUST try a completely different approach.`,
+        );
+      } else {
+        parts.push(
+          "Your last action had NO VISIBLE EFFECT on the UI. Try something different.",
+        );
+      }
+      parts.push("");
+    }
+
+    // Loop detection: identical actions repeated
+    if (this._actionHistory.length >= LOOP_DETECTION_WINDOW) {
+      const recent = this._actionHistory.slice(-LOOP_DETECTION_WINDOW);
+      const allIdentical = recent.every(
+        (r) =>
+          r.toolName === recent[0].toolName &&
+          JSON.stringify(r.input) === JSON.stringify(recent[0].input),
+      );
+      if (allIdentical) {
+        parts.push(
+          `WARNING: You've repeated the same action (${recent[0].toolName}) ${LOOP_DETECTION_WINDOW} times. Try something different.`,
+        );
+        parts.push("");
+      }
+    }
+
+    // Current screen state wrapped in markers for history compaction
+    if (obs.axTree) {
+      parts.push("<ax-tree>");
+      parts.push("CURRENT SCREEN STATE:");
+      parts.push(HostCuProxy.escapeAxTreeContent(obs.axTree));
+      parts.push("</ax-tree>");
+    }
+
+    // Screenshot metadata
+    const screenshotMeta = this.formatScreenshotMetadata(obs);
+    if (screenshotMeta.length > 0) {
+      parts.push("");
+      parts.push(...screenshotMeta);
+    }
+
+    const content = parts.join("\n").trim() || "Action executed";
+
+    // Build content blocks for screenshot
+    const contentBlocks: ContentBlock[] = [];
+    if (obs.screenshot) {
+      contentBlocks.push({
+        type: "image",
+        source: {
+          type: "base64",
+          media_type: "image/jpeg",
+          data: obs.screenshot,
+        },
+      });
+    }
+
+    const isError = obs.executionError != null;
+
+    return {
+      content: isError
+        ? `Action failed: ${obs.executionError}\n\n${content}`
+        : content,
+      isError,
+      ...(contentBlocks.length > 0 ? { contentBlocks } : {}),
+    };
+  }
+
+  // ---------------------------------------------------------------------------
+  // Dispose
+  // ---------------------------------------------------------------------------
+
+  dispose(): void {
+    for (const [_requestId, entry] of this.pending) {
+      clearTimeout(entry.timer);
+      entry.reject(
+        new AssistantError("Host CU proxy disposed", ErrorCode.INTERNAL_ERROR),
+      );
+    }
+    this.pending.clear();
+  }
+
+  // ---------------------------------------------------------------------------
+  // Private helpers
+  // ---------------------------------------------------------------------------
+
+  /** Update consecutive-unchanged tracking from an incoming observation. */
+  private updateStateFromObservation(obs: CuObservationResult): void {
+    if (this._stepCount > 0) {
+      if (
+        obs.axDiff == null &&
+        this._previousAXTree != null &&
+        obs.axTree != null
+      ) {
+        this._consecutiveUnchangedSteps++;
+      } else if (obs.axDiff != null) {
+        this._consecutiveUnchangedSteps = 0;
+      }
+    }
+
+    if (obs.axTree != null) {
+      this._previousAXTree = obs.axTree;
+    }
+  }
+
+  private formatScreenshotMetadata(obs: CuObservationResult): string[] {
+    if (!obs.screenshot) return [];
+
+    const lines: string[] = [];
+    if (obs.screenshotWidthPx != null && obs.screenshotHeightPx != null) {
+      lines.push(
+        `Screenshot metadata: ${obs.screenshotWidthPx}x${obs.screenshotHeightPx} px`,
+      );
+    }
+    if (obs.screenWidthPt != null && obs.screenHeightPt != null) {
+      lines.push(
+        `Screen metadata: ${obs.screenWidthPt}x${obs.screenHeightPt} pt`,
+      );
+    }
+    return lines;
+  }
+
+  /**
+   * Escapes literal `</ax-tree>` inside AX tree content so compaction
+   * regex does not stop prematurely.
+   */
+  static escapeAxTreeContent(content: string): string {
+    return content.replace(/<\/ax-tree>/gi, "&lt;/ax-tree&gt;");
+  }
+}