@vellumai/assistant 0.4.46 → 0.4.49

package/src/daemon/host-cu-proxy.ts

@@ -0,0 +1,401 @@
+/**
+ * Host computer-use proxy.
+ *
+ * Proxies computer-use actions to the desktop client when running as a
+ * managed assistant, following the same request/resolve pattern as
+ * HostBashProxy. Also owns CU-specific state tracking (step counting,
+ * loop detection, observation formatting) for the unified agent loop.
+ */
+
+import { v4 as uuid } from "uuid";
+
+import type { ContentBlock } from "../providers/types.js";
+import type { ToolExecutionResult } from "../tools/types.js";
+import { AssistantError, ErrorCode } from "../util/errors.js";
+import { getLogger } from "../util/logger.js";
+import type { ServerMessage } from "./message-protocol.js";
+
+const log = getLogger("host-cu-proxy");
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const REQUEST_TIMEOUT_SEC = 60;
+const MAX_STEPS = 50;
+const MAX_HISTORY_ENTRIES = 10;
+const LOOP_DETECTION_WINDOW = 3;
+const CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD = 2;
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface CuObservationResult {
+  axTree?: string;
+  axDiff?: string;
+  secondaryWindows?: string;
+  screenshot?: string; // base64 JPEG
+  screenshotWidthPx?: number;
+  screenshotHeightPx?: number;
+  screenWidthPt?: number;
+  screenHeightPt?: number;
+  executionResult?: string;
+  executionError?: string;
+  userGuidance?: string;
+}
+
+export interface ActionRecord {
+  step: number;
+  toolName: string;
+  input: Record<string, unknown>;
+  reasoning?: string;
+}
+
+interface PendingRequest {
+  resolve: (result: ToolExecutionResult) => void;
+  reject: (err: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+}
+
+// ---------------------------------------------------------------------------
+// HostCuProxy
+// ---------------------------------------------------------------------------
+
+export class HostCuProxy {
+  private pending = new Map<string, PendingRequest>();
+  private sendToClient: (msg: ServerMessage) => void;
+  private clientConnected = false;
+
+  // CU state tracking (per-conversation)
+  private _stepCount = 0;
+  private _maxSteps: number;
+  private _previousAXTree: string | undefined;
+  private _consecutiveUnchangedSteps = 0;
+  private _actionHistory: ActionRecord[] = [];
+
+  constructor(
+    sendToClient: (msg: ServerMessage) => void,
+    maxSteps = MAX_STEPS,
+  ) {
+    this.sendToClient = sendToClient;
+    this._maxSteps = maxSteps;
+  }
+
+  // ---------------------------------------------------------------------------
+  // CU state accessors (for testing / external inspection)
+  // ---------------------------------------------------------------------------
+
+  get stepCount(): number {
+    return this._stepCount;
+  }
+
+  get maxSteps(): number {
+    return this._maxSteps;
+  }
+
+  get previousAXTree(): string | undefined {
+    return this._previousAXTree;
+  }
+
+  get consecutiveUnchangedSteps(): number {
+    return this._consecutiveUnchangedSteps;
+  }
+
+  get actionHistory(): readonly ActionRecord[] {
+    return this._actionHistory;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Sender management
+  // ---------------------------------------------------------------------------
+
+  updateSender(
+    sendToClient: (msg: ServerMessage) => void,
+    clientConnected: boolean,
+  ): void {
+    this.sendToClient = sendToClient;
+    this.clientConnected = clientConnected;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Request / resolve lifecycle
+  // ---------------------------------------------------------------------------
+
+  request(
+    toolName: string,
+    input: Record<string, unknown>,
+    sessionId: string,
+    stepNumber: number,
+    reasoning?: string,
+    signal?: AbortSignal,
+  ): Promise<ToolExecutionResult> {
+    if (signal?.aborted) {
+      return Promise.resolve({
+        content: "Aborted",
+        isError: true,
+      });
+    }
+
+    // Enforce step limit before sending to client
+    if (this._stepCount > this._maxSteps) {
+      return Promise.resolve({
+        content: `Step limit (${this._maxSteps}) exceeded. Call computer_use_done to finish.`,
+        isError: true,
+      });
+    }
+
+    const requestId = uuid();
+
+    return new Promise<ToolExecutionResult>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(requestId);
+        log.warn({ requestId, toolName }, "Host CU proxy request timed out");
+        resolve({
+          content: "Host CU proxy timed out waiting for client response",
+          isError: true,
+        });
+      }, REQUEST_TIMEOUT_SEC * 1000);
+
+      this.pending.set(requestId, { resolve, reject, timer });
+
+      if (signal) {
+        const onAbort = () => {
+          if (this.pending.has(requestId)) {
+            clearTimeout(timer);
+            this.pending.delete(requestId);
+            resolve({ content: "Aborted", isError: true });
+          }
+        };
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
+
+      this.sendToClient({
+        type: "host_cu_request",
+        requestId,
+        sessionId,
+        toolName,
+        input,
+        stepNumber,
+        reasoning,
+      } as ServerMessage);
+    });
+  }
+
+  resolve(requestId: string, observation: CuObservationResult): void {
+    const entry = this.pending.get(requestId);
+    if (!entry) {
+      log.warn({ requestId }, "No pending host CU request for response");
+      return;
+    }
+    clearTimeout(entry.timer);
+    this.pending.delete(requestId);
+
+    // Update CU state from observation
+    this.updateStateFromObservation(observation);
+
+    const result = this.formatObservation(observation);
+    entry.resolve(result);
+  }
+
+  hasPendingRequest(requestId: string): boolean {
+    return this.pending.has(requestId);
+  }
+
+  // ---------------------------------------------------------------------------
+  // CU state management
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Increment step count and record an action. Call this before sending
+   * each non-terminal tool request.
+   */
+  recordAction(
+    toolName: string,
+    input: Record<string, unknown>,
+    reasoning?: string,
+  ): void {
+    this._stepCount++;
+    this._actionHistory.push({
+      step: this._stepCount,
+      toolName,
+      input,
+      reasoning,
+    });
+    // Keep history bounded
+    if (this._actionHistory.length > MAX_HISTORY_ENTRIES) {
+      this._actionHistory = this._actionHistory.slice(-MAX_HISTORY_ENTRIES);
+    }
+  }
+
+  /** Reset all CU state. Called on terminal tools (computer_use_done, etc.). */
+  reset(): void {
+    this._stepCount = 0;
+    this._previousAXTree = undefined;
+    this._consecutiveUnchangedSteps = 0;
+    this._actionHistory = [];
+  }
+
+  // ---------------------------------------------------------------------------
+  // Observation formatting
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Formats a CU observation into a ToolExecutionResult with text content
+   * (AX tree wrapped in markers, diff, warnings) and optional screenshot
+   * as an image content block.
+   */
+  formatObservation(obs: CuObservationResult): ToolExecutionResult {
+    const parts: string[] = [];
+
+    // Surface user guidance prominently so the model sees it first
+    if (obs.userGuidance) {
+      parts.push(`USER GUIDANCE: ${obs.userGuidance}`);
+      parts.push("");
+    }
+
+    if (obs.executionResult) {
+      parts.push(obs.executionResult);
+      parts.push("");
+    }
+
+    // AX tree diff / unchanged warning
+    if (obs.axDiff) {
+      parts.push(obs.axDiff);
+      parts.push("");
+    } else if (this._previousAXTree != null && obs.axTree != null) {
+      // No diff means the screen didn't change
+      if (
+        this._consecutiveUnchangedSteps >=
+        CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD
+      ) {
+        parts.push(
+          `WARNING: ${this._consecutiveUnchangedSteps} consecutive actions had NO VISIBLE EFFECT on the UI. You MUST try a completely different approach.`,
+        );
+      } else {
+        parts.push(
+          "Your last action had NO VISIBLE EFFECT on the UI. Try something different.",
+        );
+      }
+      parts.push("");
+    }
+
+    // Loop detection: identical actions repeated
+    if (this._actionHistory.length >= LOOP_DETECTION_WINDOW) {
+      const recent = this._actionHistory.slice(-LOOP_DETECTION_WINDOW);
+      const allIdentical = recent.every(
+        (r) =>
+          r.toolName === recent[0].toolName &&
+          JSON.stringify(r.input) === JSON.stringify(recent[0].input),
+      );
+      if (allIdentical) {
+        parts.push(
+          `WARNING: You've repeated the same action (${recent[0].toolName}) ${LOOP_DETECTION_WINDOW} times. Try something different.`,
+        );
+        parts.push("");
+      }
+    }
+
+    // Current screen state wrapped in markers for history compaction
+    if (obs.axTree) {
+      parts.push("<ax-tree>");
+      parts.push("CURRENT SCREEN STATE:");
+      parts.push(HostCuProxy.escapeAxTreeContent(obs.axTree));
+      parts.push("</ax-tree>");
+    }
+
+    // Screenshot metadata
+    const screenshotMeta = this.formatScreenshotMetadata(obs);
+    if (screenshotMeta.length > 0) {
+      parts.push("");
+      parts.push(...screenshotMeta);
+    }
+
+    const content = parts.join("\n").trim() || "Action executed";
+
+    // Build content blocks for screenshot
+    const contentBlocks: ContentBlock[] = [];
+    if (obs.screenshot) {
+      contentBlocks.push({
+        type: "image",
+        source: {
+          type: "base64",
+          media_type: "image/jpeg",
+          data: obs.screenshot,
+        },
+      });
+    }
+
+    const isError = obs.executionError != null;
+
+    return {
+      content: isError
+        ? `Action failed: ${obs.executionError}\n\n${content}`
+        : content,
+      isError,
+      ...(contentBlocks.length > 0 ? { contentBlocks } : {}),
+    };
+  }
+
+  // ---------------------------------------------------------------------------
+  // Dispose
+  // ---------------------------------------------------------------------------
+
+  dispose(): void {
+    for (const [_requestId, entry] of this.pending) {
+      clearTimeout(entry.timer);
+      entry.reject(
+        new AssistantError("Host CU proxy disposed", ErrorCode.INTERNAL_ERROR),
+      );
+    }
+    this.pending.clear();
+  }
+
+  // ---------------------------------------------------------------------------
+  // Private helpers
+  // ---------------------------------------------------------------------------
+
+  /** Update consecutive-unchanged tracking from an incoming observation. */
+  private updateStateFromObservation(obs: CuObservationResult): void {
+    if (this._stepCount > 0) {
+      if (
+        obs.axDiff == null &&
+        this._previousAXTree != null &&
+        obs.axTree != null
+      ) {
+        this._consecutiveUnchangedSteps++;
+      } else if (obs.axDiff != null) {
+        this._consecutiveUnchangedSteps = 0;
+      }
+    }
+
+    if (obs.axTree != null) {
+      this._previousAXTree = obs.axTree;
+    }
+  }
+
+  private formatScreenshotMetadata(obs: CuObservationResult): string[] {
+    if (!obs.screenshot) return [];
+
+    const lines: string[] = [];
+    if (obs.screenshotWidthPx != null && obs.screenshotHeightPx != null) {
+      lines.push(
+        `Screenshot metadata: ${obs.screenshotWidthPx}x${obs.screenshotHeightPx} px`,
+      );
+    }
+    if (obs.screenWidthPt != null && obs.screenHeightPt != null) {
+      lines.push(
+        `Screen metadata: ${obs.screenWidthPt}x${obs.screenHeightPt} pt`,
+      );
+    }
+    return lines;
+  }
+
+  /**
+   * Escapes literal `</ax-tree>` inside AX tree content so compaction
+   * regex does not stop prematurely.
+   */
+  static escapeAxTreeContent(content: string): string {
+    return content.replace(/<\/ax-tree>/gi, "&lt;/ax-tree&gt;");
+  }
+}

package/src/daemon/lifecycle.ts

@@ -15,6 +15,7 @@ import {
   getQdrantUrlEnv,
   getRuntimeHttpHost,
   getRuntimeHttpPort,
+  setIngressPublicBaseUrl,
   validateEnv,
 } from "../config/env.js";
 import { loadConfig } from "../config/loader.js";
@@ -22,7 +23,7 @@ import { HeartbeatService } from "../heartbeat/heartbeat-service.js";
 import { getHookManager } from "../hooks/manager.js";
 import { installTemplates } from "../hooks/templates.js";
 import { closeSentry, initSentry } from "../instrument.js";
-import { initLogfire } from "../logfire.js";
+import { disableLogfire, initLogfire } from "../logfire.js";
 import { getMcpServerManager } from "../mcp/manager.js";
 import * as attachmentsStore from "../memory/attachments-store.js";
 import {
@@ -40,6 +41,8 @@ import {
   emitNotificationSignal,
   registerBroadcastFn,
 } from "../notifications/emit-signal.js";
+import { backfillManualTokenConnections } from "../oauth/manual-token-connection.js";
+import { seedOAuthProviders } from "../oauth/seed-providers.js";
 import { ensurePromptFiles } from "../prompts/system-prompt.js";
 import { syncUpdateBulletinOnStartup } from "../prompts/update-bulletin.js";
 import { buildAssistantEvent } from "../runtime/assistant-event.js";
@@ -54,7 +57,6 @@ import {
 import { ensureVellumGuardianBinding } from "../runtime/guardian-vellum-migration.js";
 import { RuntimeHttpServer } from "../runtime/http-server.js";
 import { startScheduler } from "../schedule/scheduler.js";
-import { watchSessions } from "../tools/watch/watch-state.js";
 import { getLogger, initLogger } from "../util/logger.js";
 import {
   ensureDataDir,
@@ -94,10 +96,6 @@ import {
   registerMessagingProviders,
   registerWatcherProviders,
 } from "./providers-setup.js";
-import {
-  handleRideShotgunStart,
-  handleRideShotgunStop,
-} from "./ride-shotgun-handler.js";
 import { seedInterfaceFiles } from "./seed-files.js";
 import { DaemonServer } from "./server.js";
 import { initSlashPairingContext } from "./session-slash.js";
@@ -159,6 +157,12 @@ export async function runDaemon(): Promise<void> {
       );
     }
     initializeDb();
+    // Seed well-known OAuth provider configurations (insert-if-not-exists)
+    seedOAuthProviders();
+    // Backfill oauth_connection rows for manual-token providers (Telegram,
+    // Slack channel) that already have keychain credentials from before the
+    // oauth_connection migration. Safe to call on every startup.
+    await backfillManualTokenConnections();
     log.info("Daemon startup: DB initialized");
 
     // Ensure a vellum guardian binding exists and mint the CLI edge token
@@ -193,7 +197,9 @@ export async function runDaemon(): Promise<void> {
       const httpTokenPath = join(getRootDir(), "http-token");
       writeFileSync(httpTokenPath, credentials.accessToken, { mode: 0o600 });
       chmodSync(httpTokenPath, 0o600);
-      log.info("Daemon startup: CLI edge token written to credential store and http-token");
+      log.info(
+        "Daemon startup: CLI edge token written to credential store and http-token",
+      );
     } else {
       log.warn("No guardian principal available — CLI edge token not written");
     }
@@ -234,6 +240,19 @@ export async function runDaemon(): Promise<void> {
     log.info("Daemon startup: loading config");
     const config = loadConfig();
 
+    // Seed module-level ingress state from the workspace config so that
+    // getIngressPublicBaseUrl() returns the correct value immediately after
+    // startup (before any handleIngressConfig("set") call). Without this,
+    // code paths that read the module-level state directly (e.g. session-slash
+    // pairing info) would see undefined until an explicit set.
+    if (config.ingress.enabled && config.ingress.publicBaseUrl) {
+      setIngressPublicBaseUrl(config.ingress.publicBaseUrl);
+      log.info(
+        { url: config.ingress.publicBaseUrl },
+        "Daemon startup: seeded ingress URL from workspace config",
+      );
+    }
+
     if (config.logFile.dir) {
       initLogger({
         dir: config.logFile.dir,
@@ -251,6 +270,18 @@ export async function runDaemon(): Promise<void> {
       await closeSentry();
     }
 
+    // If Logfire observability is not explicitly enabled, disable it so
+    // wrapWithLogfire() calls during provider setup become no-ops. Logfire
+    // is initialized eagerly (before config loads) for the same reason as
+    // Sentry — but the feature flag gates whether it actually traces.
+    const logfireEnabled = isAssistantFeatureFlagEnabled(
+      "feature_flags.logfire.enabled",
+      config,
+    );
+    if (!logfireEnabled) {
+      disableLogfire();
+    }
+
     await initializeProvidersAndTools(config);
 
     // Start the DaemonServer (session manager) before Qdrant so HTTP
@@ -493,64 +524,9 @@ export async function runDaemon(): Promise<void> {
           );
         },
       },
-      getComputerUseDeps: () => {
+      getWatchDeps: () => {
         const ctx = server.getHandlerContext();
         return {
-          cuSessions: ctx.cuSessions,
-          sharedRequestTimestamps: ctx.sharedRequestTimestamps,
-          cuObservationParseSequence: ctx.cuObservationParseSequence,
-          handleRideShotgunStart: async (params) => {
-            // The handler generates its own watchId/sessionId and
-            // sends them via ctx.send as a watch_started message.
-            // We intercept send to capture the IDs before they broadcast.
-            let capturedWatchId = "";
-            let capturedSessionId = "";
-            const interceptCtx = {
-              ...ctx,
-              send: (msg: ServerMessage) => {
-                if (
-                  "type" in msg &&
-                  msg.type === "watch_started" &&
-                  "watchId" in msg &&
-                  "sessionId" in msg
-                ) {
-                  capturedWatchId = (msg as { watchId: string }).watchId;
-                  capturedSessionId = (msg as { sessionId: string }).sessionId;
-                }
-                ctx.send(msg);
-              },
-            };
-            await handleRideShotgunStart(
-              {
-                type: "ride_shotgun_start",
-                durationSeconds: params.durationSeconds,
-                intervalSeconds: params.intervalSeconds,
-                mode: params.mode,
-                targetDomain: params.targetDomain,
-                navigateDomain: params.navigateDomain,
-                autoNavigate: params.autoNavigate,
-              },
-              interceptCtx,
-            );
-            return { watchId: capturedWatchId, sessionId: capturedSessionId };
-          },
-          handleRideShotgunStop: async (watchId) => {
-            await handleRideShotgunStop(
-              { type: "ride_shotgun_stop", watchId },
-              ctx,
-            );
-          },
-          getRideShotgunStatus: (watchId) => {
-            const session = watchSessions.get(watchId);
-            if (!session) return undefined;
-            return {
-              status: session.status,
-              sessionId: session.sessionId,
-              recordingId: session.recordingId,
-              savedRecordingPath: session.savedRecordingPath,
-              bootstrapFailureReason: session.bootstrapFailureReason,
-            };
-          },
           handleWatchObservation: async (params) => {
             await handleWatchObservation(
               {

package/src/daemon/message-protocol.ts

@@ -21,6 +21,7 @@ export * from "./message-types/diagnostics.js";
 export * from "./message-types/documents.js";
 export * from "./message-types/guardian-actions.js";
 export * from "./message-types/host-bash.js";
+export * from "./message-types/host-cu.js";
 export * from "./message-types/host-file.js";
 export * from "./message-types/inbox.js";
 export * from "./message-types/integrations.js";
@@ -69,6 +70,7 @@ import type {
   _GuardianActionsServerMessages,
 } from "./message-types/guardian-actions.js";
 import type { _HostBashServerMessages } from "./message-types/host-bash.js";
+import type { _HostCuServerMessages } from "./message-types/host-cu.js";
 import type { _HostFileServerMessages } from "./message-types/host-file.js";
 import type {
   _InboxClientMessages,
@@ -180,6 +182,7 @@ export type ServerMessage =
   | _DocumentsServerMessages
   | _GuardianActionsServerMessages
   | _HostBashServerMessages
+  | _HostCuServerMessages
   | _HostFileServerMessages
   | _MemoryServerMessages
   | _WorkspaceServerMessages