@vellumai/assistant 0.4.48 → 0.4.50

package/src/daemon/host-cu-proxy.ts

@@ -0,0 +1,430 @@
+/**
+ * Host computer-use proxy.
+ *
+ * Proxies computer-use actions to the desktop client when running as a
+ * managed assistant, following the same request/resolve pattern as
+ * HostBashProxy. Also owns CU-specific state tracking (step counting,
+ * loop detection, observation formatting) for the unified agent loop.
+ */
+
+import { v4 as uuid } from "uuid";
+
+import { escapeAxTreeContent } from "../agent/loop.js";
+import type { ContentBlock } from "../providers/types.js";
+import type { ToolExecutionResult } from "../tools/types.js";
+import { AssistantError, ErrorCode } from "../util/errors.js";
+import { getLogger } from "../util/logger.js";
+import type { ServerMessage } from "./message-protocol.js";
+
+const log = getLogger("host-cu-proxy");
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const REQUEST_TIMEOUT_SEC = 60;
+const MAX_STEPS = 50;
+const MAX_HISTORY_ENTRIES = 10;
+const LOOP_DETECTION_WINDOW = 3;
+const CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD = 2;
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface CuObservationResult {
+  axTree?: string;
+  axDiff?: string;
+  secondaryWindows?: string;
+  screenshot?: string; // base64 JPEG
+  screenshotWidthPx?: number;
+  screenshotHeightPx?: number;
+  screenWidthPt?: number;
+  screenHeightPt?: number;
+  executionResult?: string;
+  executionError?: string;
+  userGuidance?: string;
+}
+
+export interface ActionRecord {
+  step: number;
+  toolName: string;
+  input: Record<string, unknown>;
+  reasoning?: string;
+}
+
+interface PendingRequest {
+  resolve: (result: ToolExecutionResult) => void;
+  reject: (err: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+}
+
+// ---------------------------------------------------------------------------
+// HostCuProxy
+// ---------------------------------------------------------------------------
+
+export class HostCuProxy {
+  private pending = new Map<string, PendingRequest>();
+  private sendToClient: (msg: ServerMessage) => void;
+  private onInternalResolve?: (requestId: string) => void;
+  private clientConnected = false;
+
+  // CU state tracking (per-conversation)
+  private _stepCount = 0;
+  private _maxSteps: number;
+  private _previousAXTree: string | undefined;
+  private _consecutiveUnchangedSteps = 0;
+  private _actionHistory: ActionRecord[] = [];
+
+  constructor(
+    sendToClient: (msg: ServerMessage) => void,
+    onInternalResolve?: (requestId: string) => void,
+    maxSteps = MAX_STEPS,
+  ) {
+    this.sendToClient = sendToClient;
+    this.onInternalResolve = onInternalResolve;
+    this._maxSteps = maxSteps;
+  }
+
+  // ---------------------------------------------------------------------------
+  // CU state accessors (for testing / external inspection)
+  // ---------------------------------------------------------------------------
+
+  get stepCount(): number {
+    return this._stepCount;
+  }
+
+  get maxSteps(): number {
+    return this._maxSteps;
+  }
+
+  get previousAXTree(): string | undefined {
+    return this._previousAXTree;
+  }
+
+  get consecutiveUnchangedSteps(): number {
+    return this._consecutiveUnchangedSteps;
+  }
+
+  get actionHistory(): readonly ActionRecord[] {
+    return this._actionHistory;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Sender management
+  // ---------------------------------------------------------------------------
+
+  updateSender(
+    sendToClient: (msg: ServerMessage) => void,
+    clientConnected: boolean,
+  ): void {
+    this.sendToClient = sendToClient;
+    this.clientConnected = clientConnected;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Request / resolve lifecycle
+  // ---------------------------------------------------------------------------
+
+  request(
+    toolName: string,
+    input: Record<string, unknown>,
+    sessionId: string,
+    stepNumber: number,
+    reasoning?: string,
+    signal?: AbortSignal,
+  ): Promise<ToolExecutionResult> {
+    if (signal?.aborted) {
+      return Promise.resolve({
+        content: "Aborted",
+        isError: true,
+      });
+    }
+
+    // Enforce step limit before sending to client
+    if (this._stepCount > this._maxSteps) {
+      return Promise.resolve({
+        content: `Step limit (${this._maxSteps}) exceeded. Call computer_use_done to finish.`,
+        isError: true,
+      });
+    }
+
+    const requestId = uuid();
+
+    return new Promise<ToolExecutionResult>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(requestId);
+        this.onInternalResolve?.(requestId);
+        log.warn({ requestId, toolName }, "Host CU proxy request timed out");
+        resolve({
+          content: "Host CU proxy timed out waiting for client response",
+          isError: true,
+        });
+      }, REQUEST_TIMEOUT_SEC * 1000);
+
+      this.pending.set(requestId, { resolve, reject, timer });
+
+      if (signal) {
+        const onAbort = () => {
+          if (this.pending.has(requestId)) {
+            clearTimeout(timer);
+            this.pending.delete(requestId);
+            this.onInternalResolve?.(requestId);
+            resolve({ content: "Aborted", isError: true });
+          }
+        };
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
+
+      this.sendToClient({
+        type: "host_cu_request",
+        requestId,
+        sessionId,
+        toolName,
+        input,
+        stepNumber,
+        reasoning,
+      } as ServerMessage);
+    });
+  }
+
+  resolve(requestId: string, observation: CuObservationResult): void {
+    const entry = this.pending.get(requestId);
+    if (!entry) {
+      log.warn({ requestId }, "No pending host CU request for response");
+      return;
+    }
+    clearTimeout(entry.timer);
+    this.pending.delete(requestId);
+
+    // Capture pre-update state so formatObservation sees the correct previous AX tree
+    const prevAXTree = this._previousAXTree;
+
+    // Update CU state from observation
+    this.updateStateFromObservation(observation);
+
+    const result = this.formatObservation(observation, prevAXTree);
+    entry.resolve(result);
+  }
+
+  hasPendingRequest(requestId: string): boolean {
+    return this.pending.has(requestId);
+  }
+
+  isAvailable(): boolean {
+    return this.clientConnected;
+  }
+
+  // ---------------------------------------------------------------------------
+  // CU state management
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Increment step count and record an action. Call this before sending
+   * each non-terminal tool request.
+   */
+  recordAction(
+    toolName: string,
+    input: Record<string, unknown>,
+    reasoning?: string,
+  ): void {
+    this._stepCount++;
+    this._actionHistory.push({
+      step: this._stepCount,
+      toolName,
+      input,
+      reasoning,
+    });
+    // Keep history bounded
+    if (this._actionHistory.length > MAX_HISTORY_ENTRIES) {
+      this._actionHistory = this._actionHistory.slice(-MAX_HISTORY_ENTRIES);
+    }
+  }
+
+  /** Reset all CU state. Called on terminal tools (computer_use_done, etc.). */
+  reset(): void {
+    this._stepCount = 0;
+    this._previousAXTree = undefined;
+    this._consecutiveUnchangedSteps = 0;
+    this._actionHistory = [];
+  }
+
+  // ---------------------------------------------------------------------------
+  // Observation formatting
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Formats a CU observation into a ToolExecutionResult with text content
+   * (AX tree wrapped in markers, diff, warnings) and optional screenshot
+   * as an image content block.
+   */
+  formatObservation(
+    obs: CuObservationResult,
+    previousAXTree?: string,
+  ): ToolExecutionResult {
+    const prevTree = previousAXTree;
+    const parts: string[] = [];
+
+    // Surface user guidance prominently so the model sees it first
+    if (obs.userGuidance) {
+      parts.push(`USER GUIDANCE: ${obs.userGuidance}`);
+      parts.push("");
+    }
+
+    if (obs.executionResult) {
+      parts.push(obs.executionResult);
+      parts.push("");
+    }
+
+    // AX tree diff / unchanged warning
+    if (obs.axDiff) {
+      parts.push(obs.axDiff);
+      parts.push("");
+    } else if (prevTree != null && obs.axTree != null) {
+      // Skip unchanged warning after wait actions — they intentionally yield no immediate change
+      const lastAction =
+        this._actionHistory.length > 0
+          ? this._actionHistory[this._actionHistory.length - 1]
+          : undefined;
+      const isWaitAction = lastAction?.toolName === "computer_use_wait";
+
+      if (!isWaitAction) {
+        // No diff means the screen didn't change
+        if (
+          this._consecutiveUnchangedSteps >=
+          CONSECUTIVE_UNCHANGED_WARNING_THRESHOLD
+        ) {
+          parts.push(
+            `WARNING: ${this._consecutiveUnchangedSteps} consecutive actions had NO VISIBLE EFFECT on the UI. You MUST try a completely different approach.`,
+          );
+        } else {
+          parts.push(
+            "Your last action had NO VISIBLE EFFECT on the UI. Try something different.",
+          );
+        }
+        parts.push("");
+      }
+    }
+
+    // Loop detection: identical actions repeated
+    if (this._actionHistory.length >= LOOP_DETECTION_WINDOW) {
+      const recent = this._actionHistory.slice(-LOOP_DETECTION_WINDOW);
+      const allIdentical = recent.every(
+        (r) =>
+          r.toolName === recent[0].toolName &&
+          JSON.stringify(r.input) === JSON.stringify(recent[0].input),
+      );
+      if (allIdentical) {
+        parts.push(
+          `WARNING: You've repeated the same action (${recent[0].toolName}) ${LOOP_DETECTION_WINDOW} times. Try something different.`,
+        );
+        parts.push("");
+      }
+    }
+
+    // Current screen state wrapped in markers for history compaction
+    if (obs.axTree) {
+      parts.push("<ax-tree>");
+      parts.push("CURRENT SCREEN STATE:");
+      parts.push(escapeAxTreeContent(obs.axTree));
+      parts.push("</ax-tree>");
+    }
+
+    // Secondary windows for cross-app awareness
+    if (obs.secondaryWindows) {
+      parts.push("");
+      parts.push(obs.secondaryWindows);
+      parts.push("");
+      parts.push(
+        "Note: The element [ID]s above are from other windows — you can reference them for context but can only interact with the focused window's elements.",
+      );
+    }
+
+    // Screenshot metadata
+    const screenshotMeta = this.formatScreenshotMetadata(obs);
+    if (screenshotMeta.length > 0) {
+      parts.push("");
+      parts.push(...screenshotMeta);
+    }
+
+    const content = parts.join("\n").trim() || "Action executed";
+
+    // Build content blocks for screenshot
+    const contentBlocks: ContentBlock[] = [];
+    if (obs.screenshot) {
+      contentBlocks.push({
+        type: "image",
+        source: {
+          type: "base64",
+          media_type: "image/jpeg",
+          data: obs.screenshot,
+        },
+      });
+    }
+
+    const isError = obs.executionError != null;
+
+    return {
+      content: isError
+        ? `Action failed: ${obs.executionError}\n\n${content}`
+        : content,
+      isError,
+      ...(contentBlocks.length > 0 ? { contentBlocks } : {}),
+    };
+  }
+
+  // ---------------------------------------------------------------------------
+  // Dispose
+  // ---------------------------------------------------------------------------
+
+  dispose(): void {
+    for (const [requestId, entry] of this.pending) {
+      clearTimeout(entry.timer);
+      this.onInternalResolve?.(requestId);
+      entry.reject(
+        new AssistantError("Host CU proxy disposed", ErrorCode.INTERNAL_ERROR),
+      );
+    }
+    this.pending.clear();
+  }
+
+  // ---------------------------------------------------------------------------
+  // Private helpers
+  // ---------------------------------------------------------------------------
+
+  /** Update consecutive-unchanged tracking from an incoming observation. */
+  private updateStateFromObservation(obs: CuObservationResult): void {
+    if (this._stepCount > 0) {
+      if (
+        obs.axDiff == null &&
+        this._previousAXTree != null &&
+        obs.axTree != null
+      ) {
+        this._consecutiveUnchangedSteps++;
+      } else if (obs.axDiff != null) {
+        this._consecutiveUnchangedSteps = 0;
+      }
+    }
+
+    if (obs.axTree != null) {
+      this._previousAXTree = obs.axTree;
+    }
+  }
+
+  private formatScreenshotMetadata(obs: CuObservationResult): string[] {
+    if (!obs.screenshot) return [];
+
+    const lines: string[] = [];
+    if (obs.screenshotWidthPx != null && obs.screenshotHeightPx != null) {
+      lines.push(
+        `Screenshot metadata: ${obs.screenshotWidthPx}x${obs.screenshotHeightPx} px`,
+      );
+    }
+    if (obs.screenWidthPt != null && obs.screenHeightPt != null) {
+      lines.push(
+        `Screen metadata: ${obs.screenWidthPt}x${obs.screenHeightPt} pt`,
+      );
+    }
+    return lines;
+  }
+}

package/src/daemon/lifecycle.ts

@@ -15,6 +15,7 @@ import {
   getQdrantUrlEnv,
   getRuntimeHttpHost,
   getRuntimeHttpPort,
+  setIngressPublicBaseUrl,
   validateEnv,
 } from "../config/env.js";
 import { loadConfig } from "../config/loader.js";
@@ -22,16 +23,21 @@ import { HeartbeatService } from "../heartbeat/heartbeat-service.js";
 import { getHookManager } from "../hooks/manager.js";
 import { installTemplates } from "../hooks/templates.js";
 import { closeSentry, initSentry } from "../instrument.js";
-import { initLogfire } from "../logfire.js";
+import { disableLogfire, initLogfire } from "../logfire.js";
 import { getMcpServerManager } from "../mcp/manager.js";
 import * as attachmentsStore from "../memory/attachments-store.js";
+import { expireAllPendingCanonicalRequests } from "../memory/canonical-guardian-store.js";
 import {
   deleteMessageById,
   getConversationThreadType,
   getMessages,
 } from "../memory/conversation-crud.js";
 import { initializeDb } from "../memory/db.js";
-import { selectEmbeddingBackend } from "../memory/embedding-backend.js";
+import {
+  selectEmbeddingBackend,
+  SPARSE_EMBEDDING_VERSION,
+} from "../memory/embedding-backend.js";
+import { enqueueMemoryJob } from "../memory/jobs-store.js";
 import { startMemoryJobsWorker } from "../memory/jobs-worker.js";
 import { initQdrantClient } from "../memory/qdrant-client.js";
 import { QdrantManager } from "../memory/qdrant-manager.js";
@@ -40,6 +46,8 @@ import {
   emitNotificationSignal,
   registerBroadcastFn,
 } from "../notifications/emit-signal.js";
+import { backfillManualTokenConnections } from "../oauth/manual-token-connection.js";
+import { seedOAuthProviders } from "../oauth/seed-providers.js";
 import { ensurePromptFiles } from "../prompts/system-prompt.js";
 import { syncUpdateBulletinOnStartup } from "../prompts/update-bulletin.js";
 import { buildAssistantEvent } from "../runtime/assistant-event.js";
@@ -54,8 +62,6 @@ import {
 import { ensureVellumGuardianBinding } from "../runtime/guardian-vellum-migration.js";
 import { RuntimeHttpServer } from "../runtime/http-server.js";
 import { startScheduler } from "../schedule/scheduler.js";
-import { migrateKeys } from "../security/credential-key.js";
-import { watchSessions } from "../tools/watch/watch-state.js";
 import { getLogger, initLogger } from "../util/logger.js";
 import {
   ensureDataDir,
@@ -95,10 +101,6 @@ import {
   registerMessagingProviders,
   registerWatcherProviders,
 } from "./providers-setup.js";
-import {
-  handleRideShotgunStart,
-  handleRideShotgunStop,
-} from "./ride-shotgun-handler.js";
 import { seedInterfaceFiles } from "./seed-files.js";
 import { DaemonServer } from "./server.js";
 import { initSlashPairingContext } from "./session-slash.js";
@@ -137,11 +139,6 @@ export async function runDaemon(): Promise<void> {
 
     ensureDataDir();
 
-    // Migrate legacy colon-delimited credential keys to the new
-    // slash-delimited format. Must run after ensureDataDir() so the
-    // secure key store is available, and before any credential reads.
-    migrateKeys();
-
     // Load (or generate + persist) the auth signing key so tokens survive
     // daemon restarts. Must happen after ensureDataDir() creates the
     // protected directory.
@@ -165,8 +162,26 @@ export async function runDaemon(): Promise<void> {
       );
     }
     initializeDb();
+    // Seed well-known OAuth provider configurations (insert-if-not-exists)
+    seedOAuthProviders();
+    // Backfill oauth_connection rows for manual-token providers (Telegram,
+    // Slack channel) that already have keychain credentials from before the
+    // oauth_connection migration. Safe to call on every startup.
+    await backfillManualTokenConnections();
     log.info("Daemon startup: DB initialized");
 
+    // Expire any pending canonical guardian requests left over from before
+    // this process started.  Their in-memory pending-interaction session
+    // references are gone, so they can never be completed.  The agent loop
+    // will re-request tool approvals on the next turn.
+    const expiredCount = expireAllPendingCanonicalRequests();
+    if (expiredCount > 0) {
+      log.info(
+        { event: "startup_expired_stale_requests", expiredCount },
+        `Expired ${expiredCount} stale pending canonical request(s) from previous process`,
+      );
+    }
+
     // Ensure a vellum guardian binding exists and mint the CLI edge token
     // as an actor token bound to the guardian principal.
     let guardianPrincipalId: string | undefined;
@@ -242,6 +257,19 @@ export async function runDaemon(): Promise<void> {
     log.info("Daemon startup: loading config");
     const config = loadConfig();
 
+    // Seed module-level ingress state from the workspace config so that
+    // getIngressPublicBaseUrl() returns the correct value immediately after
+    // startup (before any handleIngressConfig("set") call). Without this,
+    // code paths that read the module-level state directly (e.g. session-slash
+    // pairing info) would see undefined until an explicit set.
+    if (config.ingress.enabled && config.ingress.publicBaseUrl) {
+      setIngressPublicBaseUrl(config.ingress.publicBaseUrl);
+      log.info(
+        { url: config.ingress.publicBaseUrl },
+        "Daemon startup: seeded ingress URL from workspace config",
+      );
+    }
+
     if (config.logFile.dir) {
       initLogger({
         dir: config.logFile.dir,
@@ -259,6 +287,18 @@ export async function runDaemon(): Promise<void> {
       await closeSentry();
     }
 
+    // If Logfire observability is not explicitly enabled, disable it so
+    // wrapWithLogfire() calls during provider setup become no-ops. Logfire
+    // is initialized eagerly (before config loads) for the same reason as
+    // Sentry — but the feature flag gates whether it actually traces.
+    const logfireEnabled = isAssistantFeatureFlagEnabled(
+      "feature_flags.logfire.enabled",
+      config,
+    );
+    if (!logfireEnabled) {
+      disableLogfire();
+    }
+
     await initializeProvidersAndTools(config);
 
     // Start the DaemonServer (session manager) before Qdrant so HTTP
@@ -282,9 +322,9 @@ export async function runDaemon(): Promise<void> {
       await qdrantManager.start();
       const embeddingSelection = selectEmbeddingBackend(config);
       const embeddingModel = embeddingSelection.backend
-        ? `${embeddingSelection.backend.provider}:${embeddingSelection.backend.model}`
+        ? `${embeddingSelection.backend.provider}:${embeddingSelection.backend.model}:sparse-v${SPARSE_EMBEDDING_VERSION}`
         : undefined;
-      initQdrantClient({
+      const qdrantClient = initQdrantClient({
         url: qdrantUrl,
         collection: config.memory.qdrant.collection,
         vectorSize: config.memory.qdrant.vectorSize,
@@ -292,6 +332,17 @@ export async function runDaemon(): Promise<void> {
         quantization: config.memory.qdrant.quantization,
         embeddingModel,
       });
+
+      // Eagerly ensure the collection exists so we detect migrations
+      // (unnamed→named vectors, dimension/model changes) at startup.
+      // If a destructive migration occurred, enqueue a rebuild_index job
+      // to re-embed all memory items from the SQLite cache.
+      const { migrated } = await qdrantClient.ensureCollection();
+      if (migrated) {
+        enqueueMemoryJob("rebuild_index", {});
+        log.info("Qdrant collection was migrated — enqueued rebuild_index job");
+      }
+
       log.info("Qdrant vector store initialized");
     } catch (err) {
       log.warn(
@@ -501,64 +552,9 @@ export async function runDaemon(): Promise<void> {
           );
         },
       },
-      getComputerUseDeps: () => {
+      getWatchDeps: () => {
         const ctx = server.getHandlerContext();
         return {
-          cuSessions: ctx.cuSessions,
-          sharedRequestTimestamps: ctx.sharedRequestTimestamps,
-          cuObservationParseSequence: ctx.cuObservationParseSequence,
-          handleRideShotgunStart: async (params) => {
-            // The handler generates its own watchId/sessionId and
-            // sends them via ctx.send as a watch_started message.
-            // We intercept send to capture the IDs before they broadcast.
-            let capturedWatchId = "";
-            let capturedSessionId = "";
-            const interceptCtx = {
-              ...ctx,
-              send: (msg: ServerMessage) => {
-                if (
-                  "type" in msg &&
-                  msg.type === "watch_started" &&
-                  "watchId" in msg &&
-                  "sessionId" in msg
-                ) {
-                  capturedWatchId = (msg as { watchId: string }).watchId;
-                  capturedSessionId = (msg as { sessionId: string }).sessionId;
-                }
-                ctx.send(msg);
-              },
-            };
-            await handleRideShotgunStart(
-              {
-                type: "ride_shotgun_start",
-                durationSeconds: params.durationSeconds,
-                intervalSeconds: params.intervalSeconds,
-                mode: params.mode,
-                targetDomain: params.targetDomain,
-                navigateDomain: params.navigateDomain,
-                autoNavigate: params.autoNavigate,
-              },
-              interceptCtx,
-            );
-            return { watchId: capturedWatchId, sessionId: capturedSessionId };
-          },
-          handleRideShotgunStop: async (watchId) => {
-            await handleRideShotgunStop(
-              { type: "ride_shotgun_stop", watchId },
-              ctx,
-            );
-          },
-          getRideShotgunStatus: (watchId) => {
-            const session = watchSessions.get(watchId);
-            if (!session) return undefined;
-            return {
-              status: session.status,
-              sessionId: session.sessionId,
-              recordingId: session.recordingId,
-              savedRecordingPath: session.savedRecordingPath,
-              bootstrapFailureReason: session.bootstrapFailureReason,
-            };
-          },
           handleWatchObservation: async (params) => {
             await handleWatchObservation(
               {

package/src/daemon/mcp-reload-service.ts

@@ -1,8 +1,8 @@
 /**
  * Shared MCP reload business logic.
  *
- * Used by the HTTP route (`runtime/routes/mcp-routes.ts`) so the reload
- * behaviour is defined in exactly one place.
+ * Called by the ConfigWatcher when config.json changes or a reload signal
+ * file is detected, so the daemon automatically reconnects MCP servers.
  */
 
 import { getConfig, invalidateConfigCache } from "../config/loader.js";

package/src/daemon/message-protocol.ts

@@ -21,6 +21,7 @@ export * from "./message-types/diagnostics.js";
 export * from "./message-types/documents.js";
 export * from "./message-types/guardian-actions.js";
 export * from "./message-types/host-bash.js";
+export * from "./message-types/host-cu.js";
 export * from "./message-types/host-file.js";
 export * from "./message-types/inbox.js";
 export * from "./message-types/integrations.js";
@@ -69,6 +70,7 @@ import type {
   _GuardianActionsServerMessages,
 } from "./message-types/guardian-actions.js";
 import type { _HostBashServerMessages } from "./message-types/host-bash.js";
+import type { _HostCuServerMessages } from "./message-types/host-cu.js";
 import type { _HostFileServerMessages } from "./message-types/host-file.js";
 import type {
   _InboxClientMessages,
@@ -180,6 +182,7 @@ export type ServerMessage =
   | _DocumentsServerMessages
   | _GuardianActionsServerMessages
   | _HostBashServerMessages
+  | _HostCuServerMessages
   | _HostFileServerMessages
   | _MemoryServerMessages
   | _WorkspaceServerMessages