@vellumai/assistant 0.5.5 → 0.5.7

package/src/daemon/lifecycle.ts

@@ -7,7 +7,6 @@ import { reconcileCallsOnStartup } from "../calls/call-recovery.js";
 import { setRelayBroadcast } from "../calls/relay-server.js";
 import { TwilioConversationRelayProvider } from "../calls/twilio-provider.js";
 import { setVoiceBridgeDeps } from "../calls/voice-session-bridge.js";
-import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
 import {
   getQdrantHttpPortEnv,
   getQdrantUrlEnv,
@@ -16,12 +15,23 @@ import {
   setIngressPublicBaseUrl,
   validateEnv,
 } from "../config/env.js";
-import { loadConfig } from "../config/loader.js";
+import { loadConfig, mergeDefaultWorkspaceConfig } from "../config/loader.js";
+import type { AssistantConfig } from "../config/schema.js";
+import type { CesClient } from "../credential-execution/client.js";
+import { createCesClient } from "../credential-execution/client.js";
+import {
+  isCesCredentialBackendEnabled,
+  isCesToolsEnabled,
+} from "../credential-execution/feature-gates.js";
+import {
+  type CesProcessManager,
+  CesUnavailableError,
+  createCesProcessManager,
+} from "../credential-execution/process-manager.js";
 import { HeartbeatService } from "../heartbeat/heartbeat-service.js";
 import { getHookManager } from "../hooks/manager.js";
 import { installTemplates } from "../hooks/templates.js";
 import { closeSentry, initSentry, setSentryDeviceId } from "../instrument.js";
-import { disableLogfire, initLogfire } from "../logfire.js";
 import { getMcpServerManager } from "../mcp/manager.js";
 import * as attachmentsStore from "../memory/attachments-store.js";
 import { expireAllPendingCanonicalRequests } from "../memory/canonical-guardian-store.js";
@@ -30,7 +40,6 @@ import {
   getConversationType,
   getMessages,
   purgePrivateConversations,
-  sweepStaleReducerJobs,
 } from "../memory/conversation-crud.js";
 import { resolveConversationId } from "../memory/conversation-key-store.js";
 import { initializeDb } from "../memory/db.js";
@@ -52,17 +61,19 @@ import { backfillManualTokenConnections } from "../oauth/manual-token-connection
 import { seedOAuthProviders } from "../oauth/seed-providers.js";
 import { ensurePromptFiles } from "../prompts/system-prompt.js";
 import { syncUpdateBulletinOnStartup } from "../prompts/update-bulletin.js";
+import { resolveManagedProxyContext } from "../providers/managed-proxy/context.js";
 import { buildAssistantEvent } from "../runtime/assistant-event.js";
 import { assistantEventHub } from "../runtime/assistant-event-hub.js";
 import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
 import {
   initAuthSigningKey,
-  loadOrCreateSigningKey,
   mintPairingBearerToken,
+  resolveSigningKey,
 } from "../runtime/auth/token-service.js";
 import { ensureVellumGuardianBinding } from "../runtime/guardian-vellum-migration.js";
 import { RuntimeHttpServer } from "../runtime/http-server.js";
 import { startScheduler } from "../schedule/scheduler.js";
+import { setCesClient } from "../security/secure-keys.js";
 import { seedCatalogSkillMemories } from "../skills/skill-memory.js";
 import { UsageTelemetryReporter } from "../telemetry/usage-telemetry-reporter.js";
 import { getDeviceId } from "../util/device-id.js";
@@ -102,6 +113,7 @@ import {
   switchConversation,
   undoLastMessage,
 } from "./handlers/conversations.js";
+import { installAssistantSymlink } from "./install-symlink.js";
 import type { ServerMessage } from "./message-protocol.js";
 import {
   initializeProvidersAndTools,
@@ -130,6 +142,103 @@ function loadDotEnv(): void {
   dotenvConfig({ path: join(getRootDir(), ".env"), quiet: true });
 }
 
+export interface CesStartupResult {
+  client: CesClient | undefined;
+  processManager: CesProcessManager | undefined;
+  clientPromise: Promise<CesClient | undefined> | undefined;
+  abortController: AbortController | undefined;
+}
+
+/**
+ * Start the CES (Credential Execution Service) process and perform the RPC
+ * handshake. Returns a promise that resolves with the CES client and process
+ * manager. Callers can fire-and-forget — the daemon does not need to await
+ * this for startup to continue.
+ *
+ * The managed sidecar accepts exactly one bootstrap connection, so this must
+ * be called at the process level (not per-conversation).
+ */
+export async function startCesProcess(
+  config: AssistantConfig,
+): Promise<CesStartupResult> {
+  const shouldStartCes = isCesToolsEnabled(config) || isCesCredentialBackendEnabled(config);
+  if (!shouldStartCes) {
+    return {
+      client: undefined,
+      processManager: undefined,
+      clientPromise: undefined,
+      abortController: undefined,
+    };
+  }
+
+  const pm = createCesProcessManager({ assistantConfig: config });
+  const abortController = new AbortController();
+  let clientRef: CesClient | undefined;
+
+  const clientPromise = (async (): Promise<CesClient | undefined> => {
+    try {
+      const transport = await pm.start();
+      if (abortController.signal.aborted) {
+        throw new Error("CES initialization aborted during shutdown");
+      }
+      const client = createCesClient(transport);
+      clientRef = client;
+      // Resolve the assistant API key so CES can use it for platform
+      // credential materialisation. In managed mode the key is provisioned
+      // after hatch and stored in the credential store — CES can't read
+      // the env var, so we pass it via the handshake.
+      const proxyCtx = await resolveManagedProxyContext();
+      const { accepted, reason } = await client.handshake(
+        proxyCtx.assistantApiKey
+          ? { assistantApiKey: proxyCtx.assistantApiKey }
+          : undefined,
+      );
+      if (abortController.signal.aborted) {
+        client.close();
+        throw new Error("CES initialization aborted during shutdown");
+      }
+      if (accepted) {
+        log.info(
+          "CES client initialized and handshake accepted (server-level)",
+        );
+        return client;
+      }
+      log.warn(
+        { reason },
+        "CES handshake rejected — CES tools will be unavailable",
+      );
+      client.close();
+      clientRef = undefined;
+      await pm.stop();
+      return undefined;
+    } catch (err) {
+      if (err instanceof CesUnavailableError) {
+        log.info(
+          { reason: err.message },
+          "CES is not available — CES tools will be unavailable",
+        );
+      } else {
+        log.warn(
+          { error: err instanceof Error ? err.message : String(err) },
+          "Failed to initialize CES client — CES tools will be unavailable",
+        );
+      }
+      await pm.stop().catch(() => {});
+      clientRef = undefined;
+      return undefined;
+    }
+  })();
+
+  return {
+    get client() {
+      return clientRef;
+    },
+    processManager: pm,
+    clientPromise,
+    abortController,
+  };
+}
+
 // Entry point for the daemon process itself
 export async function runDaemon(): Promise<void> {
   loadDotEnv();
@@ -141,14 +250,12 @@ export async function runDaemon(): Promise<void> {
     // closeSentry() if the user has disabled it.
     initSentry();
 
-    await initLogfire();
-
     ensureDataDir();
 
     // Load (or generate + persist) the auth signing key so tokens survive
     // daemon restarts. Must happen after ensureDataDir() creates the
     // protected directory.
-    const signingKey = loadOrCreateSigningKey();
+    const signingKey = resolveSigningKey();
     initAuthSigningKey(signingKey);
 
     seedInterfaceFiles();
@@ -164,9 +271,19 @@ export async function runDaemon(): Promise<void> {
     initializeDb();
     // Seed well-known OAuth provider configurations (insert-if-not-exists)
     seedOAuthProviders();
+    log.info("Daemon startup: DB initialized");
+
+    await runWorkspaceMigrations(getWorkspaceDir(), WORKSPACE_MIGRATIONS);
+    log.info("Daemon startup: workspace migrations complete");
+
     // Backfill oauth_connection rows for manual-token providers (Telegram,
     // Slack channel) that already have keychain credentials from before the
     // oauth_connection migration. Safe to call on every startup.
+    //
+    // Must run AFTER workspace migrations so that migration 015 (which copies
+    // encrypted-store credentials to the keychain) has already executed.
+    // Otherwise syncManualTokenConnection sees no keychain credentials and
+    // incorrectly removes existing connection rows.
     try {
       await backfillManualTokenConnections();
     } catch (err) {
@@ -175,10 +292,6 @@ export async function runDaemon(): Promise<void> {
         "Manual-token connection backfill failed — continuing startup",
       );
     }
-    log.info("Daemon startup: DB initialized");
-
-    await runWorkspaceMigrations(getWorkspaceDir(), WORKSPACE_MIGRATIONS);
-    log.info("Daemon startup: workspace migrations complete");
 
     // Now that workspace migrations have run (including 003-seed-device-id
     // which may copy the legacy installationId into device.json), it is safe
@@ -213,40 +326,16 @@ export async function runDaemon(): Promise<void> {
           targetId: summaryId,
         });
       }
-      for (const obsId of deletedMemory.deletedObservationIds) {
-        enqueueMemoryJob("delete_qdrant_vectors", {
-          targetType: "observation",
-          targetId: obsId,
-        });
-      }
-      for (const chunkId of deletedMemory.deletedChunkIds) {
-        enqueueMemoryJob("delete_qdrant_vectors", {
-          targetType: "chunk",
-          targetId: chunkId,
-        });
-      }
-      for (const episodeId of deletedMemory.deletedEpisodeIds) {
-        enqueueMemoryJob("delete_qdrant_vectors", {
-          targetType: "episode",
-          targetId: episodeId,
-        });
-      }
       if (
         deletedMemory.segmentIds.length > 0 ||
         deletedMemory.orphanedItemIds.length > 0 ||
-        deletedMemory.deletedSummaryIds.length > 0 ||
-        deletedMemory.deletedObservationIds.length > 0 ||
-        deletedMemory.deletedChunkIds.length > 0 ||
-        deletedMemory.deletedEpisodeIds.length > 0
+        deletedMemory.deletedSummaryIds.length > 0
       ) {
         log.info(
           {
             segments: deletedMemory.segmentIds.length,
             orphanedItems: deletedMemory.orphanedItemIds.length,
             deletedSummaries: deletedMemory.deletedSummaryIds.length,
-            deletedObservations: deletedMemory.deletedObservationIds.length,
-            deletedChunks: deletedMemory.deletedChunkIds.length,
-            deletedEpisodes: deletedMemory.deletedEpisodeIds.length,
           },
           "Enqueued Qdrant vector cleanup jobs for purged private conversations",
         );
@@ -271,24 +360,6 @@ export async function runDaemon(): Promise<void> {
       );
     }
 
-    // Sweep dirty conversations whose tail messages are already past the
-    // idle delay — they should have been reduced while the daemon was down.
-    // Enqueue immediate reducer jobs so the memory worker picks them up.
-    try {
-      const sweepCount = sweepStaleReducerJobs();
-      if (sweepCount > 0) {
-        log.info(
-          { sweepCount },
-          `Enqueued reducer jobs for ${sweepCount} stale dirty conversation(s)`,
-        );
-      }
-    } catch (err) {
-      log.warn(
-        { err },
-        "Startup sweep for stale reducer jobs failed — continuing startup",
-      );
-    }
-
     // Ensure a vellum guardian binding exists so the identity system works
     // without requiring a manual bootstrap step.
     try {
@@ -333,6 +404,11 @@ export async function runDaemon(): Promise<void> {
       log.warn({ err }, "Call recovery failed — continuing startup");
     }
 
+    // Merge CLI-provided default config (from VELLUM_DEFAULT_WORKSPACE_CONFIG_PATH)
+    // into the workspace config file before the first loadConfig() call so
+    // onboarding preferences are persisted alongside schema defaults.
+    mergeDefaultWorkspaceConfig();
+
     log.info("Daemon startup: loading config");
     const config = loadConfig();
 
@@ -373,16 +449,34 @@ export async function runDaemon(): Promise<void> {
       log.info("Usage telemetry reporter started");
     }
 
-    // If Logfire observability is not explicitly enabled, disable it so
-    // wrapWithLogfire() calls during provider setup become no-ops. Logfire
-    // is initialized eagerly (before config loads) for the same reason as
-    // Sentry — but the feature flag gates whether it actually traces.
-    const logfireEnabled = isAssistantFeatureFlagEnabled(
-      "feature_flags.logfire.enabled",
-      config,
-    );
-    if (!logfireEnabled) {
-      disableLogfire();
+    // CES lifecycle — kick off early so CES handshake runs concurrently with
+    // provider/tool initialization. The CES sidecar accepts exactly one
+    // bootstrap connection, so startup must happen at the process level.
+    const cesStartupPromise = startCesProcess(config);
+
+    // When the credential backend flag is enabled, CES startup must complete
+    // BEFORE provider initialization so credential reads can go through CES.
+    // Block with a 3-second timeout — fall back to direct credential store
+    // on timeout.
+    if (isCesCredentialBackendEnabled(config)) {
+      const cesResult = await cesStartupPromise;
+      // startCesProcess() returns immediately — the actual handshake runs
+      // inside clientPromise. Await it (with a 3s timeout) so the CES client
+      // is available before provider initialization.
+      if (cesResult.clientPromise) {
+        const client = await Promise.race([
+          cesResult.clientPromise,
+          new Promise<undefined>((resolve) =>
+            setTimeout(() => {
+              log.warn("CES handshake timed out after 3s — falling back to direct credential store");
+              resolve(undefined);
+            }, 3000),
+          ),
+        ]);
+        if (client) {
+          setCesClient(client);
+        }
+      }
     }
 
     await initializeProvidersAndTools(config);
@@ -391,6 +485,7 @@ export async function runDaemon(): Promise<void> {
     // routes can begin accepting requests while Qdrant initializes.
     log.info("Daemon startup: starting DaemonServer");
     const server = new DaemonServer();
+    server.setCes(await cesStartupPromise);
     await server.start();
     log.info("Daemon startup: DaemonServer started");
 
@@ -414,39 +509,69 @@ export async function runDaemon(): Promise<void> {
       log.info({ qdrantUrl }, "Daemon startup: initializing Qdrant");
       const manager = new QdrantManager({ url: qdrantUrl });
       bgRefs.qdrantManager = manager;
-      try {
-        await manager.start();
-        const embeddingSelection = await selectEmbeddingBackend(config);
-        const embeddingModel = embeddingSelection.backend
-          ? `${embeddingSelection.backend.provider}:${embeddingSelection.backend.model}:sparse-v${SPARSE_EMBEDDING_VERSION}`
-          : undefined;
-        const qdrantClient = initQdrantClient({
-          url: qdrantUrl,
-          collection: config.memory.qdrant.collection,
-          vectorSize: config.memory.qdrant.vectorSize,
-          onDisk: config.memory.qdrant.onDisk,
-          quantization: config.memory.qdrant.quantization,
-          embeddingModel,
-        });
+      const QDRANT_START_MAX_ATTEMPTS = 3;
+      let qdrantStarted = false;
+      for (let attempt = 1; attempt <= QDRANT_START_MAX_ATTEMPTS; attempt++) {
+        try {
+          await manager.start();
+          qdrantStarted = true;
+          break;
+        } catch (err) {
+          if (attempt < QDRANT_START_MAX_ATTEMPTS) {
+            const backoffMs = attempt * 5_000; // 5s, 10s
+            log.warn(
+              {
+                err,
+                attempt,
+                maxAttempts: QDRANT_START_MAX_ATTEMPTS,
+                backoffMs,
+              },
+              "Qdrant startup failed, retrying",
+            );
+            await Bun.sleep(backoffMs);
+          } else {
+            log.warn(
+              { err },
+              "Qdrant failed to start after all attempts — memory features will be unavailable",
+            );
+          }
+        }
+      }
+
+      if (qdrantStarted) {
+        try {
+          const embeddingSelection = await selectEmbeddingBackend(config);
+          const embeddingModel = embeddingSelection.backend
+            ? `${embeddingSelection.backend.provider}:${embeddingSelection.backend.model}:sparse-v${SPARSE_EMBEDDING_VERSION}`
+            : undefined;
+          const qdrantClient = initQdrantClient({
+            url: qdrantUrl,
+            collection: config.memory.qdrant.collection,
+            vectorSize: config.memory.qdrant.vectorSize,
+            onDisk: config.memory.qdrant.onDisk,
+            quantization: config.memory.qdrant.quantization,
+            embeddingModel,
+          });
 
-        // Eagerly ensure the collection exists so we detect migrations
-        // (unnamed→named vectors, dimension/model changes) at startup.
-        // If a destructive migration occurred, enqueue a rebuild_index job
-        // to re-embed all memory items from the SQLite cache.
-        const { migrated } = await qdrantClient.ensureCollection();
-        if (migrated) {
-          enqueueMemoryJob("rebuild_index", {});
-          log.info(
-            "Qdrant collection was migrated — enqueued rebuild_index job",
+          // Eagerly ensure the collection exists so we detect migrations
+          // (unnamed→named vectors, dimension/model changes) at startup.
+          // If a destructive migration occurred, enqueue a rebuild_index job
+          // to re-embed all memory items from the SQLite cache.
+          const { migrated } = await qdrantClient.ensureCollection();
+          if (migrated) {
+            enqueueMemoryJob("rebuild_index", {});
+            log.info(
+              "Qdrant collection was migrated — enqueued rebuild_index job",
+            );
+          }
+
+          log.info("Qdrant vector store initialized");
+        } catch (err) {
+          log.warn(
+            { err },
+            "Qdrant client initialization failed — memory features will be degraded",
           );
         }
-
-        log.info("Qdrant vector store initialized");
-      } catch (err) {
-        log.warn(
-          { err },
-          "Qdrant failed to start — memory features will be unavailable",
-        );
       }
 
       log.info("Daemon startup: starting memory worker");
@@ -876,7 +1001,7 @@ export async function runDaemon(): Promise<void> {
         server.broadcast(msg as ServerMessage),
       );
       initSlashPairingContext(runtimeHttp.getPairingStore());
-      server.setHttpPort(httpPort);
+      server.broadcastStatus();
       log.info(
         { port: httpPort, hostname },
         "Daemon startup: runtime HTTP server listening",
@@ -892,6 +1017,14 @@ export async function runDaemon(): Promise<void> {
     writePid(process.pid);
     log.info({ pid: process.pid }, "Daemon started");
 
+    // Install the `assistant` CLI symlink idempotently on every daemon start.
+    // Non-blocking — failures are logged but don't affect startup.
+    try {
+      installAssistantSymlink();
+    } catch (err) {
+      log.warn({ err }, "Assistant symlink installation failed — continuing");
+    }
+
     const hookManager = getHookManager();
     hookManager.watch();
 

package/src/daemon/message-types/conversations.ts

@@ -221,9 +221,8 @@ export interface PongMessage {
   type: "pong";
 }
 
-export interface DaemonStatusMessage {
-  type: "daemon_status";
-  httpPort?: number;
+export interface AssistantStatusMessage {
+  type: "assistant_status";
   version?: string;
   keyFingerprint?: string;
 }
@@ -419,7 +418,7 @@ export type _ConversationsClientMessages =
 export type _ConversationsServerMessages =
   | AuthResult
   | PongMessage
-  | DaemonStatusMessage
+  | AssistantStatusMessage
   | GenerationCancelled
   | GenerationHandoff
   | ModelInfo

package/src/daemon/message-types/diagnostics.ts

@@ -1,15 +1,9 @@
-// Diagnostics, environment, blob probe, and dictation types.
+// Diagnostics, environment, and dictation types.
 
 import type { DictationContext } from "./shared.js";
 
 // === Client → Server ===
 
-export interface DiagnosticsExportRequest {
-  type: "diagnostics_export_request";
-  conversationId: string;
-  anchorMessageId?: string; // if omitted, use latest assistant message
-}
-
 export interface EnvVarsRequest {
   type: "env_vars_request";
 }
@@ -23,13 +17,6 @@ export interface DictationRequest {
 
 // === Server → Client ===
 
-export interface DiagnosticsExportResponse {
-  type: "diagnostics_export_response";
-  success: boolean;
-  filePath?: string; // path to the zip file on success
-  error?: string; // error message on failure
-}
-
 export interface EnvVarsResponse {
   type: "env_vars_response";
   vars: Record<string, string>;
@@ -46,12 +33,6 @@ export interface DictationResponse {
 
 // --- Domain-level union aliases (consumed by the barrel file) ---
 
-export type _DiagnosticsClientMessages =
-  | DiagnosticsExportRequest
-  | EnvVarsRequest
-  | DictationRequest;
+export type _DiagnosticsClientMessages = EnvVarsRequest | DictationRequest;
 
-export type _DiagnosticsServerMessages =
-  | DiagnosticsExportResponse
-  | EnvVarsResponse
-  | DictationResponse;
+export type _DiagnosticsServerMessages = EnvVarsResponse | DictationResponse;

package/src/daemon/message-types/messages.ts

@@ -13,8 +13,6 @@ export interface UserMessage {
   activeSurfaceId?: string;
   /** The page currently displayed in the WebView (e.g. "settings.html"). */
   currentPage?: string;
-  /** When true, skip the secret-ingress check. Set by the client when the user clicks "Send Anyway". */
-  bypassSecretCheck?: boolean;
   /** Originating channel identifier (e.g. 'vellum'). Defaults to 'vellum' when absent. */
   channel?: ChannelId;
   /** Originating interface identifier (e.g. 'macos'). */

package/src/daemon/message-types/upgrades.ts

@@ -7,6 +7,13 @@ export interface ServiceGroupUpdateStarting {
   expectedDowntimeSeconds: number;
 }
 
+/** Broadcast to connected clients with a progress update during an upgrade or rollback. */
+export interface ServiceGroupUpdateProgress {
+  type: "service_group_update_progress";
+  /** A short, user-friendly status message describing what's happening right now. */
+  statusMessage: string;
+}
+
 /** Broadcast to connected clients when a service group update has completed. */
 export interface ServiceGroupUpdateComplete {
   type: "service_group_update_complete";
@@ -20,4 +27,5 @@ export interface ServiceGroupUpdateComplete {
 
 export type _UpgradesServerMessages =
   | ServiceGroupUpdateStarting
+  | ServiceGroupUpdateProgress
   | ServiceGroupUpdateComplete;