@vellumai/credential-executor 0.7.0 → 0.7.1

package/src/main.ts

@@ -21,9 +21,12 @@
 
 import { mkdirSync } from "node:fs";
 import { homedir } from "node:os";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
 
-import { CES_PROTOCOL_VERSION, CesRpcMethod } from "@vellumai/service-contracts/credential-rpc";
+import {
+  CES_PROTOCOL_VERSION,
+  CesRpcMethod,
+} from "@vellumai/service-contracts/credential-rpc";
 import { StaticCredentialMetadataStore } from "@vellumai/credential-storage";
 
 import { AuditStore } from "./audit/store.js";
@@ -36,6 +39,7 @@ import {
 } from "./grants/rpc-handlers.js";
 import { TemporaryGrantStore } from "./grants/temporary-store.js";
 import { LocalMaterialiser } from "./materializers/local.js";
+import type { SecureKeyBackend } from "@vellumai/credential-storage";
 import { createLocalSecureKeyBackend } from "./materializers/local-secure-key-backend.js";
 import { createLocalOAuthLookup } from "./materializers/local-oauth-lookup.js";
 import { createLocalTokenRefreshFn } from "./materializers/local-token-refresh.js";
@@ -57,9 +61,14 @@ import {
   type RpcHandlerRegistry,
   type SessionIdRef,
 } from "./server.js";
-import { deleteBundleFromToolstore, publishBundle } from "./toolstore/publish.js";
+import {
+  deleteBundleFromToolstore,
+  publishBundle,
+} from "./toolstore/publish.js";
 import { validateSourceUrl } from "./toolstore/manifest.js";
 import { buildCesEgressHooks } from "./commands/egress-hooks.js";
+import { CES_MIGRATIONS } from "./migrations/registry.js";
+import { runCesMigrations } from "./migrations/runner.js";
 
 // ---------------------------------------------------------------------------
 // Data directory bootstrap
@@ -78,19 +87,46 @@ function ensureDataDirs(): void {
 }
 
 // ---------------------------------------------------------------------------
-// Vellum root resolution (mirrors assistant/src/util/platform.ts)
+// Path resolution
 // ---------------------------------------------------------------------------
 
-function getVellumRootDir(): string {
-  const baseDataDir = process.env["BASE_DATA_DIR"]?.trim();
-  return join(baseDataDir || homedir(), ".vellum");
+/**
+ * Resolve the workspace directory.
+ *
+ * Priority:
+ * 1. `VELLUM_WORKSPACE_DIR` env var (set by the platform template)
+ * 2. Default: `~/.vellum/workspace`
+ */
+function getWorkspaceDir(): string {
+  return (
+    process.env["VELLUM_WORKSPACE_DIR"]?.trim() ||
+    join(homedir(), ".vellum", "workspace")
+  );
+}
+
+/**
+ * Resolve the CES security directory (contains key stores, encryption data).
+ *
+ * Priority:
+ * 1. `CREDENTIAL_SECURITY_DIR` env var (set by the platform template for
+ *    the CES container — `/ces-security` in managed mode)
+ * 2. Default: `~/.vellum/protected` (local mode)
+ */
+function getSecurityDir(): string {
+  return (
+    process.env["CREDENTIAL_SECURITY_DIR"]?.trim() ||
+    join(homedir(), ".vellum", "protected")
+  );
 }
 
 // ---------------------------------------------------------------------------
 // Build RPC handler registry
 // ---------------------------------------------------------------------------
 
-function buildHandlers(sessionIdRef: SessionIdRef): RpcHandlerRegistry {
+function buildHandlers(
+  sessionIdRef: SessionIdRef,
+  secureKeyBackend: SecureKeyBackend,
+): RpcHandlerRegistry {
   // -- Grant stores ----------------------------------------------------------
   const persistentGrantStore = new PersistentGrantStore(
     getCesGrantsDir("local"),
@@ -106,10 +142,9 @@ function buildHandlers(sessionIdRef: SessionIdRef): RpcHandlerRegistry {
   // -- Credential backend (local) --------------------------------------------
   // In local mode CES shares the filesystem with the assistant and can access
   // the same credential metadata and secure-key stores.
-  const vellumRoot = getVellumRootDir();
+  const workspaceDir = getWorkspaceDir();
   const credentialMetadataPath = join(
-    vellumRoot,
-    "workspace",
+    workspaceDir,
     "data",
     "credentials",
     "metadata.json",
@@ -120,33 +155,27 @@ function buildHandlers(sessionIdRef: SessionIdRef): RpcHandlerRegistry {
 
   // Read-only OAuth connection lookup backed by the assistant's SQLite
   // database. CES opens the database in read-only mode.
-  const oauthConnections = createLocalOAuthLookup(vellumRoot);
-
-  // CES-native SecureKeyBackend that reads from the assistant's encrypted
-  // key store file. Read-only — CES never writes or deletes keys.
-  const secureKeyBackend = createLocalSecureKeyBackend(vellumRoot);
+  const oauthConnections = createLocalOAuthLookup(workspaceDir);
 
   const localMaterialiser = new LocalMaterialiser({
     secureKeyBackend,
-    tokenRefreshFn: createLocalTokenRefreshFn(vellumRoot, secureKeyBackend),
+    tokenRefreshFn: createLocalTokenRefreshFn(workspaceDir, secureKeyBackend),
   });
 
   // -- Build handler registry ------------------------------------------------
 
   // Start with the HTTP handler (make_authenticated_request)
-  const handlers = buildHandlersWithHttp(
-    {
-      persistentGrantStore,
-      temporaryGrantStore,
-      localMaterialiser,
-      localSubjectDeps: {
-        metadataStore,
-        oauthConnections,
-      },
-      auditStore,
-      sessionId: sessionIdRef,
+  const handlers = buildHandlersWithHttp({
+    persistentGrantStore,
+    temporaryGrantStore,
+    localMaterialiser,
+    localSubjectDeps: {
+      metadataStore,
+      oauthConnections,
     },
-  );
+    auditStore,
+    sessionId: sessionIdRef,
+  });
 
   // Register run_authenticated_command handler
   registerCommandExecutionHandler(handlers, {
@@ -174,7 +203,9 @@ function buildHandlers(sessionIdRef: SessionIdRef): RpcHandlerRegistry {
           }
         }
 
-        const matResult = await localMaterialiser.materialise(subjectResult.subject);
+        const matResult = await localMaterialiser.materialise(
+          subjectResult.subject,
+        );
         if (!matResult.ok) {
           return { ok: false as const, error: matResult.error };
         }
@@ -189,11 +220,19 @@ function buildHandlers(sessionIdRef: SessionIdRef): RpcHandlerRegistry {
       cesMode: "local",
       egressHooks: buildCesEgressHooks(),
     },
-    defaultWorkspaceDir: join(vellumRoot, "workspace"),
+    defaultWorkspaceDir: workspaceDir,
   });
 
   // Register manage_secure_command_tool handler
-  const toolRegistry = new Map<string, { toolName: string; credentialHandle: string; description: string; bundleDigest: string }>();
+  const toolRegistry = new Map<
+    string,
+    {
+      toolName: string;
+      credentialHandle: string;
+      description: string;
+      bundleDigest: string;
+    }
+  >();
 
   registerManageSecureCommandToolHandler(handlers, {
     downloadBundle: async (sourceUrl: string) => {
@@ -202,13 +241,17 @@ function buildHandlers(sessionIdRef: SessionIdRef): RpcHandlerRegistry {
         throw new Error(urlError);
       }
       const MAX_BUNDLE_SIZE = 100 * 1024 * 1024; // 100 MB
-      const resp = await fetch(sourceUrl, { signal: AbortSignal.timeout(60_000) });
+      const resp = await fetch(sourceUrl, {
+        signal: AbortSignal.timeout(60_000),
+      });
       if (!resp.ok) {
         throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
       }
       const contentLength = resp.headers.get("content-length");
       if (contentLength && parseInt(contentLength, 10) > MAX_BUNDLE_SIZE) {
-        throw new Error(`Bundle too large: ${contentLength} bytes (max ${MAX_BUNDLE_SIZE})`);
+        throw new Error(
+          `Bundle too large: ${contentLength} bytes (max ${MAX_BUNDLE_SIZE})`,
+        );
       }
       // Stream the body and enforce the size limit on actual bytes received,
       // since Content-Length can be absent (chunked encoding) or lie.
@@ -221,7 +264,9 @@ function buildHandlers(sessionIdRef: SessionIdRef): RpcHandlerRegistry {
       for await (const chunk of body) {
         totalBytes += chunk.byteLength;
         if (totalBytes > MAX_BUNDLE_SIZE) {
-          throw new Error(`Bundle too large: received >${MAX_BUNDLE_SIZE} bytes (max ${MAX_BUNDLE_SIZE})`);
+          throw new Error(
+            `Bundle too large: received >${MAX_BUNDLE_SIZE} bytes (max ${MAX_BUNDLE_SIZE})`,
+          );
         }
         chunks.push(chunk);
       }
@@ -232,7 +277,9 @@ function buildHandlers(sessionIdRef: SessionIdRef): RpcHandlerRegistry {
       const entry = toolRegistry.get(toolName);
       const removed = toolRegistry.delete(toolName);
       if (removed && entry?.bundleDigest) {
-        const stillInUse = Array.from(toolRegistry.values()).some(t => t.bundleDigest === entry.bundleDigest);
+        const stillInUse = Array.from(toolRegistry.values()).some(
+          (t) => t.bundleDigest === entry.bundleDigest,
+        );
         if (!stillInUse) {
           deleteBundleFromToolstore(entry.bundleDigest, "local");
         }
@@ -248,50 +295,57 @@ function buildHandlers(sessionIdRef: SessionIdRef): RpcHandlerRegistry {
   handlers[CesRpcMethod.RecordGrant] = createRecordGrantHandler({
     persistentGrantStore,
     temporaryGrantStore,
-  }) as typeof handlers[string];
+  }) as (typeof handlers)[string];
 
   handlers[CesRpcMethod.ListGrants] = createListGrantsHandler({
     persistentGrantStore,
-  }) as typeof handlers[string];
+  }) as (typeof handlers)[string];
 
   handlers[CesRpcMethod.RevokeGrant] = createRevokeGrantHandler({
     persistentGrantStore,
-  }) as typeof handlers[string];
+  }) as (typeof handlers)[string];
 
   // Register audit record handler
   handlers[CesRpcMethod.ListAuditRecords] = createListAuditRecordsHandler({
     auditStore,
-  }) as typeof handlers[string];
+  }) as (typeof handlers)[string];
 
   // Register credential CRUD handlers
   handlers[CesRpcMethod.GetCredential] = (async (req: { account: string }) => {
     const value = await secureKeyBackend.get(req.account);
     return { found: value !== undefined, value };
-  }) as typeof handlers[string];
+  }) as (typeof handlers)[string];
 
-  handlers[CesRpcMethod.SetCredential] = (async (req: { account: string; value: string }) => {
+  handlers[CesRpcMethod.SetCredential] = (async (req: {
+    account: string;
+    value: string;
+  }) => {
     const ok = await secureKeyBackend.set(req.account, req.value);
     return { ok };
-  }) as typeof handlers[string];
+  }) as (typeof handlers)[string];
 
-  handlers[CesRpcMethod.DeleteCredential] = (async (req: { account: string }) => {
+  handlers[CesRpcMethod.DeleteCredential] = (async (req: {
+    account: string;
+  }) => {
     const result = await secureKeyBackend.delete(req.account);
     return { result };
-  }) as typeof handlers[string];
+  }) as (typeof handlers)[string];
 
   handlers[CesRpcMethod.ListCredentials] = (async () => {
     const accounts = await secureKeyBackend.list();
     return { accounts };
-  }) as typeof handlers[string];
+  }) as (typeof handlers)[string];
 
-  handlers[CesRpcMethod.BulkSetCredentials] = (async (req: { credentials: Array<{ account: string; value: string }> }) => {
+  handlers[CesRpcMethod.BulkSetCredentials] = (async (req: {
+    credentials: Array<{ account: string; value: string }>;
+  }) => {
     const results = [];
     for (const { account, value } of req.credentials) {
       const ok = await secureKeyBackend.set(account, value);
       results.push({ account, ok });
     }
     return { results };
-  }) as typeof handlers[string];
+  }) as (typeof handlers)[string];
 
   return handlers;
 }
@@ -306,7 +360,9 @@ async function main(): Promise<void> {
   initLogger({ dir: getCesLogDir(), retentionDays: 30 });
   const log = getLogger("main");
 
-  log.info(`Starting CES v${CES_PROTOCOL_VERSION} (local mode, stdio transport)`);
+  log.info(
+    `Starting CES v${CES_PROTOCOL_VERSION} (local mode, stdio transport)`,
+  );
 
   const controller = new AbortController();
 
@@ -318,11 +374,25 @@ async function main(): Promise<void> {
   process.on("SIGTERM", shutdown);
   process.on("SIGINT", shutdown);
 
+  // Build the credential backend and run one-time migrations before starting
+  // the RPC server. Migrations complete synchronously before any connection
+  // is accepted — the backend is then passed to buildHandlers so it is not
+  // re-instantiated.
+  const secureKeyBackend = createLocalSecureKeyBackend(
+    dirname(getSecurityDir()),
+  );
+  await runCesMigrations(
+    getCesDataRoot("local"),
+    secureKeyBackend,
+    CES_MIGRATIONS,
+  );
+  log.info("CES local startup: migrations complete");
+
   // Build the handler registry with all available RPC implementations.
   // Use a mutable ref so audit records capture the handshake session ID
   // once it's negotiated (the handshake completes before any RPC call).
   const sessionIdRef: SessionIdRef = { current: `ces-local-${Date.now()}` };
-  const handlers = buildHandlers(sessionIdRef);
+  const handlers = buildHandlers(sessionIdRef, secureKeyBackend);
 
   const rpcLog = getLogger("rpc");
   const server = new CesRpcServer({

package/src/managed-main.ts

@@ -63,6 +63,8 @@ import type { SecureKeyBackend } from "@vellumai/credential-storage";
 import { createLocalSecureKeyBackend } from "./materializers/local-secure-key-backend.js";
 import { handleCredentialRoute, type CredentialRouteDeps } from "./http/credential-routes.js";
 import { handleLogExportRoute } from "./http/log-export-routes.js";
+import { CES_MIGRATIONS } from "./migrations/registry.js";
+import { runCesMigrations } from "./migrations/runner.js";
 
 // ---------------------------------------------------------------------------
 // Logging
@@ -543,6 +545,10 @@ async function main(): Promise<void> {
   const vellumRoot = join(assistantDataMount, ".vellum");
   const secureKeyBackend = createLocalSecureKeyBackend(vellumRoot);
 
+  // Run one-time credential store migrations before accepting connections.
+  await runCesMigrations(getCesDataRoot("managed"), secureKeyBackend, CES_MIGRATIONS);
+  log.info("CES managed startup: migrations complete");
+
   // Set up credential CRUD routes if a service token is configured.
   // The assistant and gateway use CES_SERVICE_TOKEN to authenticate
   // credential management requests over HTTP.

package/src/materializers/local-oauth-lookup.ts

@@ -65,12 +65,12 @@ function rowToRecord(row: OAuthConnectionRow): OAuthConnectionRecord {
  * Create a read-only OAuth connection lookup backed by the assistant's
  * SQLite database.
  *
- * @param vellumRoot - The Vellum root directory (e.g. `~/.vellum`).
+ * @param workspaceDir - The workspace directory (e.g. `~/.vellum/workspace`).
  */
 export function createLocalOAuthLookup(
-  vellumRoot: string,
+  workspaceDir: string,
 ): OAuthConnectionLookup {
-  const dbPath = join(vellumRoot, "workspace", "data", "db", "assistant.db");
+  const dbPath = join(workspaceDir, "data", "db", "assistant.db");
 
   return {
     getById(connectionId: string): OAuthConnectionRecord | undefined {
@@ -80,9 +80,10 @@ export function createLocalOAuthLookup(
       try {
         db = new Database(dbPath, { readonly: true });
         const row = db
-          .query<OAuthConnectionRow, [string, string]>(
-            `SELECT * FROM oauth_connections WHERE id = ? AND status = ? LIMIT 1`,
-          )
+          .query<
+            OAuthConnectionRow,
+            [string, string]
+          >(`SELECT * FROM oauth_connections WHERE id = ? AND status = ? LIMIT 1`)
           .get(connectionId, "active");
 
         if (!row) return undefined;

package/src/materializers/local-token-refresh.ts

@@ -88,20 +88,24 @@ async function resolveRefreshConfig(
 
     // 1. Look up the connection to get oauth_app_id and provider_key
     const conn = db
-      .query<OAuthConnectionRow, [string, string]>(
-        `SELECT id, oauth_app_id, provider_key FROM oauth_connections WHERE id = ? AND status = ? LIMIT 1`,
-      )
+      .query<
+        OAuthConnectionRow,
+        [string, string]
+      >(`SELECT id, oauth_app_id, provider_key FROM oauth_connections WHERE id = ? AND status = ? LIMIT 1`)
       .get(connectionId, "active");
 
     if (!conn) {
-      return { error: `No active OAuth connection found for "${connectionId}"` };
+      return {
+        error: `No active OAuth connection found for "${connectionId}"`,
+      };
     }
 
     // 2. Look up the app to get client_id and client_secret_credential_path
     const app = db
-      .query<OAuthAppRow, [string]>(
-        `SELECT id, provider_key, client_id, client_secret_credential_path FROM oauth_apps WHERE id = ? LIMIT 1`,
-      )
+      .query<
+        OAuthAppRow,
+        [string]
+      >(`SELECT id, provider_key, client_id, client_secret_credential_path FROM oauth_apps WHERE id = ? LIMIT 1`)
       .get(conn.oauth_app_id);
 
     if (!app) {
@@ -110,9 +114,10 @@ async function resolveRefreshConfig(
 
     // 3. Look up the provider to get token_url and auth method
     const provider = db
-      .query<OAuthProviderRow, [string]>(
-        `SELECT provider_key, token_url, refresh_url, token_endpoint_auth_method, token_exchange_body_format FROM oauth_providers WHERE provider_key = ? LIMIT 1`,
-      )
+      .query<
+        OAuthProviderRow,
+        [string]
+      >(`SELECT provider_key, token_url, refresh_url, token_endpoint_auth_method, token_exchange_body_format FROM oauth_providers WHERE provider_key = ? LIMIT 1`)
       .get(conn.provider_key);
 
     if (!provider) {
@@ -123,7 +128,9 @@ async function resolveRefreshConfig(
     const tokenUrl = provider.refresh_url || provider.token_url;
 
     if (!tokenUrl || !app.client_id) {
-      return { error: `Missing OAuth2 refresh config for "${conn.provider_key}"` };
+      return {
+        error: `Missing OAuth2 refresh config for "${conn.provider_key}"`,
+      };
     }
 
     // 4. Retrieve the client secret from secure storage
@@ -131,8 +138,11 @@ async function resolveRefreshConfig(
       app.client_secret_credential_path,
     );
 
-    const authMethod = (provider.token_endpoint_auth_method as TokenEndpointAuthMethod | null) ?? "client_secret_post";
-    const bodyFormat = (provider.token_exchange_body_format as "form" | "json" | null) ?? "form";
+    const authMethod =
+      (provider.token_endpoint_auth_method as TokenEndpointAuthMethod | null) ??
+      "client_secret_post";
+    const bodyFormat =
+      (provider.token_exchange_body_format as "form" | "json" | null) ?? "form";
 
     return {
       tokenUrl,
@@ -238,10 +248,10 @@ async function performTokenRefresh(
  * @param secureKeyBackend - Backend for retrieving the OAuth client secret.
  */
 export function createLocalTokenRefreshFn(
-  vellumRoot: string,
+  workspaceDir: string,
   secureKeyBackend: SecureKeyBackend,
 ): TokenRefreshFn {
-  const dbPath = join(vellumRoot, "workspace", "data", "db", "assistant.db");
+  const dbPath = join(workspaceDir, "data", "db", "assistant.db");
 
   return async (
     connectionId: string,

package/src/migrations/001-no-op.ts

@@ -0,0 +1,19 @@
+import type { CesMigration } from "./types.js";
+
+/**
+ * No-op foundation migration.
+ *
+ * Establishes the CES migration system and seeds the checkpoint file for
+ * all existing installations. The next real migration (API key → credential
+ * key rekeying) will follow this one.
+ */
+export const noOpMigration: CesMigration = {
+  id: "001-no-op",
+  description: "Seed CES migration checkpoint (no-op foundation)",
+  run(_backend): void {
+    // Intentionally empty — seeds the checkpoint file for existing installs.
+  },
+  down(_backend): void {
+    // Intentionally empty — nothing to reverse.
+  },
+};

package/src/migrations/002-api-keys-to-credentials.ts

@@ -0,0 +1,60 @@
+import type { CesMigration } from "./types.js";
+
+/**
+ * Providers whose bare API-key entries (e.g. `anthropic`) must be moved to
+ * the canonical `credential/{provider}/api_key` namespace.
+ *
+ * Note: `elevenlabs` is intentionally omitted — it was already migrated by
+ * `migrateElevenLabsToCredential()` in the Swift layer before CES migrations
+ * were introduced.
+ */
+const PROVIDERS_TO_MIGRATE = [
+  "anthropic",
+  "openai",
+  "gemini",
+  "ollama",
+  "fireworks",
+  "openrouter",
+  "brave",
+  "perplexity",
+  "deepgram",
+  "xai",
+] as const;
+
+export const apiKeyToCredentialsMigration: CesMigration = {
+  id: "002-api-keys-to-credentials",
+  description:
+    "Rekey bare provider API keys to credential/{provider}/api_key namespace",
+
+  async run(backend): Promise<void> {
+    for (const provider of PROVIDERS_TO_MIGRATE) {
+      const bareValue = await backend.get(provider);
+      if (bareValue === undefined) continue; // nothing to migrate for this provider
+
+      const credKey = `credential/${provider}/api_key`;
+      const existingCred = await backend.get(credKey);
+      if (existingCred === undefined) {
+        // Write new key first — safe to re-run if we crash after this.
+        // Skip delete if the write fails so the bare key is preserved for retry.
+        const ok = await backend.set(credKey, bareValue);
+        if (!ok) continue;
+      }
+      // Always delete old bare key (idempotent: harmless if already absent)
+      await backend.delete(provider);
+    }
+  },
+
+  async down(backend): Promise<void> {
+    for (const provider of PROVIDERS_TO_MIGRATE) {
+      const credKey = `credential/${provider}/api_key`;
+      const credValue = await backend.get(credKey);
+      if (credValue === undefined) continue;
+
+      const existingBare = await backend.get(provider);
+      if (existingBare === undefined) {
+        await backend.set(provider, credValue);
+      }
+      await backend.delete(credKey);
+    }
+  },
+};

package/src/migrations/registry.ts

@@ -0,0 +1,15 @@
+import { apiKeyToCredentialsMigration } from "./002-api-keys-to-credentials.js";
+import { noOpMigration } from "./001-no-op.js";
+import type { CesMigration } from "./types.js";
+
+/**
+ * Ordered list of all CES data migrations.
+ *
+ * New migrations are appended to the end. Never reorder or remove entries —
+ * the runner uses array position for ordering and the `id` field for
+ * checkpoint tracking.
+ */
+export const CES_MIGRATIONS: CesMigration[] = [
+  noOpMigration,
+  apiKeyToCredentialsMigration,
+];