@vellumai/cli 0.10.5 → 0.10.6-dev.202607062124.26c976d

package/src/commands/wake.ts

@@ -12,11 +12,12 @@ import {
   resetGuardianBootstrap,
   seedGuardianTokenFromSiblingEnv,
 } from "../lib/guardian-token.js";
-import { resolveProcessState, stopProcessByPidFile } from "../lib/process";
+import { resolveProcessState, stopProcessByPidFile, isProcessAlive } from "../lib/process";
 import {
   generateLocalSigningKey,
   isAssistantWatchModeAvailable,
   isGatewayWatchModeAvailable,
+  startCes,
   startLocalDaemon,
   startGateway,
 } from "../lib/local";
@@ -176,7 +177,31 @@ export async function wake(): Promise<void> {
   }
 
   if (!daemonRunning) {
-    await startLocalDaemon(watch, resources, { foreground, signingKey });
+    // Spin up CES and the daemon in parallel, the way the Docker topology
+    // brings its sibling containers up together — the assistant polls for the
+    // CES socket during startup (discoverCesWithRetry), so it tolerates CES
+    // still binding. CES's lifecycle tracks the daemon (its only consumer):
+    // restarting it under a live daemon would sever the daemon's open
+    // connection, so it is only (re)started alongside the daemon. startCes is a
+    // no-op unless CES_STANDALONE is set, in which case the assistant spawns
+    // CES itself as today.
+    await Promise.all([
+      startCes(watch, resources),
+      startLocalDaemon(watch, resources, { foreground, signingKey }),
+    ]);
+  } else {
+    // Self-heal: the daemon is already healthy, but the CES sibling may have
+    // died independently (crash, OOM kill). A dead ces.pid under a live daemon
+    // means credential operations will fail until the next wake. Relaunch the
+    // sibling so the daemon's lazy reconnect (secure-keys.ts) picks it up on
+    // the next credential read. startCes is a no-op unless CES_STANDALONE.
+    const vellumDir = join(resources.instanceDir, ".vellum");
+    const cesPidFile = join(vellumDir, "ces.pid");
+    const cesAlive = isProcessAlive(cesPidFile).alive;
+    if (!cesAlive) {
+      console.log("CES sibling not running — relaunching...");
+      await startCes(watch, resources);
+    }
   }
 
   // Start gateway
@@ -256,7 +281,11 @@ export async function wake(): Promise<void> {
     for (let attempt = 1; attempt <= maxAttempts; attempt++) {
       try {
         await resetGuardianBootstrap(loopbackUrl, bootstrapSecret);
-        await leaseGuardianToken(loopbackUrl, entry.assistantId, bootstrapSecret);
+        await leaseGuardianToken(
+          loopbackUrl,
+          entry.assistantId,
+          bootstrapSecret,
+        );
         console.log("   Re-provisioned guardian token.");
         break;
       } catch (err) {

package/src/index.ts

@@ -1,5 +1,10 @@
 #!/usr/bin/env bun
 
+import { join } from "path";
+import { readFileSync } from "node:fs";
+
+import { resolveConfigDir } from "@vellumai/local-mode";
+
 import cliPkg from "../package.json";
 import { backup } from "./commands/backup";
 import { clean } from "./commands/clean";
@@ -156,6 +161,56 @@ function applyNoColorFlags(argv: string[]): void {
   }
 }
 
+/**
+ * Load env vars from the vellum config dotenv file into `process.env` so
+ * that `vellum hatch` forwards provider API keys to containers and other
+ * commands have access to them.
+ *
+ * Reads `$XDG_CONFIG_HOME/vellum{-env}/.env` — the same config directory
+ * the CLI uses for guardian tokens and environment state. The file is
+ * written by remote-hatch scripts and can be user-managed.
+ *
+ * Existing `process.env` values take precedence (standard dotenv convention).
+ * Only KEY=VALUE lines are parsed. Lines starting with # are comments.
+ * Values may be quoted with single or double quotes.
+ */
+function loadConfigDotenv(): void {
+  const configDir = resolveConfigDir(process.env);
+  const envPath = join(configDir, ".env");
+
+  let content: string;
+  try {
+    content = readFileSync(envPath, "utf-8");
+  } catch {
+    return;
+  }
+
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+
+    const eqIndex = trimmed.indexOf("=");
+    if (eqIndex === -1) continue;
+
+    const key = trimmed.slice(0, eqIndex).trim();
+    if (!key) continue;
+
+    // Existing env vars take precedence (dotenv convention).
+    if (process.env[key] !== undefined) continue;
+
+    let value = trimmed.slice(eqIndex + 1).trim();
+    // Strip surrounding quotes.
+    if (
+      (value.startsWith('"') && value.endsWith('"')) ||
+      (value.startsWith("'") && value.endsWith("'"))
+    ) {
+      value = value.slice(1, -1);
+    }
+
+    process.env[key] = value;
+  }
+}
+
 /**
  * If a running assistant is detected, launch the TUI client and return true.
  * Otherwise return false so the caller can fall back to help text.
@@ -181,6 +236,10 @@ async function tryLaunchClient(): Promise<boolean> {
 }
 
 async function main() {
+  // Load $XDG_CONFIG_HOME/vellum/.env before any command runs so
+  // provider API keys and other config are available to hatch, exec, etc.
+  loadConfigDotenv();
+
   const args = process.argv.slice(2);
 
   // Must run before any command or terminal-capabilities usage

package/src/lib/docker.ts

@@ -1626,7 +1626,45 @@ async function waitForGatewayAndLease(opts: {
     log(`  Container: ${containerName}`);
     log("");
     log(`Stop with: vellum retire ${instanceName}`);
-    return { ready: true };
+
+    // Lease a guardian token even in detached mode so that `vellum ps`,
+    // `vellum exec`, and other CLI commands can authenticate to the
+    // gateway. Skip the /readyz readiness poll (the caller asked to detach
+    // and not block) but retry the lease itself since the gateway may need
+    // a moment to accept connections after the container starts.
+    const leaseStart = Date.now();
+    const leaseDeadline = containersUpAt + DOCKER_READY_TIMEOUT_MS;
+    let guardianAccessToken: string | undefined;
+    while (Date.now() < leaseDeadline) {
+      try {
+        const tokenData = await leaseGuardianToken(
+          runtimeUrl,
+          instanceName,
+          bootstrapSecret,
+        );
+        guardianAccessToken = tokenData.accessToken;
+        const leaseElapsed = ((Date.now() - leaseStart) / 1000).toFixed(1);
+        log(
+          `Guardian token lease: success after ${leaseElapsed}s (principalId=${tokenData.guardianPrincipalId})`,
+        );
+        break;
+      } catch (err) {
+        const elapsed = ((Date.now() - leaseStart) / 1000).toFixed(0);
+        const msg = err instanceof Error ? err.message : String(err);
+        log(
+          `Guardian token lease: attempt failed after ${elapsed}s (${msg.split("\n")[0]}), retrying...`,
+        );
+        await new Promise((r) => setTimeout(r, 2000));
+      }
+    }
+    if (!guardianAccessToken) {
+      log(
+        `⚠️  Guardian token lease failed after ${DOCKER_READY_TIMEOUT_MS / 1000}s.\n` +
+          `   The assistant is running but CLI commands (vellum ps, vellum exec) will not authenticate.\n` +
+          `   Re-hatch or run \`vellum setup\` to recover.`,
+      );
+    }
+    return { ready: true, guardianAccessToken };
   }
 
   log(`  Container: ${containerName}`);

package/src/lib/hatch-local.ts

@@ -25,6 +25,7 @@ import type { Species } from "./constants.js";
 import { buildHatchConfigValues, writeInitialConfig } from "./config-utils.js";
 import {
   generateLocalSigningKey,
+  startCes,
   startLocalDaemon,
   startGateway,
   stopLocalProcesses,
@@ -225,10 +226,17 @@ export async function hatchLocal(
   reporter.progress(3, 6, "Starting assistant...");
   const signingKey = generateLocalSigningKey();
   const bootstrapSecret = generateLocalSigningKey();
-  await startLocalDaemon(watch, resources, {
-    defaultWorkspaceConfigPath,
-    signingKey,
-  });
+  // Launch the CES sibling alongside the daemon, in parallel — matching the
+  // Docker topology. Under the opt-in the assistant does not spawn its own CES,
+  // so a freshly hatched instance would otherwise come up with CES unavailable.
+  // startCes is a no-op unless CES_STANDALONE is set.
+  await Promise.all([
+    startCes(watch, resources),
+    startLocalDaemon(watch, resources, {
+      defaultWorkspaceConfigPath,
+      signingKey,
+    }),
+  ]);
 
   reporter.progress(4, 6, "Starting gateway...");
   let runtimeUrl = `http://127.0.0.1:${resources.gatewayPort}`;

package/src/lib/local.ts

@@ -11,6 +11,8 @@ import { createRequire } from "module";
 import { homedir, networkInterfaces, platform, tmpdir } from "os";
 import { basename, dirname, join } from "path";
 
+import { isValidReleaseVersion } from "@vellumai/local-mode";
+
 import {
   getDaemonPidPath,
   type LocalInstanceResources,
@@ -143,11 +145,36 @@ function localRuntimeGatewayDir(
   return isGatewaySourceDir(candidate) ? candidate : undefined;
 }
 
+function localRuntimeCesDir(
+  resources: LocalInstanceResources | undefined,
+): string | undefined {
+  const installDir = resources?.runtimeInstallDir;
+  if (!installDir) return undefined;
+  const candidate = packagePath(
+    installDir,
+    "@vellumai/credential-executor",
+    "",
+  );
+  return isCesSourceDir(candidate) ? candidate : undefined;
+}
+
 export function ensureLocalRuntime(
   resources: LocalInstanceResources,
   version: string,
   options: { force?: boolean } = {},
 ): LocalRuntimeInstall {
+  // Reject anything that is not a trusted release identifier BEFORE it becomes
+  // a filesystem path segment or a `bun install` dependency spec. Without this,
+  // a package-manager spec (npm alias, tarball/git URL) or a `../`-laden string
+  // reaching this sink would install and then execute arbitrary attacker code
+  // as the local assistant runtime. Shares the validator with the host-bridge
+  // boundary guard (`runUpgrade`) so the two can never drift.
+  if (!isValidReleaseVersion(version)) {
+    throw new Error(
+      `Invalid runtime version '${version}': expected a release tag like v1.2.3 or 'latest'.`,
+    );
+  }
+
   const normalizedVersion = normalizeRuntimeVersion(version);
   const displayVersion =
     normalizedVersion === "latest" ? "latest" : `v${normalizedVersion}`;
@@ -333,6 +360,18 @@ function isGatewaySourceDir(dir: string): boolean {
   }
 }
 
+function isCesSourceDir(dir: string): boolean {
+  const pkgPath = join(dir, "package.json");
+  if (!existsSync(pkgPath) || !existsSync(join(dir, "src", "main.ts")))
+    return false;
+  try {
+    const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
+    return pkg.name === "@vellumai/credential-executor";
+  } catch {
+    return false;
+  }
+}
+
 function findGatewaySourceFromCwd(): string | undefined {
   let current = process.cwd();
   while (true) {
@@ -526,6 +565,14 @@ function applyDaemonEnvOverrides(
     env.VELLUM_DEFAULT_WORKSPACE_CONFIG_PATH =
       options.defaultWorkspaceConfigPath;
   }
+  // When the CLI launches CES as a sibling (CES_STANDALONE), pin the daemon to
+  // the exact socket the sibling binds so the two agree regardless of any stale
+  // CES_LOCAL_SOCKET inherited from the parent environment. The assistant then
+  // connects to the sibling instead of spawning its own CES.
+  if (isCesSiblingOptIn()) {
+    env.CES_STANDALONE = "1";
+    env.CES_LOCAL_SOCKET = resolveCesSocketPath(resources);
+  }
   applyIpcSocketDirOverride(env);
 }
 
@@ -717,6 +764,168 @@ function resolveGatewayDir(resources?: LocalInstanceResources): string {
   }
 }
 
+function resolveCesDir(resources?: LocalInstanceResources): string {
+  const runtimeCesDir = localRuntimeCesDir(resources);
+  if (runtimeCesDir) return runtimeCesDir;
+
+  // Source tree / npm sibling: cli/src/lib/ → ../../.. → credential-executor/
+  const sourceDir = join(
+    import.meta.dir,
+    "..",
+    "..",
+    "..",
+    "credential-executor",
+  );
+  if (isCesSourceDir(sourceDir)) {
+    return sourceDir;
+  }
+
+  // npm-installed elsewhere on disk: resolve via the package entry.
+  try {
+    const pkgPath = _require.resolve(
+      "@vellumai/credential-executor/package.json",
+    );
+    return dirname(pkgPath);
+  } catch {
+    throw new Error(
+      "credential-executor not found. Ensure @vellumai/credential-executor is installed or run from the source tree.",
+    );
+  }
+}
+
+/**
+ * Resolve the Unix socket path the CLI-launched CES sibling binds and the
+ * daemon connects to. Both sides read `CES_LOCAL_SOCKET`, which the CLI sets to
+ * this exact path so they agree. On macOS, long workspace paths are relocated
+ * to a short tmpdir override (the same one the IPC sockets use) to stay under
+ * the AF_UNIX path limit.
+ */
+function resolveCesSocketPath(resources?: LocalInstanceResources): string {
+  const workspaceDir = resources
+    ? join(resources.instanceDir, ".vellum", "workspace")
+    : join(homedir(), ".vellum", "workspace");
+  const override = computeIpcSocketDirOverride(workspaceDir);
+  const socketDir = override ?? workspaceDir;
+  mkdirSync(socketDir, { recursive: true });
+  return join(socketDir, "ces.sock");
+}
+
+/**
+ * Whether the CLI should launch CES as an independent sibling process instead
+ * of leaving the assistant to spawn it as an stdio child. Temporary opt-in
+ * (`CES_STANDALONE=1`) while local CES converges onto the sibling model that
+ * containerized homes already use.
+ */
+function isCesSiblingOptIn(): boolean {
+  return process.env.CES_STANDALONE === "1";
+}
+
+/**
+ * Launch the local CES sibling over a Unix socket (opted into via
+ * `CES_STANDALONE=1`). No-op unless the opt-in is set, in which case the
+ * assistant continues to spawn CES itself as today.
+ *
+ * The sibling runs with `CES_STANDALONE=1` so its lifecycle is anchored to
+ * SIGTERM rather than stdin EOF, mirroring the gateway: a CLI-owned process
+ * with a PID file under `.vellum/ces.pid`, started by `wake` and stopped by
+ * `sleep`.
+ */
+export async function startCes(
+  watch: boolean = false,
+  resources?: LocalInstanceResources,
+): Promise<void> {
+  if (!isCesSiblingOptIn()) return;
+
+  const vellumDir = resources
+    ? join(resources.instanceDir, ".vellum")
+    : join(homedir(), ".vellum");
+  const cesPidFile = join(vellumDir, "ces.pid");
+
+  // Kill any existing sibling first — a stale CES holds the socket and would
+  // corrupt the shared credential store if a second copy also bound it.
+  await stopProcessByPidFile(cesPidFile, "credential-executor");
+
+  console.log("🔐 Starting credential-executor sibling...");
+
+  const socketPath = resolveCesSocketPath(resources);
+  // A stale socket file from an unclean shutdown blocks re-bind; CES unlinks it
+  // on startup, but remove it here too so a leftover never masks a launch bug.
+  try {
+    unlinkSync(socketPath);
+  } catch {
+    /* no stale socket — fine */
+  }
+
+  const securityDir = resources
+    ? join(resources.instanceDir, ".vellum", "protected")
+    : join(homedir(), ".vellum", "protected");
+  const workspaceDir = resources
+    ? join(resources.instanceDir, ".vellum", "workspace")
+    : join(homedir(), ".vellum", "workspace");
+  mkdirSync(securityDir, { recursive: true });
+
+  const cesEnv: Record<string, string | undefined> = {
+    ...process.env,
+    CES_STANDALONE: "1",
+    CES_LOCAL_SOCKET: socketPath,
+    CREDENTIAL_SECURITY_DIR: securityDir,
+    VELLUM_WORKSPACE_DIR: workspaceDir,
+  };
+
+  let ces;
+  const runtimeCesDir = !watch ? localRuntimeCesDir(resources) : undefined;
+  const cesBinary = join(dirname(process.execPath), "credential-executor");
+  if (!runtimeCesDir && existsSync(cesBinary) && !watch) {
+    // Compiled binary alongside the CLI (desktop app / compiled CLI).
+    const cesLogFd = openLogFile("hatch.log");
+    ces = spawn(cesBinary, [], {
+      detached: true,
+      stdio: ["ignore", "pipe", "pipe"],
+      env: cesEnv,
+    });
+    pipeToLogFile(ces, cesLogFd, "credential-executor");
+  } else {
+    // Source tree / bunx: run the CES entry point via bun.
+    const cesDir = runtimeCesDir ?? resolveCesDir(resources);
+    const bunArgs = watch
+      ? ["--watch", "run", "src/main.ts"]
+      : ["run", "src/main.ts"];
+    const cesLogFd = openLogFile("hatch.log");
+    ces = spawn(resolveBunExecutable(), bunArgs, {
+      cwd: cesDir,
+      detached: true,
+      stdio: ["ignore", "pipe", "pipe"],
+      env: envWithBunPath(cesEnv),
+    });
+    pipeToLogFile(ces, cesLogFd, "credential-executor");
+    if (watch) {
+      console.log("   credential-executor started in watch mode (bun --watch)");
+    }
+  }
+
+  ces.unref();
+
+  if (ces.pid) {
+    mkdirSync(vellumDir, { recursive: true });
+    writeFileSync(cesPidFile, String(ces.pid), "utf-8");
+  }
+
+  // Wait for the socket to appear so the daemon's discovery finds it on the
+  // first probe rather than burning its retry budget.
+  const deadline = Date.now() + 10_000;
+  while (Date.now() < deadline) {
+    if (existsSync(socketPath)) break;
+    await new Promise((r) => setTimeout(r, 50));
+  }
+  if (!existsSync(socketPath)) {
+    console.warn(
+      "⚠ credential-executor started but its socket did not appear within 10s",
+    );
+  } else {
+    console.log("✅ credential-executor started\n");
+  }
+}
+
 /**
  * Check if the daemon is responsive by hitting its HTTP `/healthz` endpoint.
  * This replaces the socket-based `isSocketResponsive()` check.
@@ -1407,6 +1616,12 @@ export async function stopLocalProcesses(
   const gatewayPidFile = join(vellumDir, "gateway.pid");
   await stopProcessByPidFile(gatewayPidFile, "gateway", undefined, 7000);
 
+  // Stop the CES sibling if one was launched (CES_STANDALONE). No-op when the
+  // PID file is absent, so this is safe on the default topology where the
+  // assistant owns CES as an stdio child.
+  const cesPidFile = join(vellumDir, "ces.pid");
+  await stopProcessByPidFile(cesPidFile, "credential-executor");
+
   // Kill ngrok directly by PID rather than using stopProcessByPidFile, because
   // isVellumProcess() won't match the ngrok binary — resulting in a no-op that
   // leaves ngrok running. Verify the PID still belongs to ngrok before killing

package/src/lib/process.ts

@@ -15,7 +15,7 @@ export function isVellumProcess(pid: number): boolean {
       timeout: 3000,
       stdio: ["ignore", "pipe", "ignore"],
     }).trim();
-    return /vellum-daemon|vellum-cli|vellum-gateway|@vellumai|\/\.?vellum\/|\/daemon\/main|\/\.vellum\/.*qdrant\/bin\/qdrant/.test(
+    return /vellum-daemon|vellum-cli|vellum-gateway|credential-executor|@vellumai|\/\.?vellum\/|\/daemon\/main|\/\.vellum\/.*qdrant\/bin\/qdrant/.test(
       output,
     );
   } catch {

package/src/lib/retire-local.ts

@@ -66,6 +66,18 @@ export async function retireLocal(
   const gatewayPidFile = join(vellumDir, "gateway.pid");
   await stopProcessByPidFile(gatewayPidFile, "gateway", undefined, 7000);
 
+  // Stop the CES sibling if one was launched (CES_STANDALONE). No-op when the
+  // PID file is absent — on the default topology the assistant owns CES as an
+  // stdio child and it exits with the daemon.
+  const cesPidFile = join(vellumDir, "ces.pid");
+  const cesStopped = await stopProcessByPidFile(
+    cesPidFile,
+    "credential-executor",
+  );
+  if (cesStopped) {
+    reporter.log("credential-executor stopped.");
+  }
+
   // Stop Qdrant — the daemon's graceful shutdown tries to stop it via
   // qdrantManager.stop(), but if the daemon was SIGKILL'd (after 2s timeout)
   // Qdrant may still be running as an orphan. Check both the current PID file

package/tsconfig.json

@@ -10,7 +10,7 @@
     "resolveJsonModule": true,
     "noEmit": true,
     "jsx": "react-jsx",
-    "types": ["bun-types"]
+    "types": ["bun"]
   },
   "include": ["src/**/*", "src/**/*.tsx"],
   "exclude": ["node_modules"]