@vellumai/cli 0.6.3 → 0.6.4

package/src/lib/__tests__/docker.test.ts

@@ -0,0 +1,78 @@
+import { describe, test, expect } from "bun:test";
+import {
+  ASSISTANT_INTERNAL_PORT,
+  dockerResourceNames,
+  serviceDockerRunArgs,
+  type ServiceName,
+} from "../docker.js";
+
+const instanceName = "test-instance";
+const imageTags: Record<ServiceName, string> = {
+  assistant: "vellumai/vellum-assistant:test",
+  "credential-executor": "vellumai/vellum-credential-executor:test",
+  gateway: "vellumai/vellum-gateway:test",
+};
+
+function buildAssistantArgs(): string[] {
+  const res = dockerResourceNames(instanceName);
+  const builders = serviceDockerRunArgs({
+    gatewayPort: 7830,
+    imageTags,
+    instanceName,
+    res,
+  });
+  return builders.assistant();
+}
+
+describe("serviceDockerRunArgs — assistant", () => {
+  test("runs privileged so the inner dockerd can manage cgroups/iptables/overlayfs", () => {
+    const args = buildAssistantArgs();
+    expect(args).toContain("--privileged");
+  });
+
+  test("mounts a dedicated named volume at /var/lib/docker for the inner dockerd data store", () => {
+    const args = buildAssistantArgs();
+    const spec = `${instanceName}-dockerd-data:/var/lib/docker`;
+    const mountIndex = args.indexOf(spec);
+    expect(mountIndex).toBeGreaterThan(0);
+    expect(args[mountIndex - 1]).toBe("-v");
+  });
+
+  test("does NOT bind-mount the host Docker socket (DinD replaces host-socket access)", () => {
+    const args = buildAssistantArgs();
+    expect(args).not.toContain("/var/run/docker.sock:/var/run/docker.sock");
+  });
+
+  test("does NOT set VELLUM_WORKSPACE_VOLUME_NAME (legacy Phase 1.8 hint, no longer needed in DinD)", () => {
+    const args = buildAssistantArgs();
+    expect(
+      args.some((a) => a.startsWith("VELLUM_WORKSPACE_VOLUME_NAME=")),
+    ).toBe(false);
+  });
+
+  test("keeps existing workspace and socket volume mounts intact", () => {
+    const args = buildAssistantArgs();
+    expect(args).toContain(`${instanceName}-workspace:/workspace`);
+    expect(args).toContain(`${instanceName}-socket:/run/ces-bootstrap`);
+  });
+
+  test("preserves existing required env vars", () => {
+    const args = buildAssistantArgs();
+    expect(args).toContain("IS_CONTAINERIZED=true");
+    expect(args).toContain("VELLUM_WORKSPACE_DIR=/workspace");
+    expect(args).toContain(`VELLUM_ASSISTANT_NAME=${instanceName}`);
+  });
+
+  test("publishes the assistant HTTP port on all host interfaces so sibling bot containers can reach the daemon via host.docker.internal on both Docker Desktop and Linux", () => {
+    const args = buildAssistantArgs();
+    // The port mapping is expressed as two adjacent args: "-p" then the spec.
+    // Bound to all interfaces (no `127.0.0.1:` prefix) because on vanilla
+    // Linux Docker, host.docker.internal:host-gateway resolves to the Docker
+    // bridge gateway IP — packets arrive at the bridge interface, not
+    // loopback, so a 127.0.0.1 DNAT rule would not match.
+    const portSpec = `${ASSISTANT_INTERNAL_PORT}:${ASSISTANT_INTERNAL_PORT}`;
+    const portIndex = args.indexOf(portSpec);
+    expect(portIndex).toBeGreaterThan(0);
+    expect(args[portIndex - 1]).toBe("-p");
+  });
+});

package/src/lib/assistant-config.ts

@@ -8,7 +8,7 @@ import {
   writeFileSync,
 } from "fs";
 import { homedir } from "os";
-import { join } from "path";
+import { dirname, join } from "path";
 
 import {
   DAEMON_INTERNAL_ASSISTANT_ID,
@@ -16,8 +16,13 @@ import {
   DEFAULT_DAEMON_PORT,
   DEFAULT_GATEWAY_PORT,
   DEFAULT_QDRANT_PORT,
-  LOCKFILE_NAMES,
 } from "./constants.js";
+import {
+  getLockfilePath,
+  getLockfilePaths,
+  getMultiInstanceDir,
+} from "./environments/paths.js";
+import { getCurrentEnvironment } from "./environments/resolve.js";
 import { probePort } from "./port-probe.js";
 
 /**
@@ -27,10 +32,11 @@ import { probePort } from "./port-probe.js";
  */
 export interface LocalInstanceResources {
   /**
-   * Instance-specific data root. The first local assistant uses `~` (home
-   * directory) with default ports. Subsequent instances are placed under
-   * `~/.local/share/vellum/assistants/<name>/`.
-   * The daemon's `.vellum/` directory lives inside it.
+   * Instance-specific data root. New local assistants are placed under
+   * `$XDG_DATA_HOME/vellum{-env}/assistants/<name>/`. Legacy entries
+   * (pre env-data-layout) may still point at `~` — the read path honors
+   * whatever `instanceDir` is stored. The daemon's `.vellum/` directory
+   * lives inside it.
    */
   instanceDir: string;
   /** HTTP port for the daemon runtime server */
@@ -84,18 +90,17 @@ export interface AssistantEntry {
   resources?: LocalInstanceResources;
   /** PID of the file watcher process for docker instances hatched with --watch. */
   watcherPid?: number;
-  /** Last-known version of the service group, populated at hatch and updated by health checks. */
-  serviceGroupVersion?: string;
   /** Docker image metadata for rollback. Only present for docker topology entries. */
   containerInfo?: ContainerInfo;
-  /** The service group version that was running before the last upgrade. */
-  previousServiceGroupVersion?: string;
   /** Docker image metadata from before the last upgrade. Enables rollback to the prior version. */
   previousContainerInfo?: ContainerInfo;
   /** Path to the .vbundle backup created for the most recent upgrade. Used by rollback to restore
    *  only the backup from the specific upgrade being rolled back — never a stale backup from a
    *  previous upgrade cycle. */
   preUpgradeBackupPath?: string;
+  /** Running version of the service group at the time of the last upgrade, as reported by
+   *  the health endpoint.  Used by saved-state rollback for logging / broadcast events. */
+  previousVersion?: string;
   /** Pre-upgrade DB migration version — used by rollback to know how far back to revert. */
   previousDbMigrationVersion?: number;
   /** Pre-upgrade workspace migration ID — used by rollback to know how far back to revert. */
@@ -114,15 +119,8 @@ export function getBaseDir(): string {
   return process.env.BASE_DATA_DIR?.trim() || homedir();
 }
 
-/** The lockfile always lives under the home directory. */
-function getLockfileDir(): string {
-  return process.env.VELLUM_LOCKFILE_DIR?.trim() || homedir();
-}
-
 function readLockfile(): LockfileData {
-  const base = getLockfileDir();
-  const candidates = LOCKFILE_NAMES.map((name) => join(base, name));
-  for (const lockfilePath of candidates) {
+  for (const lockfilePath of getLockfilePaths(getCurrentEnvironment())) {
     if (!existsSync(lockfilePath)) continue;
     try {
       const raw = readFileSync(lockfilePath, "utf-8");
@@ -138,7 +136,8 @@ function readLockfile(): LockfileData {
 }
 
 function writeLockfile(data: LockfileData): void {
-  const lockfilePath = join(getLockfileDir(), LOCKFILE_NAMES[0]);
+  const lockfilePath = getLockfilePath(getCurrentEnvironment());
+  mkdirSync(dirname(lockfilePath), { recursive: true });
   const tmpPath = `${lockfilePath}.${randomBytes(4).toString("hex")}.tmp`;
   try {
     writeFileSync(tmpPath, JSON.stringify(data, null, 2) + "\n");
@@ -187,6 +186,7 @@ export function migrateLegacyEntry(raw: Record<string, unknown>): boolean {
     return false;
   }
 
+  const env = getCurrentEnvironment();
   let mutated = false;
 
   // Migrate top-level `baseDataDir` → `resources.instanceDir`
@@ -208,11 +208,7 @@ export function migrateLegacyEntry(raw: Record<string, unknown>): boolean {
     const gatewayPort =
       parsePortFromUrl(raw.runtimeUrl) ?? DEFAULT_GATEWAY_PORT;
     const instanceDir = join(
-      homedir(),
-      ".local",
-      "share",
-      "vellum",
-      "assistants",
+      getMultiInstanceDir(env),
       typeof raw.assistantId === "string"
         ? raw.assistantId
         : DAEMON_INTERNAL_ASSISTANT_ID,
@@ -231,11 +227,7 @@ export function migrateLegacyEntry(raw: Record<string, unknown>): boolean {
     const res = raw.resources as Record<string, unknown>;
     if (!res.instanceDir) {
       res.instanceDir = join(
-        homedir(),
-        ".local",
-        "share",
-        "vellum",
-        "assistants",
+        getMultiInstanceDir(env),
         typeof raw.assistantId === "string"
           ? raw.assistantId
           : DAEMON_INTERNAL_ASSISTANT_ID,
@@ -394,23 +386,6 @@ export function saveAssistantEntry(entry: AssistantEntry): void {
   writeAssistants(entries);
 }
 
-/**
- * Update just the serviceGroupVersion field on a lockfile entry.
- * Reads the current entry, updates the version if changed, and writes back.
- * No-op if the entry doesn't exist or the version hasn't changed.
- */
-export function updateServiceGroupVersion(
-  assistantId: string,
-  version: string,
-): void {
-  const entries = readAssistants();
-  const entry = entries.find((e) => e.assistantId === assistantId);
-  if (!entry) return;
-  if (entry.serviceGroupVersion === version) return;
-  entry.serviceGroupVersion = version;
-  writeAssistants(entries);
-}
-
 /**
  * Scan upward from `basePort` to find an available port. A port is considered
  * available when `probePort()` returns false (nothing listening). Scans up to
@@ -434,58 +409,32 @@ async function findAvailablePort(
 
 /**
  * Allocate an isolated set of resources for a named local instance.
- * The first local assistant uses the home directory with default ports.
- * Subsequent assistants are placed under
- * `~/.local/share/vellum/assistants/<name>/` with scanned ports.
+ * Every new local assistant is allocated under
+ * `$XDG_DATA_HOME/vellum{-env}/assistants/<name>/`. The legacy `~/.vellum/`
+ * path is only reached via existing lockfile entries from before this change
+ * — the read path honors whatever `resources.instanceDir` is stored, so
+ * production users' existing first-local assistants keep their `~/.vellum/`
+ * roots unchanged.
  */
 export async function allocateLocalResources(
   instanceName: string,
 ): Promise<LocalInstanceResources> {
-  // First local assistant gets the home directory with default ports.
-  // Respect BASE_DATA_DIR when set (e.g. in e2e tests) so the daemon,
-  // gateway, and credential store all resolve paths under the same root.
-  const existingLocals = loadAllAssistants().filter((e) => e.cloud === "local");
-  if (existingLocals.length === 0) {
-    const baseDir = getBaseDir();
-    const vellumDir = join(baseDir, ".vellum");
-    return {
-      instanceDir: baseDir,
-      daemonPort: DEFAULT_DAEMON_PORT,
-      gatewayPort: DEFAULT_GATEWAY_PORT,
-      qdrantPort: DEFAULT_QDRANT_PORT,
-      cesPort: DEFAULT_CES_PORT,
-      pidFile: join(vellumDir, "vellum.pid"),
-    };
-  }
-
-  const instanceDir = join(
-    homedir(),
-    ".local",
-    "share",
-    "vellum",
-    "assistants",
-    instanceName,
-  );
+  const env = getCurrentEnvironment();
+  const instanceDir = join(getMultiInstanceDir(env), instanceName);
   mkdirSync(instanceDir, { recursive: true });
 
   // Collect ports already assigned to other local instances in the lockfile.
-  // Even if those instances are stopped, we must avoid reusing their ports
-  // to prevent binding collisions when both are woken.
   const reservedPorts: number[] = [];
   for (const entry of loadAllAssistants()) {
-    if (entry.cloud !== "local") continue;
-    if (entry.resources) {
-      reservedPorts.push(
-        entry.resources.daemonPort,
-        entry.resources.gatewayPort,
-        entry.resources.qdrantPort,
-        entry.resources.cesPort,
-      );
-    }
+    if (entry.cloud !== "local" || !entry.resources) continue;
+    reservedPorts.push(
+      entry.resources.daemonPort,
+      entry.resources.gatewayPort,
+      entry.resources.qdrantPort,
+      entry.resources.cesPort,
+    );
   }
 
-  // Allocate ports sequentially to avoid overlapping ranges assigning the
-  // same port to multiple services (e.g. daemon 7821-7920 overlaps gateway 7830-7929).
   const daemonPort = await findAvailablePort(
     DEFAULT_DAEMON_PORT,
     reservedPorts,
@@ -516,6 +465,18 @@ export async function allocateLocalResources(
   };
 }
 
+/**
+ * Return `platformBaseUrl` from the lockfile, if set. This is the value
+ * persisted by {@link syncConfigToLockfile} the last time the active
+ * assistant was hatched/waked, and is the source of truth for "which
+ * platform does the currently-active assistant target".
+ */
+export function getLockfilePlatformBaseUrl(): string | undefined {
+  const url = readLockfile().platformBaseUrl;
+  if (typeof url === "string" && url.trim()) return url.trim();
+  return undefined;
+}
+
 /**
  * Read the assistant config file and sync client-relevant values to the
  * lockfile. This lets external tools (e.g. vel) discover the platform URL

package/src/lib/aws.ts

@@ -411,7 +411,18 @@ export async function hatchAws(
       }
     }
 
-    const sshUser = userInfo().username;
+    let sshUser: string;
+    try {
+      sshUser = userInfo().username;
+    } catch {
+      sshUser = process.env.USER ?? "";
+    }
+    if (!sshUser) {
+      console.error(
+        "Error: Could not determine SSH username. Set the USER environment variable and try again.",
+      );
+      process.exit(1);
+    }
     const hatchedBy = process.env.VELLUM_HATCHED_BY;
     const providerApiKeys: Record<string, string> = {};
     for (const [, envVar] of Object.entries(PROVIDER_ENV_VAR_NAMES)) {

package/src/lib/constants.ts

@@ -16,16 +16,6 @@ export const DEFAULT_GATEWAY_PORT = 7830;
 export const DEFAULT_QDRANT_PORT = 6333;
 export const DEFAULT_CES_PORT = 8090;
 
-/**
- * Lockfile candidate filenames, checked in priority order.
- * `.vellum.lock.json` is the current name; `.vellum.lockfile.json` is the
- * legacy name kept for backwards compatibility with older installs.
- */
-export const LOCKFILE_NAMES = [
-  ".vellum.lock.json",
-  ".vellum.lockfile.json",
-] as const;
-
 export const VALID_REMOTE_HOSTS = [
   "local",
   "gcp",

package/src/lib/docker.ts

@@ -39,13 +39,17 @@ export const DOCKERHUB_IMAGES: Record<ServiceName, string> = {
 };
 
 /** Internal ports exposed by each service's Dockerfile. */
-export const ASSISTANT_INTERNAL_PORT = 3001;
+export const ASSISTANT_INTERNAL_PORT = 7821;
 export const GATEWAY_INTERNAL_PORT = 7830;
 
 /** Max time to wait for the assistant container to emit the readiness sentinel. */
 export const DOCKER_READY_TIMEOUT_MS = 3 * 60 * 1000;
 
+/** Default memory (GiB) allocated to the Colima VM. */
+const COLIMA_DEFAULT_MEMORY_GIB = 8;
+
 /** Directory for user-local binary installs (no sudo required). */
+
 const LOCAL_BIN_DIR = join(
   process.env.HOME || process.env.USERPROFILE || ".",
   ".local",
@@ -294,7 +298,11 @@ async function ensureDockerInstalled(): Promise<void> {
 
     console.log("🚀 Docker daemon not running. Starting Colima...");
     try {
-      await exec("colima", ["start"]);
+      await exec("colima", [
+        "start",
+        "--memory",
+        String(COLIMA_DEFAULT_MEMORY_GIB),
+      ]);
     } catch {
       // Colima may fail if a previous VM instance is in a corrupt state.
       // Attempt to delete the stale instance and retry once.
@@ -311,7 +319,11 @@ async function ensureDockerInstalled(): Promise<void> {
 
       try {
         console.log("🔄 Retrying colima start...");
-        await exec("colima", ["start"]);
+        await exec("colima", [
+          "start",
+          "--memory",
+          String(COLIMA_DEFAULT_MEMORY_GIB),
+        ]);
       } catch (retryErr) {
         const message =
           retryErr instanceof Error ? retryErr.message : String(retryErr);
@@ -329,6 +341,7 @@ export function dockerResourceNames(instanceName: string) {
     assistantContainer: `${instanceName}-assistant`,
     cesContainer: `${instanceName}-credential-executor`,
     cesSecurityVolume: `${instanceName}-ces-sec`,
+    dockerdDataVolume: `${instanceName}-dockerd-data`,
     gatewayContainer: `${instanceName}-gateway`,
     gatewaySecurityVolume: `${instanceName}-gateway-sec`,
     network: `${instanceName}-net`,
@@ -388,6 +401,7 @@ export async function retireDocker(name: string): Promise<void> {
     res.workspaceVolume,
     res.cesSecurityVolume,
     res.gatewaySecurityVolume,
+    res.dockerdDataVolume,
   ]) {
     try {
       await exec("docker", ["volume", "rm", vol]);
@@ -551,19 +565,53 @@ export function serviceDockerRunArgs(opts: {
   } = opts;
   return {
     assistant: () => {
+      // Run the assistant container in Docker-in-Docker (DinD) mode: the
+      // container runs its own `dockerd` so the Meet subsystem can spawn
+      // sibling meet-bot containers without needing access to the host's
+      // Docker engine. This requires:
+      //   - `--privileged` so the inner dockerd can manage cgroups, iptables,
+      //     overlayfs mounts, etc.
+      //   - A dedicated named volume mounted at `/var/lib/docker` so the
+      //     inner Docker image cache and container state survive restarts of
+      //     the assistant container.
+      // The host's `/var/run/docker.sock` is intentionally NOT mounted — all
+      // Meet-bot spawning happens against the inner dockerd.
       const args: string[] = [
         "run",
         "--init",
         "-d",
+        "--privileged",
         "--name",
         res.assistantContainer,
         `--network=${res.network}`,
         "-p",
         `${gatewayPort}:${GATEWAY_INTERNAL_PORT}`,
+        // Published so the Meet subsystem's sibling bot containers can reach
+        // the daemon's internal HTTP API at host.docker.internal:<port>.
+        //
+        // Published on all host interfaces (no `127.0.0.1:` prefix) because on
+        // vanilla Linux Docker, `host.docker.internal:host-gateway` resolves
+        // to the Docker bridge gateway IP (e.g. 172.17.0.1), not loopback.
+        // Packets from sibling containers arrive at the host's bridge
+        // interface, and an iptables DNAT rule keyed on dest=127.0.0.1 would
+        // not match — causing connection refused. Docker Desktop (macOS/
+        // Windows) still works because its VM proxy forwards to the same
+        // published port regardless of the binding address.
+        //
+        // Security tradeoff: the daemon HTTP API is now reachable from the
+        // host's LAN (any device that can hit the host IP on this port).
+        // This matches the gateway port's existing posture and is acceptable
+        // for single-user self-hosted Docker mode per the Phase 1.8 security
+        // note. Managed/multi-tenant deployments are out of scope and would
+        // require a different design.
+        "-p",
+        `${ASSISTANT_INTERNAL_PORT}:${ASSISTANT_INTERNAL_PORT}`,
         "-v",
         `${res.workspaceVolume}:/workspace`,
         "-v",
         `${res.socketVolume}:/run/ces-bootstrap`,
+        "-v",
+        `${res.dockerdDataVolume}:/var/lib/docker`,
         "-e",
         "IS_CONTAINERIZED=true",
         "-e",
@@ -575,6 +623,10 @@ export function serviceDockerRunArgs(opts: {
         "-e",
         "VELLUM_WORKSPACE_DIR=/workspace",
         "-e",
+        "VELLUM_BACKUP_DIR=/workspace/.backups",
+        "-e",
+        "VELLUM_BACKUP_KEY_PATH=/workspace/.backup.key",
+        "-e",
         "CES_CREDENTIAL_URL=http://localhost:8090",
         "-e",
         `GATEWAY_INTERNAL_URL=http://localhost:${GATEWAY_INTERNAL_PORT}`,
@@ -596,6 +648,7 @@ export function serviceDockerRunArgs(opts: {
       }
       for (const envVar of [
         ...Object.values(PROVIDER_ENV_VAR_NAMES),
+        "VELLUM_ENVIRONMENT",
         "VELLUM_PLATFORM_URL",
       ]) {
         if (process.env[envVar]) {
@@ -644,6 +697,9 @@ export function serviceDockerRunArgs(opts: {
       ...(opts.bootstrapSecret
         ? ["-e", `GUARDIAN_BOOTSTRAP_SECRET=${opts.bootstrapSecret}`]
         : []),
+      ...(process.env.VELLUM_ENVIRONMENT
+        ? ["-e", `VELLUM_ENVIRONMENT=${process.env.VELLUM_ENVIRONMENT}`]
+        : []),
       ...(process.env.VELLUM_PLATFORM_URL
         ? ["-e", `VELLUM_PLATFORM_URL=${process.env.VELLUM_PLATFORM_URL}`]
         : []),
@@ -700,6 +756,16 @@ export async function startContainers(
   },
   log: (msg: string) => void,
 ): Promise<void> {
+  // Ensure the inner dockerd's data volume exists before mounting it.
+  // For instances hatched on Phase 1.10+, this is created in hatchDocker and
+  // is a no-op here. For instances that pre-date Phase 1.10 (DinD) and are
+  // upgrading in place, Docker would otherwise auto-create the volume on
+  // first `-v` mount without our standard ownership/labeling. Creating it
+  // explicitly keeps volume provenance consistent across fresh and upgraded
+  // instances. `docker volume create` is idempotent for an existing volume
+  // of the same name, so this is safe to run on every start.
+  await exec("docker", ["volume", "create", opts.res.dockerdDataVolume]);
+
   const runArgs = serviceDockerRunArgs(opts);
   for (const service of SERVICE_START_ORDER) {
     log(`🚀 Starting ${service} container...`);
@@ -1110,6 +1176,7 @@ export async function hatchDocker(
     await exec("docker", ["volume", "create", res.workspaceVolume]);
     await exec("docker", ["volume", "create", res.cesSecurityVolume]);
     await exec("docker", ["volume", "create", res.gatewaySecurityVolume]);
+    await exec("docker", ["volume", "create", res.dockerdDataVolume]);
 
     // Set workspace volume ownership so non-root containers (UID 1001) can write.
     await exec("docker", [
@@ -1165,7 +1232,6 @@ export async function hatchDocker(
       cloud: "docker",
       species,
       hatchedAt: new Date().toISOString(),
-      serviceGroupVersion: cliPkg.version ? `v${cliPkg.version}` : undefined,
       containerInfo: {
         assistantImage: imageTags.assistant,
         gatewayImage: imageTags.gateway,