@vellumai/cli 0.4.55 → 0.4.57

package/src/lib/docker.ts

@@ -1,13 +1,20 @@
-import { spawn as nodeSpawn } from "child_process";
-import { existsSync } from "fs";
-import { createRequire } from "module";
+import { existsSync, watch as fsWatch } from "fs";
 import { dirname, join } from "path";
 
-import { saveAssistantEntry, setActiveAssistant } from "./assistant-config";
+// Direct import — bun embeds this at compile time so it works in compiled binaries.
+import cliPkg from "../../package.json";
+
+import {
+  findAssistantByName,
+  saveAssistantEntry,
+  setActiveAssistant,
+} from "./assistant-config";
 import type { AssistantEntry } from "./assistant-config";
 import { DEFAULT_GATEWAY_PORT } from "./constants";
 import type { Species } from "./constants";
-import { generateRandomSuffix } from "./random-name";
+import { leaseGuardianToken } from "./guardian-token";
+import { isVellumProcess, stopProcess } from "./process";
+import { generateInstanceName } from "./random-name";
 import { exec, execOutput } from "./step-runner";
 import {
   closeLogFile,
@@ -16,7 +23,21 @@ import {
   writeToLogFile,
 } from "./xdg-log";
 
-const _require = createRequire(import.meta.url);
+export type ServiceName = "assistant" | "credential-executor" | "gateway";
+
+const DOCKERHUB_ORG = "vellumai";
+export const DOCKERHUB_IMAGES: Record<ServiceName, string> = {
+  assistant: `${DOCKERHUB_ORG}/vellum-assistant`,
+  "credential-executor": `${DOCKERHUB_ORG}/vellum-credential-executor`,
+  gateway: `${DOCKERHUB_ORG}/vellum-gateway`,
+};
+
+/** Internal ports exposed by each service's Dockerfile. */
+export const ASSISTANT_INTERNAL_PORT = 3001;
+export const GATEWAY_INTERNAL_PORT = 7830;
+
+/** Max time to wait for the assistant container to emit the readiness sentinel. */
+export const DOCKER_READY_TIMEOUT_MS = 3 * 60 * 1000;
 
 /**
  * Checks whether the `docker` CLI and daemon are available on the system.
@@ -114,367 +135,711 @@ async function ensureDockerInstalled(): Promise<void> {
   }
 }
 
-interface DockerRoot {
-  /** Directory to use as the Docker build context */
-  root: string;
-  /** Relative path from root to the directory containing the Dockerfiles */
-  dockerfileDir: string;
+/** Derive the Docker resource names from the instance name. */
+export function dockerResourceNames(instanceName: string) {
+  return {
+    assistantContainer: `${instanceName}-assistant`,
+    cesContainer: `${instanceName}-credential-executor`,
+    dataVolume: `vellum-data-${instanceName}`,
+    gatewayContainer: `${instanceName}-gateway`,
+    network: `vellum-net-${instanceName}`,
+    socketVolume: `vellum-ces-bootstrap-${instanceName}`,
+  };
 }
 
-/**
- * Locate the directory containing the Dockerfile. In the source tree the
- * Dockerfiles live under `meta/`, but when installed as an npm package they
- * are at the package root.
- */
-function findDockerRoot(developmentMode: boolean = false): DockerRoot {
-  // Source tree: cli/src/lib/ -> repo root (Dockerfiles in meta/)
-  const sourceTreeRoot = join(import.meta.dir, "..", "..", "..");
-  if (existsSync(join(sourceTreeRoot, "meta", "Dockerfile"))) {
-    return { root: sourceTreeRoot, dockerfileDir: "meta" };
+/** Silently attempt to stop and remove a Docker container. */
+export async function removeContainer(containerName: string): Promise<void> {
+  try {
+    await exec("docker", ["stop", containerName]);
+  } catch {
+    // container may not exist or already stopped
   }
+  try {
+    await exec("docker", ["rm", containerName]);
+  } catch {
+    // container may not exist or already removed
+  }
+}
+
+export async function retireDocker(name: string): Promise<void> {
+  console.log(`\u{1F5D1}\ufe0f  Stopping Docker containers for '${name}'...\n`);
+
+  // Stop the file watcher process if one is tracked for this instance.
+  const entry = findAssistantByName(name);
+  const watcherPid =
+    typeof entry?.watcherPid === "number" ? entry.watcherPid : null;
+  if (watcherPid !== null) {
+    if (isVellumProcess(watcherPid)) {
+      await stopProcess(watcherPid, "file-watcher");
+    } else {
+      console.log(
+        `PID ${watcherPid} is not a vellum process — skipping stale file-watcher PID.`,
+      );
+    }
+  }
+
+  const res = dockerResourceNames(name);
+
+  await removeContainer(res.cesContainer);
+  await removeContainer(res.gatewayContainer);
+  await removeContainer(res.assistantContainer);
 
-  // bunx layout: @vellumai/cli/src/lib/ -> ../../../.. -> node_modules -> vellum/
-  const bunxRoot = join(import.meta.dir, "..", "..", "..", "..", "vellum");
-  if (existsSync(join(bunxRoot, "Dockerfile"))) {
-    return { root: bunxRoot, dockerfileDir: "." };
+  // Also clean up a legacy single-container instance if it exists
+  await removeContainer(name);
+
+  // Remove shared network and volumes
+  try {
+    await exec("docker", ["network", "rm", res.network]);
+  } catch {
+    // network may not exist
+  }
+  for (const vol of [res.dataVolume, res.socketVolume]) {
+    try {
+      await exec("docker", ["volume", "rm", vol]);
+    } catch {
+      // volume may not exist
+    }
   }
 
-  // Walk up from cwd looking for meta/Dockerfile (source checkout)
-  let dir = process.cwd();
+  console.log(`\u2705 Docker instance retired.`);
+}
+
+/**
+ * Walk up from `startDir` looking for a directory that contains
+ * `assistant/Dockerfile`. Returns the path if found, otherwise `undefined`.
+ */
+function walkUpForRepoRoot(startDir: string): string | undefined {
+  let dir = startDir;
   while (true) {
-    if (existsSync(join(dir, "meta", "Dockerfile"))) {
-      return { root: dir, dockerfileDir: "meta" };
+    if (existsSync(join(dir, "assistant", "Dockerfile"))) {
+      return dir;
     }
     const parent = dirname(dir);
     if (parent === dir) break;
     dir = parent;
   }
+  return undefined;
+}
 
-  // In development mode, walk up from the executable path to find the repo
-  // root. This handles the macOS app bundle case where the binary lives inside
-  // the repo at e.g. clients/macos/dist/Vellum.app/Contents/MacOS/.
-  if (developmentMode) {
-    let execDir = dirname(process.execPath);
-    while (true) {
-      if (existsSync(join(execDir, "meta", "Dockerfile.development"))) {
-        return { root: execDir, dockerfileDir: "meta" };
-      }
-      const parent = dirname(execDir);
-      if (parent === execDir) break;
-      execDir = parent;
-    }
+/**
+ * Locate the repository root by walking up from `cli/src/lib/` until we
+ * find a directory containing the expected Dockerfiles.
+ */
+function findRepoRoot(): string {
+  // cli/src/lib/ -> repo root (works when running from source via bun)
+  const sourceTreeRoot = join(import.meta.dir, "..", "..", "..");
+  if (existsSync(join(sourceTreeRoot, "assistant", "Dockerfile"))) {
+    return sourceTreeRoot;
   }
 
-  // macOS app bundle: Contents/MacOS/vellum-cli -> Contents/Resources/Dockerfile
-  const appResourcesDir = join(dirname(process.execPath), "..", "Resources");
-  if (existsSync(join(appResourcesDir, "Dockerfile"))) {
-    return { root: appResourcesDir, dockerfileDir: "." };
+  // Walk up from the compiled binary's location. When the CLI is bundled
+  // inside the macOS app (e.g. .../dist/Vellum.app/Contents/MacOS/vellum-cli),
+  // the binary still lives inside the repo tree, so walking up will
+  // eventually reach the repo root.
+  const execRoot = walkUpForRepoRoot(dirname(process.execPath));
+  if (execRoot) {
+    return execRoot;
   }
 
-  // Fall back to Node module resolution for the `vellum` package
-  try {
-    const vellumPkgPath = _require.resolve("vellum/package.json");
-    const vellumDir = dirname(vellumPkgPath);
-    if (existsSync(join(vellumDir, "Dockerfile"))) {
-      return { root: vellumDir, dockerfileDir: "." };
-    }
-  } catch {
-    // resolution failed
+  // Walk up from cwd as a final fallback
+  const cwdRoot = walkUpForRepoRoot(process.cwd());
+  if (cwdRoot) {
+    return cwdRoot;
   }
 
   throw new Error(
-    "Could not find Dockerfile. Run this command from within the " +
-      "vellum-assistant repository, or ensure the vellum package is installed.",
+    "Could not find repository root containing assistant/Dockerfile. " +
+      "Run this command from within the vellum-assistant repository.",
+  );
+}
+
+interface ServiceImageConfig {
+  context: string;
+  dockerfile: string;
+  tag: string;
+}
+
+async function buildImage(config: ServiceImageConfig): Promise<void> {
+  await exec(
+    "docker",
+    ["build", "-f", config.dockerfile, "-t", config.tag, "."],
+    { cwd: config.context },
+  );
+}
+
+function serviceImageConfigs(
+  repoRoot: string,
+  imageTags: Record<ServiceName, string>,
+): Record<ServiceName, ServiceImageConfig> {
+  return {
+    assistant: {
+      context: repoRoot,
+      dockerfile: "assistant/Dockerfile",
+      tag: imageTags.assistant,
+    },
+    "credential-executor": {
+      context: repoRoot,
+      dockerfile: "credential-executor/Dockerfile",
+      tag: imageTags["credential-executor"],
+    },
+    gateway: {
+      context: join(repoRoot, "gateway"),
+      dockerfile: "Dockerfile",
+      tag: imageTags.gateway,
+    },
+  };
+}
+
+async function buildAllImages(
+  repoRoot: string,
+  imageTags: Record<ServiceName, string>,
+  log: (msg: string) => void,
+): Promise<void> {
+  const configs = serviceImageConfigs(repoRoot, imageTags);
+  log("🔨 Building all images in parallel...");
+  await Promise.all(
+    Object.entries(configs).map(async ([name, config]) => {
+      await buildImage(config);
+      log(`✅ ${name} built`);
+    }),
   );
 }
 
 /**
- * Creates a line-buffered output prefixer that prepends `[docker]` to each
- * line from the container's stdout/stderr. Calls `onLine` for each complete
- * line so the caller can detect sentinel output (e.g. hatch completion).
+ * Returns a function that builds the `docker run` arguments for a given
+ * service. Each container joins a shared Docker bridge network so they
+ * can be restarted independently.
  */
-function createLinePrefixer(
-  stream: NodeJS.WritableStream,
-  onLine?: (line: string) => void,
-): { write(data: Buffer): void; flush(): void } {
-  let remainder = "";
+export function serviceDockerRunArgs(opts: {
+  extraAssistantEnv?: Record<string, string>;
+  gatewayPort: number;
+  imageTags: Record<ServiceName, string>;
+  instanceName: string;
+  res: ReturnType<typeof dockerResourceNames>;
+}): Record<ServiceName, () => string[]> {
+  const { extraAssistantEnv, gatewayPort, imageTags, instanceName, res } = opts;
   return {
-    write(data: Buffer) {
-      const text = remainder + data.toString();
-      const lines = text.split("\n");
-      remainder = lines.pop() ?? "";
-      for (const line of lines) {
-        stream.write(`   [docker] ${line}\n`);
-        onLine?.(line);
+    assistant: () => {
+      const args: string[] = [
+        "run",
+        "--init",
+        "-d",
+        "--name",
+        res.assistantContainer,
+        `--network=${res.network}`,
+        "-v",
+        `${res.dataVolume}:/data`,
+        "-v",
+        `${res.socketVolume}:/run/ces-bootstrap`,
+        "-e",
+        `VELLUM_ASSISTANT_NAME=${instanceName}`,
+        "-e",
+        "RUNTIME_HTTP_HOST=0.0.0.0",
+      ];
+      for (const envVar of ["ANTHROPIC_API_KEY", "VELLUM_PLATFORM_URL"]) {
+        if (process.env[envVar]) {
+          args.push("-e", `${envVar}=${process.env[envVar]}`);
+        }
       }
-    },
-    flush() {
-      if (remainder) {
-        stream.write(`   [docker] ${remainder}\n`);
-        onLine?.(remainder);
-        remainder = "";
+      if (extraAssistantEnv) {
+        for (const [key, value] of Object.entries(extraAssistantEnv)) {
+          args.push("-e", `${key}=${value}`);
+        }
       }
+      args.push(imageTags.assistant);
+      return args;
     },
+    gateway: () => [
+      "run",
+      "--init",
+      "-d",
+      "--name",
+      res.gatewayContainer,
+      `--network=${res.network}`,
+      "-p",
+      `${gatewayPort}:${GATEWAY_INTERNAL_PORT}`,
+      "-v",
+      `${res.dataVolume}:/data`,
+      "-e",
+      "BASE_DATA_DIR=/data",
+      "-e",
+      `GATEWAY_PORT=${GATEWAY_INTERNAL_PORT}`,
+      "-e",
+      `ASSISTANT_HOST=${res.assistantContainer}`,
+      "-e",
+      `RUNTIME_HTTP_PORT=${ASSISTANT_INTERNAL_PORT}`,
+      "-e",
+      "RUNTIME_PROXY_ENABLED=true",
+      imageTags.gateway,
+    ],
+    "credential-executor": () => [
+      "run",
+      "--init",
+      "-d",
+      "--name",
+      res.cesContainer,
+      `--network=${res.network}`,
+      "-v",
+      `${res.socketVolume}:/run/ces-bootstrap`,
+      "-v",
+      `${res.dataVolume}:/data:ro`,
+      "-e",
+      "CES_MODE=managed",
+      "-e",
+      "CES_BOOTSTRAP_SOCKET_DIR=/run/ces-bootstrap",
+      "-e",
+      "CES_ASSISTANT_DATA_MOUNT=/data",
+      imageTags["credential-executor"],
+    ],
   };
 }
 
-async function fetchRemoteBearerToken(
-  containerName: string,
-): Promise<string | null> {
-  try {
-    const remoteCmd =
-      'cat ~/.vellum.lock.json 2>/dev/null || cat ~/.vellum.lockfile.json 2>/dev/null || echo "{}"';
-    const output = await execOutput("docker", [
-      "exec",
-      containerName,
-      "sh",
-      "-c",
-      remoteCmd,
-    ]);
-    const data = JSON.parse(output.trim());
-    const assistants = data.assistants;
-    if (Array.isArray(assistants) && assistants.length > 0) {
-      const token = assistants[0].bearerToken;
-      if (typeof token === "string" && token) {
-        return token;
-      }
-    }
-    return null;
-  } catch {
-    return null;
+/** The order in which services must be started. */
+export const SERVICE_START_ORDER: ServiceName[] = [
+  "assistant",
+  "gateway",
+  "credential-executor",
+];
+
+/** Start all three containers in dependency order. */
+export async function startContainers(
+  opts: {
+    extraAssistantEnv?: Record<string, string>;
+    gatewayPort: number;
+    imageTags: Record<ServiceName, string>;
+    instanceName: string;
+    res: ReturnType<typeof dockerResourceNames>;
+  },
+  log: (msg: string) => void,
+): Promise<void> {
+  const runArgs = serviceDockerRunArgs(opts);
+  for (const service of SERVICE_START_ORDER) {
+    log(`🚀 Starting ${service} container...`);
+    await exec("docker", runArgs[service]());
   }
 }
 
-export async function retireDocker(name: string): Promise<void> {
-  console.log(`\u{1F5D1}\ufe0f  Stopping Docker container '${name}'...\n`);
+/** Stop and remove all three containers (ignoring errors). */
+export async function stopContainers(
+  res: ReturnType<typeof dockerResourceNames>,
+): Promise<void> {
+  await removeContainer(res.cesContainer);
+  await removeContainer(res.gatewayContainer);
+  await removeContainer(res.assistantContainer);
+}
 
-  try {
-    await exec("docker", ["stop", name]);
-  } catch (error) {
-    console.warn(
-      `\u26a0\ufe0f  Failed to stop container: ${error instanceof Error ? error.message : error}`,
-    );
+/**
+ * Determine which services are affected by a changed file path relative
+ * to the repository root.
+ */
+function affectedServices(
+  filePath: string,
+  repoRoot: string,
+): Set<ServiceName> {
+  const rel = filePath.startsWith(repoRoot)
+    ? filePath.slice(repoRoot.length + 1)
+    : filePath;
+
+  const affected = new Set<ServiceName>();
+
+  if (rel.startsWith("assistant/")) {
+    affected.add("assistant");
+  }
+  if (rel.startsWith("credential-executor/")) {
+    affected.add("credential-executor");
+  }
+  if (rel.startsWith("gateway/")) {
+    affected.add("gateway");
+  }
+  // Shared packages affect both assistant and credential-executor
+  if (rel.startsWith("packages/")) {
+    affected.add("assistant");
+    affected.add("credential-executor");
   }
 
-  try {
-    await exec("docker", ["rm", name]);
-  } catch (error) {
-    console.warn(
-      `\u26a0\ufe0f  Failed to remove container: ${error instanceof Error ? error.message : error}`,
-    );
+  return affected;
+}
+
+/**
+ * Watch for file changes in the assistant, gateway, credential-executor,
+ * and packages directories. When changes are detected, rebuild the affected
+ * images and restart their containers.
+ */
+function startFileWatcher(opts: {
+  gatewayPort: number;
+  imageTags: Record<ServiceName, string>;
+  instanceName: string;
+  repoRoot: string;
+  res: ReturnType<typeof dockerResourceNames>;
+}): () => void {
+  const { gatewayPort, imageTags, instanceName, repoRoot, res } = opts;
+
+  const watchDirs = [
+    join(repoRoot, "assistant"),
+    join(repoRoot, "credential-executor"),
+    join(repoRoot, "gateway"),
+    join(repoRoot, "packages"),
+  ];
+
+  let debounceTimer: ReturnType<typeof setTimeout> | null = null;
+  let pendingServices = new Set<ServiceName>();
+  let rebuilding = false;
+
+  const configs = serviceImageConfigs(repoRoot, imageTags);
+  const runArgs = serviceDockerRunArgs({
+    gatewayPort,
+    imageTags,
+    instanceName,
+    res,
+  });
+  const containerForService: Record<ServiceName, string> = {
+    assistant: res.assistantContainer,
+    "credential-executor": res.cesContainer,
+    gateway: res.gatewayContainer,
+  };
+
+  async function rebuildAndRestart(): Promise<void> {
+    if (rebuilding) return;
+    rebuilding = true;
+
+    const services = pendingServices;
+    pendingServices = new Set();
+
+    const serviceNames = [...services].join(", ");
+    console.log(`\n🔄 Changes detected — rebuilding: ${serviceNames}`);
+
+    try {
+      await Promise.all(
+        [...services].map(async (service) => {
+          console.log(`🔨 Building ${service}...`);
+          await buildImage(configs[service]);
+          console.log(`✅ ${service} built`);
+        }),
+      );
+
+      for (const service of services) {
+        const container = containerForService[service];
+        console.log(`🔄 Restarting ${container}...`);
+        await removeContainer(container);
+        await exec("docker", runArgs[service]());
+      }
+
+      console.log("✅ Rebuild complete — watching for changes...\n");
+    } catch (err) {
+      console.error(
+        `❌ Rebuild failed: ${err instanceof Error ? err.message : err}`,
+      );
+      console.log("   Watching for changes...\n");
+    } finally {
+      rebuilding = false;
+      if (pendingServices.size > 0) {
+        rebuildAndRestart();
+      }
+    }
   }
 
-  console.log(`\u2705 Docker instance retired.`);
+  const watchers: ReturnType<typeof fsWatch>[] = [];
+
+  for (const dir of watchDirs) {
+    if (!existsSync(dir)) continue;
+    const watcher = fsWatch(dir, { recursive: true }, (_event, filename) => {
+      if (!filename) return;
+      if (
+        filename.includes("node_modules") ||
+        filename.includes(".env") ||
+        filename.startsWith(".")
+      ) {
+        return;
+      }
+
+      const fullPath = join(dir, filename);
+      const services = affectedServices(fullPath, repoRoot);
+      if (services.size === 0) return;
+
+      for (const s of services) {
+        pendingServices.add(s);
+      }
+
+      if (debounceTimer) clearTimeout(debounceTimer);
+      debounceTimer = setTimeout(() => {
+        debounceTimer = null;
+        rebuildAndRestart();
+      }, 500);
+    });
+    watchers.push(watcher);
+  }
+
+  console.log("👀 Watching for file changes in:");
+  console.log("   assistant/, gateway/, credential-executor/, packages/");
+  console.log("");
+
+  return () => {
+    for (const watcher of watchers) {
+      watcher.close();
+    }
+    if (debounceTimer) clearTimeout(debounceTimer);
+  };
 }
 
 export async function hatchDocker(
   species: Species,
   detached: boolean,
   name: string | null,
-  watch: boolean,
+  watch: boolean = false,
 ): Promise<void> {
   resetLogFile("hatch.log");
 
-  await ensureDockerInstalled();
+  let logFd = openLogFile("hatch.log");
+  const log = (msg: string): void => {
+    console.log(msg);
+    writeToLogFile(logFd, `${new Date().toISOString()} ${msg}\n`);
+  };
 
-  let repoRoot: string;
-  let dockerfileDir: string;
   try {
-    ({ root: repoRoot, dockerfileDir } = findDockerRoot(watch));
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    const logFd = openLogFile("hatch.log");
-    writeToLogFile(
+    await ensureDockerInstalled();
+
+    const instanceName = generateInstanceName(species, name);
+    const gatewayPort = DEFAULT_GATEWAY_PORT;
+
+    const imageTags: Record<ServiceName, string> = {
+      assistant: "",
+      "credential-executor": "",
+      gateway: "",
+    };
+
+    let repoRoot: string | undefined;
+
+    if (watch) {
+      repoRoot = findRepoRoot();
+      const localTag = `local-${instanceName}`;
+      imageTags.assistant = `vellum-assistant:${localTag}`;
+      imageTags.gateway = `vellum-gateway:${localTag}`;
+      imageTags["credential-executor"] =
+        `vellum-credential-executor:${localTag}`;
+
+      log(`🥚 Hatching Docker assistant: ${instanceName}`);
+      log(`   Species: ${species}`);
+      log(`   Mode: development (watch)`);
+      log(`   Repo: ${repoRoot}`);
+      log(`   Images (local build):`);
+      log(`     assistant:            ${imageTags.assistant}`);
+      log(`     gateway:              ${imageTags.gateway}`);
+      log(`     credential-executor:  ${imageTags["credential-executor"]}`);
+      log("");
+
+      await buildAllImages(repoRoot, imageTags, log);
+      log("✅ Docker images built");
+    } else {
+      const version = cliPkg.version;
+      const versionTag = version ? `v${version}` : "latest";
+      imageTags.assistant = `${DOCKERHUB_IMAGES.assistant}:${versionTag}`;
+      imageTags.gateway = `${DOCKERHUB_IMAGES.gateway}:${versionTag}`;
+      imageTags["credential-executor"] =
+        `${DOCKERHUB_IMAGES["credential-executor"]}:${versionTag}`;
+
+      log(`🥚 Hatching Docker assistant: ${instanceName}`);
+      log(`   Species: ${species}`);
+      log(`   Images:`);
+      log(`     assistant:            ${imageTags.assistant}`);
+      log(`     gateway:              ${imageTags.gateway}`);
+      log(`     credential-executor:  ${imageTags["credential-executor"]}`);
+      log("");
+
+      log("📦 Pulling Docker images...");
+      await exec("docker", ["pull", imageTags.assistant]);
+      await exec("docker", ["pull", imageTags.gateway]);
+      await exec("docker", ["pull", imageTags["credential-executor"]]);
+      log("✅ Docker images pulled");
+    }
+
+    const res = dockerResourceNames(instanceName);
+
+    log("📁 Creating shared network and volumes...");
+    await exec("docker", ["network", "create", res.network]);
+    await exec("docker", ["volume", "create", res.dataVolume]);
+    await exec("docker", ["volume", "create", res.socketVolume]);
+
+    await startContainers({ gatewayPort, imageTags, instanceName, res }, log);
+
+    const runtimeUrl = `http://localhost:${gatewayPort}`;
+    const dockerEntry: AssistantEntry = {
+      assistantId: instanceName,
+      runtimeUrl,
+      cloud: "docker",
+      species,
+      hatchedAt: new Date().toISOString(),
+      volume: res.dataVolume,
+    };
+    saveAssistantEntry(dockerEntry);
+    setActiveAssistant(instanceName);
+
+    const { ready } = await waitForGatewayAndLease({
+      containerName: res.assistantContainer,
+      detached: watch ? false : detached,
+      instanceName,
       logFd,
-      `[docker-hatch] ${new Date().toISOString()} ERROR\n${message}\n`,
-    );
-    closeLogFile(logFd);
-    console.error(message);
-    throw err;
-  }
+      runtimeUrl,
+    });
 
-  const instanceName = name ?? `${species}-${generateRandomSuffix()}`;
-  const dockerfileName = watch ? "Dockerfile.development" : "Dockerfile";
-  const dockerfile = join(dockerfileDir, dockerfileName);
-  const dockerfilePath = join(repoRoot, dockerfile);
+    if (!ready && !(watch && repoRoot)) {
+      throw new Error("Timed out waiting for assistant to become ready");
+    }
 
-  if (!existsSync(dockerfilePath)) {
-    const message = `Error: ${dockerfile} not found at ${dockerfilePath}`;
-    const logFd = openLogFile("hatch.log");
-    writeToLogFile(
-      logFd,
-      `[docker-hatch] ${new Date().toISOString()} ERROR\n${message}\n`,
-    );
+    if (watch && repoRoot) {
+      saveAssistantEntry({ ...dockerEntry, watcherPid: process.pid });
+
+      const stopWatcher = startFileWatcher({
+        gatewayPort,
+        imageTags,
+        instanceName,
+        repoRoot,
+        res,
+      });
+
+      await new Promise<void>((resolve) => {
+        const cleanup = async () => {
+          log("\n🛑 Shutting down...");
+          stopWatcher();
+          await stopContainers(res);
+          saveAssistantEntry({ ...dockerEntry, watcherPid: undefined });
+          log("✅ Docker instance stopped.");
+          resolve();
+        };
+
+        // SIGINT (Ctrl+C): full cleanup including stopping containers.
+        process.on("SIGINT", () => void cleanup());
+
+        // SIGTERM (from `vellum retire`): exit quickly — the caller
+        // handles container teardown, so we only need to close the
+        // file watchers and let the process terminate.
+        process.on("SIGTERM", () => {
+          stopWatcher();
+          saveAssistantEntry({ ...dockerEntry, watcherPid: undefined });
+          resolve();
+        });
+      });
+    }
+  } finally {
     closeLogFile(logFd);
-    console.error(message);
-    process.exit(1);
+    logFd = "ignore";
   }
+}
 
-  console.log(`🥚 Hatching Docker assistant: ${instanceName}`);
-  console.log(`   Species: ${species}`);
-  console.log(`   Dockerfile: ${dockerfile}`);
-  if (watch) {
-    console.log(`   Mode: development (watch)`);
-  }
-  console.log("");
+/**
+ * In detached mode, print instance details and return immediately.
+ * Otherwise, poll the gateway health check until it responds, then
+ * lease a guardian token.
+ */
+async function waitForGatewayAndLease(opts: {
+  containerName: string;
+  detached: boolean;
+  instanceName: string;
+  logFd: number | "ignore";
+  runtimeUrl: string;
+}): Promise<{ ready: boolean }> {
+  const { containerName, detached, instanceName, logFd, runtimeUrl } = opts;
+
+  const log = (msg: string): void => {
+    console.log(msg);
+    writeToLogFile(logFd, `${new Date().toISOString()} ${msg}\n`);
+  };
 
-  const imageTag = `vellum-assistant:${instanceName}`;
-  const logFd = openLogFile("hatch.log");
-  console.log("🔨 Building Docker image...");
-  try {
-    await exec("docker", ["build", "-f", dockerfile, "-t", imageTag, "."], {
-      cwd: repoRoot,
-    });
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    writeToLogFile(
-      logFd,
-      `[docker-build] ${new Date().toISOString()} ERROR\n${message}\n`,
-    );
-    closeLogFile(logFd);
-    throw err;
+  if (detached) {
+    log("\n✅ Docker assistant hatched!\n");
+    log("Instance details:");
+    log(`  Name: ${instanceName}`);
+    log(`  Runtime: ${runtimeUrl}`);
+    log(`  Container: ${containerName}`);
+    log("");
+    log(`Stop with: vellum retire ${instanceName}`);
+    return { ready: true };
   }
-  closeLogFile(logFd);
-  console.log("✅ Docker image built\n");
-
-  const gatewayPort = DEFAULT_GATEWAY_PORT;
-  const runArgs: string[] = [
-    "run",
-    "--init",
-    "--name",
-    instanceName,
-    "-p",
-    `${gatewayPort}:${gatewayPort}`,
-  ];
 
-  // Pass through environment variables the assistant needs
-  for (const envVar of ["ANTHROPIC_API_KEY", "VELLUM_PLATFORM_URL"]) {
-    if (process.env[envVar]) {
-      runArgs.push("-e", `${envVar}=${process.env[envVar]}`);
+  log(`  Container: ${containerName}`);
+  log(`  Runtime: ${runtimeUrl}`);
+  log("");
+  log("Waiting for assistant to become ready...");
+
+  const readyUrl = `${runtimeUrl}/readyz`;
+  const start = Date.now();
+  let ready = false;
+
+  while (Date.now() - start < DOCKER_READY_TIMEOUT_MS) {
+    try {
+      const resp = await fetch(readyUrl, {
+        signal: AbortSignal.timeout(5000),
+      });
+      if (resp.ok) {
+        ready = true;
+        break;
+      }
+      const body = await resp.text();
+      let detail = "";
+      try {
+        const json = JSON.parse(body);
+        const parts = [json.status];
+        if (json.upstream != null) parts.push(`upstream=${json.upstream}`);
+        detail = ` — ${parts.join(", ")}`;
+      } catch {}
+      log(`Readiness check: ${resp.status}${detail} (retrying...)`);
+    } catch {
+      // Connection refused / timeout — not up yet
     }
+    await new Promise((r) => setTimeout(r, 1000));
   }
 
-  // Pass the instance name so the inner hatch uses the same assistant ID
-  // instead of generating a new random one.
-  runArgs.push("-e", `VELLUM_ASSISTANT_NAME=${instanceName}`);
-
-  // Mount source volumes in watch mode for hot reloading
-  if (watch) {
-    runArgs.push(
-      "-v",
-      `${join(repoRoot, "assistant", "src")}:/app/assistant/src`,
-      "-v",
-      `${join(repoRoot, "gateway", "src")}:/app/gateway/src`,
-      "-v",
-      `${join(repoRoot, "cli", "src")}:/app/cli/src`,
-    );
+  if (!ready) {
+    log("");
+    log(`   \u26a0\ufe0f  Timed out waiting for assistant to become ready.`);
+    log(`   The container is still running.`);
+    log(`   Check logs with: docker logs -f ${containerName}`);
+    log("");
+    return { ready: false };
   }
 
-  // Docker containers bind to 0.0.0.0 so localhost always works. Skip
-  // mDNS/LAN discovery — the .local hostname often fails to resolve on the
-  // host machine itself (mDNS is designed for cross-device discovery).
-  const runtimeUrl = `http://localhost:${gatewayPort}`;
-  const dockerEntry: AssistantEntry = {
-    assistantId: instanceName,
-    runtimeUrl,
-    cloud: "docker",
-    species,
-    hatchedAt: new Date().toISOString(),
-  };
-  saveAssistantEntry(dockerEntry);
-  setActiveAssistant(instanceName);
-
-  // The Dockerfiles already define a CMD that runs `vellum hatch --keep-alive`.
-  // Only override CMD when a non-default species is specified, since that
-  // requires an extra argument the Dockerfile doesn't include.
-  const containerCmd: string[] =
-    species !== "vellum"
-      ? [
-          "vellum",
-          "hatch",
-          species,
-          ...(watch ? ["--watch"] : []),
-          "--keep-alive",
-        ]
-      : [];
-
-  // Always start the container detached so it keeps running after the CLI exits.
-  runArgs.push("-d");
-  console.log("🚀 Starting Docker container...");
-  await exec("docker", [...runArgs, imageTag, ...containerCmd], {
-    cwd: repoRoot,
-  });
+  const elapsedSec = ((Date.now() - start) / 1000).toFixed(1);
+  log(`Assistant ready after ${elapsedSec}s`);
 
-  if (detached) {
-    console.log("\n✅ Docker assistant hatched!\n");
-    console.log("Instance details:");
-    console.log(`  Name: ${instanceName}`);
-    console.log(`  Runtime: ${runtimeUrl}`);
-    console.log(`  Container: ${instanceName}`);
-    console.log("");
-    console.log(`Stop with: docker stop ${instanceName}`);
-  } else {
-    console.log(`  Container: ${instanceName}`);
-    console.log(`  Runtime: ${runtimeUrl}`);
-    console.log("");
-
-    // Tail container logs until the inner hatch completes, then exit and
-    // leave the container running in the background.
-    await new Promise<void>((resolve, reject) => {
-      const child = nodeSpawn("docker", ["logs", "-f", instanceName], {
-        stdio: ["ignore", "pipe", "pipe"],
-      });
+  // Lease guardian token. The /readyz check confirms both gateway and
+  // assistant are reachable. Retry with backoff in case there is a brief
+  // window where readiness passes but the guardian endpoint is not yet ready.
+  log(`Guardian token lease: starting for ${instanceName} at ${runtimeUrl}`);
+  const leaseStart = Date.now();
+  const leaseDeadline = start + DOCKER_READY_TIMEOUT_MS;
+  let leaseSuccess = false;
+  let lastLeaseError: string | undefined;
 
-      const handleLine = (line: string): void => {
-        if (line.includes("Local assistant hatched!")) {
-          process.nextTick(async () => {
-            const remoteBearerToken =
-              await fetchRemoteBearerToken(instanceName);
-            if (remoteBearerToken) {
-              dockerEntry.bearerToken = remoteBearerToken;
-              saveAssistantEntry(dockerEntry);
-            }
-
-            console.log("");
-            console.log(`\u2705 Docker container is up and running!`);
-            console.log(`   Name: ${instanceName}`);
-            console.log(`   Runtime: ${runtimeUrl}`);
-            console.log("");
-            child.kill();
-            resolve();
-          });
-        }
-      };
-
-      const stdoutPrefixer = createLinePrefixer(process.stdout, handleLine);
-      const stderrPrefixer = createLinePrefixer(process.stderr, handleLine);
-
-      child.stdout?.on("data", (data: Buffer) => stdoutPrefixer.write(data));
-      child.stderr?.on("data", (data: Buffer) => stderrPrefixer.write(data));
-      child.stdout?.on("end", () => stdoutPrefixer.flush());
-      child.stderr?.on("end", () => stderrPrefixer.flush());
-
-      child.on("close", (code) => {
-        // The log tail may exit if the container stops before the sentinel
-        // is seen, or we killed it after detecting the sentinel.
-        if (
-          code === 0 ||
-          code === null ||
-          code === 130 ||
-          code === 137 ||
-          code === 143
-        ) {
-          resolve();
-        } else {
-          reject(new Error(`Docker container exited with code ${code}`));
-        }
-      });
-      child.on("error", reject);
+  while (Date.now() < leaseDeadline) {
+    try {
+      const tokenData = await leaseGuardianToken(runtimeUrl, instanceName);
+      const leaseElapsed = ((Date.now() - leaseStart) / 1000).toFixed(1);
+      log(
+        `Guardian token lease: success after ${leaseElapsed}s (principalId=${tokenData.guardianPrincipalId}, expiresAt=${tokenData.accessTokenExpiresAt})`,
+      );
+      leaseSuccess = true;
+      break;
+    } catch (err) {
+      lastLeaseError =
+        err instanceof Error ? (err.stack ?? err.message) : String(err);
+      // Log periodically so the user knows we're still trying
+      const elapsed = ((Date.now() - leaseStart) / 1000).toFixed(0);
+      log(
+        `Guardian token lease: attempt failed after ${elapsed}s (${lastLeaseError.split("\n")[0]}), retrying...`,
+      );
+    }
+    await new Promise((r) => setTimeout(r, 2000));
+  }
 
-      process.on("SIGINT", () => {
-        child.kill();
-        resolve();
-      });
-    });
+  if (!leaseSuccess) {
+    log(
+      `\u26a0\ufe0f  Guardian token lease: FAILED after ${((Date.now() - leaseStart) / 1000).toFixed(1)}s — ${lastLeaseError ?? "unknown error"}`,
+    );
   }
+
+  log("");
+  log(`\u2705 Docker containers are up and running!`);
+  log(`   Name: ${instanceName}`);
+  log(`   Runtime: ${runtimeUrl}`);
+  log("");
+  return { ready: true };
 }