@mycadet/cli 0.12.7 → 0.12.9-dev.202610131452.9acab64

package/src/lib/docker.ts

@@ -27,6 +27,7 @@ import {
   ASSISTANT_INTERNAL_PORT,
   getDefaultPorts,
 } from "./environments/paths.js";
+import { green, yellow } from "./color.js";
 import { getCurrentEnvironment } from "./environments/resolve.js";
 import { leaseGuardianToken } from "./guardian-token";
 import { logHatchNextSteps } from "./hatch-next-steps.js";
@@ -37,11 +38,11 @@ import {
   isLocalBuildRef,
   loadImageViaHost,
 } from "./host-image-loader.js";
-import { resolveOwnImageRefs } from "./platform-releases.js";
+import { resolveOwnImageRefs } from "./cloud-releases.js";
 import { isDevVersion } from "./version-compat.js";
 import {
+  collectProviderApiKey,
   configureHatchProviderApiKey,
-  formatProviderName,
   promptProviderChoice,
   resolveHatchProvider,
   shouldPromptForHatchProvider,
@@ -340,14 +341,96 @@ export function resolveDockerProviderCredentialSetupAction(options: {
   provider: string | null | undefined;
   guardianAccessToken?: string;
   detached: boolean;
+  /**
+   * Key — or an explicit skip ("") — collected from the user right after
+   * they picked the provider. When present (even empty), the user already
+   * answered at the prompt, so detached hatches configure (or report the
+   * missing key) instead of deferring to a second `cadet setup` run.
+   */
+  precollectedKey?: string;
 }): DockerProviderCredentialSetupAction {
   if (options.provider === undefined) return "skip";
   if (options.provider === null) return options.detached ? "skip" : "configure";
-  if (options.detached) return "defer";
+  if (options.detached) {
+    if (options.precollectedKey !== undefined) {
+      return options.guardianAccessToken ? "configure" : "missing-token";
+    }
+    return "defer";
+  }
   if (!options.guardianAccessToken) return "missing-token";
   return "configure";
 }
 
+/** Time to wait for the Docker daemon to answer `docker info` after a Colima start. */
+const COLIMA_SETTLE_TIMEOUT_MS = 15_000;
+
+/** Poll `docker info` until the daemon answers or the timeout elapses. */
+async function waitForDockerDaemon(
+  execFn: typeof exec,
+  timeoutMs: number,
+): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    try {
+      await execFn("docker", ["info"]);
+      return true;
+    } catch {
+      if (Date.now() >= deadline) return false;
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+    }
+  }
+}
+
+/**
+ * Start Colima and wait for the Docker daemon to actually answer.
+ *
+ * `colima start` exiting 0 only means Colima's own state file says the VM is
+ * up — that state can be stale (zombie VM after a host sleep or crash), in
+ * which case start is a no-op ("already running, ignoring") while the daemon
+ * inside stays dead. So success is decided by re-probing `docker info`, and a
+ * failed probe triggers the manual-fix cycle: force-stop and start fresh.
+ * Never `colima delete` — the VM disk holds every container image and volume.
+ * `startArgs` carries extra flags (e.g. `--memory`) for a first start; the
+ * wake path starts an existing instance bare.
+ */
+async function startColimaAndAwaitDaemon(
+  execFn: typeof exec,
+  log: (msg: string) => void,
+  opts: { startArgs?: string[]; settleTimeoutMs?: number } = {},
+): Promise<void> {
+  const startArgs = opts.startArgs ?? [];
+  const settleTimeoutMs = opts.settleTimeoutMs ?? COLIMA_SETTLE_TIMEOUT_MS;
+
+  try {
+    await execFn("colima", ["start", ...startArgs]);
+  } catch (err) {
+    // exec captures the child's output and only surfaces it in the error —
+    // print it so a failed start is never silent.
+    log(`⚠️  colima start failed: ${err instanceof Error ? err.message : err}`);
+  }
+
+  if (await waitForDockerDaemon(execFn, settleTimeoutMs)) return;
+
+  log(
+    "🔄 Colima state looks stale — restarting the VM (colima stop --force, then start)...",
+  );
+  await execFn("colima", ["stop", "--force"]).catch(() => {});
+  try {
+    await execFn("colima", ["start", ...startArgs]);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(
+      `Failed to restart Colima. Please run 'colima stop --force && colima start' manually.\n${message}`,
+    );
+  }
+  if (!(await waitForDockerDaemon(execFn, settleTimeoutMs))) {
+    throw new Error(
+      "Docker daemon is still not reachable after restarting Colima. " +
+        "Check `colima status` and `docker info`, then try again.",
+    );
+  }
+}
+
 /**
  * Checks whether the `docker` CLI and daemon are available on the system.
  * Installs Colima and Docker via direct binary download if missing (no sudo
@@ -418,41 +501,9 @@ async function ensureDockerInstalled(): Promise<void> {
     }
 
     console.log("🚀 Docker daemon not running. Starting Colima...");
-    try {
-      await exec("colima", [
-        "start",
-        "--memory",
-        String(COLIMA_DEFAULT_MEMORY_GIB),
-      ]);
-    } catch {
-      // Colima may fail if a previous VM instance is in a corrupt state.
-      // Attempt to delete the stale instance and retry once.
-      console.log(
-        "⚠️  Colima start failed — attempting to reset stale VM state...",
-      );
-      try {
-        await exec("colima", ["stop", "--force"]).catch(() => {});
-        await exec("colima", ["delete", "--force"]);
-      } catch {
-        // If delete also fails, fall through to the retry which will
-        // produce a clear error message.
-      }
-
-      try {
-        console.log("🔄 Retrying colima start...");
-        await exec("colima", [
-          "start",
-          "--memory",
-          String(COLIMA_DEFAULT_MEMORY_GIB),
-        ]);
-      } catch (retryErr) {
-        const message =
-          retryErr instanceof Error ? retryErr.message : String(retryErr);
-        throw new Error(
-          `Failed to start Colima after resetting stale VM state. Please run 'colima start' manually.\n${message}`,
-        );
-      }
-    }
+    await startColimaAndAwaitDaemon(exec, console.log, {
+      startArgs: ["--memory", String(COLIMA_DEFAULT_MEMORY_GIB)],
+    });
   }
 }
 
@@ -486,6 +537,56 @@ export async function removeContainer(containerName: string): Promise<void> {
   }
 }
 
+/**
+ * Error signatures Docker Hub returns when the host has STALE Docker Hub
+ * credentials: an expired personal access token saved by a past
+ * `docker login` gets attached to the pull, and Hub rejects the request
+ * instead of falling back to anonymous access for a public image.
+ */
+export function isExpiredDockerHubCredentialError(err: unknown): boolean {
+  const message = err instanceof Error ? err.message : String(err);
+  return /personal access token is expired|authentication required|unauthorized/i.test(
+    message,
+  );
+}
+
+/**
+ * Pull an image, retrying once without stale credentials when Docker Hub
+ * rejects the pull with an expired-token/auth error — public pulls need no
+ * login, so `docker logout` (clears the stored Docker Hub credentials the
+ * docker CLI attaches to pulls) costs nothing. Other failures propagate raw;
+ * a still-failing auth retry is rethrown with actionable copy.
+ */
+export async function pullImage(
+  ref: string,
+  log: (msg: string) => void = console.log,
+  execFn: typeof exec = exec,
+): Promise<void> {
+  try {
+    await execFn("docker", ["pull", ref]);
+    return;
+  } catch (err) {
+    if (!isExpiredDockerHubCredentialError(err)) throw err;
+    log(
+      `⚠️  Docker Hub rejected the pull of ${ref} with an expired-credential ` +
+        `error — clearing stale Docker Hub login and retrying (public images ` +
+        `need no credentials)...`,
+    );
+    await execFn("docker", ["logout"]).catch(() => {});
+  }
+  try {
+    await execFn("docker", ["pull", ref]);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(
+      `Failed to pull ${ref}.\n${message}\n` +
+        `If the Docker daemon is not running, start it, e.g. 'colima start' on macOS. ` +
+        `If the error mentions authentication, run 'docker logout' and check your ` +
+        `Docker Hub credentials — public image pulls need none.`,
+    );
+  }
+}
+
 export async function retireDocker(name: string): Promise<void> {
   console.log(`\u{1F5D1}\ufe0f  Stopping Docker containers for '${name}'...\n`);
 
@@ -785,6 +886,113 @@ export async function wakeContainers(
   }
 }
 
+/** The `docker start` failure for a container that was never created. */
+export function isNoSuchContainerError(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return msg.includes("No such container");
+}
+
+/**
+ * Recreate a docker instance's service group from its lockfile entry — the
+ * wake path for containers that don't exist yet. A migrated install has its
+ * data in the volumes but no containers: nothing ever ran `docker run` for
+ * it (v0.0.10 run feedback: `wake --all` died on "No such container:
+ * <name>-assistant").
+ *
+ * Network and volume creates are idempotent; the three service containers
+ * come up from the entry's recorded images (the migrator points those at
+ * the installed cadet's release). The guardian bootstrap secret rides
+ * through from the entry; the signing key and CES token are deliberately
+ * NOT invented — with no env value the gateway and assistant load the
+ * persisted trust material from their security volumes (a migrated install
+ * keeps its own), matching the gateway's resolution order (env → file →
+ * generate). Empty volumes just get fresh state on first boot.
+ */
+export async function recreateDockerContainers(entry: AssistantEntry): Promise<void> {
+  const info = entry.containerInfo;
+  if (!info?.assistantImage || !info.gatewayImage || !info.cesImage) {
+    throw new Error(
+      `no container images recorded for '${entry.assistantId}' — re-hatch or run \`cadet upgrade\` to set them`,
+    );
+  }
+  const res = dockerResourceNames(entry.assistantId);
+  const log = (msg: string): void => console.log(msg);
+
+  // Clean slate for the create: an existing container (even a stopped one)
+  // would collide with `docker run --name` (the upgrade path does the same).
+  await stopContainers(res);
+
+  try {
+    await exec("docker", ["network", "inspect", res.network]);
+  } catch {
+    await exec("docker", ["network", "create", res.network]);
+  }
+  for (const vol of [
+    res.socketVolume,
+    res.assistantIpcVolume,
+    res.gatewayIpcVolume,
+    res.workspaceVolume,
+    res.gatewaySecurityVolume,
+    res.cesSecurityVolume,
+  ]) {
+    try {
+      await exec("docker", ["volume", "inspect", vol]);
+    } catch {
+      await exec("docker", ["volume", "create", vol]);
+    }
+  }
+
+  // The recorded images may not be on the machine yet (a migrated install
+  // gets its refs written before anything pulled them) — a pull of an
+  // already-present image is a cheap no-op.
+  for (const image of [info.assistantImage, info.gatewayImage, info.cesImage]) {
+    log(`📦 Pulling ${image}...`);
+    await pullImage(image, log);
+  }
+
+  const gatewayPort = parseGatewayPort(entry.runtimeUrl) ??
+    getDefaultPorts(getCurrentEnvironment()).gateway;
+  const assistantPort = info.assistantPort ?? ASSISTANT_INTERNAL_PORT;
+  const device = getOrCreateHostDeviceId();
+  await startContainers(
+    {
+      bootstrapSecret: entry.guardianBootstrapSecret,
+      extraAssistantEnv: { CADET_DEVICE_ID: device },
+      extraGatewayEnv: { CADET_DEVICE_ID: device },
+      gatewayPort,
+      assistantPort,
+      imageTags: {
+        assistant: info.assistantImage,
+        gateway: info.gatewayImage,
+        "credential-executor": info.cesImage,
+      },
+      instanceName: entry.assistantId,
+      res,
+    },
+    log,
+  );
+
+  // Persist what the recreation decided (the gateway port in the runtime
+  // url, the assistant port when it defaulted) so plain `docker start`
+  // wakes work and clients reach the right endpoints.
+  saveAssistantEntry({
+    ...entry,
+    runtimeUrl: `http://localhost:${gatewayPort}`,
+    containerInfo: { ...info, assistantPort },
+  });
+}
+
+/** The port of the CLI's own runtime url form (`http://localhost:7830`). */
+function parseGatewayPort(runtimeUrl: string): number | null {
+  try {
+    const port = new URL(runtimeUrl).port;
+    const parsed = parseInt(port, 10);
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Ensure a Docker daemon is reachable before waking an instance's containers
  * (macOS only — on Linux the daemon runs natively).
@@ -805,6 +1013,7 @@ export async function wakeContainers(
  */
 export async function ensureDockerDaemonRunning(
   execFn: typeof exec = exec,
+  opts: { settleTimeoutMs?: number } = {},
 ): Promise<void> {
   ensureLocalBinOnPath();
 
@@ -817,12 +1026,10 @@ export async function ensureDockerDaemonRunning(
     // No daemon reachable — fall through to the Colima fallback below.
   }
 
-  try {
-    await execFn("colima", ["status"]);
-  } catch {
-    console.log("🚀 Docker daemon not running. Starting Colima...");
-    await execFn("colima", ["start"]);
-  }
+  // Deliberately no `colima status` probe: it trusts Colima's own state file,
+  // which can be stale (zombie VM) — daemon reachability is the oracle.
+  console.log("🚀 Docker daemon not running. Starting Colima...");
+  await startColimaAndAwaitDaemon(execFn, console.log, opts);
 }
 
 /**
@@ -1127,6 +1334,7 @@ export async function hatchDocker(params: HatchDockerParams): Promise<void> {
   let provider = setupProviderCredentials
     ? resolveHatchProvider(configValues)
     : undefined;
+  let precollectedApiKey: string | undefined;
 
   if (
     shouldPromptForHatchProvider({
@@ -1139,6 +1347,9 @@ export async function hatchDocker(params: HatchDockerParams): Promise<void> {
     const picked = await promptProviderChoice();
     if (picked) {
       provider = picked;
+      // Collect the key immediately, while the choice is fresh — the
+      // startup logs are quiet now, so nothing separates the two prompts.
+      precollectedApiKey = await collectProviderApiKey(picked);
     }
   }
 
@@ -1320,7 +1531,7 @@ export async function hatchDocker(params: HatchDockerParams): Promise<void> {
           const MAX_PULL_RETRIES = 3;
           for (let attempt = 1; attempt <= MAX_PULL_RETRIES; attempt++) {
             try {
-              await exec("docker", ["pull", ref]);
+              await pullImage(ref, log);
               break;
             } catch (err) {
               if (attempt === MAX_PULL_RETRIES) throw err;
@@ -1333,7 +1544,7 @@ export async function hatchDocker(params: HatchDockerParams): Promise<void> {
           }
         }
       }
-      log("✅ Docker images acquired");
+      log(green("✅ Docker images acquired", process.stdout));
     }
 
     const res = dockerResourceNames(instanceName);
@@ -1434,9 +1645,9 @@ export async function hatchDocker(params: HatchDockerParams): Promise<void> {
       : ownSecret;
 
     emitProgress(4, 6, "Starting containers...");
-    if (flagEnvVars.CADET_DISABLE_PLATFORM) {
-      extraAssistantEnv.CADET_DISABLE_PLATFORM =
-        flagEnvVars.CADET_DISABLE_PLATFORM;
+    if (flagEnvVars.CADET_DISABLE_CLOUD) {
+      extraAssistantEnv.CADET_DISABLE_CLOUD =
+        flagEnvVars.CADET_DISABLE_CLOUD;
     }
     const hostDeviceId = getOrCreateHostDeviceId();
     extraAssistantEnv.CADET_DEVICE_ID = hostDeviceId;
@@ -1478,8 +1689,6 @@ export async function hatchDocker(params: HatchDockerParams): Promise<void> {
     );
     const containersUpAt = Date.now();
 
-    const imageDigests = await captureImageRefs(res);
-
     const runtimeUrl = `http://localhost:${gatewayPort}`;
     const dockerEntry: AssistantEntry = {
       assistantId: instanceName,
@@ -1491,10 +1700,6 @@ export async function hatchDocker(params: HatchDockerParams): Promise<void> {
         assistantImage: imageTags.assistant,
         gatewayImage: imageTags.gateway,
         cesImage: imageTags["credential-executor"],
-        assistantDigest: imageDigests?.assistant,
-        gatewayDigest: imageDigests?.gateway,
-        cesDigest: imageDigests?.["credential-executor"],
-        networkName: res.network,
         assistantPort,
       },
     };
@@ -1530,6 +1735,7 @@ export async function hatchDocker(params: HatchDockerParams): Promise<void> {
         provider,
         guardianAccessToken,
         detached: waitDetached,
+        precollectedKey: precollectedApiKey,
       });
 
       if (providerSetupAction === "defer" && provider !== null) {
@@ -1539,23 +1745,22 @@ export async function hatchDocker(params: HatchDockerParams): Promise<void> {
         );
       } else if (providerSetupAction === "missing-token" && provider !== null) {
         log(
-          `⚠️  Provider credential setup skipped because the guardian token was not leased.\n` +
-            `   The assistant is still hatched. Run \`cadet setup --provider ${provider}\` after fixing the connection.`,
+          yellow(
+            `⚠️  Provider credential setup skipped because the guardian token was not leased.\n` +
+              `   The assistant is still hatched. Run \`cadet setup --provider ${provider}\` after fixing the connection.`,
+            process.stdout,
+          ),
         );
       } else if (
         providerSetupAction === "configure" &&
         provider !== undefined
       ) {
-        log(
-          provider === null
-            ? "Checking provider credentials..."
-            : `Checking ${formatProviderName(provider)} credentials...`,
-        );
         await configureHatchProviderApiKey({
           gatewayUrl: runtimeUrl,
           provider,
           bearerToken: guardianAccessToken,
           env: process.env,
+          precollectedKey: precollectedApiKey,
           log,
         });
       }
@@ -1648,25 +1853,12 @@ async function waitForGatewayAndLease(opts: {
   };
 
   if (detached) {
-    log("\n✅ Docker assistant hatched!\n");
-    log("Instance details:");
-    printInstanceDetails(
-      {
-        name: instanceName,
-        runtime: runtimeUrl,
-        container: containerName,
-      },
-      log,
-    );
-    log("");
-    log(`Stop with: cadet retire ${instanceName}`);
-
-    // Lease a guardian token even in detached mode so that `cadet ps`,
-    // `cadet exec`, and other CLI commands can authenticate to the
-    // gateway. Skip the /readyz readiness poll (the caller asked to detach
-    // and not block) but retry the lease itself since the gateway may need
-    // a moment to accept connections after the container starts.
-    const leaseStart = Date.now();
+    // Lease a guardian token BEFORE declaring the assistant hatched so the
+    // banner means `cadet ps`, `cadet exec`, and other CLI commands can
+    // already authenticate. Skip the /readyz readiness poll (the caller
+    // asked to detach and not block) but retry the lease silently since
+    // the gateway may need a moment to accept connections after the
+    // container starts; only a final failure surfaces to the user.
     const leaseDeadline = containersUpAt + DOCKER_READY_TIMEOUT_MS;
     let guardianAccessToken: string | undefined;
     while (Date.now() < leaseDeadline) {
@@ -1677,27 +1869,33 @@ async function waitForGatewayAndLease(opts: {
           bootstrapSecret,
         );
         guardianAccessToken = tokenData.accessToken;
-        const leaseElapsed = ((Date.now() - leaseStart) / 1000).toFixed(1);
-        log(
-          `Guardian token lease: success after ${leaseElapsed}s (principalId=${tokenData.guardianPrincipalId})`,
-        );
         break;
-      } catch (err) {
-        const elapsed = ((Date.now() - leaseStart) / 1000).toFixed(0);
-        const msg = err instanceof Error ? err.message : String(err);
-        log(
-          `Guardian token lease: attempt failed after ${elapsed}s (${msg.split("\n")[0]}), retrying...`,
-        );
+      } catch {
         await new Promise((r) => setTimeout(r, 2000));
       }
     }
     if (!guardianAccessToken) {
       log(
-        `⚠️  Guardian token lease failed after ${DOCKER_READY_TIMEOUT_MS / 1000}s.\n` +
-          `   The assistant is running but CLI commands (cadet ps, cadet exec) will not authenticate.\n` +
-          `   Re-hatch or run \`cadet setup\` to recover.`,
+        yellow(
+          `⚠️  Guardian token lease failed after ${DOCKER_READY_TIMEOUT_MS / 1000}s.\n` +
+            `   The assistant is running but CLI commands (cadet ps, cadet exec) will not authenticate.\n` +
+            `   Re-hatch or run \`cadet setup\` to recover.`,
+          process.stdout,
+        ),
       );
     }
+
+    log(`\n${green("✅ Docker assistant hatched!", process.stdout)}\n`);
+    log("Instance details:");
+    printInstanceDetails(
+      {
+        name: instanceName,
+        runtime: runtimeUrl,
+        container: containerName,
+      },
+      log,
+    );
+    log("");
     return { ready: true, guardianAccessToken };
   }
 
@@ -1778,7 +1976,6 @@ async function waitForGatewayAndLease(opts: {
   // Lease guardian token. The /readyz check confirms both gateway and
   // assistant are reachable. Retry with backoff in case there is a brief
   // window where readiness passes but the guardian endpoint is not yet ready.
-  log(`Guardian token lease: starting for ${instanceName} at ${runtimeUrl}`);
   const leaseStart = Date.now();
   const leaseDeadline = start + DOCKER_READY_TIMEOUT_MS;
   let leaseSuccess = false;
@@ -1792,38 +1989,30 @@ async function waitForGatewayAndLease(opts: {
         instanceName,
         bootstrapSecret,
       );
-      const leaseElapsed = ((Date.now() - leaseStart) / 1000).toFixed(1);
-      log(
-        `Guardian token lease: success after ${leaseElapsed}s (principalId=${tokenData.guardianPrincipalId}, expiresAt=${tokenData.accessTokenExpiresAt})`,
-      );
       guardianAccessToken = tokenData.accessToken;
       leaseSuccess = true;
       break;
     } catch (err) {
       lastLeaseError =
         err instanceof Error ? (err.stack ?? err.message) : String(err);
-      // Log periodically so the user knows we're still trying
-      const elapsed = ((Date.now() - leaseStart) / 1000).toFixed(0);
-      log(
-        `Guardian token lease: attempt failed after ${elapsed}s (${
-          lastLeaseError.split("\n")[0]
-        }), retrying...`,
-      );
     }
     await new Promise((r) => setTimeout(r, 2000));
   }
 
   if (!leaseSuccess) {
     log(
-      `\u26a0\ufe0f  Guardian token lease: FAILED after ${(
-        (Date.now() - leaseStart) /
-        1000
-      ).toFixed(1)}s — ${lastLeaseError ?? "unknown error"}`,
+      yellow(
+        `\u26a0\ufe0f  Guardian token lease: FAILED after ${(
+          (Date.now() - leaseStart) /
+          1000
+        ).toFixed(1)}s — ${lastLeaseError ?? "unknown error"}`,
+        process.stdout,
+      ),
     );
   }
 
   log("");
-  log(`\u2705 Docker containers are up and running!`);
+  log(green(`\u2705 Docker containers are up and running!`, process.stdout));
   log(`   Name: ${instanceName}`);
   log(`   Runtime: ${runtimeUrl}`);
   log("");