@mycadet/cli 0.12.10 → 0.13.0-dev.202610171617.9ea363e

package/src/lib/hatch-local.ts

@@ -1,3 +1,4 @@
+import { persistRootSigningKey } from "./root-signing-key.js";
 import {
   existsSync,
   lstatSync,
@@ -52,7 +53,6 @@ import {
 import { logHatchNextSteps } from "./hatch-next-steps.js";
 import { printInstanceDetails } from "./instance-details.js";
 import { checkProviderApiKey } from "./api-key-check.js";
-import { loopbackSafeFetch } from "./loopback-fetch.js";
 import {
   describeLoginError,
   ensureCloudLoginInteractive,
@@ -156,13 +156,13 @@ export interface HatchLocalParams {
   name?: string | null;
   /** Build from a local source tree and hot-reload on change. */
   watch?: boolean;
-  /** Stay attached after hatch and poll the gateway until it stops. */
-  keepAlive?: boolean;
   /** Hatch-time config values (key → value). */
   configValues?: Record<string, string>;
   /** Extra env vars forwarded into the assistant process. */
   flagEnvVars?: Record<string, string>;
   setupProviderCredentials?: boolean;
+  /** Whole-installation state installed before the first service startup. */
+  initialArchive?: string;
   /**
    * Show the per-service startup logs (assistant, credential executor,
    * gateway) and the extended next steps. Off by default — the happy path
@@ -192,13 +192,7 @@ export interface HatchLocalResult {
 export async function hatchLocal(
   params: HatchLocalParams = {},
 ): Promise<HatchLocalResult> {
-  const {
-    name,
-    watch = false,
-    keepAlive = false,
-    configValues = {},
-    flagEnvVars = {},
-  } = params;
+  const { name, watch = false, configValues = {}, flagEnvVars = {} } = params;
   const reporter = params.reporter ?? consoleLifecycleReporter;
   const verbose = params.verbose ?? false;
   const setupProviderCredentials = params.setupProviderCredentials !== false;
@@ -300,7 +294,20 @@ export async function hatchLocal(
       const defaultWorkspaceConfigPath = writeInitialConfig(hatchConfigValues);
 
       reporter.progress(3, 6, "Starting assistant...");
-      const signingKey = generateLocalSigningKey();
+      const signingKey = params.initialArchive
+        ? await (
+            await import("./root-backup.js")
+          ).initializeRootArchive(
+            {
+              assistantId: instanceName,
+              cloud: "local",
+              runtimeUrl: "",
+              resources,
+            },
+            params.initialArchive,
+          )
+        : generateLocalSigningKey();
+      persistRootSigningKey(resources.instanceDir, signingKey);
       const bootstrapSecret = generateLocalSigningKey();
       let runtimeUrl = `http://127.0.0.1:${resources.gatewayPort}`;
       try {
@@ -454,46 +461,5 @@ export async function hatchLocal(
     guardianAccessToken,
   };
 
-  if (keepAlive) {
-    const healthUrl = `http://127.0.0.1:${resources.gatewayPort}/healthz`;
-    const healthTarget = "Gateway";
-    const POLL_INTERVAL_MS = 5000;
-    const MAX_FAILURES = 3;
-    let consecutiveFailures = 0;
-
-    const shutdown = async (): Promise<void> => {
-      reporter.log("\nShutting down local processes...");
-      await stopLocalProcesses(resources);
-      process.exit(0);
-    };
-
-    process.on("SIGTERM", () => void shutdown());
-    process.on("SIGINT", () => void shutdown());
-
-    // Poll the health endpoint until it stops responding.
-    while (true) {
-      await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
-      try {
-        const res = await loopbackSafeFetch(healthUrl, {
-          signal: AbortSignal.timeout(3000),
-        });
-        if (res.ok) {
-          consecutiveFailures = 0;
-        } else {
-          consecutiveFailures++;
-        }
-      } catch {
-        consecutiveFailures++;
-      }
-      if (consecutiveFailures >= MAX_FAILURES) {
-        reporter.log(
-          `\n⚠️  ${healthTarget} stopped responding — shutting down.`,
-        );
-        await stopLocalProcesses(resources);
-        process.exit(1);
-      }
-    }
-  }
-
   return result;
 }

package/src/lib/hatch-next-steps.ts

@@ -1,8 +1,21 @@
+import { getCurrentEnvironment } from "./environments/resolve.js";
+
+function cliCommand(): string {
+  // Production keeps the bare `cadet` binary; every non-production
+  // environment installs as `cadet-<env>` (the same rule as the config and
+  // state dirs — see environments/paths.ts).
+  const env = getCurrentEnvironment();
+  return env.name === "production" ? "cadet" : `cadet-${env.name}`;
+}
+
 export function logHatchNextSteps(log: (message: string) => void): void {
+  const command = cliCommand();
   log("Next steps:");
-  log("  - Run `cadet client` to interact with your cadet via the terminal");
   log(
-    "  - Run `cadet tunnel` to make your cadet publicly accessible and interact with it via the browser",
+    `  - Run \`${command} client\` to interact with your cadet via the terminal`,
+  );
+  log(
+    `  - Run \`${command} tunnel\` to make your cadet publicly accessible and interact with it via the browser`,
   );
   log("");
 }

package/src/lib/host-teleport.ts

@@ -0,0 +1,506 @@
+import { rm } from "node:fs/promises";
+import { z } from "zod";
+import cliPkg from "../../package.json";
+import type { AssistantEntry } from "./assistant-config.js";
+import {
+  findAssistantByName,
+  removeAssistantEntry,
+  setActiveAssistant,
+} from "./assistant-config.js";
+import { createBackup, pruneOldBackups } from "./backup-ops.js";
+import {
+  dockerResourceNames,
+  hatchDocker,
+  retireDocker,
+  startContainers,
+} from "./docker.js";
+import { hatchLocal } from "./hatch-local.js";
+import { retireLocal } from "./retire-local.js";
+import { validateAssistantName } from "./retire-archive.js";
+import {
+  DockerRootStorage,
+  LocalRootStorage,
+  restoreRootBackup,
+  stageRootArchive,
+  STATE_ROOTS,
+  withRootOperationLock,
+} from "./root-backup.js";
+import { DOCKER_STATEFUL_SET_SPEC } from "./statefulset.js";
+import { execOutput } from "./step-runner.js";
+import { buildReplayState } from "./upgrade-lifecycle.js";
+import { fetchCurrentVersion } from "./http-client.js";
+import { compareVersions } from "./version-compat.js";
+import { sleepAssistant } from "../commands/sleep.js";
+import { generateInstanceName } from "./random-name.js";
+import { GATEWAY_INTERNAL_PORT } from "./environments/paths.js";
+
+type HostStorage = LocalRootStorage | DockerRootStorage;
+type HostEnvironment = "local" | "docker";
+
+export class TeleportRecoveryError extends AggregateError {}
+
+export interface HostTeleportOps {
+  storage(entry: AssistantEntry): HostStorage;
+  sleep(entry: AssistantEntry): Promise<void>;
+  backup(entry: AssistantEntry): Promise<string>;
+  version(entry: AssistantEntry): Promise<string | undefined>;
+  hatch(
+    env: HostEnvironment,
+    name: string | undefined,
+    archive: string,
+  ): Promise<AssistantEntry>;
+  restore(
+    entry: AssistantEntry,
+    archive: string,
+    recovery: string,
+  ): Promise<void>;
+  retire(entry: AssistantEntry): Promise<void>;
+}
+
+function storageFor(entry: AssistantEntry): HostStorage {
+  return entry.cloud === "docker"
+    ? new DockerRootStorage(entry)
+    : new LocalRootStorage(entry);
+}
+
+const defaultOps: HostTeleportOps = {
+  storage: storageFor,
+  sleep: (entry) =>
+    sleepAssistant(entry, { force: false, wait: false, waitMs: null }),
+  backup: async (entry) => {
+    const archive = await createBackup(entry.runtimeUrl, entry.assistantId, {
+      prefix: `${entry.assistantId}-pre-teleport`,
+    });
+    if (!archive) {
+      throw new Error(
+        `Could not capture '${entry.assistantId}'; teleport aborted.`,
+      );
+    }
+    console.log(`Backup saved: ${archive}`);
+    return archive;
+  },
+  version: readHostRuntimeVersion,
+  hatch: async (env, name, initialArchive) => {
+    const instanceName = generateInstanceName(name ?? null);
+    if (findAssistantByName(instanceName)) {
+      throw new Error(`Assistant '${instanceName}' already exists.`);
+    }
+    try {
+      if (env === "docker") {
+        await hatchDocker({
+          name: instanceName,
+          setupProviderCredentials: false,
+          initialArchive,
+        });
+      } else {
+        await hatchLocal({
+          name: instanceName,
+          setupProviderCredentials: false,
+          initialArchive,
+        });
+      }
+      const entry = findAssistantByName(instanceName);
+      if (!entry) {
+        throw new Error("Could not find the newly hatched teleport target.");
+      }
+      return entry;
+    } catch (error) {
+      const entry = findAssistantByName(instanceName);
+      if (entry) {
+        await defaultOps.sleep(entry);
+        await defaultOps.retire(entry);
+      }
+      throw error;
+    }
+  },
+  restore: restoreHostTeleportTarget,
+  retire: async (entry) => {
+    if (entry.cloud === "docker") {
+      await retireDocker(entry.assistantId);
+    } else {
+      await retireLocal(entry.assistantId, entry);
+    }
+    removeAssistantEntry(entry.assistantId);
+  },
+};
+
+export async function readHostRuntimeVersion(
+  entry: AssistantEntry,
+): Promise<string | undefined> {
+  if (entry.cloud !== "docker") {
+    return (
+      (await fetchCurrentVersion(entry.runtimeUrl)) ??
+      entry.resources?.runtimeVersion
+    );
+  }
+  try {
+    const container = dockerResourceNames(entry.assistantId).gatewayContainer;
+    const state = await new DockerRootStorage(entry).inspect(container);
+    const configured = state.Config.Env.find((value) =>
+      value.startsWith("GATEWAY_PORT="),
+    )?.slice("GATEWAY_PORT=".length);
+    const port = configured ? Number(configured) : GATEWAY_INTERNAL_PORT;
+    if (!Number.isSafeInteger(port) || port < 1 || port > 65535) {
+      throw new Error("Invalid Docker gateway health port.");
+    }
+    const result = await execOutput(
+      "docker",
+      [
+        "exec",
+        container,
+        "bun",
+        "--eval",
+        "const r=await fetch(process.argv.at(-1),{signal:AbortSignal.timeout(1500)});if(!r.ok)process.exit(1);console.log(JSON.stringify(await r.json()))",
+        `http://127.0.0.1:${port}/healthz`,
+      ],
+      { timeoutMs: 3000 },
+    );
+    const parsed = z
+      .object({ version: z.string() })
+      .safeParse(JSON.parse(result));
+    if (parsed.success) {
+      return parsed.data.version;
+    }
+  } catch {
+    // Sleeping containers still carry their recorded release tag.
+  }
+  return entry.containerInfo?.gatewayImage.match(
+    /:v?(\d+\.\d+\.\d+(?:-[\w.-]+)?)$/,
+  )?.[1];
+}
+
+function checkVersion(
+  source: string | undefined,
+  target: string | undefined,
+): void {
+  const comparison = source && target ? compareVersions(target, source) : null;
+  if (comparison !== null && comparison < 0) {
+    throw new Error(
+      `Target runtime ${target} is older than source ${source}. Upgrade the target before teleporting.`,
+    );
+  }
+}
+
+async function startAndCheck(storage: HostStorage): Promise<void> {
+  storage.initialRunning =
+    storage instanceof DockerRootStorage
+      ? new Set(Object.values(storage.containers))
+      : new Set(["ces", "assistant", "gateway"]);
+  await storage.resume();
+}
+
+export async function teleportHostInstallation(
+  source: AssistantEntry,
+  targetEnv: HostEnvironment,
+  targetName: string | undefined,
+  options: { dryRun: boolean; keepSource: boolean },
+  ops = defaultOps,
+): Promise<void> {
+  const existing = targetName ? findAssistantByName(targetName) : null;
+  const work = () =>
+    performHostTeleport(source, targetEnv, targetName, existing, options, ops);
+  if (options.dryRun) {
+    return work();
+  }
+  return withRootOperationLock(source, () =>
+    existing ? withRootOperationLock(existing, work) : work(),
+  );
+}
+
+async function performHostTeleport(
+  source: AssistantEntry,
+  targetEnv: HostEnvironment,
+  targetName: string | undefined,
+  existing: AssistantEntry | null,
+  options: { dryRun: boolean; keepSource: boolean },
+  ops: HostTeleportOps,
+): Promise<void> {
+  if (existing && existing.cloud !== targetEnv) {
+    throw new Error(
+      `Assistant '${targetName}' is a ${existing.cloud} assistant, not ${targetEnv}.`,
+    );
+  }
+  if (targetName && !existing) {
+    validateAssistantName(targetName);
+  }
+  if (options.dryRun) {
+    console.log(
+      `Would move ${STATE_ROOTS.join(", ")} from '${source.assistantId}' to ${existing ? `'${existing.assistantId}'` : `a new ${targetEnv} assistant`}.`,
+    );
+    console.log(
+      "Would gracefully sleep services, save root tar safety backups, restore all roots, and verify the destination.",
+    );
+    console.log(
+      options.keepSource
+        ? "Would keep the source stopped."
+        : "Would retire the source only after the destination is ready.",
+    );
+    console.log("Dry run complete. No services stopped or archives created.");
+    return;
+  }
+
+  const sourceVersion = await ops.version(source);
+  checkVersion(
+    sourceVersion,
+    existing ? await ops.version(existing) : cliPkg.version,
+  );
+  const sourceStorage = ops.storage(source);
+  const sourceWasRunning = await sourceStorage.running();
+  let target: AssistantEntry | undefined;
+  let ready = false;
+  try {
+    await ops.sleep(source);
+    if (await sourceStorage.running()) {
+      throw new Error("Source services are still running; teleport aborted.");
+    }
+    // This exact snapshot is both the safety backup and the transferred state.
+    const archive = await ops.backup(source);
+    if (existing) {
+      target = existing;
+      const targetStorage = ops.storage(target);
+      const targetWasRunning = await targetStorage.running();
+      try {
+        await ops.sleep(target);
+        if (await targetStorage.running()) {
+          throw new Error(
+            "Target services are still running; teleport aborted.",
+          );
+        }
+        const recovery = await ops.backup(target);
+        await ops.restore(target, archive, recovery);
+      } catch (error) {
+        if (targetWasRunning && !(error instanceof TeleportRecoveryError)) {
+          await targetStorage.resume();
+        }
+        throw error;
+      }
+    } else {
+      target = await ops.hatch(targetEnv, targetName, archive);
+      checkVersion(sourceVersion, await ops.version(target));
+      await startAndCheck(ops.storage(target));
+    }
+    ready = true;
+    setActiveAssistant(target.assistantId);
+    if (options.keepSource) {
+      console.log(
+        `Source '${source.assistantId}' retained, stopped, via --keep-source.`,
+      );
+    } else {
+      if (await sourceStorage.running()) {
+        throw new Error(
+          "Source was restarted during teleport. Destination is ready; source was retained to preserve newer writes.",
+        );
+      }
+      await ops.retire(source);
+    }
+    pruneOldBackups(source.assistantId, 3, "pre-teleport");
+    if (existing) {
+      pruneOldBackups(existing.assistantId, 3, "pre-teleport");
+    }
+    console.log(
+      `Teleport complete: ${source.assistantId} → ${target.assistantId}`,
+    );
+  } catch (error) {
+    if (!ready) {
+      if (target && !existing) {
+        await ops.sleep(target);
+        await ops.retire(target);
+      }
+      if (sourceWasRunning) {
+        await sourceStorage.resume();
+      }
+    }
+    throw error;
+  }
+}
+
+const DockerConfiguration = z.object({
+  Id: z.string().optional(),
+  Image: z.string().min(1),
+  Config: z.object({ Env: z.array(z.string()), User: z.string() }),
+  HostConfig: z.object({ NetworkMode: z.string() }),
+  Mounts: z.array(
+    z.object({
+      Type: z.string(),
+      Name: z.string().optional(),
+      Destination: z.string(),
+      RW: z.boolean(),
+    }),
+  ),
+});
+
+/** Recreate standard CLI containers so immutable env cannot override imported identity. */
+async function dockerReplacementOptions(
+  entry: AssistantEntry,
+): Promise<Parameters<typeof startContainers>[0]> {
+  const res = dockerResourceNames(entry.assistantId);
+  const states = new Map<string, z.infer<typeof DockerConfiguration>>();
+  for (const container of [
+    res.assistantContainer,
+    res.gatewayContainer,
+    res.cesContainer,
+  ]) {
+    const parsed = z
+      .array(DockerConfiguration)
+      .length(1)
+      .safeParse(
+        JSON.parse(await execOutput("docker", ["inspect", container])),
+      );
+    if (!parsed.success) {
+      throw new Error(`Cannot read Docker configuration for ${container}.`);
+    }
+    states.set(container, parsed.data[0]!);
+  }
+  const containerByService = {
+    assistant: res.assistantContainer,
+    gateway: res.gatewayContainer,
+    "credential-executor": res.cesContainer,
+  };
+  for (const spec of DOCKER_STATEFUL_SET_SPEC.containers) {
+    const state = states.get(containerByService[spec.internalName])!;
+    const expectedNetwork =
+      spec.network === "bridge"
+        ? res.network
+        : `container:${res.assistantContainer}`;
+    const assistantId = states.get(res.assistantContainer)!.Id;
+    if (
+      state.HostConfig.NetworkMode !== expectedNetwork &&
+      !(
+        spec.network === "container" &&
+        assistantId &&
+        state.HostConfig.NetworkMode === `container:${assistantId}`
+      )
+    ) {
+      throw new Error(
+        "Teleport into a customized Docker network requires a newly hatched target.",
+      );
+    }
+    if (state.Mounts.length !== spec.volumeMounts.length) {
+      throw new Error(
+        "Teleport into customized Docker mounts requires a newly hatched target.",
+      );
+    }
+    for (const mount of spec.volumeMounts) {
+      const volume = DOCKER_STATEFUL_SET_SPEC.volumeClaimTemplates.find(
+        (volume) => volume.name === mount.volumeName,
+      )!;
+      const found = state.Mounts.find(
+        (item) => item.Destination === mount.mountPath,
+      );
+      if (
+        !found ||
+        found.Type !== "volume" ||
+        found.Name !== volume.dockerVolume(entry.assistantId) ||
+        found.RW !== !mount.readOnly
+      ) {
+        throw new Error(
+          "Teleport into customized Docker mounts requires a newly hatched target.",
+        );
+      }
+    }
+  }
+  const env = (container: string) =>
+    Object.fromEntries(
+      states.get(container)!.Config.Env.map((value) => {
+        const index = value.indexOf("=");
+        return [value.slice(0, index), value.slice(index + 1)];
+      }),
+    );
+  const replay = buildReplayState(
+    env(res.assistantContainer),
+    env(res.gatewayContainer),
+  );
+  const gatewayPort = Number(new URL(entry.runtimeUrl).port);
+  const assistantPort = entry.containerInfo?.assistantPort;
+  if (!Number.isSafeInteger(gatewayPort) || gatewayPort < 1 || !assistantPort) {
+    throw new Error(
+      "Docker target is missing its published ports. Use a newly hatched target.",
+    );
+  }
+  return {
+    ...replay,
+    gatewayPort,
+    assistantPort,
+    imageTags: {
+      assistant: states.get(res.assistantContainer)!.Image,
+      gateway: states.get(res.gatewayContainer)!.Image,
+      "credential-executor": states.get(res.cesContainer)!.Image,
+    },
+    instanceName: entry.assistantId,
+    res,
+    createOnly: true,
+  };
+}
+
+export async function restoreHostTeleportTarget(
+  entry: AssistantEntry,
+  archive: string,
+  recovery: string,
+): Promise<void> {
+  const stage = await stageRootArchive(archive);
+  await rm(stage, { recursive: true, force: true });
+  const previous =
+    entry.cloud === "docker"
+      ? await dockerReplacementOptions(entry)
+      : undefined;
+  if (await storageFor(entry).running()) {
+    throw new Error("Target services must be stopped before teleport restore.");
+  }
+  const originalResources = entry.resources;
+  const replaceContainers = async (
+    opts: Parameters<typeof startContainers>[0],
+  ) => {
+    const storage = new DockerRootStorage(entry);
+    if (await storage.running()) {
+      throw new Error(
+        "Docker services must be stopped before container replacement.",
+      );
+    }
+    for (const container of [
+      opts.res.assistantContainer,
+      opts.res.gatewayContainer,
+      opts.res.cesContainer,
+    ]) {
+      await execOutput("docker", ["rm", container]);
+    }
+    await startContainers(opts, (message) => console.log(message));
+  };
+  try {
+    if (previous) {
+      await replaceContainers({ ...previous, signingKey: undefined });
+    }
+    await restoreRootBackup(entry, archive, { resume: false });
+    await startAndCheck(storageFor(entry));
+  } catch (error) {
+    try {
+      await defaultOps.sleep(entry);
+      if (previous) {
+        // Failed creation can leave an incomplete service group.
+        for (const container of [
+          previous.res.assistantContainer,
+          previous.res.gatewayContainer,
+          previous.res.cesContainer,
+        ]) {
+          try {
+            await execOutput("docker", ["rm", container]);
+          } catch (removeError) {
+            if (
+              !String(removeError).toLowerCase().includes("no such container")
+            ) {
+              throw removeError;
+            }
+          }
+        }
+        await startContainers(previous, (message) => console.log(message));
+      }
+      entry.resources = originalResources;
+      await restoreRootBackup(entry, recovery, { resume: false });
+    } catch (recoveryError) {
+      throw new TeleportRecoveryError(
+        [error, recoveryError],
+        `Teleport target recovery failed; keep services stopped. Restore ${recovery}.`,
+      );
+    }
+    throw error;
+  }
+}

package/src/lib/http-client.ts

@@ -1,3 +1,4 @@
+import { z } from "zod";
 import { loopbackSafeFetch } from "./loopback-fetch.js";
 
 export type HttpProbeEndpoint = "healthz" | "readyz";
@@ -141,3 +142,27 @@ export async function waitForDaemonMigrationsReady(
     await new Promise((r) => setTimeout(r, 250));
   }
 }
+
+const GatewayHealthSchema = z.object({ version: z.string().min(1).optional() });
+
+/**
+ * Best-effort fetch of the running service group version from the gateway
+ * `/healthz` endpoint.  Returns `undefined` when the endpoint is
+ * unreachable or does not include a version field.
+ */
+export async function fetchCurrentVersion(
+  runtimeUrl: string,
+): Promise<string | undefined> {
+  try {
+    const resp = await loopbackSafeFetch(`${runtimeUrl}/healthz`, {
+      signal: AbortSignal.timeout(5000),
+    });
+    if (resp.ok) {
+      const parsed = GatewayHealthSchema.safeParse(await resp.json());
+      return parsed.success ? parsed.data.version : undefined;
+    }
+  } catch {
+    // Best-effort
+  }
+  return undefined;
+}