@mycadet/cli 0.12.10 → 0.13.0

package/src/lib/host-teleport.ts

@@ -0,0 +1,506 @@
+import { rm } from "node:fs/promises";
+import { z } from "zod";
+import cliPkg from "../../package.json";
+import type { AssistantEntry } from "./assistant-config.js";
+import {
+  findAssistantByName,
+  removeAssistantEntry,
+  setActiveAssistant,
+} from "./assistant-config.js";
+import { createBackup, pruneOldBackups } from "./backup-ops.js";
+import {
+  dockerResourceNames,
+  hatchDocker,
+  retireDocker,
+  startContainers,
+} from "./docker.js";
+import { hatchLocal } from "./hatch-local.js";
+import { retireLocal } from "./retire-local.js";
+import { validateAssistantName } from "./retire-archive.js";
+import {
+  DockerRootStorage,
+  LocalRootStorage,
+  restoreRootBackup,
+  stageRootArchive,
+  STATE_ROOTS,
+  withRootOperationLock,
+} from "./root-backup.js";
+import { DOCKER_STATEFUL_SET_SPEC } from "./statefulset.js";
+import { execOutput } from "./step-runner.js";
+import { buildReplayState } from "./upgrade-lifecycle.js";
+import { fetchCurrentVersion } from "./http-client.js";
+import { compareVersions } from "./version-compat.js";
+import { sleepAssistant } from "../commands/sleep.js";
+import { generateInstanceName } from "./random-name.js";
+import { GATEWAY_INTERNAL_PORT } from "./environments/paths.js";
+
+type HostStorage = LocalRootStorage | DockerRootStorage;
+type HostEnvironment = "local" | "docker";
+
+export class TeleportRecoveryError extends AggregateError {}
+
+export interface HostTeleportOps {
+  storage(entry: AssistantEntry): HostStorage;
+  sleep(entry: AssistantEntry): Promise<void>;
+  backup(entry: AssistantEntry): Promise<string>;
+  version(entry: AssistantEntry): Promise<string | undefined>;
+  hatch(
+    env: HostEnvironment,
+    name: string | undefined,
+    archive: string,
+  ): Promise<AssistantEntry>;
+  restore(
+    entry: AssistantEntry,
+    archive: string,
+    recovery: string,
+  ): Promise<void>;
+  retire(entry: AssistantEntry): Promise<void>;
+}
+
+function storageFor(entry: AssistantEntry): HostStorage {
+  return entry.cloud === "docker"
+    ? new DockerRootStorage(entry)
+    : new LocalRootStorage(entry);
+}
+
+const defaultOps: HostTeleportOps = {
+  storage: storageFor,
+  sleep: (entry) =>
+    sleepAssistant(entry, { force: false, wait: false, waitMs: null }),
+  backup: async (entry) => {
+    const archive = await createBackup(entry.runtimeUrl, entry.assistantId, {
+      prefix: `${entry.assistantId}-pre-teleport`,
+    });
+    if (!archive) {
+      throw new Error(
+        `Could not capture '${entry.assistantId}'; teleport aborted.`,
+      );
+    }
+    console.log(`Backup saved: ${archive}`);
+    return archive;
+  },
+  version: readHostRuntimeVersion,
+  hatch: async (env, name, initialArchive) => {
+    const instanceName = generateInstanceName(name ?? null);
+    if (findAssistantByName(instanceName)) {
+      throw new Error(`Assistant '${instanceName}' already exists.`);
+    }
+    try {
+      if (env === "docker") {
+        await hatchDocker({
+          name: instanceName,
+          setupProviderCredentials: false,
+          initialArchive,
+        });
+      } else {
+        await hatchLocal({
+          name: instanceName,
+          setupProviderCredentials: false,
+          initialArchive,
+        });
+      }
+      const entry = findAssistantByName(instanceName);
+      if (!entry) {
+        throw new Error("Could not find the newly hatched teleport target.");
+      }
+      return entry;
+    } catch (error) {
+      const entry = findAssistantByName(instanceName);
+      if (entry) {
+        await defaultOps.sleep(entry);
+        await defaultOps.retire(entry);
+      }
+      throw error;
+    }
+  },
+  restore: restoreHostTeleportTarget,
+  retire: async (entry) => {
+    if (entry.cloud === "docker") {
+      await retireDocker(entry.assistantId);
+    } else {
+      await retireLocal(entry.assistantId, entry);
+    }
+    removeAssistantEntry(entry.assistantId);
+  },
+};
+
+export async function readHostRuntimeVersion(
+  entry: AssistantEntry,
+): Promise<string | undefined> {
+  if (entry.cloud !== "docker") {
+    return (
+      (await fetchCurrentVersion(entry.runtimeUrl)) ??
+      entry.resources?.runtimeVersion
+    );
+  }
+  try {
+    const container = dockerResourceNames(entry.assistantId).gatewayContainer;
+    const state = await new DockerRootStorage(entry).inspect(container);
+    const configured = state.Config.Env.find((value) =>
+      value.startsWith("GATEWAY_PORT="),
+    )?.slice("GATEWAY_PORT=".length);
+    const port = configured ? Number(configured) : GATEWAY_INTERNAL_PORT;
+    if (!Number.isSafeInteger(port) || port < 1 || port > 65535) {
+      throw new Error("Invalid Docker gateway health port.");
+    }
+    const result = await execOutput(
+      "docker",
+      [
+        "exec",
+        container,
+        "bun",
+        "--eval",
+        "const r=await fetch(process.argv.at(-1),{signal:AbortSignal.timeout(1500)});if(!r.ok)process.exit(1);console.log(JSON.stringify(await r.json()))",
+        `http://127.0.0.1:${port}/healthz`,
+      ],
+      { timeoutMs: 3000 },
+    );
+    const parsed = z
+      .object({ version: z.string() })
+      .safeParse(JSON.parse(result));
+    if (parsed.success) {
+      return parsed.data.version;
+    }
+  } catch {
+    // Sleeping containers still carry their recorded release tag.
+  }
+  return entry.containerInfo?.gatewayImage.match(
+    /:v?(\d+\.\d+\.\d+(?:-[\w.-]+)?)$/,
+  )?.[1];
+}
+
+function checkVersion(
+  source: string | undefined,
+  target: string | undefined,
+): void {
+  const comparison = source && target ? compareVersions(target, source) : null;
+  if (comparison !== null && comparison < 0) {
+    throw new Error(
+      `Target runtime ${target} is older than source ${source}. Upgrade the target before teleporting.`,
+    );
+  }
+}
+
+async function startAndCheck(storage: HostStorage): Promise<void> {
+  storage.initialRunning =
+    storage instanceof DockerRootStorage
+      ? new Set(Object.values(storage.containers))
+      : new Set(["ces", "assistant", "gateway"]);
+  await storage.resume();
+}
+
+export async function teleportHostInstallation(
+  source: AssistantEntry,
+  targetEnv: HostEnvironment,
+  targetName: string | undefined,
+  options: { dryRun: boolean; keepSource: boolean },
+  ops = defaultOps,
+): Promise<void> {
+  const existing = targetName ? findAssistantByName(targetName) : null;
+  const work = () =>
+    performHostTeleport(source, targetEnv, targetName, existing, options, ops);
+  if (options.dryRun) {
+    return work();
+  }
+  return withRootOperationLock(source, () =>
+    existing ? withRootOperationLock(existing, work) : work(),
+  );
+}
+
+async function performHostTeleport(
+  source: AssistantEntry,
+  targetEnv: HostEnvironment,
+  targetName: string | undefined,
+  existing: AssistantEntry | null,
+  options: { dryRun: boolean; keepSource: boolean },
+  ops: HostTeleportOps,
+): Promise<void> {
+  if (existing && existing.cloud !== targetEnv) {
+    throw new Error(
+      `Assistant '${targetName}' is a ${existing.cloud} assistant, not ${targetEnv}.`,
+    );
+  }
+  if (targetName && !existing) {
+    validateAssistantName(targetName);
+  }
+  if (options.dryRun) {
+    console.log(
+      `Would move ${STATE_ROOTS.join(", ")} from '${source.assistantId}' to ${existing ? `'${existing.assistantId}'` : `a new ${targetEnv} assistant`}.`,
+    );
+    console.log(
+      "Would gracefully sleep services, save root tar safety backups, restore all roots, and verify the destination.",
+    );
+    console.log(
+      options.keepSource
+        ? "Would keep the source stopped."
+        : "Would retire the source only after the destination is ready.",
+    );
+    console.log("Dry run complete. No services stopped or archives created.");
+    return;
+  }
+
+  const sourceVersion = await ops.version(source);
+  checkVersion(
+    sourceVersion,
+    existing ? await ops.version(existing) : cliPkg.version,
+  );
+  const sourceStorage = ops.storage(source);
+  const sourceWasRunning = await sourceStorage.running();
+  let target: AssistantEntry | undefined;
+  let ready = false;
+  try {
+    await ops.sleep(source);
+    if (await sourceStorage.running()) {
+      throw new Error("Source services are still running; teleport aborted.");
+    }
+    // This exact snapshot is both the safety backup and the transferred state.
+    const archive = await ops.backup(source);
+    if (existing) {
+      target = existing;
+      const targetStorage = ops.storage(target);
+      const targetWasRunning = await targetStorage.running();
+      try {
+        await ops.sleep(target);
+        if (await targetStorage.running()) {
+          throw new Error(
+            "Target services are still running; teleport aborted.",
+          );
+        }
+        const recovery = await ops.backup(target);
+        await ops.restore(target, archive, recovery);
+      } catch (error) {
+        if (targetWasRunning && !(error instanceof TeleportRecoveryError)) {
+          await targetStorage.resume();
+        }
+        throw error;
+      }
+    } else {
+      target = await ops.hatch(targetEnv, targetName, archive);
+      checkVersion(sourceVersion, await ops.version(target));
+      await startAndCheck(ops.storage(target));
+    }
+    ready = true;
+    setActiveAssistant(target.assistantId);
+    if (options.keepSource) {
+      console.log(
+        `Source '${source.assistantId}' retained, stopped, via --keep-source.`,
+      );
+    } else {
+      if (await sourceStorage.running()) {
+        throw new Error(
+          "Source was restarted during teleport. Destination is ready; source was retained to preserve newer writes.",
+        );
+      }
+      await ops.retire(source);
+    }
+    pruneOldBackups(source.assistantId, 3, "pre-teleport");
+    if (existing) {
+      pruneOldBackups(existing.assistantId, 3, "pre-teleport");
+    }
+    console.log(
+      `Teleport complete: ${source.assistantId} → ${target.assistantId}`,
+    );
+  } catch (error) {
+    if (!ready) {
+      if (target && !existing) {
+        await ops.sleep(target);
+        await ops.retire(target);
+      }
+      if (sourceWasRunning) {
+        await sourceStorage.resume();
+      }
+    }
+    throw error;
+  }
+}
+
+const DockerConfiguration = z.object({
+  Id: z.string().optional(),
+  Image: z.string().min(1),
+  Config: z.object({ Env: z.array(z.string()), User: z.string() }),
+  HostConfig: z.object({ NetworkMode: z.string() }),
+  Mounts: z.array(
+    z.object({
+      Type: z.string(),
+      Name: z.string().optional(),
+      Destination: z.string(),
+      RW: z.boolean(),
+    }),
+  ),
+});
+
+/** Recreate standard CLI containers so immutable env cannot override imported identity. */
+async function dockerReplacementOptions(
+  entry: AssistantEntry,
+): Promise<Parameters<typeof startContainers>[0]> {
+  const res = dockerResourceNames(entry.assistantId);
+  const states = new Map<string, z.infer<typeof DockerConfiguration>>();
+  for (const container of [
+    res.assistantContainer,
+    res.gatewayContainer,
+    res.cesContainer,
+  ]) {
+    const parsed = z
+      .array(DockerConfiguration)
+      .length(1)
+      .safeParse(
+        JSON.parse(await execOutput("docker", ["inspect", container])),
+      );
+    if (!parsed.success) {
+      throw new Error(`Cannot read Docker configuration for ${container}.`);
+    }
+    states.set(container, parsed.data[0]!);
+  }
+  const containerByService = {
+    assistant: res.assistantContainer,
+    gateway: res.gatewayContainer,
+    "credential-executor": res.cesContainer,
+  };
+  for (const spec of DOCKER_STATEFUL_SET_SPEC.containers) {
+    const state = states.get(containerByService[spec.internalName])!;
+    const expectedNetwork =
+      spec.network === "bridge"
+        ? res.network
+        : `container:${res.assistantContainer}`;
+    const assistantId = states.get(res.assistantContainer)!.Id;
+    if (
+      state.HostConfig.NetworkMode !== expectedNetwork &&
+      !(
+        spec.network === "container" &&
+        assistantId &&
+        state.HostConfig.NetworkMode === `container:${assistantId}`
+      )
+    ) {
+      throw new Error(
+        "Teleport into a customized Docker network requires a newly hatched target.",
+      );
+    }
+    if (state.Mounts.length !== spec.volumeMounts.length) {
+      throw new Error(
+        "Teleport into customized Docker mounts requires a newly hatched target.",
+      );
+    }
+    for (const mount of spec.volumeMounts) {
+      const volume = DOCKER_STATEFUL_SET_SPEC.volumeClaimTemplates.find(
+        (volume) => volume.name === mount.volumeName,
+      )!;
+      const found = state.Mounts.find(
+        (item) => item.Destination === mount.mountPath,
+      );
+      if (
+        !found ||
+        found.Type !== "volume" ||
+        found.Name !== volume.dockerVolume(entry.assistantId) ||
+        found.RW !== !mount.readOnly
+      ) {
+        throw new Error(
+          "Teleport into customized Docker mounts requires a newly hatched target.",
+        );
+      }
+    }
+  }
+  const env = (container: string) =>
+    Object.fromEntries(
+      states.get(container)!.Config.Env.map((value) => {
+        const index = value.indexOf("=");
+        return [value.slice(0, index), value.slice(index + 1)];
+      }),
+    );
+  const replay = buildReplayState(
+    env(res.assistantContainer),
+    env(res.gatewayContainer),
+  );
+  const gatewayPort = Number(new URL(entry.runtimeUrl).port);
+  const assistantPort = entry.containerInfo?.assistantPort;
+  if (!Number.isSafeInteger(gatewayPort) || gatewayPort < 1 || !assistantPort) {
+    throw new Error(
+      "Docker target is missing its published ports. Use a newly hatched target.",
+    );
+  }
+  return {
+    ...replay,
+    gatewayPort,
+    assistantPort,
+    imageTags: {
+      assistant: states.get(res.assistantContainer)!.Image,
+      gateway: states.get(res.gatewayContainer)!.Image,
+      "credential-executor": states.get(res.cesContainer)!.Image,
+    },
+    instanceName: entry.assistantId,
+    res,
+    createOnly: true,
+  };
+}
+
+export async function restoreHostTeleportTarget(
+  entry: AssistantEntry,
+  archive: string,
+  recovery: string,
+): Promise<void> {
+  const stage = await stageRootArchive(archive);
+  await rm(stage, { recursive: true, force: true });
+  const previous =
+    entry.cloud === "docker"
+      ? await dockerReplacementOptions(entry)
+      : undefined;
+  if (await storageFor(entry).running()) {
+    throw new Error("Target services must be stopped before teleport restore.");
+  }
+  const originalResources = entry.resources;
+  const replaceContainers = async (
+    opts: Parameters<typeof startContainers>[0],
+  ) => {
+    const storage = new DockerRootStorage(entry);
+    if (await storage.running()) {
+      throw new Error(
+        "Docker services must be stopped before container replacement.",
+      );
+    }
+    for (const container of [
+      opts.res.assistantContainer,
+      opts.res.gatewayContainer,
+      opts.res.cesContainer,
+    ]) {
+      await execOutput("docker", ["rm", container]);
+    }
+    await startContainers(opts, (message) => console.log(message));
+  };
+  try {
+    if (previous) {
+      await replaceContainers({ ...previous, signingKey: undefined });
+    }
+    await restoreRootBackup(entry, archive, { resume: false });
+    await startAndCheck(storageFor(entry));
+  } catch (error) {
+    try {
+      await defaultOps.sleep(entry);
+      if (previous) {
+        // Failed creation can leave an incomplete service group.
+        for (const container of [
+          previous.res.assistantContainer,
+          previous.res.gatewayContainer,
+          previous.res.cesContainer,
+        ]) {
+          try {
+            await execOutput("docker", ["rm", container]);
+          } catch (removeError) {
+            if (
+              !String(removeError).toLowerCase().includes("no such container")
+            ) {
+              throw removeError;
+            }
+          }
+        }
+        await startContainers(previous, (message) => console.log(message));
+      }
+      entry.resources = originalResources;
+      await restoreRootBackup(entry, recovery, { resume: false });
+    } catch (recoveryError) {
+      throw new TeleportRecoveryError(
+        [error, recoveryError],
+        `Teleport target recovery failed; keep services stopped. Restore ${recovery}.`,
+      );
+    }
+    throw error;
+  }
+}

package/src/lib/http-client.ts

@@ -1,3 +1,4 @@
+import { z } from "zod";
 import { loopbackSafeFetch } from "./loopback-fetch.js";
 
 export type HttpProbeEndpoint = "healthz" | "readyz";
@@ -141,3 +142,27 @@ export async function waitForDaemonMigrationsReady(
     await new Promise((r) => setTimeout(r, 250));
   }
 }
+
+const GatewayHealthSchema = z.object({ version: z.string().min(1).optional() });
+
+/**
+ * Best-effort fetch of the running service group version from the gateway
+ * `/healthz` endpoint.  Returns `undefined` when the endpoint is
+ * unreachable or does not include a version field.
+ */
+export async function fetchCurrentVersion(
+  runtimeUrl: string,
+): Promise<string | undefined> {
+  try {
+    const resp = await loopbackSafeFetch(`${runtimeUrl}/healthz`, {
+      signal: AbortSignal.timeout(5000),
+    });
+    if (resp.ok) {
+      const parsed = GatewayHealthSchema.safeParse(await resp.json());
+      return parsed.success ? parsed.data.version : undefined;
+    }
+  } catch {
+    // Best-effort
+  }
+  return undefined;
+}

package/src/lib/live-voice/tunnel.test.ts

@@ -23,7 +23,7 @@ describe("tunnelHostForCloudUrl", () => {
       "tunnel.mycadet.ai",
     );
     expect(tunnelHostForCloudUrl("https://dev-cloud.mycadet.ai")).toBe(
-      "tunnel-dev.mycadet.ai",
+      "dev-tunnel.mycadet.ai",
     );
   });
 

package/src/lib/live-voice/tunnel.ts

@@ -28,7 +28,7 @@
  * mint again rather than reuse what resolution returned.
  */
 
-import { tunnelHostForPlatformHost } from "@mycadet/service-contracts/ingress";
+import { tunnelHostForCloudHost } from "@mycadet/service-contracts/ingress";
 
 import { authHeaders } from "../cloud-client.js";
 
@@ -55,7 +55,7 @@ export class TunnelWsTokenError extends Error {
  * reached: its tunnel is a loopback port that no naming convention predicts.
  * Otherwise the deployment convention maps the cloud host onto its sibling
  * (`cloud.mycadet.ai` to `tunnel.mycadet.ai`, `dev-cloud.mycadet.ai` to
- * `tunnel-dev.mycadet.ai`). A host outside that convention, including localhost,
+ * `dev-tunnel.mycadet.ai`). A host outside that convention, including localhost,
  * yields the production default, because guessing a tunnel for it would be
  * worse than dialing the one host that certainly exists.
  */
@@ -66,7 +66,7 @@ export function tunnelHostForCloudUrl(cloudUrl: string): string {
   }
   try {
     return (
-      tunnelHostForPlatformHost(new URL(cloudUrl).host) ??
+      tunnelHostForCloudHost(new URL(cloudUrl).host) ??
       DEFAULT_TUNNEL_HOST
     );
   } catch {

package/src/lib/local.ts

@@ -76,10 +76,11 @@ import {
   pathListDelimiter,
   resolveProcessState,
   stopProcessByPidFile,
+  stopProcessGracefully,
 } from "./process.js";
 import { stopTunnelProcess } from "./tunnel.js";
 import { stripVersionPrefix } from "./version-compat.js";
-import { openLogFile, pipeToLogFile } from "./xdg-log.js";
+import { openLogFile, openRuntimeLogFile, pipeToLogFile } from "./xdg-log.js";
 
 const _require = createRequire(import.meta.url);
 
@@ -839,7 +840,7 @@ async function startDaemonFromSource(
         windowsHide: true,
       })
     : (() => {
-        const daemonLogFd = openLogFile("hatch.log");
+        const daemonLogFd = openRuntimeLogFile(resources?.instanceDir ?? spawnStateRoot(), "assistant");
         const c = spawn(bunPath, ["run", resolve(daemonMainPath)], {
           cwd: spawnStateRoot(),
           detached: true,
@@ -912,7 +913,7 @@ async function startDaemonWatchFromSource(
   // detect the in-progress spawn and wait instead of racing.
   writeFileSync(pidFile, "starting", "utf-8");
 
-  const daemonLogFd = openLogFile("hatch.log");
+  const daemonLogFd = openRuntimeLogFile(resources?.instanceDir ?? spawnStateRoot(), "assistant");
   const child = spawn(
     resolveBunExecutable(),
     ["--watch", "run", resolve(mainPath)],
@@ -1699,7 +1700,7 @@ export async function startLocalDaemon(
             windowsHide: true,
           })
         : (() => {
-            const daemonLogFd = openLogFile("hatch.log");
+            const daemonLogFd = openRuntimeLogFile(resources?.instanceDir ?? spawnStateRoot(), "assistant");
             const c = spawn(daemonBinary, [], {
               cwd: spawnStateRoot(),
               detached: true,
@@ -1905,7 +1906,7 @@ export async function startGateway(
     // Use the compiled gateway binary when available (desktop app or compiled
     // CLI invoked from the terminal). In watch mode, skip the bundled binary
     // and use source (bun --watch only works with source files).
-    const gatewayLogFd = openLogFile("hatch.log");
+    const gatewayLogFd = openRuntimeLogFile(resources?.instanceDir ?? spawnStateRoot(), "gateway");
     gateway = spawn(gatewayBinary, [], {
       cwd: spawnStateRoot(),
       detached: true,
@@ -1921,7 +1922,7 @@ export async function startGateway(
     const bunArgs = watch
       ? ["--watch", "run", gatewayEntry, "--cadet-gateway"]
       : ["run", gatewayEntry, "--cadet-gateway"];
-    const gwLogFd = openLogFile("hatch.log");
+    const gwLogFd = openRuntimeLogFile(resources?.instanceDir ?? spawnStateRoot(), "gateway");
     gateway = spawn(resolveBunExecutable(), bunArgs, {
       cwd: spawnStateRoot(),
       detached: true,
@@ -1981,15 +1982,16 @@ export async function stopLocalProcesses(
     "daemon",
     undefined,
     DAEMON_STOP_TIMEOUT_MS,
+    stopProcessGracefully,
   );
 
   const gatewayPidFile = join(cadetDir, "gateway.pid");
-  await stopProcessByPidFile(gatewayPidFile, "gateway", undefined, 7000);
+  await stopProcessByPidFile(gatewayPidFile, "gateway", undefined, 7000, stopProcessGracefully);
 
   // Stop the CES sibling if one was launched. No-op when the
   // PID file is absent.
   const cesPidFile = join(cadetDir, "ces.pid");
-  await stopProcessByPidFile(cesPidFile, "credential-executor");
+  await stopProcessByPidFile(cesPidFile, "credential-executor", undefined, DAEMON_STOP_TIMEOUT_MS, stopProcessGracefully);
 
   // Stop the tunnel agent this CLI spawned (root lockfile `tunnel` entry),
   // verifying the pid still belongs to the provider's binary before signaling