@mycadet/cli 0.12.11 → 0.13.0

package/src/lib/root-signing-key.ts

@@ -0,0 +1,31 @@
+import { randomUUID } from "node:crypto";
+import {
+  lstatSync,
+  mkdirSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+
+/** Persist gateway identity in the service root without following an existing file link. */
+export function persistRootSigningKey(root: string, key: string): void {
+  if (!/^[a-f0-9]{64}$/i.test(key)) {
+    throw new Error("Invalid gateway signing key.");
+  }
+  const directory = join(root, "gateway-security");
+  mkdirSync(directory, { recursive: true, mode: 0o700 });
+  if (!lstatSync(directory).isDirectory()) {
+    throw new Error("Gateway security root must be a directory.");
+  }
+  const temporary = join(directory, `.signing-key-${randomUUID()}`);
+  try {
+    writeFileSync(temporary, Buffer.from(key, "hex"), {
+      mode: 0o600,
+      flag: "wx",
+    });
+    renameSync(temporary, join(directory, "actor-token-signing-key"));
+  } finally {
+    rmSync(temporary, { force: true });
+  }
+}

package/src/lib/runtime-url.ts

@@ -4,35 +4,6 @@ import { hostname } from "node:os";
 import { getLocalLanIPv4 } from "./local";
 import type { AssistantEntry } from "./assistant-config.js";
 
-/**
- * Resolve the URL for a runtime migration endpoint, taking the assistant's
- * topology into account.
- *
- * - For local/docker assistants, `runtimeUrl` is the loopback gateway and
- *   the runtime serves `/v1/migrations/<subpath>` directly. The CLI hits
- *   that path with guardian-token bearer auth.
- * - For cloud-managed (cloud="cadet") assistants, `runtimeUrl` is the
- *   platform host (e.g. `https://cloud.mycadet.ai`). The cloud's
- *   `MigrationViewSet` does NOT expose `export-to-gcs` or arbitrary runtime
- *   migration paths under `/v1/migrations/...`. The wildcard runtime proxy
- *   at `/v1/assistants/<id>/<path:rest>` is what forwards arbitrary runtime
- *   paths to the cloud runtime — vembda's unified proxy bootstraps the
- *   guardian token internally for the runtime call. From the CLI side it's
- *   user-session auth.
- *
- * The `subpath` is appended to the migrations namespace verbatim
- * (e.g. `"export-to-gcs"`, `"import-from-gcs"`, `\`jobs/${jobId}\``).
- */
-export function resolveRuntimeMigrationUrl(
-  entry: Pick<AssistantEntry, "cloud" | "runtimeUrl" | "assistantId">,
-  subpath: string,
-): string {
-  if (isCloudHome(entry.cloud)) {
-    return `${entry.runtimeUrl}/v1/assistants/${entry.assistantId}/migrations/${subpath}`;
-  }
-  return `${entry.runtimeUrl}/v1/migrations/${subpath}`;
-}
-
 /**
  * Resolve the URL for a generic runtime endpoint under `/v1/<subpath>`,
  * taking the assistant's topology into account.

package/src/lib/teleport-backup.ts

@@ -1,17 +1,4 @@
-/**
- * Platform (cloud="cadet") half of the pre-teleport safety backup.
- *
- * A platform source is snapshotted through Django's user-facing backup
- * endpoint, which cuts a PVC VolumeSnapshot via vembda. The POST returns as
- * soon as the snapshot object exists; it becomes restorable asynchronously,
- * so {@link createPlatformBackup} polls the listing until the new snapshot
- * reports `ready_to_use` before returning. A snapshot that never becomes
- * ready within the timeout is a backup failure, not a success.
- *
- * Local and docker sources are backed up host-side by `backup-ops` instead,
- * so the restore point survives the source's retirement. Policy shared with
- * the web teleport lives in `@mycadet/local-mode/teleport-backup-policy`.
- */
+/** Cloud API safety backups use session auth and wait for a restorable root tar. */
 
 import {
   MANAGED_BACKUP_READY_POLL_INTERVAL_MS,
@@ -93,7 +80,7 @@ async function fetchPlatformBackups(
 }
 
 /**
- * `created_at` of every restorable PVC snapshot of a cloud assistant:
+ * `created_at` of every restorable root archive of a cloud assistant:
  * `GET /v1/assistants/<id>/backups/`.
  */
 export async function listPlatformBackups(
@@ -109,7 +96,7 @@ export interface CreatePlatformBackupOptions {
 }
 
 /**
- * Take a PVC snapshot of a cloud assistant and wait until it is
+ * Capture a root archive of a cloud assistant and wait until it is
  * restorable: `POST /v1/assistants/<id>/backups/`, then re-list until the
  * returned `snapshot_name` reports `ready_to_use`. Throws if the POST fails
  * or the snapshot is not ready within `timeoutMs`.

package/src/lib/upgrade-lifecycle.ts

@@ -20,18 +20,16 @@ import {
   pullImage,
   startContainers,
   stopContainers,
+  sleepContainers,
+  wakeContainers,
 } from "./docker.js";
 import { getStateDir } from "./environments/paths.js";
 import { getCurrentEnvironment } from "./environments/resolve.js";
 import { loadGuardianToken } from "./guardian-token.js";
 import { classifyReadyzResponse } from "./http-client.js";
-import { fetchReleases, resolveImageRefs } from "./cloud-releases.js";
+import { resolveImageRefs } from "./cloud-releases.js";
 import { z } from "zod";
 
-const GatewayHealthSchema = z.object({
-  version: z.string().optional(),
-});
-
 const MigrationGateBodySchema = z.object({
   status: z.string().optional(),
   reason: z.string().optional(),
@@ -42,7 +40,7 @@ import {
   type ServiceName,
 } from "./statefulset.js";
 import { execOutput } from "./step-runner.js";
-import { compareVersions, stripVersionPrefix } from "./version-compat.js";
+import { compareVersions } from "./version-compat.js";
 import { loopbackSafeFetch } from "./loopback-fetch.js";
 
 // ---------------------------------------------------------------------------
@@ -231,7 +229,7 @@ export function buildReplayEnv(
 interface ReplayState {
   bootstrapSecret: string | undefined;
   cesServiceToken: string;
-  signingKey: string;
+  signingKey?: string;
   extraAssistantEnv: Record<string, string>;
   extraGatewayEnv: Record<string, string>;
 }
@@ -239,8 +237,9 @@ interface ReplayState {
 /**
  * Derive the secrets and replay env for replacement containers from
  * already-captured assistant/gateway envs. GUARDIAN_BOOTSTRAP_SECRET is only
- * set on the gateway; CES_SERVICE_TOKEN and ACTOR_TOKEN_SIGNING_KEY fall back
- * to fresh values for instances that predate them. CADET_DEVICE_ID is
+ * set on the gateway; CES_SERVICE_TOKEN can be generated for older instances.
+ * An absent signing-key override stays absent so replacements use the persisted
+ * gateway root rather than rotate its identity. CADET_DEVICE_ID is
  * backfilled on the gateway from the host, and the assistant inherits the
  * gateway's value (captured values win) so the pair always matches.
  */
@@ -259,7 +258,8 @@ export function buildReplayState(
     cesServiceToken:
       capturedEnv["CES_SERVICE_TOKEN"] || randomBytes(32).toString("hex"),
     signingKey:
-      capturedEnv["ACTOR_TOKEN_SIGNING_KEY"] || randomBytes(32).toString("hex"),
+      gatewayEnv["ACTOR_TOKEN_SIGNING_KEY"] ||
+      capturedEnv["ACTOR_TOKEN_SIGNING_KEY"],
     extraAssistantEnv,
     extraGatewayEnv,
   };
@@ -287,28 +287,6 @@ export async function captureReplayState(
   return buildReplayState(capturedEnv, gatewayEnv);
 }
 
-/**
- * Best-effort fetch of the running service group version from the gateway
- * `/healthz` endpoint.  Returns `undefined` when the endpoint is
- * unreachable or does not include a version field.
- */
-export async function fetchCurrentVersion(
-  runtimeUrl: string,
-): Promise<string | undefined> {
-  try {
-    const resp = await loopbackSafeFetch(`${runtimeUrl}/healthz`, {
-      signal: AbortSignal.timeout(5000),
-    });
-    if (resp.ok) {
-      const parsed = GatewayHealthSchema.safeParse(await resp.json());
-      return parsed.success ? parsed.data.version : undefined;
-    }
-  } catch {
-    // Best-effort
-  }
-  return undefined;
-}
-
 /**
  * Best-effort fetch of the assistant's configured public ingress URL from the
  * gateway `integrations/ingress/config` endpoint.  Returns `undefined` when
@@ -344,41 +322,6 @@ export async function fetchAssistantIngressUrl(
   return undefined;
 }
 
-/**
- * Determine the version that was running before the current one.
- *
- * Checks (in order):
- *  1. `entry.previousVersion` (saved by the upgrade flow from health).
- *  2. The releases list from the cloud API — finds the version
- *     immediately before `currentVersion`.
- *
- * Returns `undefined` when neither source yields a result.
- */
-export async function fetchPreviousVersion(
-  currentVersion: string | undefined,
-  previousVersionFromLockfile: string | undefined,
-): Promise<string | undefined> {
-  // 1. Lockfile-cached value (written during upgrade from health endpoint)
-  if (previousVersionFromLockfile) return previousVersionFromLockfile;
-
-  // 2. Derive from releases list
-  if (!currentVersion) return undefined;
-  const releases = await fetchReleases();
-  if (!releases) return undefined;
-
-  const normalizedCurrent = stripVersionPrefix(currentVersion);
-
-  // Releases are ordered newest-first; find the entry right after the
-  // current version (i.e. the one that was running before the upgrade).
-  const idx = releases.findIndex(
-    (r) => stripVersionPrefix(r.version ?? "") === normalizedCurrent,
-  );
-  if (idx >= 0 && idx + 1 < releases.length) {
-    return releases[idx + 1].version;
-  }
-  return undefined;
-}
-
 /**
  * Poll the gateway `/readyz` endpoint until the assistant reports migrations
  * ready or the timeout elapses. Returns whether the assistant became ready.
@@ -577,12 +520,8 @@ export async function rollbackMigrations(
 }
 
 /**
- * Recovery step for a reverted stack that reports terminally failed
- * migrations: import the backup (the failed-state migrations/import
- * exemption admits it, and the import replaces the DB wholesale), then
- * restart the containers — the failed readiness latch only clears on
- * restart — and re-check readiness. Shared by the upgrade failure branch
- * and performDockerRollback's auto-rollback.
+ * Restore the host root archive for a reverted stack with failed migrations,
+ * restart its containers, and check readiness before resuming the installation.
  */
 export async function attemptFailedStateRestore(opts: {
   runtimeUrl: string;
@@ -601,7 +540,7 @@ export async function attemptFailedStateRestore(opts: {
     opts.runtimeUrl,
     opts.assistantId,
     opts.backupPath,
-    { kind: "docker", assistantId: opts.assistantId },
+    { resume: false },
   );
   if (!restored) {
     return { restored: false, ready: false };
@@ -609,6 +548,7 @@ export async function attemptFailedStateRestore(opts: {
   console.log(
     "   Backup imported — restarting containers to complete recovery...",
   );
+  await sleepContainers(opts.res);
   await stopContainers(opts.res);
   await startContainers(opts.containerOptions, (msg) => console.log(msg));
   return { restored: true, ready: await waitForReady(opts.runtimeUrl) };
@@ -619,28 +559,29 @@ export async function attemptFailedStateRestore(opts: {
 // ---------------------------------------------------------------------------
 
 export interface PerformDockerRollbackOptions {
+  /** Root archive must be installed before the target runtime starts. */
+  backupPath: string;
   /** Specific version to roll back to. */
   targetVersion?: string;
 }
 
 /**
- * Perform a Docker rollback to a target version. Reusable by both `rollback.ts`
- * (targeted version rollback) and `restore.ts` (version + data restore).
+ * Switch Docker runtime versions as part of a versioned backup restore.
  *
  * This function handles the full lifecycle:
  * - Version validation (target must be older than current)
  * - Image resolution and pulling
- * - Migration ceiling lookup and pre-swap rollback
+ * - Post-swap migration rollback to the target runtime's registry ceiling
  * - Container stop/start with target images
  * - Readiness check
- * - Lockfile update with rollback state
+ * - Lockfile update with the running images
  * - Auto-rollback on failure
  */
 export async function performDockerRollback(
   entry: AssistantEntry,
   options: PerformDockerRollbackOptions,
 ): Promise<void> {
-  const { targetVersion } = options;
+  const { targetVersion, backupPath } = options;
 
   if (!targetVersion) {
     throw new Error("targetVersion is required for performDockerRollback");
@@ -708,21 +649,6 @@ export async function performDockerRollback(
     }
   }
 
-  // Persist rollback state to lockfile BEFORE any destructive changes
-  if (entry.containerInfo) {
-    const rollbackEntry: AssistantEntry = {
-      ...entry,
-      previousContainerInfo: { ...entry.containerInfo },
-      previousVersion: currentVersion,
-      previousDbMigrationVersion: preMigrationState.dbVersion,
-      previousWorkspaceMigrationId: preMigrationState.lastWorkspaceMigrationId,
-    };
-    saveAssistantEntry(rollbackEntry);
-    if (currentVersion) {
-      console.log(`   Saved rollback state: ${currentVersion}\n`);
-    }
-  }
-
   // Record rollback start in workspace git history
   await commitWorkspaceViaGateway(
     entry.runtimeUrl,
@@ -747,15 +673,17 @@ export async function performDockerRollback(
     entry.runtimeUrl,
     entry.assistantId,
     {
-      prefix: `${entry.assistantId}-pre-upgrade`,
+      prefix: `${entry.assistantId}-pre-rollback`,
       description: `Pre-rollback snapshot before ${currentVersion ?? "unknown"} → ${targetVersion}`,
     },
   );
   if (preRollbackBackupPath) {
     console.log(`   Backup saved: ${preRollbackBackupPath}\n`);
-    pruneOldBackups(entry.assistantId, 3);
+    pruneOldBackups(entry.assistantId, 3, "pre-rollback");
   } else {
-    console.warn("⚠️  Pre-rollback backup failed, continuing with rollback\n");
+    throw new Error(
+      "Pre-rollback root backup failed; rollback aborted before replacing services.",
+    );
   }
 
   const {
@@ -831,46 +759,47 @@ export async function performDockerRollback(
 
   // Stop containers, migrate security files, start with target images
   console.log("🛑 Stopping existing containers...");
+  await sleepContainers(res);
   await stopContainers(res);
   console.log("✅ Containers stopped\n");
 
-  console.log("🚀 Starting containers with target version...");
-  await startContainers(
-    {
-      signingKey,
-      bootstrapSecret,
-      cesServiceToken,
-      extraAssistantEnv,
-      extraGatewayEnv,
-      gatewayPort,
-      assistantPort,
-      imageTags: targetImageTags,
-      instanceName,
-      res,
-    },
-    (msg) => console.log(msg),
-  );
-  console.log("✅ Containers started\n");
-
-  // Wait for readiness
-  console.log("Waiting for assistant to become ready...");
-  const ready = await waitForReady(entry.runtimeUrl);
+  let ready = false;
+  try {
+    console.log("Creating stopped containers with the target version...");
+    await startContainers(
+      {
+        signingKey,
+        bootstrapSecret,
+        cesServiceToken,
+        extraAssistantEnv,
+        extraGatewayEnv,
+        gatewayPort,
+        assistantPort,
+        imageTags: targetImageTags,
+        instanceName,
+        res,
+        createOnly: true,
+      },
+      (msg) => console.log(msg),
+    );
+    if (
+      !(await restoreBackup(entry.runtimeUrl, entry.assistantId, backupPath, {
+        resume: false,
+      }))
+    )
+      throw new Error("Target backup roots could not be restored.");
+    await wakeContainers(res);
+    console.log("Waiting for restored assistant to become ready...");
+    ready = await waitForReady(entry.runtimeUrl);
+  } catch (error) {
+    console.error(
+      `Versioned restore failed: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
 
   if (ready) {
     // Success path
 
-    // Post-swap migration rollback: ask the now-running old daemon to roll
-    // back any migrations above its own registry ceiling.
-    await rollbackMigrations(
-      entry.runtimeUrl,
-      entry.assistantId,
-      undefined,
-      undefined,
-      true,
-    );
-
-    // Capture new digests from the rolled-back containers
-    // Swap current/previous state to enable "rollback the rollback"
     const updatedEntry: AssistantEntry = {
       ...entry,
       containerInfo: {
@@ -879,10 +808,6 @@ export async function performDockerRollback(
         cesImage: targetImageTags["credential-executor"],
         assistantPort,
       },
-      previousContainerInfo: entry.containerInfo,
-      previousDbMigrationVersion: preMigrationState.dbVersion,
-      previousWorkspaceMigrationId: preMigrationState.lastWorkspaceMigrationId,
-      preUpgradeBackupPath: undefined,
     };
     saveAssistantEntry(updatedEntry);
 
@@ -950,6 +875,7 @@ export async function performDockerRollback(
           );
         }
 
+        await sleepContainers(res);
         await stopContainers(res);
 
         await startContainers(
@@ -1013,13 +939,13 @@ export async function performDockerRollback(
               entry.runtimeUrl,
               entry.assistantId,
               preRollbackBackupPath,
-              { kind: "docker", assistantId: entry.assistantId },
+              {},
             );
             if (restored) {
               console.log("   ✅ Data restored successfully\n");
             } else {
-              console.warn(
-                "   ⚠️  Data restore failed (auto-rollback continues without data restoration)\n",
+              throw new Error(
+                `Root restore failed; rollback recovery is incomplete. Backup: ${preRollbackBackupPath}`,
               );
             }
           }
@@ -1038,10 +964,6 @@ export async function performDockerRollback(
                 currentImageRefs["credential-executor"],
               assistantPort,
             },
-            previousContainerInfo: undefined,
-            previousDbMigrationVersion: undefined,
-            previousWorkspaceMigrationId: undefined,
-            preUpgradeBackupPath: undefined,
           };
           saveAssistantEntry(revertedEntry);
 

package/src/lib/web-dist.ts

@@ -1,4 +1,7 @@
-import { existsSync } from "node:fs";
+import { stripVersionPrefix } from "./version-compat.js";
+import { z } from "zod";
+import { isValidReleaseVersion } from "@mycadet/local-mode";
+import { existsSync, readFileSync } from "node:fs";
 import { createRequire } from "node:module";
 import { dirname, join } from "node:path";
 
@@ -46,3 +49,22 @@ export function findWebDistDir(options: WebDistOptions = {}): string | null {
   }
   return null;
 }
+
+const WebReleaseSchema = z.object({
+  version: z
+    .string()
+    .min(1)
+    .refine((value) => value !== "latest" && isValidReleaseVersion(value)),
+});
+
+export function readWebReleaseVersion(distDir: string): string | undefined {
+  try {
+    const payload: unknown = JSON.parse(
+      readFileSync(join(distDir, "release.json"), "utf8"),
+    );
+    const parsed = WebReleaseSchema.safeParse(payload);
+    return parsed.success ? stripVersionPrefix(parsed.data.version) : undefined;
+  } catch {
+    return undefined;
+  }
+}