@vellumai/cli 0.10.6 → 0.10.7-dev.202607072228.38de414

package/node_modules/@vellumai/local-mode/src/status.ts

@@ -5,9 +5,7 @@ import path from "node:path";
 
 import { SEEDS } from "@vellumai/environments";
 
-import type {
-  LockfileAssistant,
-} from "./lockfile-contract";
+import type { LockfileAssistant } from "./lockfile-contract";
 import { getLockfileData } from "./lockfile";
 
 const HEALTH_TIMEOUT_MS = 1_500;
@@ -20,6 +18,8 @@ const DEFAULT_PORTS = {
 
 export type LocalAssistantRuntimeState =
   | "healthy"
+  /** Alive and serving, but DB migrations failed — restart to recover. */
+  | "unhealthy"
   | "upgrading"
   | "sleeping"
   | "starting"
@@ -77,10 +77,7 @@ function readPidState(pidFile: string): PidState {
   }
 }
 
-function isFreshPidState(
-  pidState: PidState,
-  observedAtMs: number,
-): boolean {
+function isFreshPidState(pidState: PidState, observedAtMs: number): boolean {
   return (
     "updatedAtMs" in pidState &&
     observedAtMs - pidState.updatedAtMs <= STARTING_GRACE_MS
@@ -129,6 +126,63 @@ function httpHealthCheck(port: number): Promise<boolean> {
   });
 }
 
+type DaemonReadyzClassification =
+  "ready" | "migrating" | "failed" | "no_answer";
+
+/**
+ * Classify the daemon's DB migration readiness from the `/readyz` BODY.
+ * `/healthz` is a static probe that never carries migration state, and the
+ * `/readyz` status code stays 200 while migrations run (the k8s keep-the-pod
+ * contract) — so only the body distinguishes ready / migrating / failed.
+ * Legacy daemons' strict 503 startup body (no `dbMigrations` field) classifies
+ * as "migrating", which renders as the starting state.
+ */
+function probeDaemonReadyz(port: number): Promise<DaemonReadyzClassification> {
+  return new Promise((resolve) => {
+    const req = http.get(
+      {
+        hostname: "127.0.0.1",
+        port,
+        path: "/readyz",
+        timeout: HEALTH_TIMEOUT_MS,
+      },
+      (res) => {
+        const chunks: Buffer[] = [];
+        res.on("data", (chunk: Buffer) => chunks.push(chunk));
+        res.on("end", () => {
+          let body: {
+            ready?: boolean;
+            dbMigrations?: { state?: string };
+          } | null = null;
+          try {
+            body = JSON.parse(Buffer.concat(chunks).toString()) as {
+              ready?: boolean;
+              dbMigrations?: { state?: string };
+            };
+          } catch {
+            // Non-JSON body — classify from the status code alone.
+          }
+          if (body?.dbMigrations?.state === "failed") {
+            resolve("failed");
+            return;
+          }
+          if (res.statusCode === 200 && body?.ready !== false) {
+            resolve("ready");
+            return;
+          }
+          resolve("migrating");
+        });
+      },
+    );
+
+    req.on("timeout", () => {
+      req.destroy();
+      resolve("no_answer");
+    });
+    req.on("error", () => resolve("no_answer"));
+  });
+}
+
 function localOnlyEntry(
   entry: LockfileAssistant | undefined,
 ): LockfileAssistant | null {
@@ -309,6 +363,29 @@ async function runtimeStatusForEntry(
     };
   }
 
+  // Both processes answer their health checks, but the daemon's DB migrations
+  // may still be running (in-progress startup) or terminally failed (every
+  // DB-backed route 503s until a restart). Classify from the /readyz body so
+  // desktop/web hosts don't report a broken daemon as healthy.
+  const readiness = await probeDaemonReadyz(resources.daemonPort);
+  if (readiness === "failed") {
+    return {
+      ok: true,
+      state: "unhealthy",
+      pid: assistantPid.pid,
+      detail: "assistant database migrations failed — restart to recover",
+    };
+  }
+  if (readiness === "migrating") {
+    return {
+      ok: true,
+      state: "starting",
+      pid: assistantPid.pid,
+      detail: "assistant database migrations are running",
+    };
+  }
+  // "no_answer" while /healthz just passed is a transient hiccup — keep the
+  // health check authoritative rather than flapping to a worse state.
   return { ok: true, state: "healthy", pid: assistantPid.pid };
 }
 

package/node_modules/@vellumai/local-mode/tsconfig.json

@@ -9,7 +9,7 @@
     "forceConsistentCasingInFileNames": true,
     "resolveJsonModule": true,
     "noEmit": true,
-    "types": ["bun-types"]
+    "types": ["bun"]
   },
   "include": ["src/**/*"],
   "exclude": ["node_modules"]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/cli",
-  "version": "0.10.6",
+  "version": "0.10.7-dev.202607072228.38de414",
   "description": "CLI tools for vellum-assistant",
   "type": "module",
   "exports": {
@@ -22,7 +22,6 @@
     "format:check": "prettier --check .",
     "lint": "eslint",
     "lint:unused": "knip --include files,dependencies,unlisted",
-    "prepack": "node ../scripts/prepack-bundled-deps.mjs",
     "compile:check": "bun build --compile --external react-devtools-core src/index.ts --outfile /dev/null",
     "test": "bun test",
     "typecheck": "bunx tsc --noEmit"

package/src/__tests__/daemon-readiness-probe.test.ts

@@ -0,0 +1,158 @@
+/**
+ * Tests for the CLI's migration-aware daemon readiness probe.
+ *
+ * `/readyz` keeps its HTTP status at 200 while DB migrations run (the k8s
+ * contract — the pod must stay in service), so the CLI classifies readiness
+ * from the response BODY instead: ready / migrating / failed / unreachable.
+ * The `failed` state is terminal for the daemon process, so the polling wait
+ * must return immediately instead of burning its full deadline.
+ */
+
+import { afterEach, describe, expect, test } from "bun:test";
+
+import {
+  probeDaemonReadiness,
+  waitForDaemonMigrationsReady,
+} from "../lib/http-client.js";
+
+type ReadyzResponder = () => Response;
+
+let server: ReturnType<typeof Bun.serve> | null = null;
+
+function serveReadyz(responder: ReadyzResponder): number {
+  server = Bun.serve({
+    port: 0,
+    hostname: "127.0.0.1",
+    fetch(req) {
+      if (new URL(req.url).pathname === "/readyz") return responder();
+      return new Response("not found", { status: 404 });
+    },
+  });
+  const { port } = server;
+  if (port === undefined) throw new Error("test server did not bind a port");
+  return port;
+}
+
+afterEach(() => {
+  server?.stop(true);
+  server = null;
+});
+
+describe("probeDaemonReadiness", () => {
+  test("classifies the ready body as ready", async () => {
+    const port = serveReadyz(() =>
+      Response.json({ status: "ok", ready: true }),
+    );
+    expect(await probeDaemonReadiness(port)).toBe("ready");
+  });
+
+  test("classifies a 200 migrating body as migrating", async () => {
+    const port = serveReadyz(() =>
+      Response.json({
+        status: "migrating",
+        ready: false,
+        dbMigrations: { ready: false, state: "running" },
+      }),
+    );
+    expect(await probeDaemonReadiness(port)).toBe("migrating");
+  });
+
+  test("classifies the 503 failed body as failed", async () => {
+    const port = serveReadyz(() =>
+      Response.json(
+        {
+          status: "error",
+          ready: false,
+          reason: "db_migrations_failed",
+          dbMigrations: { ready: false, state: "failed" },
+        },
+        { status: 503 },
+      ),
+    );
+    expect(await probeDaemonReadiness(port)).toBe("failed");
+  });
+
+  test("treats a legacy 200 body without a ready field as ready", async () => {
+    // Daemons that predate the migration-state body return { status: "ok" }.
+    const port = serveReadyz(() => Response.json({ status: "ok" }));
+    expect(await probeDaemonReadiness(port)).toBe("ready");
+  });
+
+  test("classifies a legacy strict-503 startup body as migrating, not unreachable", async () => {
+    // Pre-migration-body daemons return 503 { status, ready, notReady }
+    // throughout startup. The daemon ANSWERED, so it is alive — classifying
+    // it unreachable would make callers treat a starting daemon as dead.
+    const port = serveReadyz(() =>
+      Response.json(
+        { status: "unready", ready: false, notReady: ["startup", "db"] },
+        { status: 503 },
+      ),
+    );
+    expect(await probeDaemonReadiness(port)).toBe("migrating");
+  });
+
+  test("reports unreachable when nothing is listening", async () => {
+    const port = serveReadyz(() => Response.json({ ready: true }));
+    server?.stop(true);
+    server = null;
+    expect(await probeDaemonReadiness(port)).toBe("unreachable");
+  });
+});
+
+describe("waitForDaemonMigrationsReady", () => {
+  test("returns failed immediately without waiting out the deadline", async () => {
+    const port = serveReadyz(() =>
+      Response.json(
+        {
+          status: "error",
+          ready: false,
+          dbMigrations: { ready: false, state: "failed" },
+        },
+        { status: 503 },
+      ),
+    );
+    const started = Date.now();
+    const readiness = await waitForDaemonMigrationsReady(
+      port,
+      Date.now() + 60_000,
+    );
+    expect(readiness).toBe("failed");
+    // Terminal state short-circuits: nowhere near the 60s deadline.
+    expect(Date.now() - started).toBeLessThan(5_000);
+  });
+
+  test("keeps polling through migrating and resolves ready", async () => {
+    let calls = 0;
+    const port = serveReadyz(() => {
+      calls++;
+      return calls < 3
+        ? Response.json({
+            status: "migrating",
+            ready: false,
+            dbMigrations: { ready: false, state: "running" },
+          })
+        : Response.json({ status: "ok", ready: true });
+    });
+    const readiness = await waitForDaemonMigrationsReady(
+      port,
+      Date.now() + 10_000,
+    );
+    expect(readiness).toBe("ready");
+    expect(calls).toBeGreaterThanOrEqual(3);
+  });
+
+  test("returns migrating when the deadline passes mid-migration", async () => {
+    const port = serveReadyz(() =>
+      Response.json({
+        status: "migrating",
+        ready: false,
+        dbMigrations: { ready: false, state: "running" },
+      }),
+    );
+    const readiness = await waitForDaemonMigrationsReady(
+      port,
+      Date.now() + 600,
+    );
+    expect(readiness).toBe("migrating");
+  });
+});

package/src/__tests__/recover.test.ts

@@ -30,12 +30,14 @@ const realLocal = {
   generateLocalSigningKey: localModule.generateLocalSigningKey,
   startLocalDaemon: localModule.startLocalDaemon,
   startGateway: localModule.startGateway,
+  startCes: localModule.startCes,
 };
 const realExec = stepRunnerModule.exec;
 
-// Prevent real daemon / gateway from starting
+// Prevent real daemon / gateway / CES from starting
 const startLocalDaemonMock = mock(async () => {});
 const startGatewayMock = mock(async () => {});
+const startCesMock = mock(async () => {});
 
 // Capture exec calls without running real tar
 const execMock = mock(async (_cmd: string, _args: string[]) => {});
@@ -45,6 +47,7 @@ beforeAll(() => {
     generateLocalSigningKey: () => "deadbeefdeadbeefdeadbeefdeadbeef",
     startLocalDaemon: startLocalDaemonMock,
     startGateway: startGatewayMock,
+    startCes: startCesMock,
   }));
   mock.module("../lib/step-runner.js", () => ({ exec: execMock }));
 });
@@ -128,6 +131,7 @@ beforeEach(() => {
   execMock.mockClear();
   startLocalDaemonMock.mockClear();
   startGatewayMock.mockClear();
+  startCesMock.mockClear();
 });
 
 afterEach(() => {

package/src/__tests__/upgrade-local.test.ts

@@ -119,6 +119,7 @@ const startGatewayMock = mock<typeof local.startGateway>(
 const stopLocalProcessesMock = mock<typeof local.stopLocalProcesses>(
   async () => {},
 );
+const startCesMock = mock<typeof local.startCes>(async () => {});
 
 mock.module("../lib/local.js", () => ({
   ...realLocal,
@@ -127,6 +128,7 @@ mock.module("../lib/local.js", () => ({
   startLocalDaemon: startLocalDaemonMock,
   startGateway: startGatewayMock,
   stopLocalProcesses: stopLocalProcessesMock,
+  startCes: startCesMock,
 }));
 
 const loopbackSafeFetchMock = mock<typeof loopbackFetch.loopbackSafeFetch>(
@@ -256,6 +258,8 @@ beforeEach(() => {
   startGatewayMock.mockResolvedValue("http://127.0.0.1:7830");
   stopLocalProcessesMock.mockReset();
   stopLocalProcessesMock.mockResolvedValue(undefined);
+  startCesMock.mockReset();
+  startCesMock.mockResolvedValue(undefined);
   loopbackSafeFetchMock.mockReset();
   loopbackSafeFetchMock.mockResolvedValue({
     ok: true,

package/src/__tests__/upgrade-wait-for-ready.test.ts

@@ -0,0 +1,103 @@
+/**
+ * Tests for the docker upgrade/rollback readiness gate.
+ *
+ * `waitForReady` classifies readiness from the gateway `/readyz` BODY, not
+ * the status code: the forwarded assistant body is 200 with `ready: false`
+ * while DB migrations run (the k8s keep-the-pod contract) and while the
+ * gateway's own startup gate is closed. Simplifying this back to
+ * `if (resp.ok) return true` would make `vellum upgrade` commit and report
+ * success mid-migration and make the rollback/restore path unreachable —
+ * these tests pin the body semantics.
+ */
+
+import { afterEach, describe, expect, test } from "bun:test";
+
+import { waitForReady } from "../lib/upgrade-lifecycle.js";
+
+type ReadyzResponder = () => Response;
+
+let server: ReturnType<typeof Bun.serve> | null = null;
+
+function serveReadyz(responder: ReadyzResponder): string {
+  server = Bun.serve({
+    port: 0,
+    hostname: "127.0.0.1",
+    fetch(req) {
+      if (new URL(req.url).pathname === "/readyz") return responder();
+      return new Response("not found", { status: 404 });
+    },
+  });
+  const { port } = server;
+  if (port === undefined) throw new Error("test server did not bind a port");
+  return `http://127.0.0.1:${port}`;
+}
+
+afterEach(() => {
+  server?.stop(true);
+  server = null;
+});
+
+describe("waitForReady", () => {
+  test("returns true when the stack reports ready", async () => {
+    const url = serveReadyz(() => Response.json({ status: "ok", ready: true }));
+    expect(await waitForReady(url)).toBe(true);
+  });
+
+  test("does NOT return true on a bare 200 while migrations run", async () => {
+    // 200 { ready: false } is the migrating body — resp.ok alone must not
+    // count as ready. The responder flips to ready after a few polls so the
+    // test proves waitForReady kept waiting rather than returning early.
+    let polls = 0;
+    const url = serveReadyz(() => {
+      polls++;
+      return polls < 3
+        ? Response.json({
+            status: "migrating",
+            ready: false,
+            dbMigrations: { ready: false, state: "running" },
+          })
+        : Response.json({ status: "ok", ready: true });
+    });
+
+    expect(await waitForReady(url)).toBe(true);
+    expect(polls).toBeGreaterThanOrEqual(3);
+  });
+
+  test("keeps waiting through the gateway's starting gate body", async () => {
+    let polls = 0;
+    const url = serveReadyz(() => {
+      polls++;
+      return polls < 2
+        ? Response.json({ status: "starting", ready: false })
+        : Response.json({ status: "ok", ready: true });
+    });
+
+    expect(await waitForReady(url)).toBe(true);
+    expect(polls).toBeGreaterThanOrEqual(2);
+  });
+
+  test("fails fast on terminally failed migrations so rollback can run", async () => {
+    const url = serveReadyz(() =>
+      Response.json(
+        {
+          status: "upstream_unhealthy",
+          upstream: 503,
+          ready: false,
+          dbMigrations: { ready: false, state: "failed" },
+        },
+        { status: 503 },
+      ),
+    );
+
+    const started = Date.now();
+    expect(await waitForReady(url)).toBe(false);
+    // Terminal state short-circuits — nowhere near the 5-minute timeout.
+    expect(Date.now() - started).toBeLessThan(5_000);
+  });
+
+  test("treats a legacy 200 body without a ready field as ready", async () => {
+    // Pre-migration-body gateways returned a fixed { status: "ok" }.
+    const url = serveReadyz(() => Response.json({ status: "ok" }));
+    expect(await waitForReady(url)).toBe(true);
+  });
+});

package/src/commands/restore.ts

@@ -695,15 +695,18 @@ export async function restore(): Promise<void> {
 
     let response: Response;
     try {
-      response = await loopbackSafeFetch(`${entry.runtimeUrl}/v1/migrations/import`, {
-        method: "POST",
-        headers: {
-          Authorization: `Bearer ${accessToken}`,
-          "Content-Type": "application/octet-stream",
+      response = await loopbackSafeFetch(
+        `${entry.runtimeUrl}/v1/migrations/import`,
+        {
+          method: "POST",
+          headers: {
+            Authorization: `Bearer ${accessToken}`,
+            "Content-Type": "application/octet-stream",
+          },
+          body: bundleData,
+          signal: AbortSignal.timeout(120_000),
         },
-        body: bundleData,
-        signal: AbortSignal.timeout(120_000),
-      });
+      );
     } catch (err) {
       if (err instanceof Error && err.name === "TimeoutError") {
         console.error("Error: Import request timed out after 2 minutes.");
@@ -752,6 +755,26 @@ export async function restore(): Promise<void> {
     console.log(`  Files skipped:     ${summary.files_skipped}`);
     console.log(`  Backups created:   ${summary.backups_created}`);
 
+    // A daemon in the terminal failed-migrations state accepts this import
+    // (the migration-repair exemption), but its readiness latch only clears
+    // on restart — without one it keeps refusing DB-backed routes.
+    try {
+      const readyz = await loopbackSafeFetch(`${entry.runtimeUrl}/readyz`, {
+        signal: AbortSignal.timeout(3_000),
+      });
+      const readyzBody = (await readyz.json().catch(() => null)) as {
+        dbMigrations?: { state?: string };
+      } | null;
+      if (readyzBody?.dbMigrations?.state === "failed") {
+        console.warn(
+          "\n⚠️  The assistant is in a failed-migrations state; the restored data takes effect after a restart:",
+        );
+        console.warn(`   vellum sleep ${name} && vellum wake ${name}`);
+      }
+    } catch {
+      // Best-effort advisory — never fail a completed restore over it.
+    }
+
     // Print warnings if any
     const warnings = result.warnings ?? [];
     if (warnings.length > 0) {

package/src/commands/upgrade.ts

@@ -45,6 +45,7 @@ import { emitCliError, categorizeUpgradeError } from "../lib/cli-error.js";
 import { exec } from "../lib/step-runner.js";
 import {
   broadcastUpgradeEvent,
+  attemptFailedStateRestore,
   buildCompleteEvent,
   buildProgressEvent,
   buildStartingEvent,
@@ -605,13 +606,43 @@ async function upgradeDocker(
           (msg) => console.log(msg),
         );
 
-        const rollbackReady = await waitForReady(entry.runtimeUrl);
+        let rollbackReady = await waitForReady(entry.runtimeUrl);
+        let restoredViaFailedState = false;
+        if (!rollbackReady && backupPath) {
+          const recovery = await attemptFailedStateRestore({
+            runtimeUrl: entry.runtimeUrl,
+            assistantId: entry.assistantId,
+            backupPath,
+            backupLabel: "pre-upgrade",
+            res,
+            containerOptions: {
+              signingKey,
+              bootstrapSecret,
+              cesServiceToken,
+              extraAssistantEnv,
+              extraGatewayEnv,
+              gatewayPort,
+              assistantPort,
+              imageTags: previousImageRefs,
+              instanceName,
+              res,
+            },
+          });
+          restoredViaFailedState = recovery.restored;
+          if (recovery.restored) {
+            rollbackReady = recovery.ready;
+          }
+        }
         if (rollbackReady) {
           // Restore data from the backup created for THIS upgrade attempt.
           // Only use the specific backupPath — never scan for the latest
           // backup on disk, which could be from a previous upgrade cycle
           // and contain stale data.
-          if (backupPath) {
+          if (restoredViaFailedState) {
+            console.log(
+              "   ✅ Data restored during failed-state recovery above\n",
+            );
+          } else if (backupPath) {
             await broadcastUpgradeEvent(
               entry.runtimeUrl,
               entry.assistantId,