@vellumai/assistant 0.10.3-dev.202606291342.c7a83ea → 0.10.3-dev.202606291517.f0c68f7

package/openapi.yaml

@@ -12,9 +12,26 @@ paths:
   /healthz:
     get:
       operationId: healthz_get
+      summary: Liveness probe
+      description:
+        Trivial liveness/startup probe. Returns { status, version } the instant the HTTP server is up, with zero
+        DB/CES/lifecycle access.
       responses:
         "200":
           description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  status:
+                    type: string
+                  version:
+                    type: string
+                required:
+                  - status
+                  - version
+                additionalProperties: false
   /pages/{id}:
     get:
       operationId: pages__id__get
@@ -30,9 +47,52 @@ paths:
   /readyz:
     get:
       operationId: readyz_get
+      summary: Readiness probe
+      description:
+        Strict readiness probe. Returns 503 with { status, ready, notReady } until the daemon's synchronous startup
+        latches (isStartupComplete() && isDbReady()) are set, then a stable 200. notReady lists the gates still failing
+        ("startup", "db"). CES is informational and never gates readiness.
       responses:
         "200":
           description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  status:
+                    type: string
+                  ready:
+                    type: boolean
+                  notReady:
+                    type: array
+                    items:
+                      type: string
+                required:
+                  - status
+                  - ready
+                  - notReady
+                additionalProperties: false
+        "503":
+          description: Not ready — startup incomplete or database not ready.
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  status:
+                    type: string
+                  ready:
+                    type: boolean
+                  notReady:
+                    type: array
+                    items:
+                      type: string
+                required:
+                  - status
+                  - ready
+                  - notReady
+                additionalProperties: false
   /v1/acp/{id}/cancel:
     post:
       operationId: acp_by_id_cancel_post

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.10.3-dev.202606291342.c7a83ea",
+  "version": "0.10.3-dev.202606291517.f0c68f7",
   "license": "MIT",
   "type": "module",
   "exports": {

package/scripts/generate-openapi.ts

@@ -216,13 +216,69 @@ async function collectRoutesFromModules(): Promise<RouteEntry[]> {
   return routes;
 }
 
+/**
+ * Trivial liveness/startup probe response. `/healthz` is the k8s startup +
+ * liveness target and stays intentionally minimal: a static `{ status, version }`
+ * answered the instant the HTTP server is up, with zero DB/CES/lifecycle access.
+ */
+const trivialHealthSchema = z.object({
+  status: z.string(),
+  version: z.string(),
+});
+
+/**
+ * Strict readiness probe response. `/readyz` gates on the daemon's synchronous
+ * startup latches (`isStartupComplete() && isDbReady()`), returning 503 with
+ * `ready: false` until both are set, then a stable 200. `notReady` lists the
+ * gates still failing (`"startup"`, `"db"`). CES is never consulted.
+ */
+const readyzSchema = z.object({
+  status: z.string(),
+  ready: z.boolean(),
+  notReady: z.array(z.string()),
+});
+
 /**
  * Top-level routes outside the /v1/ namespace.
  * These are added to the spec separately.
  */
-const NON_V1_ROUTES: Array<{ method: string; path: string }> = [
-  { method: "GET", path: "/healthz" },
-  { method: "GET", path: "/readyz" },
+const NON_V1_ROUTES: Array<{
+  method: string;
+  path: string;
+  summary?: string;
+  description?: string;
+  responseBody?: z.ZodType;
+  additionalResponses?: Record<
+    string,
+    { description: string; schema?: unknown }
+  >;
+}> = [
+  {
+    method: "GET",
+    path: "/healthz",
+    summary: "Liveness probe",
+    description:
+      "Trivial liveness/startup probe. Returns { status, version } the instant " +
+      "the HTTP server is up, with zero DB/CES/lifecycle access.",
+    responseBody: trivialHealthSchema,
+  },
+  {
+    method: "GET",
+    path: "/readyz",
+    summary: "Readiness probe",
+    description:
+      "Strict readiness probe. Returns 503 with { status, ready, notReady } until " +
+      "the daemon's synchronous startup latches (isStartupComplete() && isDbReady()) " +
+      "are set, then a stable 200. notReady lists the gates still failing " +
+      '("startup", "db"). CES is informational and never gates readiness.',
+    responseBody: readyzSchema,
+    additionalResponses: {
+      "503": {
+        description: "Not ready — startup incomplete or database not ready.",
+        schema: readyzSchema,
+      },
+    },
+  },
   { method: "GET", path: "/pages/{id}" },
 ];
 
@@ -322,7 +378,16 @@ function buildSpec(
         path: r.path,
         method: r.method,
         endpoint: r.path,
-        entry: { method: r.method, endpoint: r.path },
+        entry: {
+          method: r.method,
+          endpoint: r.path,
+          ...(r.summary ? { summary: r.summary } : {}),
+          ...(r.description ? { description: r.description } : {}),
+          ...(r.responseBody ? { responseBody: r.responseBody } : {}),
+          ...(r.additionalResponses
+            ? { additionalResponses: r.additionalResponses }
+            : {}),
+        },
       });
     }
   }

package/src/__tests__/identity-routes.test.ts

@@ -22,13 +22,20 @@ mock.module("../util/logger.js", () => ({
     }),
 }));
 
+import {
+  resetReadinessForTest,
+  setDbReady,
+  setStartupComplete,
+} from "../daemon/daemon-readiness.js";
 import {
   handleDetailedHealth,
+  handleHealth,
   handleReadyz,
   ROUTES,
 } from "../runtime/routes/identity-routes.js";
 import { setCesClient } from "../security/secure-keys.js";
 import { getWorkspaceDir } from "../util/platform.js";
+import { APP_VERSION } from "../version.js";
 import {
   getHatchedSidecarPath,
   resolveHatchedAtReadOnly,
@@ -196,31 +203,6 @@ describe("identity routes — health endpoint", () => {
       setCesClient(undefined);
     });
 
-    test("readyz returns 200 and logs warning when CES is unavailable", () => {
-      const res = handleReadyz();
-      expect(res.status).toBe(200);
-    });
-
-    test("readyz returns 200 when CES is connected and ready", () => {
-      const mockClient = {
-        isReady: () => true,
-        close: () => {},
-      } as unknown as import("../credential-execution/client.js").CesClient;
-      setCesClient(mockClient);
-      const res = handleReadyz();
-      expect(res.status).toBe(200);
-    });
-
-    test("readyz returns 200 when CES client exists but is not ready", () => {
-      const mockClient = {
-        isReady: () => false,
-        close: () => {},
-      } as unknown as import("../credential-execution/client.js").CesClient;
-      setCesClient(mockClient);
-      const res = handleReadyz();
-      expect(res.status).toBe(200);
-    });
-
     test("/v1/health reports ces.connected=true when CES is ready", async () => {
       const mockClient = {
         isReady: () => true,
@@ -407,6 +389,112 @@ describe("identity routes — health endpoint", () => {
   });
 });
 
+describe("identity routes — trivial /healthz liveness probe", () => {
+  test("returns 200 with { status: 'ok', version } carrying APP_VERSION", async () => {
+    const res = handleHealth();
+    expect(res.status).toBe(200);
+
+    const body = (await res.json()) as Record<string, unknown>;
+    expect(body.status).toBe("ok");
+    expect(typeof body.version).toBe("string");
+    expect(body.version).not.toBe("");
+    expect(body.version).toBe(APP_VERSION);
+  });
+
+  test("never touches CES/lifecycle state — a throwing CES client is never consulted", async () => {
+    // If the trivial probe touched CES at all this would throw.
+    const explodingClient = {
+      isReady: () => {
+        throw new Error("handleHealth must not access CES");
+      },
+      close: () => {},
+    } as unknown as import("../credential-execution/client.js").CesClient;
+    setCesClient(explodingClient);
+
+    const res = handleHealth();
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as Record<string, unknown>;
+    expect(body).toEqual({ status: "ok", version: APP_VERSION });
+
+    setCesClient(undefined);
+  });
+
+  test("answers with CES uninitialized", async () => {
+    setCesClient(undefined);
+    const res = handleHealth();
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as Record<string, unknown>;
+    expect(body).toEqual({ status: "ok", version: APP_VERSION });
+  });
+});
+
+describe("identity routes — /readyz readiness gate", () => {
+  afterEach(() => {
+    resetReadinessForTest();
+    setCesClient(undefined);
+  });
+
+  test("returns 503 with notReady:['startup','db'] before startup", async () => {
+    resetReadinessForTest();
+    const res = handleReadyz();
+    expect(res.status).toBe(503);
+    const body = (await res.json()) as Record<string, unknown>;
+    expect(body).toEqual({
+      status: "unready",
+      ready: false,
+      notReady: ["startup", "db"],
+    });
+  });
+
+  test("returns 503 with notReady:['startup'] when db is ready but startup isn't", async () => {
+    resetReadinessForTest();
+    setDbReady(true);
+    const res = handleReadyz();
+    expect(res.status).toBe(503);
+    const body = (await res.json()) as Record<string, unknown>;
+    expect(body.ready).toBe(false);
+    expect(body.notReady).toEqual(["startup"]);
+  });
+
+  test("returns 503 with notReady:['db'] when started but db not ready", async () => {
+    resetReadinessForTest();
+    setStartupComplete();
+    const res = handleReadyz();
+    expect(res.status).toBe(503);
+    const body = (await res.json()) as Record<string, unknown>;
+    expect(body.ready).toBe(false);
+    expect(body.notReady).toEqual(["db"]);
+  });
+
+  test("returns 200 when started and db ready even if CES is down", async () => {
+    resetReadinessForTest();
+    setStartupComplete();
+    setDbReady(true);
+    const explodingClient = {
+      isReady: () => {
+        throw new Error("/readyz must not consult CES");
+      },
+      close: () => {},
+    } as unknown as import("../credential-execution/client.js").CesClient;
+    setCesClient(explodingClient);
+
+    const res = handleReadyz();
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as Record<string, unknown>;
+    expect(body.ready).toBe(true);
+  });
+
+  test("returns 200 with the ok body when fully ready", async () => {
+    resetReadinessForTest();
+    setStartupComplete();
+    setDbReady(true);
+    const res = handleReadyz();
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as Record<string, unknown>;
+    expect(body).toEqual({ status: "ok", ready: true, notReady: [] });
+  });
+});
+
 describe("identity routes — createdAt selection", () => {
   test("falls back to mtime when birthtime is the Unix epoch", () => {
     const mtime = new Date("2026-05-01T14:49:47.519Z");

package/src/daemon/__tests__/daemon-readiness.test.ts

@@ -0,0 +1,51 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import {
+  isDbReady,
+  isStartupComplete,
+  resetReadinessForTest,
+  setDbReady,
+  setStartupComplete,
+} from "../daemon-readiness.js";
+
+describe("daemon-readiness", () => {
+  beforeEach(() => {
+    resetReadinessForTest();
+  });
+
+  afterEach(() => {
+    resetReadinessForTest();
+  });
+
+  test("defaults are false", () => {
+    expect(isDbReady()).toBe(false);
+    expect(isStartupComplete()).toBe(false);
+  });
+
+  test("setDbReady flips state both ways", () => {
+    setDbReady(true);
+    expect(isDbReady()).toBe(true);
+    setDbReady(false);
+    expect(isDbReady()).toBe(false);
+  });
+
+  test("setStartupComplete latches startup state", () => {
+    expect(isStartupComplete()).toBe(false);
+    setStartupComplete();
+    expect(isStartupComplete()).toBe(true);
+  });
+
+  test("setStartupComplete is monotonic", () => {
+    setStartupComplete();
+    setStartupComplete();
+    expect(isStartupComplete()).toBe(true);
+  });
+
+  test("resetReadinessForTest clears both latches", () => {
+    setDbReady(true);
+    setStartupComplete();
+    resetReadinessForTest();
+    expect(isDbReady()).toBe(false);
+    expect(isStartupComplete()).toBe(false);
+  });
+});

package/src/daemon/daemon-readiness.ts

@@ -0,0 +1,33 @@
+// Module-level readiness state for the daemon, readable synchronously from any
+// request handler with no `await`. The `/readyz` probe gates on these latches
+// to report whether the daemon can actually serve requests.
+//
+// CES is intentionally not latched here: it is read live
+// (`getCesClient()?.isReady()`) only when reported in a response body, and must
+// never gate readiness.
+
+let dbReady = false;
+let startupComplete = false;
+
+export function setDbReady(v: boolean): void {
+  dbReady = v;
+}
+
+export function isDbReady(): boolean {
+  return dbReady;
+}
+
+// One-way latch: once startup completes it stays complete for the process
+// lifetime.
+export function setStartupComplete(): void {
+  startupComplete = true;
+}
+
+export function isStartupComplete(): boolean {
+  return startupComplete;
+}
+
+export function resetReadinessForTest(): void {
+  dbReady = false;
+  startupComplete = false;
+}

package/src/daemon/lifecycle.ts

@@ -95,6 +95,7 @@ import { startAppSourceWatcher } from "./app-source-watcher.js";
 import { startConfigWatcher } from "./config-watcher.js";
 import { startConversationEvictor } from "./conversation-store.js";
 import { writePid } from "./daemon-control.js";
+import { setDbReady, setStartupComplete } from "./daemon-readiness.js";
 import {
   evaluateDiskPressureNow,
   startDiskPressureGuard,
@@ -270,15 +271,35 @@ export async function runDaemon(): Promise<void> {
   // depend on DB migrations having run (e.g. the inline-attachment-to-disk
   // backfill that populates attachment filePaths).
   //
-  // If DB initialization fails (e.g. a migration error), the daemon
-  // continues in a degraded state — DB-dependent features won't work but
-  // the HTTP server and config-based subsystems still start so the process
-  // remains reachable for health checks and diagnostics.
+  // The daemon continues in a degraded state on either DB failure mode:
+  // (a) initializeDb() throws (e.g. the DB can't be opened), or (b) the DB
+  // opens but one or more migrations failed (initializeDb resolves with
+  // migrationsOk:false rather than throwing, per the daemon-never-blocks
+  // philosophy). In both cases DB-dependent features won't work, but the
+  // HTTP server and config-based subsystems still start so the process
+  // remains reachable for diagnostics. The trivial /healthz and detailed
+  // /v1/health(z) endpoints still answer 200, but setDbReady(true) runs only
+  // when migrations all applied, so the readiness latch stays unset and
+  // /readyz reports not-ready (503) for the rest of the process lifetime.
+  // That 503 is intentional: a DB-broken daemon should fail readiness so it
+  // is not routed user traffic.
+  //
+  // The local `dbReady` and the module-level readiness latch intentionally
+  // diverge on the migration-failure path: `dbReady` stays true to allow the
+  // downstream best-effort seeding / workspace migrations, while the latch
+  // stays unset to keep /readyz 503.
   let dbReady = false;
   try {
-    await initializeDb();
+    const { migrationsOk } = await initializeDb();
     dbReady = true;
-    log.info("Daemon startup: DB initialized");
+    if (migrationsOk) {
+      setDbReady(true);
+      log.info("Daemon startup: DB initialized");
+    } else {
+      log.error(
+        "Daemon startup: DB opened but one or more migrations failed — /readyz will remain unready (degraded mode)",
+      );
+    }
   } catch (err) {
     log.error(
       { err },
@@ -1166,6 +1187,12 @@ export async function runDaemon(): Promise<void> {
 
   installShutdownHandlers({ server });
 
+  // The critical startup await-chain has completed and the daemon can serve
+  // requests, so latch readiness before logging "Daemon started". Any fatal
+  // failure earlier in startup propagates out of runDaemon before this line,
+  // so the latch is never set on a failed start.
+  setStartupComplete();
+
   log.info(
     {
       durationMs: Date.now() - startupStartedAt,

package/src/persistence/__tests__/db-init-migrations-ok.test.ts

@@ -0,0 +1,20 @@
+/**
+ * Tests for initializeDb()'s readiness-gating return contract.
+ *
+ * The daemon's readiness latch (setDbReady) is gated on initializeDb resolving
+ * with migrationsOk:true, so a clean init must report all migrations applied.
+ * Migration-step failures are caught-and-logged inside the runner (not thrown),
+ * so the boolean is the only signal lifecycle has to keep /readyz unready on a
+ * partially-migrated schema — see assistant/src/daemon/lifecycle.ts.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import { initializeDb } from "../db-init.js";
+
+describe("initializeDb — migrationsOk return contract", () => {
+  test("resolves to { migrationsOk: true } on a clean init", async () => {
+    const result = await initializeDb();
+    expect(result).toEqual({ migrationsOk: true });
+  });
+});

package/src/persistence/db-init.ts

@@ -215,9 +215,10 @@ export async function checkpointWalBeforeOpen(): Promise<void> {
 
 // ---------------------------------------------------------------------------
 
-export async function initializeDb(): Promise<void> {
+export async function initializeDb(): Promise<{ migrationsOk: boolean }> {
   if (process.env.BUN_TEST === "1" && tryRestoreTemplate()) {
-    return;
+    // A restored template is fully migrated.
+    return { migrationsOk: true };
   }
 
   // Fold any post-crash WAL back into the database off the main event loop
@@ -260,13 +261,22 @@ export async function initializeDb(): Promise<void> {
     );
   }
 
+  // A passing post-run validation is part of readiness: validateMigrationState
+  // flags schema inconsistencies (e.g. a completed step missing a declared
+  // dependsOn checkpoint) that no individual step body surfaces as a failure.
+  let validationOk = true;
   try {
     validateMigrationState(database, migrationSteps);
   } catch (err) {
+    validationOk = false;
     log.error({ err }, "validateMigrationState failed");
   }
 
   if (process.env.BUN_TEST === "1") {
     saveTemplate();
   }
+
+  // migrationsOk reflects BOTH no failed migration steps AND a passing
+  // post-run validation, so an inconsistent schema keeps /readyz at 503.
+  return { migrationsOk: failed.length === 0 && validationOk };
 }

package/src/runtime/routes/identity-routes.ts

@@ -8,6 +8,7 @@ import { availableParallelism, cpus, totalmem } from "node:os";
 import { z } from "zod";
 
 import { getCpuLimit, getIsPlatform } from "../../config/env-registry.js";
+import { isDbReady, isStartupComplete } from "../../daemon/daemon-readiness.js";
 import { parseIdentityFields } from "../../daemon/handlers/identity.js";
 import { getProfilerRuntimeStatus } from "../../daemon/profiler-run-store.js";
 import { getMaxRollbackVersion } from "../../persistence/migrations/run-migrations.js";
@@ -17,7 +18,6 @@ import {
   getDiskUsageInfo,
   parseK8sMemoryBytes,
 } from "../../util/disk-usage.js";
-import { getLogger } from "../../util/logger.js";
 import { getWorkspacePromptPath } from "../../util/platform.js";
 import { APP_VERSION } from "../../version.js";
 import { resolveHatchedAtReadOnly } from "../../workspace/hatched-date.js";
@@ -314,8 +314,16 @@ function getCpuInfo(): CpuInfo {
   };
 }
 
+/**
+ * Trivial liveness/startup probe (`GET /healthz`).
+ *
+ * This is the k8s startup + liveness probe target: it must answer the instant
+ * the HTTP server is up and must NEVER touch DB, CES, migrations, or any other
+ * lifecycle state. Keep it to a static `{ status, version }` payload — no
+ * syscalls, no disk/memory/cpu reads, no async work.
+ */
 export function handleHealth(): Response {
-  return Response.json({ status: "ok" });
+  return Response.json({ status: "ok", version: APP_VERSION });
 }
 
 function getDetailedHealth() {
@@ -354,17 +362,28 @@ export function handleDetailedHealth(): Response {
   return Response.json(getDetailedHealth());
 }
 
+/**
+ * Strict readiness probe (`GET /readyz`).
+ *
+ * Reports whether the daemon can actually serve requests, gating on two
+ * monotonic startup latches read SYNCHRONOUSLY (no `await`, DB query, or
+ * subprocess) so the probe stays cheap on a ~10s interval. Returns 503 until
+ * both latches are set, then a stable 200 for the rest of the process lifetime.
+ *
+ * CES is intentionally never consulted: it is informational only. Gating on it
+ * would leave a pod permanently NotReady whenever CES never handshakes, even
+ * though the daemon serves degraded with an encrypted-store fallback.
+ */
 export function handleReadyz(): Response {
-  const cesClient = getCesClient();
-  if (!cesClient?.isReady()) {
-    // TODO: Return 503 once we confirm via logs that this won't cause
-    // regressions in the K8s readinessProbe.
-    getLogger("health").warn(
-      { reason: cesClient ? "ces_not_ready" : "ces_unavailable" },
-      "CES not ready — pod would be unready if 503 were enabled",
-    );
-  }
-  return Response.json({ status: "ok" });
+  const notReady: string[] = [];
+  if (!isStartupComplete()) notReady.push("startup");
+  if (!isDbReady()) notReady.push("db");
+  const ready = notReady.length === 0;
+
+  return Response.json(
+    { status: ready ? "ok" : "unready", ready, notReady },
+    { status: ready ? 200 : 503 },
+  );
 }
 
 function getIdentity() {