@bitkyc08/opencodex 2.10.2 → 2.11.0

package/src/server/proxy-liveness.ts

@@ -17,6 +17,8 @@ export interface HealthzIdentity {
   version?: unknown;
   uptime?: unknown;
   pid?: unknown;
+  port?: unknown;
+  restartCapability?: unknown;
 }
 
 export interface LivenessIo {
@@ -47,6 +49,9 @@ export interface LivenessIo {
   nowFn?: () => number;
 }
 
+/** Default per-probe fetch ceiling shared by liveness and readiness probes. */
+export const DEFAULT_PROBE_TIMEOUT_MS = 750;
+
 /** Default probe options for service stop / orphan cleanup — a just-bound proxy can miss a single 750ms probe. */
 export const SERVICE_STOP_LIVENESS: Pick<LivenessIo, "timeoutMs" | "attempts"> = {
   timeoutMs: 1500,
@@ -95,7 +100,7 @@ export async function proxyIdentityAt(
   const fetchFn = io.fetchFn ?? fetch;
   const sleepFn = io.sleepFn ?? ((ms: number) => new Promise<void>(r => setTimeout(r, ms)));
   const nowFn = io.nowFn ?? Date.now;
-  const baseTimeoutMs = io.timeoutMs ?? 750;
+  const baseTimeoutMs = io.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
   const requestedAttempts = Math.trunc(io.attempts ?? 1);
   const attempts = Number.isNaN(requestedAttempts)
     ? 1
@@ -138,6 +143,11 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise<LiveProxy | nu
   const verifyPidFn = io.verifyPidFn ?? verifyPidIdentity;
   const readRuntimeFn = io.readRuntimeFn ?? readRuntimePort;
   const configFn = io.configFn ?? loadConfig;
+  const nowFn = io.nowFn ?? Date.now;
+  const deadlineAt = io.deadlineAt;
+  const probeIo: LivenessIo = io;
+  const budgetExhausted = (): boolean =>
+    deadlineAt !== undefined && nowFn() >= deadlineAt;
 
   // The cheap pid is discovery-only. Before it can appear in a returned (killable) result
   // it must pass the full identity check AND the verifier must echo the exact candidate —
@@ -160,8 +170,9 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise<LiveProxy | nu
   if (pid) {
     const runtime = readRuntimeFn(pid);
     if (runtime?.port) {
+      if (budgetExhausted()) return null;
       probedPort = runtime.port;
-      const identity = await proxyIdentityAt(runtime.port, { hostname: runtime.hostname, expectedPid: pid }, io);
+      const identity = await proxyIdentityAt(runtime.port, { hostname: runtime.hostname, expectedPid: pid }, probeIo);
       if (identity) {
         // healthz confirmed the pid itself → trusted; a pidless legacy body did not,
         // so the cheap pid must pass full identity verification before it is returned.
@@ -176,8 +187,9 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise<LiveProxy | nu
   // identity-probe it so ensure/update/stop see the live proxy instead of shadowing it.
   const record = readRuntimeFn();
   if (record?.port && record.port !== probedPort) {
+    if (budgetExhausted()) return null;
     const expectedPid = typeof record.pid === "number" ? record.pid : undefined;
-    const identity = await proxyIdentityAt(record.port, { hostname: record.hostname, expectedPid }, io);
+    const identity = await proxyIdentityAt(record.port, { hostname: record.hostname, expectedPid }, probeIo);
     // Only the healthz-reported pid is authoritative here. The record's pid may be stale
     // (its process dead, the port reused by a pidless legacy proxy) — synthesizing it
     // would hand destructive callers (stopProxy → kill fallback) a reusable pid.
@@ -188,7 +200,8 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise<LiveProxy | nu
 
   const config = configFn();
   const port = config.port ?? 10100;
-  const identity = await proxyIdentityAt(port, { hostname: config.hostname }, io);
+  if (budgetExhausted()) return null;
+  const identity = await proxyIdentityAt(port, { hostname: config.hostname }, probeIo);
   if (identity) {
     return {
       pid: verifiedReportedPid(identity.pid) ?? killablePid(pid),
@@ -199,3 +212,115 @@ export async function findLiveProxy(io: LivenessIo = {}): Promise<LiveProxy | nu
   }
   return null;
 }
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Readiness (/readyz) strict probe.
+//
+// Liveness (/healthz) confirms the process answers; readiness confirms the
+// post-startup Codex sync has settled. A readiness probe is identity-checked the
+// same way liveness is, AND additionally enforces the full /readyz contract so
+// an adversarial or malformed body can never count as ready:
+//
+//  - HTTP 200 is required for status="ready"; HTTP 503 is required for pending
+//    or failed. Any other HTTP/body-status pairing is an invalid contract.
+//  - body.service must be exactly "opencodex".
+//  - body.version must be a non-empty string.
+//  - body.uptime must be a finite nonnegative number.
+//  - body.pid must be a positive integer; when `expectedPid` is supplied it must
+//    match exactly.
+//  - body.port must be an integer in 1..65535 and equal the probed port.
+//  - body.status must be exactly one of pending|ready|failed.
+//
+// Any unreachable, foreign, legacy, malformed, mismatched, or self-inconsistent
+// response returns `null` so callers can never treat an invalid identity/contract
+// as ready.
+// ─────────────────────────────────────────────────────────────────────────────
+
+interface ReadyzBody {
+  service?: unknown;
+  version?: unknown;
+  uptime?: unknown;
+  pid?: unknown;
+  port?: unknown;
+  status?: unknown;
+}
+
+export interface ReadinessProbeResult {
+  /** True ONLY for a valid 200 + status="ready" body with a matching pid. */
+  ready: boolean;
+  /** Fixed sanitized status. A foreign/unreadable body yields a `null` RESULT, never a `null` status. */
+  status: "ready" | "pending" | "failed";
+  /** Positive integer pid from a valid body. */
+  pid: number;
+  /** Integer port from a valid body. */
+  port: number;
+}
+
+export interface ReadinessProbeIo {
+  fetchFn?: typeof fetch;
+  timeoutMs?: number;
+}
+
+const READYZ_STATUS_VALUES = new Set<"ready" | "pending" | "failed">(["ready", "pending", "failed"]);
+
+/**
+ * Validate a parsed /readyz body against the strict contract. Returns the
+ * sanitized probe result, or `null` when the body is foreign, legacy,
+ * malformed, or fails the pid/port checks. Pure (no I/O) so it is fully
+ * deterministic and unit-testable.
+ */
+export function validateReadyzBody(
+  body: unknown,
+  port: number,
+  opts: { expectedPid?: number } = {},
+): ReadinessProbeResult | null {
+  if (!body || typeof body !== "object") return null;
+  const b = body as ReadyzBody;
+  if (b.service !== "opencodex") return null;
+  if (typeof b.version !== "string" || b.version.length === 0) return null;
+  if (typeof b.uptime !== "number" || !Number.isFinite(b.uptime) || b.uptime < 0) return null;
+  if (typeof b.pid !== "number" || !Number.isInteger(b.pid) || b.pid <= 0) return null;
+  if (
+    typeof b.port !== "number"
+    || !Number.isInteger(b.port)
+    || b.port < 1
+    || b.port > 65535
+    || b.port !== port
+  ) return null;
+  if (typeof b.status !== "string" || !READYZ_STATUS_VALUES.has(b.status as "ready" | "pending" | "failed")) return null;
+  const status = b.status as "ready" | "pending" | "failed";
+  if (opts.expectedPid !== undefined && b.pid !== opts.expectedPid) return null;
+  return { ready: status === "ready", status, pid: b.pid, port: b.port };
+}
+
+/**
+ * Identity- and contract-checked /readyz probe. Returns `null` when the
+ * endpoint is unreachable or the body fails the strict contract (foreign 200,
+ * legacy health-only body, non-JSON, missing/malformed/mismatched fields,
+ * wrong port/pid, or an HTTP/body-status inconsistency). Returns
+ * `{ready:false, ...}` when the body is ours but pending or failed. Returns
+ * `{ready:true, ...}` ONLY for a valid 200 body with `status:"ready"` and (when
+ * requested) a matching pid.
+ */
+export async function probeReadiness(
+  port: number,
+  opts: { hostname?: string; expectedPid?: number } = {},
+  io: ReadinessProbeIo = {},
+): Promise<ReadinessProbeResult | null> {
+  const fetchFn = io.fetchFn ?? fetch;
+  try {
+    const res = await fetchFn(`http://${probeHostname(opts.hostname)}:${port}/readyz`, {
+      signal: AbortSignal.timeout(io.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS),
+    });
+    // Parse even on 503: /readyz returns JSON with a sanitized status while pending.
+    const body = (await res.json().catch(() => null)) as unknown;
+    const parsed = validateReadyzBody(body, port, opts);
+    if (!parsed) return null;
+    // HTTP/body-status consistency: ready requires 200; pending/failed require 503.
+    if (parsed.status === "ready" && res.status !== 200) return null;
+    if (parsed.status !== "ready" && res.status !== 503) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}

package/src/server/readiness.ts

@@ -0,0 +1,99 @@
+/**
+ * Per-server readiness gate for the opencodex proxy.
+ *
+ * `GET /healthz` answers "is the process alive and serving HTTP?" the instant the
+ * listener binds. Readiness is stricter: the proxy is "ready" only after the
+ * post-startup Codex catalog/config sync (`syncModelsToCodex`) has settled with
+ * `ok=true` and no catalog-sync warning. Until then the process is live (Codex can
+ * open a socket) but not ready (a request would race the sync or hit a stale
+ * catalog), so clients should back off.
+ *
+ * Design contract (per P1 review):
+ *  - NO module-global mutable state. Each `startServer` invocation gets its own
+ *    private gate via `createReadinessGate()`, captured by that listener's
+ *    closure. Starting/failing a second server in the same process can never
+ *    reset or mutate the first server's gate.
+ *  - Only the fixed sanitized status enum `pending | ready | failed` is stored
+ *    and exposed. There is no `changedAt`, no free-form failure reason, no sync
+ *    message, no warning text, no catalog path, no provider output, and no
+ *    account data — those are private diagnostic data and are never exposed by
+ *    `/readyz`.
+ */
+
+/** Sanitized readiness state. Exactly these three values, nothing else. */
+export type ReadinessStatus = "pending" | "ready" | "failed";
+
+/**
+ * Private per-server readiness controller. The status starts at `pending` and
+ * transitions at most once (to `ready` or `failed`) when the post-startup sync
+ * settles. The gate is owned by the listener closure that requested it.
+ */
+export interface ReadinessGate {
+  /** Current sanitized status. */
+  getStatus(): ReadinessStatus;
+  /** Mark the proxy ready (post-startup sync settled cleanly). */
+  markReady(): void;
+  /** Mark the proxy failed. No reason is stored or exposed. */
+  markFailed(): void;
+}
+
+/**
+ * Create a fresh private gate for one `startServer` invocation. The returned
+ * gate is the only way to read or mutate this server's readiness.
+ */
+export function createReadinessGate(): ReadinessGate {
+  let status: ReadinessStatus = "pending";
+  return {
+    getStatus: () => status,
+    markReady: () => {
+      if (status === "pending") status = "ready";
+    },
+    markFailed: () => {
+      if (status === "pending") status = "failed";
+    },
+  };
+}
+
+/** Minimal shape of the post-startup sync outcome the gate cares about. */
+export interface SyncOutcomeLike {
+  ok?: boolean;
+  warning?: string;
+  /** #1046: whether the sync actually rewrote the on-disk catalog/cache. */
+  catalogWritten?: boolean;
+  cacheSynced?: boolean;
+}
+
+/**
+ * Drive the gate from the post-startup sync. Awaits `syncFn`; the gate goes to
+ * `ready` ONLY on `ok=true` with no nonempty warning. A throw, `null`, `ok=false`,
+ * or a nonempty warning transitions to `failed`. Used directly by `handleStart`
+ * so the startup transition is unit-testable without spawning the proxy. Returns
+ * the raw sync outcome so a caller that also needs the #1046 write flags (did the
+ * sync actually write the catalog/cache?) can keep them without a second call.
+ */
+export async function runStartupReadinessSync(
+  gate: ReadinessGate,
+  syncFn: () => Promise<SyncOutcomeLike | null>,
+): Promise<SyncOutcomeLike | null> {
+  let result: SyncOutcomeLike | null;
+  try {
+    result = await syncFn();
+  } catch {
+    gate.markFailed();
+    return null;
+  }
+  if (result === null) {
+    gate.markFailed();
+    return null;
+  }
+  if (result.ok !== true) {
+    gate.markFailed();
+    return result;
+  }
+  if (result.warning !== undefined && result.warning !== "") {
+    gate.markFailed();
+    return result;
+  }
+  gate.markReady();
+  return result;
+}