@iislee/opencodex 2.11.0 → 2.31.0

package/src/server/management/system-restart.ts

@@ -17,45 +17,123 @@
  * - If detached spawn fails (sync throw or pre-start `error`): exit(1) without
  *   markRecycling — after drain the listen socket is already closed, so a latch
  *   reset cannot recover serving. Clear inherited `OCX_SERVICE` so exit cleanup
- *   can restore Codex/Grok fences (ensure/tray daemons set the marker without a
- *   real supervisor). Log only a stable errno code — never the raw message
+ *   can restore Codex/Grok fences when a stale service marker has no viable
+ *   supervisor. Log only a stable errno code — never the raw message
  *   (paths in ENOENT often include the OS username).
  */
 import { spawn } from "node:child_process";
 import {
+  beginShutdownDrain,
   drainAndShutdown,
   getActiveTurnCount,
   getServerListenPort,
   isDraining,
+  isShutdownDraining,
   markRecyclingForExit,
-  setDraining,
+  stopServerListener,
 } from "../lifecycle";
 import { isServiceViable } from "../../service";
 import { readRuntimePort } from "../../config";
+import { withProcessRuntimeProvenance } from "../../lib/bun-runtime";
+import { selfLaunchArgv } from "../../lib/self-launch-argv";
+import {
+  MEMORY_DRAIN_RESTART_MS,
+  REPLACEMENT_READY_TIMEOUT_MS,
+} from "../../lib/system-restart-contract";
+import { findLiveProxy } from "../proxy-liveness";
+
+export { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "../../lib/system-restart-contract";
+export const DEADLINE_LISTENER_STOP_TIMEOUT_MS = 5_000;
+const REPLACEMENT_READY_POLL_MS = 150;
 
-/** Fixed v1 drain window for the memory-card action (not config-driven). */
-export const MEMORY_DRAIN_RESTART_MS = 60_000;
+export interface ReplacementReadinessIo {
+  findLive?: typeof findLiveProxy;
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+}
 
 export interface SystemRestartIo {
   drainAndShutdown?: typeof drainAndShutdown;
   /** True when a background service can actually respawn this process after exit(1). */
   isServiceViable?: () => boolean;
   isSupervisedServiceChild?: () => boolean;
-  /** Must resolve only after the replacement process has actually started. */
-  spawnStart?: (port?: number) => void | Promise<void>;
+  /** Ordinary start; deadline handoff may defer health until parent exit releases OS locks. */
+  spawnStart?: (port?: number, waitForHealthBeforeParentExit?: boolean) => void | Promise<void>;
+  /** Idempotent listener close; must settle before an ordinary start is spawned. */
+  stopListener?: () => void | Promise<void>;
   markRecycling?: () => void;
   exitProcess?: (code: number) => void;
   schedule?: (fn: () => void | Promise<void>, ms: number) => void;
+  scheduleDeadline?: (fn: () => void, ms: number) => () => void;
   isDraining?: () => boolean;
+  isShutdownDraining?: () => boolean;
+  beginShutdownDrain?: () => boolean;
   setDraining?: (value: boolean) => void;
   getActiveTurnCount?: () => number;
   listenPort?: () => number | undefined;
+  now?: () => number;
 }
 
 let restartIo: SystemRestartIo = {};
 /** Prevents double-scheduling in the 200ms window before drainAndShutdown sets draining. */
 let restartAccepted = false;
 
+type RestartDrainOutcome = "completed" | "rejected" | "deadline";
+type BoundedSettlementOutcome = "completed" | "rejected" | "deadline";
+
+function waitForRestartDrain(
+  drainPromise: Promise<void>,
+  deadlineMs: number,
+  now: () => number,
+  scheduleDeadline: NonNullable<SystemRestartIo["scheduleDeadline"]>,
+): Promise<RestartDrainOutcome> {
+  const remainingMs = Math.max(0, deadlineMs - now());
+  if (remainingMs === 0) {
+    // Observe any late rejection even though orchestration is already terminal.
+    void drainPromise.catch(() => {});
+    return Promise.resolve("deadline");
+  }
+  return new Promise<RestartDrainOutcome>((resolve) => {
+    let settled = false;
+    let cancelDeadline: (() => void) | undefined;
+    const finish = (outcome: RestartDrainOutcome) => {
+      if (settled) return;
+      settled = true;
+      cancelDeadline?.();
+      resolve(outcome);
+    };
+    cancelDeadline = scheduleDeadline(() => finish("deadline"), remainingMs);
+    if (settled) cancelDeadline();
+    void drainPromise.then(
+      () => finish("completed"),
+      () => finish("rejected"),
+    );
+  });
+}
+
+function waitForBoundedSettlement(
+  promise: Promise<void>,
+  timeoutMs: number,
+  scheduleDeadline: NonNullable<SystemRestartIo["scheduleDeadline"]>,
+): Promise<BoundedSettlementOutcome> {
+  return new Promise<BoundedSettlementOutcome>((resolve) => {
+    let settled = false;
+    let cancelDeadline: (() => void) | undefined;
+    const finish = (outcome: BoundedSettlementOutcome) => {
+      if (settled) return;
+      settled = true;
+      cancelDeadline?.();
+      resolve(outcome);
+    };
+    cancelDeadline = scheduleDeadline(() => finish("deadline"), timeoutMs);
+    if (settled) cancelDeadline();
+    void promise.then(
+      () => finish("completed"),
+      () => finish("rejected"),
+    );
+  });
+}
+
 /** Test seam — reset between tests. */
 export function setSystemRestartIoForTests(io: SystemRestartIo = {}): void {
   restartIo = io;
@@ -86,42 +164,180 @@ function spawnFailureCode(err: unknown): string {
   return "spawn_failed";
 }
 
-function spawnDetachedStart(port?: number): Promise<void> {
-  const args = [process.argv[1], "start"];
-  if (typeof port === "number" && Number.isFinite(port) && port > 0 && port <= 65535) {
-    args.push("--port", String(Math.trunc(port)));
+function handoffError(code: string): NodeJS.ErrnoException {
+  const error = new Error(code) as NodeJS.ErrnoException;
+  error.code = code;
+  return error;
+}
+
+export async function waitForReplacementReady(
+  expectedPid: number | undefined,
+  parentPid: number,
+  expectedPort: number | undefined,
+  io: ReplacementReadinessIo = {},
+): Promise<boolean> {
+  const findLive = io.findLive ?? findLiveProxy;
+  const now = io.now ?? Date.now;
+  const sleep = io.sleep ?? Bun.sleep;
+  const deadline = now() + REPLACEMENT_READY_TIMEOUT_MS;
+  while (now() < deadline) {
+    try {
+      const live = await findLive({
+        deadlineAt: deadline,
+        nowFn: now,
+        sleepFn: sleep,
+      });
+      // A probe that began within budget can still return after it. Never accept
+      // delayed health as proof once the shared absolute handoff budget expired.
+      if (now() >= deadline) return false;
+      if (
+        live
+        && live.pid !== null
+        && live.pid !== parentPid
+        && (expectedPid === undefined || live.pid === expectedPid)
+        && (expectedPort === undefined || live.port === expectedPort)
+      ) {
+        return true;
+      }
+    } catch {
+      // A not-yet-bound replacement is indistinguishable from a transient
+      // liveness failure here; keep polling inside the one bounded window.
+    }
+    const remainingMs = deadline - now();
+    if (remainingMs <= 0) break;
+    await sleep(Math.min(REPLACEMENT_READY_POLL_MS, remainingMs));
   }
+  return false;
+}
+
+function spawnDetachedStart(
+  port?: number,
+  waitForHealthBeforeParentExit = true,
+): Promise<void> {
+  const args = ["start"];
+  const expectedPort = typeof port === "number" && Number.isFinite(port) && port > 0 && port <= 65535
+    ? Math.trunc(port)
+    : undefined;
+  if (expectedPort !== undefined) {
+    args.push("--port", String(expectedPort));
+  }
+  const launchArgs = selfLaunchArgv(args);
   return new Promise<void>((resolve, reject) => {
     let child: ReturnType<typeof spawn>;
     try {
-      child = spawn(process.execPath, args, {
+      const env: NodeJS.ProcessEnv = { ...process.env };
+      delete env.OCX_SERVICE;
+      child = spawn(process.execPath, launchArgs, {
         detached: true,
         stdio: "ignore",
         windowsHide: true,
-        env: { ...process.env, OCX_SERVICE: "1" },
+        env: withProcessRuntimeProvenance(env),
       });
     } catch (err) {
       reject(err);
       return;
     }
     let settled = false;
-    const finish = (fn: () => void) => {
+    const cleanup = () => {
+      child.off("error", onError);
+      child.off("exit", onExit);
+      child.off("spawn", onSpawn);
+    };
+    const finish = (error?: unknown) => {
       if (settled) return;
       settled = true;
-      fn();
+      cleanup();
+      if (error !== undefined) {
+        if (child.exitCode === null && child.signalCode === null) {
+          try { child.kill(); } catch { /* best-effort failed-start cleanup */ }
+        }
+        try { child.unref(); } catch { /* best-effort */ }
+        reject(error);
+        return;
+      }
+      child.unref();
+      resolve();
     };
-    child.once("error", (err) => {
-      finish(() => reject(err));
-    });
-    child.once("spawn", () => {
-      finish(() => {
-        child.unref();
-        resolve();
-      });
-    });
+    const onError = (err: Error) => { finish(err); };
+    const onExit = () => { finish(handoffError("child_exit")); };
+    const onSpawn = () => {
+      if (!waitForHealthBeforeParentExit) {
+        // A deadline may have been caused by native-main ownership cleanup.
+        // Let the ordinary child survive parent exit, which releases those OS locks.
+        finish();
+        return;
+      }
+      void waitForReplacementReady(child.pid, process.pid, expectedPort).then(
+        ready => {
+          if (!ready) {
+            console.warn(
+              "Drain-and-restart replacement is still alive after the readiness window; allowing it to continue after parent exit",
+            );
+          }
+          // Never kill a live ordinary start at its valid reclaim boundary. Parent
+          // exit is the final resource release the child may still be waiting for.
+          finish();
+        },
+        err => finish(err),
+      );
+    };
+    child.once("error", onError);
+    child.once("exit", onExit);
+    child.once("spawn", onSpawn);
   });
 }
 
+async function completeDeferredParentExitHandoff(
+  io: SystemRestartIo,
+  exitProcess: (code: number) => void,
+  port: number | undefined,
+  phase: "deadline" | "listener-stop fallback",
+): Promise<void> {
+  try {
+    await (io.spawnStart ?? spawnDetachedStart)(port, false);
+  } catch (err) {
+    console.warn(
+      `Drain-and-restart ${phase} spawn failed (${spawnFailureCode(err)}); exiting without replacement`,
+    );
+    delete process.env.OCX_SERVICE;
+    exitProcess(1);
+    return;
+  }
+  (io.markRecycling ?? markRecyclingForExit)();
+  exitProcess(0);
+}
+
+async function completeDeadlineRestartHandoff(
+  io: SystemRestartIo,
+  exitProcess: (code: number) => void,
+  port: number | undefined,
+  scheduleDeadline: NonNullable<SystemRestartIo["scheduleDeadline"]>,
+): Promise<void> {
+  const supervised = (io.isSupervisedServiceChild ?? (() => isSupervisedServiceChild(io)))();
+  if (supervised) {
+    // Failure-only supervisors ignore exit(0); intentional non-zero triggers respawn.
+    exitProcess(1);
+    return;
+  }
+
+  const stopPromise = Promise.resolve().then(
+    () => (io.stopListener ?? (() => stopServerListener()))(),
+  );
+  const stopOutcome = await waitForBoundedSettlement(
+    stopPromise,
+    DEADLINE_LISTENER_STOP_TIMEOUT_MS,
+    scheduleDeadline,
+  );
+  if (stopOutcome === "rejected") {
+    console.warn("Drain-and-restart deadline listener stop failed; continuing parent-exit handoff");
+  } else if (stopOutcome === "deadline") {
+    console.warn("Drain-and-restart deadline listener stop timed out; continuing parent-exit handoff");
+  }
+  // The ordinary child must survive parent exit: a failed/pending socket close
+  // or overdue cleanup is completed by process teardown, without a hidden mode.
+  await completeDeferredParentExitHandoff(io, exitProcess, port, "deadline");
+}
+
 /**
  * Accept a drain-and-restart request. Returns immediately; the drain +
  * respawn runs on a short timer so the HTTP response can flush first.
@@ -133,34 +349,74 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo): {
   activeTurnCount: number;
   drainTimeoutMs: number;
 } {
-  const alreadyDraining = restartAccepted || (io.isDraining ?? isDraining)();
+  const shutdownActive = io.isShutdownDraining
+    ?? io.isDraining
+    ?? isShutdownDraining;
+  const alreadyDraining = restartAccepted || shutdownActive();
   const activeTurnCount = (io.getActiveTurnCount ?? getActiveTurnCount)();
   const schedule = io.schedule ?? ((fn, ms) => { setTimeout(() => { void fn(); }, ms); });
 
   if (!alreadyDraining) {
     restartAccepted = true;
+    const now = io.now ?? Date.now;
+    const restartDeadlineMs = now() + MEMORY_DRAIN_RESTART_MS;
     // Reject new data-plane traffic immediately (503), before the 200ms response-flush delay.
-    (io.setDraining ?? setDraining)(true);
+    if (io.beginShutdownDrain) io.beginShutdownDrain();
+    else if (io.setDraining) io.setDraining(true);
+    else beginShutdownDrain();
     schedule(async () => {
+      // Preserve the live binding before drainAndShutdown (or its deadline race)
+      // closes the listener and makes both the server ref and runtime metadata stale.
+      const restartPort = (io.listenPort ?? resolveListenPort)();
       const drain = io.drainAndShutdown ?? drainAndShutdown;
-      await drain(undefined, MEMORY_DRAIN_RESTART_MS);
+      const remainingMs = Math.max(0, restartDeadlineMs - now());
+      const drainPromise = Promise.resolve().then(() => drain(undefined, remainingMs));
+      const scheduleDeadline = io.scheduleDeadline ?? ((fn, ms) => {
+        const timer = setTimeout(fn, ms);
+        return () => clearTimeout(timer);
+      });
+      const drainOutcome = await waitForRestartDrain(drainPromise, restartDeadlineMs, now, scheduleDeadline);
+      const exitProcess = io.exitProcess ?? ((code: number) => { process.exit(code); });
+      if (drainOutcome === "deadline") {
+        console.warn("Drain-and-restart deadline expired; forcing terminal restart handoff");
+        await completeDeadlineRestartHandoff(io, exitProcess, restartPort, scheduleDeadline);
+        return;
+      }
+      if (drainOutcome === "rejected") {
+        // drainAndShutdown stops the listener in finally. Even if ancillary cleanup
+        // rejects, an accepted restart must still reach replacement or terminal exit.
+        console.warn("Drain-and-restart cleanup failed; continuing terminal restart handoff");
+      }
       const supervised = (io.isSupervisedServiceChild ?? (() => isSupervisedServiceChild(io)))();
       if (supervised) {
         // Failure-only supervisors ignore exit(0); intentional non-zero triggers respawn.
         (io.exitProcess ?? ((code: number) => { process.exit(code); }))(1);
         return;
       }
-      const port = (io.listenPort ?? resolveListenPort)();
-      const exitProcess = io.exitProcess ?? ((code: number) => { process.exit(code); });
       try {
-        await (io.spawnStart ?? spawnDetachedStart)(port);
+        await (io.stopListener ?? (() => stopServerListener()))();
+      } catch {
+        console.warn("Drain-and-restart listener stop failed; continuing parent-exit handoff");
+        await completeDeferredParentExitHandoff(
+          io,
+          exitProcess,
+          restartPort,
+          "listener-stop fallback",
+        );
+        return;
+      }
+      try {
+        // A rejected drain has uncertain cleanup ownership, so it uses the same
+        // parent-exit handoff as a deadline. Only a fully completed drain waits
+        // for replacement health in the old process.
+        await (io.spawnStart ?? spawnDetachedStart)(restartPort, drainOutcome === "completed");
       } catch (err) {
         console.warn(
           `⚠️  Drain-and-restart spawn failed (${spawnFailureCode(err)}); exiting without replacement`,
         );
         // Listen socket is already stopped; do not markRecycling — no child to inherit fences.
-        // ensure/tray children inherit OCX_SERVICE=1 without an installed service; clear it so
-        // syncCleanup can restore Codex/Grok fences instead of leaving clients pointed at a dead port.
+        // No replacement inherited the routing. Clear a stale service marker so
+        // this unsupervised parent restores clients after the failed handoff.
         delete process.env.OCX_SERVICE;
         exitProcess(1);
         return;

package/src/server/management/system-routes.ts

@@ -22,12 +22,26 @@
  * dashboard drain-and-restart confirm UX — never request bodies or IDs.
  */
 import { selectEagerPath } from "../../lib/bun-stream-caps";
+import { reportedBunRuntimeSource } from "../../lib/bun-runtime";
 import { getActiveTurnCount, isDraining } from "../lifecycle";
 import { getActiveMemoryWatchdog, observedMemoryCounter } from "../memory-watchdog";
 import { responseStateMetrics } from "../../responses/state";
 import { appOwnedBytesSnapshot } from "../../lib/app-owned-memory";
+import { readWindowsReplaceRetryCounters } from "../../lib/windows-atomic-replace";
+import {
+  SYSTEM_RESTART_EXPECTED_PID_HEADER,
+  parseExpectedSystemRestartPid,
+} from "../../lib/system-restart-contract";
+import {
+  CODEX_APP_SERVER_STATE_PATH,
+  CODEX_RESTART_PATH,
+} from "../../lib/codex-restart-contract";
 import { jsonResponse } from "../auth-cors";
 import { getInspectionCounters } from "../relay";
+import type {
+  performCodexRestart,
+  readCodexAppServerState,
+} from "../../codex/app-server-restart-service";
 import type { ManagementContext } from "./context";
 import { acceptSystemRestart } from "./system-restart";
 
@@ -78,6 +92,9 @@ export async function handleSystemRoutes(ctx: ManagementContext): Promise<Respon
       pid: process.pid,
       bunVersion: Bun.version,
       bunRevision: Bun.revision,
+      // Recorded at launch, not resolved now: absent means "this service predates the
+      // marker", which callers must report as unknown rather than guess.
+      bunRuntimeSource: reportedBunRuntimeSource(),
       platform: process.platform,
       uptimeSeconds: process.uptime(),
       rss: usage.rss,
@@ -99,7 +116,37 @@ export async function handleSystemRoutes(ctx: ManagementContext): Promise<Respon
     });
   }
 
+  /**
+   * Windows atomic-replace retry counters.
+   *
+   * A sibling route rather than a field on /api/system/memory: that payload is
+   * memory-shaped, and appending unrelated filesystem counters to it makes both
+   * harder to consume.
+   *
+   * Keys are `publisher:CODE` where publisher is a closed union
+   * (ReplacePublisher) and never a path — a path could carry a username. The
+   * type is what enforces that; a text scan cannot see a runtime value.
+   */
+  if (url.pathname === "/api/system/windows-replace-retries" && req.method === "GET") {
+    return jsonResponse({ counters: readWindowsReplaceRetryCounters() });
+  }
   if (url.pathname === "/api/system/restart" && req.method === "POST") {
+    const expectedPid = parseExpectedSystemRestartPid(
+      req.headers.get(SYSTEM_RESTART_EXPECTED_PID_HEADER),
+    );
+    if (expectedPid.kind === "invalid") {
+      return jsonResponse({
+        success: false,
+        error: "Invalid restart target identity.",
+      }, 400, req, config);
+    }
+    if (expectedPid.kind === "present" && expectedPid.pid !== process.pid) {
+      return jsonResponse({
+        success: false,
+        error: "Restart target identity changed.",
+      }, 409, req, config);
+    }
+
     // Longer informed drain than /api/stop; does not tear down Codex/Grok injection.
     const result = acceptSystemRestart();
     return jsonResponse({
@@ -113,5 +160,35 @@ export async function handleSystemRoutes(ctx: ManagementContext): Promise<Respon
     }, 202, req, config);
   }
 
+  if (
+    (url.pathname === CODEX_APP_SERVER_STATE_PATH && req.method === "GET")
+    || (url.pathname === CODEX_RESTART_PATH && req.method === "POST")
+  ) {
+    // Resolved inside the path check, not at the top of this function: every
+    // /api/system/* request runs through here, and an unconditional import would
+    // pull the platform process-enumeration helpers into requests that never
+    // touch them.
+    // An explicit branch rather than `??`: the seam is an optional property, and
+    // narrowing through a nullish default keeps its `undefined` in the union.
+    let service: {
+      readState: typeof readCodexAppServerState;
+      performRestart: typeof performCodexRestart;
+    };
+    const injected = ctx.deps.codexRestartService;
+    if (injected) {
+      service = injected;
+    } else {
+      const module = await import("../../codex/app-server-restart-service");
+      service = {
+        readState: module.readCodexAppServerState,
+        performRestart: module.performCodexRestart,
+      };
+    }
+    if (req.method === "GET") {
+      return jsonResponse(service.readState(), 200, req, config);
+    }
+    return jsonResponse(await service.performRestart(), 200, req, config);
+  }
+
   return null;
 }

package/src/server/management/usage-summary-cache.ts

@@ -8,9 +8,17 @@ export type CachedUsageSummary = UsageSummary & {
   entriesDropped: number;
 };
 
-interface UsageSummaryCacheEntry {
+export interface UsageSummaryCacheEntry {
   revisionKey: string;
+  /** path/dev/ino/birthtime only; appends keep this stable. */
+  identityKey: string;
+  maxReadBytes: number;
+  /** userCostOverlayVersion() when the summary was computed; overlay edits invalidate the entry. */
+  overlayVersion: number;
   expiresAt: number;
+  /** Generation freshness: ignore size/mtime until this instant. */
+  freshUntil: number;
+  lastSeenSize: number;
   summary: CachedUsageSummary;
   revisionReadAt: number;
   sizeBytes: number;