@bitkyc08/opencodex 2.67.0 → 2.68.0

package/src/codex/routing-healer.ts

@@ -0,0 +1,419 @@
+import { readFileSync } from "node:fs";
+import { loadConfig } from "../config";
+import { readRuntimePort } from "../config/process-state";
+import { readClientConnectionState } from "../client/state";
+import { appendDebugLogLine } from "../lib/debug-log-buffer";
+import { redactUserPath } from "../lib/redact";
+import { isRecyclingForExit, isShutdownDraining } from "../server/lifecycle";
+import { probeEndpointLiveness, type EndpointLiveness } from "../server/proxy-liveness";
+import type { OcxConfig } from "../types";
+import { shouldSyncCodexOnStart } from "./desired-state";
+import { injectCodexConfig, type CodexInjectResult, type InjectCodexOptions } from "./inject";
+import { standaloneCodexRoutingTarget } from "./inject/routing-target";
+import { journaledInjectedOpenaiBaseUrl, journaledInjectedRealtimeWsBaseUrl, journalOwner } from "./journal";
+import { effectiveLoopbackListenerPort, shouldInjectApiAuthHeader } from "./loopback-target";
+import { CODEX_CONFIG_PATH } from "./paths";
+import {
+  detectCodexRoutingDrift,
+  type CodexRoutingDrift,
+  type CodexRoutingDriftTarget,
+  type JournaledCodexRouting,
+} from "./routing-drift";
+import { siblingOfLivePort } from "./sibling-start";
+
+/**
+ * The live owner re-points Codex routing that a dead instance left on another loopback port.
+ *
+ * Startup sync is the last time an owner looks at `config.toml`. When something else re-points the
+ * opencodex-owned routing at its own port and then dies, every new Codex thread fails with
+ * "Connection refused" while this proxy keeps serving its own port. This loop closes that gap.
+ *
+ * Cost: one read of `config.toml` every 10 s from an unref'd timer, and nothing else while the
+ * bytes are unchanged. Probes, gates and the journal run only while owned routing names a port this
+ * process does not serve. Nothing here touches the request path.
+ *
+ * It writes only when every foreign target has been dead on at least two probes spanning 20 s, a
+ * final probe still says dead, and every gate is open. The write is the ordinary injector with no
+ * catalog path (so no catalog gather and the current `model_catalog_json` stays), a short lock
+ * timeout, and a synchronous guard that re-reads `config.toml` under the lock and aborts when the
+ * routing changed since the probe. A refusal that raced another write of `config.toml` is the same
+ * abort, not a refusal. A live foreign opencodex is never fought; an unknown answer never advances
+ * the streak. The caps pause the loop for at most an hour and never stop it. Detection, gates and
+ * probes read the journal read-only, so watching never deletes an unreadable journal (a heal write
+ * goes through the injector's ordinary journal handling). Started only from `handleStart`'s owner
+ * path and stopped first in its exit cleanup.
+ */
+
+export const ROUTING_HEAL_TICK_MS = 10_000;
+export const ROUTING_HEAL_MIN_DEAD_PROBES = 2;
+export const ROUTING_HEAL_MIN_DEAD_SPAN_MS = 20_000;
+export const ROUTING_HEAL_LOCK_TIMEOUT_MS = 1_000;
+/** While the bytes stay equal, a live foreign owner, an unknown answer or a closed gate is looked at again only this often. */
+export const ROUTING_HEAL_RECHECK_MS = 30_000;
+export const ROUTING_HEAL_REFUSED_BACKOFF_MS = 10 * 60_000;
+export const ROUTING_HEAL_WINDOW_MS = 60 * 60_000;
+/** Rate cap: write attempts (busy ones included) per rolling window; more pause until the oldest leaves it. */
+export const ROUTING_HEAL_MAX_ATTEMPTS = 6;
+/**
+ * Flap cap: this many successful heals in one window means another writer keeps re-pointing it, so a
+ * further heal waits until the oldest leaves the window, then needs a fresh dead streak.
+ */
+export const ROUTING_HEAL_MAX_HEALS = 3;
+
+export type CodexRoutingHealGate =
+  | "sibling"
+  | "exiting"
+  | "runtime-record"
+  | "codex-off"
+  | "admission-token"
+  | "target-not-served"
+  | "client";
+
+export interface CodexRoutingHealGates {
+  siblingOfLivePort(): number | null;
+  /** True while the process is recycling for exit or draining for shutdown. */
+  exiting(): boolean;
+  /** The port in this home's runtime record when that record names this process. */
+  runtimePortOfThisProcess(): number | null;
+  loadConfig(): OcxConfig;
+  /** Any client-connection state other than disconnected (connected, link, invalid). */
+  clientConnected(): boolean;
+  /** The journal names a connected client as its owner (read-only). */
+  clientJournalOwner(): boolean;
+}
+
+export type CodexRoutingHealGateResult =
+  | { readonly open: true; readonly config: OcxConfig; readonly targetPort: number }
+  | { readonly open: false; readonly gate: CodexRoutingHealGate };
+
+/** Cheapest first; `loadConfig` and the journal only when the in-memory gates are open. */
+export function evaluateCodexRoutingHealGates(
+  port: number,
+  ownPorts: ReadonlySet<number>,
+  gates: CodexRoutingHealGates,
+): CodexRoutingHealGateResult {
+  if (gates.siblingOfLivePort() !== null) return { open: false, gate: "sibling" };
+  if (gates.exiting()) return { open: false, gate: "exiting" };
+  if (gates.runtimePortOfThisProcess() !== port) return { open: false, gate: "runtime-record" };
+  const config = gates.loadConfig();
+  // Codex ON and not hub-gated; the injector re-checks this under its lock.
+  if (!shouldSyncCodexOnStart(config)) return { open: false, gate: "codex-off" };
+  if (shouldInjectApiAuthHeader(config)) return { open: false, gate: "admission-token" };
+  const targetPort = Number(new URL(standaloneCodexRoutingTarget(port, config).baseUrl).port);
+  if (!ownPorts.has(targetPort)) return { open: false, gate: "target-not-served" };
+  if (gates.clientConnected() || gates.clientJournalOwner()) return { open: false, gate: "client" };
+  return { open: true, config, targetPort };
+}
+
+export interface CodexRoutingHealRecord {
+  readonly at: string;
+  readonly fromUrls: readonly string[];
+  readonly toPort: number;
+  readonly probes: number;
+  readonly deadForMs: number;
+}
+
+export interface CodexRoutingHealerHandle {
+  stop(): void;
+  lastHeal(): CodexRoutingHealRecord | null;
+}
+
+export interface CodexRoutingHealerDeps {
+  /** Schedule `fn` after `ms`; defaults to an unref'd setTimeout. */
+  scheduleFn?: (fn: () => void, ms: number) => { cancel(): void };
+  /** Monotonic milliseconds. */
+  now?: () => number;
+  /** `config.toml` text, or null when absent or unreadable. */
+  readConfig?: () => string | null;
+  readJournaled?: () => JournaledCodexRouting;
+  probe?: (target: { port: number; hostname: string }) => Promise<EndpointLiveness>;
+  inject?: (port: number, config: OcxConfig, options: InjectCodexOptions) => Promise<CodexInjectResult>;
+  gates?: Partial<CodexRoutingHealGates>;
+  log?: Pick<Console, "warn">;
+  debugLine?: (line: string) => void;
+}
+
+/** Thrown by the under-lock guard: the routing changed after it was proven dead. */
+class RoutingHealAborted extends Error {
+  constructor() {
+    super("Codex routing changed before the heal could be written.");
+    this.name = "RoutingHealAborted";
+  }
+}
+
+function defaultSchedule(fn: () => void, ms: number): { cancel(): void } {
+  const timer = setTimeout(fn, ms);
+  if (typeof timer.unref === "function") timer.unref();
+  return { cancel: () => clearTimeout(timer) };
+}
+
+function readCodexConfigText(): string | null {
+  try {
+    return readFileSync(CODEX_CONFIG_PATH, "utf8");
+  } catch {
+    return null;
+  }
+}
+
+const defaultGates: CodexRoutingHealGates = {
+  siblingOfLivePort,
+  exiting: () => isRecyclingForExit() || isShutdownDraining(),
+  runtimePortOfThisProcess: () => readRuntimePort(process.pid)?.port ?? null,
+  loadConfig,
+  clientConnected: () => readClientConnectionState().kind !== "disconnected",
+  clientJournalOwner: () => journalOwner({ readOnly: true })?.kind === "client",
+};
+
+/** The first line of an injector or error message, with home paths and secret-shaped values masked. */
+function logDetail(message: string): string {
+  return redactUserPath(message.split("\n")[0] ?? "");
+}
+
+function endpointLabel(target: CodexRoutingDriftTarget): string {
+  return `${target.hostname.includes(":") ? `[${target.hostname}]` : target.hostname}:${target.port}`;
+}
+
+function endpointKey(target: Pick<CodexRoutingDriftTarget, "hostname" | "port">): string {
+  return JSON.stringify([target.hostname, target.port]);
+}
+
+export function startCodexRoutingHealer(options: {
+  port: number;
+  config: OcxConfig;
+  deps?: CodexRoutingHealerDeps;
+}): CodexRoutingHealerHandle {
+  const { port, config } = options;
+  const deps = options.deps ?? {};
+  let stopped = false;
+  let pending: { cancel(): void } | undefined;
+  let last: CodexRoutingHealRecord | null = null;
+  const handle: CodexRoutingHealerHandle = {
+    stop() {
+      stopped = true;
+      pending?.cancel();
+      pending = undefined;
+    },
+    lastHeal: () => last,
+  };
+  // Admission-token (non-loopback) routing is never healed, and the bind scope is fixed for the
+  // process lifetime, so such an owner starts no timer at all.
+  if (shouldInjectApiAuthHeader(config)) return handle;
+
+  const listenerPort = effectiveLoopbackListenerPort(config, port);
+  const ownPorts: ReadonlySet<number> = new Set(listenerPort === null ? [port] : [port, listenerPort]);
+  const scheduleFn = deps.scheduleFn ?? defaultSchedule;
+  const clock = deps.now ?? (() => performance.now());
+  const readConfig = deps.readConfig ?? readCodexConfigText;
+  const readJournaled = deps.readJournaled ?? ((): JournaledCodexRouting => ({
+    openaiBaseUrl: journaledInjectedOpenaiBaseUrl({ readOnly: true }),
+    realtimeWsBaseUrl: journaledInjectedRealtimeWsBaseUrl({ readOnly: true }),
+  }));
+  const probe = deps.probe ?? (target => probeEndpointLiveness(target));
+  const inject = deps.inject ?? injectCodexConfig;
+  const gates: CodexRoutingHealGates = { ...defaultGates, ...deps.gates };
+  const log = deps.log ?? console;
+  const debugLine = deps.debugLine ?? appendDebugLogLine;
+
+  // The verdict for the last bytes read. Unchanged bytes with a non-foreign verdict cost nothing.
+  let cache: { content: string; kind: CodexRoutingDrift["kind"] } | null = null;
+  let streak: { key: string; firstDeadAt: number; probes: number } | null = null;
+  let recheckAt = 0;
+  /** After a real refusal; cleared as soon as routing is not foreign any more. */
+  let backoffUntil = 0;
+  /** While a cap is reached: until the oldest counted attempt or heal leaves the window. */
+  let pausedUntil = 0;
+  const announcedLive = new Set<string>();
+  const attempts: number[] = [];
+  const heals: number[] = [];
+  let rateCapAnnounced = false;
+  let flapCapAnnounced = false;
+
+  const warn = (line: string) => {
+    try { log.warn(line); } catch { /* logging never breaks the loop */ }
+    try { debugLine(line); } catch { /* the debug buffer is best-effort */ }
+  };
+
+  const detect = (content: string): CodexRoutingDrift =>
+    detectCodexRoutingDrift(content, { ownPorts, journaled: readJournaled });
+
+  /** "live" wins over "unknown", which wins over "dead": all must be dead to proceed. */
+  const probeAll = async (targets: readonly CodexRoutingDriftTarget[]): Promise<{ verdict: EndpointLiveness; liveEndpoints: CodexRoutingDriftTarget[] }> => {
+    const byEndpoint = new Map<string, CodexRoutingDriftTarget>();
+    for (const target of targets) byEndpoint.set(endpointKey(target), target);
+    const liveEndpoints: CodexRoutingDriftTarget[] = [];
+    let unknown = false;
+    for (const target of byEndpoint.values()) {
+      let result: EndpointLiveness;
+      try { result = await probe({ port: target.port, hostname: target.hostname }); }
+      catch { result = "unknown"; }
+      if (result === "live") liveEndpoints.push(target);
+      else if (result === "unknown") unknown = true;
+    }
+    return { verdict: liveEndpoints.length > 0 ? "live" : unknown ? "unknown" : "dead", liveEndpoints };
+  };
+
+  const standDownForLive = (liveEndpoints: readonly CodexRoutingDriftTarget[]) => {
+    streak = null;
+    recheckAt = clock() + ROUTING_HEAL_RECHECK_MS;
+    for (const target of liveEndpoints) {
+      const key = endpointKey(target);
+      if (announcedLive.has(key)) continue;
+      announcedLive.add(key);
+      warn(`Codex routing points at another running opencodex on ${endpointLabel(target)}; leaving it.`);
+    }
+  };
+
+  /** A cap pauses probing and writing until `until`; the streak starts over once it has passed. */
+  const pauseUntil = (until: number) => {
+    streak = null;
+    pausedUntil = until;
+  };
+
+  const heal = async (
+    content: string,
+    targets: readonly CodexRoutingDriftTarget[],
+    proven: { firstDeadAt: number; probes: number },
+  ) => {
+    const now = clock();
+    while (attempts.length > 0 && now - attempts[0]! >= ROUTING_HEAL_WINDOW_MS) attempts.shift();
+    while (heals.length > 0 && now - heals[0]! >= ROUTING_HEAL_WINDOW_MS) heals.shift();
+    if (heals.length >= ROUTING_HEAL_MAX_HEALS) {
+      pauseUntil(heals[0]! + ROUTING_HEAL_WINDOW_MS);
+      if (!flapCapAnnounced) warn(`Codex routing was re-pointed at a dead port ${heals.length} times within an hour; the self-heal pauses until that hour has passed so it does not fight another writer. Run 'ocx sync' to repair routing now.`);
+      flapCapAnnounced = true;
+      return;
+    }
+    flapCapAnnounced = false;
+    if (attempts.length >= ROUTING_HEAL_MAX_ATTEMPTS) {
+      pauseUntil(attempts[0]! + ROUTING_HEAL_WINDOW_MS);
+      if (!rateCapAnnounced) warn("Codex routing self-heal paused after repeated attempts; it resumes within the hour. Run 'ocx sync' to repair now.");
+      rateCapAnnounced = true;
+      return;
+    }
+    rateCapAnnounced = false;
+    // One final look right before the write, then the gates again: the probe may have taken seconds.
+    const final = await probeAll(targets);
+    if (stopped) return; // the exit cleanup ran while the probe was out
+    if (final.verdict === "live") { standDownForLive(final.liveEndpoints); return; }
+    if (final.verdict !== "dead") { recheckAt = clock() + ROUTING_HEAL_RECHECK_MS; return; }
+    const gate = evaluateCodexRoutingHealGates(port, ownPorts, gates);
+    if (!gate.open) {
+      streak = null;
+      if (gate.gate === "sibling") stopped = true;
+      return;
+    }
+    attempts.push(clock());
+    const deadEndpoints = new Set(targets.map(endpointKey));
+    const guard = () => {
+      const current = readConfig();
+      const drift = current === null ? null : detect(current);
+      if (current !== content || drift?.kind !== "foreign"
+        || drift.targets.some(target => !deadEndpoints.has(endpointKey(target)))) {
+        throw new RoutingHealAborted();
+      }
+    };
+    /*
+     * Whether a failed write raced another writer. A coordinated home re-reads its admission under
+     * the lock BEFORE this guard runs, so a config.toml rewritten between the plan and the lock
+     * comes back as a plain refusal. Moved bytes are the same abort as the guard's; only a refusal
+     * over the very bytes that were proven dead backs off.
+     */
+    const raced = (): boolean => {
+      const current = readConfig();
+      if (current !== content) return true;
+      const drift = detect(current);
+      return drift.kind !== "foreign" || drift.targets.some(target => !deadEndpoints.has(endpointKey(target)));
+    };
+    let result: CodexInjectResult;
+    try {
+      result = await inject(port, gate.config, { lockTimeoutMs: ROUTING_HEAL_LOCK_TIMEOUT_MS, beforeClientWrite: guard });
+    } catch (error) {
+      streak = null;
+      cache = null;
+      if (error instanceof RoutingHealAborted || raced()) return;
+      backoffUntil = clock() + ROUTING_HEAL_REFUSED_BACKOFF_MS;
+      warn(`Codex routing self-heal failed (${logDetail(error instanceof Error ? error.message : String(error))}); retrying in 10 minutes. Run 'ocx sync' to repair now.`);
+      return;
+    }
+    if (result.success && result.status !== "skipped" && result.configApplied !== false) {
+      heals.push(clock());
+      streak = null;
+      cache = null;
+      const fromUrls = [...new Set(targets.map(target => target.url))];
+      last = {
+        at: new Date().toISOString(),
+        fromUrls,
+        toPort: gate.targetPort,
+        probes: proven.probes + 1,
+        deadForMs: Math.round(clock() - proven.firstDeadAt),
+      };
+      const from = [...new Set(targets.map(endpointLabel))].join(", ");
+      warn(`Codex routing pointed at ${from}, where no opencodex answers; re-pointed it at this proxy on port ${gate.targetPort}. Codex threads opened meanwhile keep the old address until you reopen them.`);
+      return;
+    }
+    // Busy lock: keep the proven streak so the next tick re-probes and tries again.
+    if (result.retryable) return;
+    streak = null;
+    cache = null;
+    // Skipped under the lock (Codex OFF, hub, sibling), an external provider appeared, or a race.
+    if (result.success || raced()) return;
+    backoffUntil = clock() + ROUTING_HEAL_REFUSED_BACKOFF_MS;
+    warn(`Codex routing self-heal was refused (${logDetail(result.message)}); retrying in 10 minutes. Run 'ocx sync' to repair now.`);
+  };
+
+  const evaluate = async () => {
+    if (gates.siblingOfLivePort() !== null) { stopped = true; return; }
+    const content = readConfig();
+    if (content === null) { cache = null; streak = null; return; }
+    const unchanged = cache !== null && cache.content === content;
+    if (unchanged && cache!.kind !== "foreign") return;
+    const drift = detect(content);
+    cache = { content, kind: drift.kind };
+    if (drift.kind !== "foreign") {
+      streak = null;
+      recheckAt = 0;
+      backoffUntil = 0;
+      announcedLive.clear();
+      return;
+    }
+    const now = clock();
+    if (now < backoffUntil || now < pausedUntil) return;
+    if (unchanged && now < recheckAt) return;
+    const gate = evaluateCodexRoutingHealGates(port, ownPorts, gates);
+    if (!gate.open) {
+      streak = null;
+      recheckAt = now + ROUTING_HEAL_RECHECK_MS;
+      if (gate.gate === "sibling") stopped = true;
+      return;
+    }
+    const probed = await probeAll(drift.targets);
+    if (stopped) return;
+    if (probed.verdict === "live") { standDownForLive(probed.liveEndpoints); return; }
+    // Unknown never advances the streak and never resets it; it is asked again at the recheck pace.
+    if (probed.verdict !== "dead") { recheckAt = clock() + ROUTING_HEAL_RECHECK_MS; return; }
+    recheckAt = 0;
+    const probedAt = clock();
+    const key = JSON.stringify([...new Set(drift.targets.map(endpointKey))].sort());
+    if (!streak || streak.key !== key) {
+      streak = { key, firstDeadAt: probedAt, probes: 1 };
+      return;
+    }
+    streak.probes += 1;
+    if (streak.probes < ROUTING_HEAL_MIN_DEAD_PROBES || probedAt - streak.firstDeadAt < ROUTING_HEAL_MIN_DEAD_SPAN_MS) return;
+    await heal(content, drift.targets, streak);
+  };
+
+  const tick = async () => {
+    pending = undefined;
+    if (stopped) return;
+    try { await evaluate(); } catch {
+      // A background check never throws into the timer; an unreadable config.json is looked at
+      // again at the slower recheck pace rather than on every tick.
+      recheckAt = clock() + ROUTING_HEAL_RECHECK_MS;
+    }
+    if (!stopped) pending = scheduleFn(() => { void tick(); }, ROUTING_HEAL_TICK_MS);
+  };
+
+  pending = scheduleFn(() => { void tick(); }, ROUTING_HEAL_TICK_MS);
+  return handle;
+}

package/src/codex/routing.ts

@@ -1,3 +1,4 @@
+import { clearIdleWindowSteering, pickIdleWindowAccount } from "./routing/idle-window";
 import { getEffectiveCodexAutoSwitchThreshold } from "./account-auto-switch";
 import { codexQuotaHasFreshUsage } from "./quota-observation-freshness";
 import { saveConfigPreservingClaudeCode } from "../config";
@@ -234,6 +235,7 @@ export function clearCodexUpstreamHealth(): void {
   // reset points. Leaving them behind lets a selection from one context suppress the
   // automatic cursor in the next one.
   clearAllManualPreferences();
+  clearIdleWindowSteering();
   clearUpstreamHealthState();
   forgetRuntimeActiveCodexAccount();
   // The reconcile watermark is part of this state, not something that outlives it. Keeping
@@ -786,6 +788,10 @@ export function previewCodexAccountForRequest(
     if (lineagePreview) return lineagePreview.accountId;
   }
 
+  const idlePick = !entry && !peekPendingReleaseReason(threadId)
+    && !(threadId && getModelDetourAffinity(threadId, modelId, quotaScope))
+    ? pickIdleWindowAccount(config, threadId, now, false, quotaScope, selectionOptions) : null;
+  if (idlePick) return idlePick;
   const strategyPick = pickUnboundStrategyAccount(
     config,
     threadId,
@@ -881,6 +887,7 @@ export function resolveCodexAccountForThreadDetailed(
   // cold one: every branch that follows -- detour reuse, transient hold, quota re-eval --
   // should treat it as the continuing conversation it is. No-op on a fresh process.
   if (threadId) adoptLegacyLineageAffinity(threadId, lineage, now, quotaScope, modelId);
+  const hadModelAffinity = !!(threadId && getModelDetourAffinity(threadId, modelId, quotaScope));
 
   if (threadId && modelScopedSelection) {
     const detourEntry = getModelDetourAffinity(threadId, modelId, quotaScope);
@@ -1088,6 +1095,9 @@ export function resolveCodexAccountForThreadDetailed(
     }
   }
 
+  const idlePick = !entry && !releaseReason && !hadModelAffinity
+    ? pickIdleWindowAccount(config, threadId, now, true, quotaScope, selectionOptions) : null;
+  if (idlePick) return { status: "selected", accountId: idlePick, affinity: affinityAfterRelease(threadId, releaseReason) };
   // A request-scoped roster may still contain unhealthy candidates. Non-quota strategies return
   // before the quota/failover helpers below, so prefer only shared-healthy roster members here;
   // otherwise RR/fill-first can immediately re-pick a known failing account even when another