@bitkyc08/opencodex 2.13.0 → 2.14.0

package/src/server/management/provider-routes.ts

@@ -2,8 +2,10 @@ import { randomUUID } from "node:crypto";
 import { readFileSync } from "node:fs";
 import type { CatalogModel } from "../../codex/catalog";
 import { catalogModelSlug, invalidateCodexModelsCache, nativeModelRows, uniqueCatalogModelsForPublicList } from "../../codex/catalog";
+import { clearGatherRoutedModelsInflight } from "../../codex/catalog/provider-fetch";
 import {
   DEFAULT_SUBAGENT_MODELS,
+  adoptPersistedProviderIntoLiveConfig,
   codexAutoStartEnabled,
   hasOwnProvider,
   isValidProviderName,
@@ -12,6 +14,7 @@ import {
   normalizeNonBlankStringArray,
   providerBaseUrlConfigError,
   providerHeadersConfigError,
+  readConfigAdmissionSnapshot,
   saveConfigPreservingClaudeCode,
   withConfigMutationLockSync,
 } from "../../config";
@@ -39,12 +42,13 @@ import {
   resolveProviderModelDiscovery,
 } from "../../providers/model-discovery";
 import { routedSlug, slugEquals } from "../../providers/slug-codec";
-import { clearProviderQuotaCache, fetchProviderQuotaReports } from "../../providers/quota";
+import { clearAccountQuotaCache, clearProviderQuotaCache, fetchProviderQuotaReports } from "../../providers/quota";
+import { clearKeyCooldowns } from "../../providers/key-failover";
 import { CODEX_FORWARD_BASE_URL, isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers";
 import { codexAccountNamespaceProviderCollisionError } from "../../codex/account-namespace-match";
 import { clearThreadAccountMap } from "../../codex/routing";
 import { primeCodexPoolQuotas } from "../../codex/auth-api";
-import { getProviderDiscoveryStatus } from "../../codex/model-cache";
+import { clearModelCache, getProviderDiscoveryStatus } from "../../codex/model-cache";
 import { DEFAULT_PROVIDER_CONTEXT_CAP, globalContextCapValue, providerContextCap, providerContextCaps, setAllProviderContextCaps, setGlobalContextCapValue, setProviderContextCap } from "../../providers/context-cap";
 import { resolveCodexHomeDir } from "../../codex/home";
 import { readUsageEntries } from "../../usage/log";
@@ -68,6 +72,11 @@ import { estimateComboCost, estimateRequestCost, normalizeCostTokens, tokensPerS
 import type { PersistedUsageAttempt } from "../../usage/log";
 import { isAllowedRequestOrigin, jsonResponse, providerManagementConfigError, publicProviderBaseUrl, safeConfigDTO } from "../auth-cors";
 import { applySystemEnvToggle } from "../system-env";
+import {
+  LOCAL_PROVIDER_RELOAD_NAME_HEADER,
+  LOCAL_PROVIDER_RELOAD_PATH,
+} from "../../lib/local-provider-reload-contract";
+import { refreshUserCostOverlays } from "../../usage/user-cost-overlays";
 
 import { isPlainRecord, parseDebugLogQuery, tokPerSecondResult, unavailableCostReason, costResult, requestLogDto, stripRegistryOnlyStaticHeaders, fetchAllModels } from "./shared";
 import type { MetricUnavailableReason, TokPerSecondResult, CostEstimateReason, CostResult, MetricSource } from "./shared";
@@ -288,7 +297,7 @@ function applyProviderPatchFields(
 }
 
 export async function handleProviderRoutes(ctx: ManagementContext): Promise<Response | null> {
-  const { req, url, config, deps, convergeCodexCatalog, syncClaudeAgentDefsBestEffort } = ctx;
+  const { req, url, config, deps, principal, convergeCodexCatalog, syncClaudeAgentDefsBestEffort } = ctx;
 
   if (url.pathname === "/api/provider-quotas" && req.method === "GET") {
     const forceRefresh = url.searchParams.get("refresh") === "1" || url.searchParams.get("refresh") === "true";
@@ -315,6 +324,78 @@ export async function handleProviderRoutes(ctx: ManagementContext): Promise<Resp
     })));
   }
 
+  if (url.pathname === LOCAL_PROVIDER_RELOAD_PATH && req.method === "POST") {
+    if (principal !== "local-provider-reload-capability") {
+      return jsonResponse({ error: "provider reload capability required" }, 403);
+    }
+    const name = req.headers.get(LOCAL_PROVIDER_RELOAD_NAME_HEADER) ?? "";
+    if (!isValidProviderName(name)) return jsonResponse({ error: "invalid provider reload target" }, 400);
+
+    const admitted = readConfigAdmissionSnapshot();
+    if (
+      admitted.kind !== "read"
+      || admitted.diagnostics.source !== "file"
+      || admitted.diagnostics.error !== null
+    ) {
+      return jsonResponse({ error: "provider reload source unavailable" }, 409);
+    }
+    const diskConfig = admitted.diagnostics.config;
+    if (!hasOwnProvider(diskConfig.providers, name)) {
+      return jsonResponse({ error: "provider reload target unavailable" }, 404);
+    }
+    const provider = diskConfig.providers[name]!;
+    const providerError = providerManagementConfigError(name, provider);
+    if (providerError) return jsonResponse({ error: "provider reload target invalid" }, 409);
+    const namespaceCollision = codexAccountNamespaceProviderCollisionError(
+      diskConfig.codexAccountNamespaces,
+      name,
+    );
+    if (namespaceCollision) return jsonResponse({ error: "provider reload target conflicts with routing" }, 409);
+    const allowBenchmarkAddresses = name === "openai" && isCanonicalOpenAiForwardProvider(provider);
+    const resolvedError = await providerDestinationResolvedError(name, provider, { allowBenchmarkAddresses });
+    if (resolvedError) return jsonResponse({ error: "provider reload target rejected" }, 409);
+
+    // Destination validation awaits DNS. A cooperating writer holds the same SQLite
+    // mutation lock, so the final exact-byte check and live adoption happen as one
+    // synchronous authority decision. The route does not save or reserialize disk.
+    let currentDiskConfig: OcxConfig | null = null;
+    let sourceChanged = false;
+    withConfigMutationLockSync(() => {
+      const current = readConfigAdmissionSnapshot();
+      if (
+        current.kind !== "read"
+        || current.diagnostics.source !== "file"
+        || current.diagnostics.error !== null
+        || current.contentSha256 !== admitted.contentSha256
+      ) {
+        sourceChanged = true;
+        return;
+      }
+      currentDiskConfig = current.diagnostics.config;
+      adoptPersistedProviderIntoLiveConfig(
+        config,
+        name,
+        current.diagnostics.config.providers[name]!,
+        current.diagnostics.config,
+      );
+    });
+    if (sourceChanged || currentDiskConfig === null) {
+      return jsonResponse({ error: "provider reload source changed" }, 409);
+    }
+    reconcileLiveStateStores();
+    // The complete disk snapshot owns display overlays, including providers that this
+    // live routing instance deliberately does not adopt.
+    refreshUserCostOverlays(currentDiskConfig);
+    clearGatherRoutedModelsInflight();
+    (deps.clearProviderQuotaCache ?? clearProviderQuotaCache)();
+    clearAccountQuotaCache(name);
+    clearKeyCooldowns(name);
+    clearModelCache(name);
+    if (name === "openai") (deps.clearThreadAccountMap ?? clearThreadAccountMap)();
+    const catalogRefresh = await convergeCodexCatalog();
+    return jsonResponse({ success: true, name, catalogRefresh });
+  }
+
   // Add (or overwrite) a single provider. Merges into the live in-memory config and
   // persists — existing providers' real keys are never round-tripped (unlike PUT /api/config,
   // which would re-save the masked keys from GET). Live routing picks it up immediately.
@@ -350,6 +431,12 @@ export async function handleProviderRoutes(ctx: ManagementContext): Promise<Resp
     }
     // Catalog providers (e.g. ollama-cloud) carry a models + vision/reasoning classification the GUI
     // doesn't send — merge it in so the sidecars are gated correctly.
+    // Sample request ownership BEFORE enrichment. Enrichment fills absent fields from the
+    // registry seed, after which "the client omitted this" and "the registry supplied it" are
+    // indistinguishable — so a carry-over guard written as `prov.x === undefined` after this
+    // call can never fire.
+    const submittedContextWindow = Object.hasOwn(prov, "contextWindow");
+    const submittedModelContextWindows = Object.hasOwn(prov, "modelContextWindows");
     enrichProviderFromCatalog(name, prov);
     const { saveConfigPreservingClaudeCode: save } = await import("../../config");
     // Overwriting an existing provider must not drop its multi-key pool: carry it over, then
@@ -361,6 +448,23 @@ export async function handleProviderRoutes(ctx: ManagementContext): Promise<Resp
     // erase hand-edited per-model prices from Logs/Usage estimates.
     const existingCosts = config.providers[name]?.modelCosts;
     if (existingCosts && !prov.modelCosts) prov.modelCosts = existingCosts;
+    // ...and to hand-edited context windows. `ProviderPayload` (gui/src/provider-payload.ts)
+    // has no member for either field, so the add/edit form structurally cannot send them:
+    // absence in the request means "not carried", never "the user deleted it". Deletion goes
+    // through PATCH with an explicit null (#1409).
+    const existing = config.providers[name];
+    if (!submittedContextWindow && existing?.contextWindow !== undefined) {
+      prov.contextWindow = existing.contextWindow;
+    }
+    if (existing?.modelContextWindows) {
+      // When the client did send a map, its keys win and the user's other keys survive. When
+      // it did not, the stored value is the user's map alone: merging the registry seed in
+      // would persist seed keys into user config as a side effect of an unrelated save, and
+      // router.ts already fills registry values beneath user entries at resolve time.
+      prov.modelContextWindows = submittedModelContextWindows
+        ? { ...existing.modelContextWindows, ...(prov.modelContextWindows ?? {}) }
+        : { ...existing.modelContextWindows };
+    }
     config.providers[name] = stripRegistryOnlyStaticHeaders(name, prov);
     if (body.setDefault === true) config.defaultProvider = name;
     save(config);

package/src/server/management-auth.ts

@@ -29,6 +29,16 @@ import {
   parseExpectedSystemRestartPid,
   verifySystemRestartCapability,
 } from "../lib/system-restart-contract";
+import {
+  LOCAL_PROVIDER_RELOAD_CAPABILITY_HEADER,
+  LOCAL_PROVIDER_RELOAD_EXPECTED_PID_HEADER,
+  LOCAL_PROVIDER_RELOAD_EXPIRES_AT_HEADER,
+  LOCAL_PROVIDER_RELOAD_NAME_HEADER,
+  LOCAL_PROVIDER_RELOAD_NONCE_HEADER,
+  LOCAL_PROVIDER_RELOAD_PATH,
+  parseExpectedLocalProviderReloadPid,
+  verifyLocalProviderReloadCapability,
+} from "../lib/local-provider-reload-contract";
 import { forgetEphemeralSecretPath, forgetHardenedSecretPath, hardenSecretDir, hardenSecretPath } from "../lib/windows-secret-acl";
 import type { OcxConfig } from "../types";
 import {
@@ -45,6 +55,9 @@ const GUI_SESSION_LIMIT = 128;
 const LOCAL_READ_REPLAY_LIMIT = 256;
 const consumedLocalReadCapabilities = new Map<string, number>();
 const admittedLocalReadRequests = new WeakSet<Request>();
+const LOCAL_PROVIDER_RELOAD_REPLAY_LIMIT = 256;
+const consumedLocalProviderReloadCapabilities = new Map<string, number>();
+const admittedLocalProviderReloadRequests = new WeakSet<Request>();
 
 interface GuiSessionRecord {
   csrfToken: string;
@@ -266,12 +279,13 @@ export function issueGuiSession(
  * rather than off request headers, which the token holder can forge freely.
  * The capability principals are process-scoped HMACs bound to the current process
  * PID and listening port. Local reads are accepted only for two exact GET paths;
- * restart remains a separate wire contract for its exact POST.
+ * restart and provider reload remain separate wire contracts for their exact POSTs.
  */
 export type ManagementPrincipal =
   | "admin-token"
   | "gui-session"
   | "local-read-capability"
+  | "local-provider-reload-capability"
   | "system-restart-capability";
 
 export interface LocalManagementAuthContext {
@@ -354,6 +368,54 @@ function hasLocalReadCapability(
   return true;
 }
 
+function hasLocalProviderReloadCapability(
+  req: Request,
+  local: LocalManagementAuthContext | undefined,
+): boolean {
+  if (admittedLocalProviderReloadRequests.has(req)) return true;
+  if (!local || req.method !== "POST") return false;
+  let url: URL;
+  try {
+    url = new URL(req.url);
+  } catch {
+    return false;
+  }
+  if (url.pathname !== LOCAL_PROVIDER_RELOAD_PATH || url.search !== "") return false;
+  const contentLength = req.headers.get("content-length");
+  if (contentLength !== "0" || req.headers.has("transfer-encoding")) return false;
+  const expectedPid = parseExpectedLocalProviderReloadPid(
+    req.headers.get(LOCAL_PROVIDER_RELOAD_EXPECTED_PID_HEADER),
+  );
+  if (expectedPid.kind !== "present" || expectedPid.pid !== local.pid) return false;
+  const expiresAtRaw = req.headers.get(LOCAL_PROVIDER_RELOAD_EXPIRES_AT_HEADER);
+  if (!expiresAtRaw || !/^[1-9]\d*$/.test(expiresAtRaw)) return false;
+  const expiresAt = Number(expiresAtRaw);
+  if (!Number.isSafeInteger(expiresAt)) return false;
+  const name = req.headers.get(LOCAL_PROVIDER_RELOAD_NAME_HEADER);
+  const capability = req.headers.get(LOCAL_PROVIDER_RELOAD_CAPABILITY_HEADER);
+  const now = Date.now();
+  if (!verifyLocalProviderReloadCapability(
+    local.attestationSecret,
+    req.headers.get(LOCAL_PROVIDER_RELOAD_NONCE_HEADER),
+    req.method,
+    url.pathname,
+    name,
+    local.pid,
+    local.port,
+    expiresAt,
+    capability,
+    now,
+  )) return false;
+  for (const [consumed, retainedUntil] of consumedLocalProviderReloadCapabilities) {
+    if (retainedUntil <= now) consumedLocalProviderReloadCapabilities.delete(consumed);
+  }
+  if (!capability || consumedLocalProviderReloadCapabilities.has(capability)) return false;
+  if (consumedLocalProviderReloadCapabilities.size >= LOCAL_PROVIDER_RELOAD_REPLAY_LIMIT) return false;
+  consumedLocalProviderReloadCapabilities.set(capability, expiresAt);
+  admittedLocalProviderReloadRequests.add(req);
+  return true;
+}
+
 /**
  * The principal for a request that already passed `requireManagementAuth`. Kept as a
  * separate resolution (rather than a changed return type) so every existing caller
@@ -368,6 +430,7 @@ export function managementPrincipal(
   local?: LocalManagementAuthContext,
 ): ManagementPrincipal | null {
   if (hasSystemRestartCapability(req, local)) return "system-restart-capability";
+  if (hasLocalProviderReloadCapability(req, local)) return "local-provider-reload-capability";
   if (hasLocalReadCapability(req, local)) return "local-read-capability";
   if (!state.available) return null;
   const actual = req.headers.get("x-opencodex-api-key")?.trim()
@@ -386,6 +449,7 @@ export function requireManagementAuth(
   local?: LocalManagementAuthContext,
 ): Response | null {
   if (hasSystemRestartCapability(req, local)) return null;
+  if (hasLocalProviderReloadCapability(req, local)) return null;
   if (hasLocalReadCapability(req, local)) return null;
   if (!state.available) {
     return Response.json({

package/src/server/proxy-liveness.ts

@@ -20,6 +20,7 @@ export interface HealthzIdentity {
   pid?: unknown;
   port?: unknown;
   restartCapability?: unknown;
+  providerReloadCapability?: unknown;
 }
 
 export interface LivenessIo {

package/src/server/responses/agent-task-recovery-cache.ts

@@ -0,0 +1,143 @@
+const MAX_CACHE_BYTES = 8 * 1024 * 1024;
+const MAX_CONCURRENT_RECOVERIES = 32;
+const CACHE_TTL_MS = 15 * 60 * 1000;
+
+interface RecoveryCacheEntry {
+  assignment: string;
+  bytes: number;
+  expiresAt: number;
+  expiryTimer: ReturnType<typeof setTimeout> | null;
+}
+
+interface RecoveryFlight {
+  controller: AbortController;
+  promise: Promise<string | null>;
+  waiters: number;
+  settled: boolean;
+}
+
+const RECOVERY_CACHE = new Map<string, RecoveryCacheEntry>();
+const RECOVERY_FLIGHTS = new Map<string, RecoveryFlight>();
+let recoveryCacheBytes = 0;
+
+function deleteRecoveryCacheEntry(key: string, expected?: RecoveryCacheEntry): void {
+  const entry = RECOVERY_CACHE.get(key);
+  if (!entry || (expected && entry !== expected)) return;
+  RECOVERY_CACHE.delete(key);
+  if (entry.expiryTimer) clearTimeout(entry.expiryTimer);
+  recoveryCacheBytes = Math.max(0, recoveryCacheBytes - entry.bytes);
+}
+
+function sweepRecoveryCache(now: number, maxEntries: number): void {
+  for (const [key, entry] of RECOVERY_CACHE) {
+    if (entry.expiresAt > now) continue;
+    deleteRecoveryCacheEntry(key, entry);
+  }
+  while (RECOVERY_CACHE.size > maxEntries || recoveryCacheBytes > MAX_CACHE_BYTES) {
+    const oldest = RECOVERY_CACHE.keys().next().value;
+    if (oldest === undefined) break;
+    deleteRecoveryCacheEntry(oldest);
+  }
+}
+
+function insertRecoveryCacheEntry(key: string, assignment: string, maxEntries: number): void {
+  const replaced = RECOVERY_CACHE.get(key);
+  if (replaced) deleteRecoveryCacheEntry(key, replaced);
+  const insertedAt = Date.now();
+  const entry: RecoveryCacheEntry = {
+    assignment,
+    bytes: Buffer.byteLength(assignment),
+    expiresAt: insertedAt + CACHE_TTL_MS,
+    expiryTimer: null,
+  };
+  entry.expiryTimer = setTimeout(
+    () => deleteRecoveryCacheEntry(key, entry),
+    CACHE_TTL_MS,
+  );
+  entry.expiryTimer.unref?.();
+  RECOVERY_CACHE.set(key, entry);
+  recoveryCacheBytes += entry.bytes;
+  sweepRecoveryCache(insertedAt, maxEntries);
+}
+
+function startRecoveryFlight(
+  key: string,
+  maxEntries: number,
+  request: (signal: AbortSignal) => Promise<string | null>,
+): RecoveryFlight | null {
+  const active = RECOVERY_FLIGHTS.get(key);
+  if (active) return active;
+  if (RECOVERY_FLIGHTS.size >= MAX_CONCURRENT_RECOVERIES) return null;
+
+  const controller = new AbortController();
+  const flight: RecoveryFlight = {
+    controller,
+    promise: Promise.resolve(null),
+    waiters: 0,
+    settled: false,
+  };
+  flight.promise = request(controller.signal)
+    .then((assignment) => {
+      if (!assignment || controller.signal.aborted) return null;
+      insertRecoveryCacheEntry(key, assignment, maxEntries);
+      return assignment;
+    })
+    .finally(() => {
+      flight.settled = true;
+      if (RECOVERY_FLIGHTS.get(key) === flight) RECOVERY_FLIGHTS.delete(key);
+    });
+  RECOVERY_FLIGHTS.set(key, flight);
+  return flight;
+}
+
+async function waitForRecoveryFlight(
+  flight: RecoveryFlight,
+  abortSignal?: AbortSignal,
+): Promise<string | null> {
+  if (abortSignal?.aborted) return null;
+  flight.waiters += 1;
+  let onAbort: (() => void) | undefined;
+  try {
+    if (!abortSignal) return await flight.promise;
+    const cancelled = new Promise<null>((resolve) => {
+      onAbort = () => resolve(null);
+      abortSignal.addEventListener("abort", onAbort, { once: true });
+      if (abortSignal.aborted) onAbort();
+    });
+    return await Promise.race([flight.promise, cancelled]);
+  } finally {
+    if (onAbort) abortSignal?.removeEventListener("abort", onAbort);
+    flight.waiters = Math.max(0, flight.waiters - 1);
+    if (flight.waiters === 0 && !flight.settled) {
+      flight.controller.abort(new DOMException("All recovery callers cancelled", "AbortError"));
+    }
+  }
+}
+
+export async function resolveCachedAgentTaskRecovery(
+  key: string,
+  maxEntries: number,
+  request: (signal: AbortSignal) => Promise<string | null>,
+  abortSignal?: AbortSignal,
+): Promise<string | null> {
+  if (abortSignal?.aborted) return null;
+  sweepRecoveryCache(Date.now(), maxEntries);
+  const cached = RECOVERY_CACHE.get(key)?.assignment;
+  if (cached) return cached;
+  const flight = startRecoveryFlight(key, maxEntries, request);
+  return flight ? waitForRecoveryFlight(flight, abortSignal) : null;
+}
+
+export function resetAgentTaskRecoveryCache(): void {
+  for (const flight of RECOVERY_FLIGHTS.values()) {
+    flight.controller.abort(new DOMException("Recovery state reset", "AbortError"));
+  }
+  RECOVERY_FLIGHTS.clear();
+  for (const key of [...RECOVERY_CACHE.keys()]) deleteRecoveryCacheEntry(key);
+}
+
+export function agentTaskRecoveryWaiterCountForTests(): number {
+  let count = 0;
+  for (const flight of RECOVERY_FLIGHTS.values()) count += flight.waiters;
+  return count;
+}