@bitkyc08/opencodex 2.10.2 → 2.11.0

package/src/codex/upstream-host-health.ts

@@ -1,67 +1,365 @@
 /**
- * Observational ledger for pre-connection upstream reachability failures,
- * keyed by (provider, host). Records ONLY — no circuit breaker, no admission
- * change (issue #914 scope). Rotation decisions stay with account health;
- * this ledger exists so a host-wide outage is visible as host-wide.
+ * Provider-origin reachability health and the optional logical-request circuit.
  *
- * Retention: bounded at 128 entries; on overflow the stalest entries by
- * last-touch are pruned before insert, and failure timestamps older than the
- * window are reconciled away on the next record — repeated provider/base-URL
- * churn cannot grow the map for the process lifetime.
+ * Issue #914 owns the observational ledger. The circuit is deliberately opt-in:
+ * a threshold of 0 preserves the ledger-only behavior. Only proven
+ * `connect_neutral` failures may settle a lease as host failure; timeout, reset,
+ * HTTP, redirect, authentication, and local failures remain outside this module.
  */
 
 export const UPSTREAM_HOST_HEALTH_MAX_ENTRIES = 128;
 export const UPSTREAM_HOST_FAILURE_WINDOW_MS = 10 * 60_000;
+export const UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS = 30_000;
+export const UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD = 20;
 
 export type UpstreamHostHealthEntry = {
   consecutiveFailures: number;
   lastFailureAt: number;
   lastFailureCode?: string;
+  cooldownUntil?: number;
   /** Recency marker for stalest-first pruning (not health semantics). */
   lastTouch: number;
 };
 
-const hostHealth = new Map<string, UpstreamHostHealthEntry>();
+/** Opaque ownership token for one admitted logical request. */
+export type UpstreamHostAdmissionLease = Readonly<{
+  key: string;
+  leaseId: symbol;
+  generation: number;
+  halfOpen: boolean;
+}>;
+
+export type UpstreamHostAdmission =
+  | { kind: "admitted"; lease: UpstreamHostAdmissionLease | null }
+  | { kind: "blocked"; retryAfterSeconds: number };
+
+type InternalUpstreamHostHealth = UpstreamHostHealthEntry & {
+  generation: number;
+  activeLeaseIds: Set<symbol>;
+  halfOpenLeaseId?: symbol;
+  /**
+   * Leases admitted in the generation that immediately preceded the current
+   * cooldown. They are stale for failure settlement, but a real HTTP response
+   * from one still proves the origin is reachable and may close this cooldown.
+   */
+  cooldownSuccessLeaseIds?: Set<symbol>;
+  /** True only when the entry is owned by opt-in circuit admissions. */
+  circuitManaged: boolean;
+};
+
+const hostHealth = new Map<string, InternalUpstreamHostHealth>();
+let nextGenerationValue = 0;
 
 export function upstreamHostHealthKey(provider: string, host: string): string {
   return `${provider}|${host.toLowerCase()}`;
 }
 
-function pruneForInsert(): void {
-  if (hostHealth.size < UPSTREAM_HOST_HEALTH_MAX_ENTRIES) return;
-  const entries = [...hostHealth.entries()].sort((a, b) => a[1].lastTouch - b[1].lastTouch);
-  for (const [key] of entries) {
-    if (hostHealth.size < UPSTREAM_HOST_HEALTH_MAX_ENTRIES) return;
+export function normalizeUpstreamHostCircuitThreshold(value: unknown): number {
+  if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) return 0;
+  return Math.min(value, UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD);
+}
+
+function nextGeneration(): number {
+  nextGenerationValue = nextGenerationValue >= Number.MAX_SAFE_INTEGER ? 1 : nextGenerationValue + 1;
+  return nextGenerationValue;
+}
+
+function snapshot(entry: InternalUpstreamHostHealth): UpstreamHostHealthEntry {
+  return {
+    consecutiveFailures: entry.consecutiveFailures,
+    lastFailureAt: entry.lastFailureAt,
+    lastTouch: entry.lastTouch,
+    ...(entry.lastFailureCode !== undefined ? { lastFailureCode: entry.lastFailureCode } : {}),
+    ...(entry.cooldownUntil !== undefined ? { cooldownUntil: entry.cooldownUntil } : {}),
+  };
+}
+
+function newEntry(now: number, circuitManaged: boolean): InternalUpstreamHostHealth {
+  return {
+    consecutiveFailures: 0,
+    lastFailureAt: 0,
+    lastTouch: now,
+    generation: nextGeneration(),
+    activeLeaseIds: new Set(),
+    circuitManaged,
+  };
+}
+
+function advanceGeneration(entry: InternalUpstreamHostHealth): void {
+  entry.generation = nextGeneration();
+  entry.activeLeaseIds.clear();
+  delete entry.halfOpenLeaseId;
+  delete entry.cooldownSuccessLeaseIds;
+}
+
+function removeExpiredUnleased(now: number): void {
+  for (const [key, entry] of hostHealth) {
+    if (entry.activeLeaseIds.size > 0) continue;
+    if (
+      entry.consecutiveFailures === 0
+      || (entry.cooldownUntil === undefined
+        && now - entry.lastFailureAt > UPSTREAM_HOST_FAILURE_WINDOW_MS)
+    ) {
+      hostHealth.delete(key);
+    }
+  }
+}
+
+function oldestUnleasedKey(now: number): string | undefined {
+  let preferred: [string, number] | undefined;
+  let cooling: [string, number] | undefined;
+  for (const [key, entry] of hostHealth) {
+    if (entry.activeLeaseIds.size > 0) continue;
+    const candidate: [string, number] = [key, entry.lastTouch];
+    if (entry.cooldownUntil !== undefined && entry.cooldownUntil > now) {
+      if (!cooling || candidate[1] < cooling[1]) cooling = candidate;
+    } else if (!preferred || candidate[1] < preferred[1]) {
+      preferred = candidate;
+    }
+  }
+  return preferred?.[0] ?? cooling?.[0];
+}
+
+function pruneTo(limit: number, now: number): void {
+  removeExpiredUnleased(now);
+  while (hostHealth.size > limit) {
+    const key = oldestUnleasedKey(now);
+    if (!key) return; // Active leases may temporarily exceed the retention cap.
     hostHealth.delete(key);
   }
 }
 
+function makeRoom(now: number): void {
+  pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES - 1, now);
+}
+
+function issueLease(
+  key: string,
+  entry: InternalUpstreamHostHealth,
+  halfOpen: boolean,
+  now: number,
+): UpstreamHostAdmissionLease {
+  const leaseId = Symbol(halfOpen ? "upstream-host-half-open" : "upstream-host-admission");
+  entry.activeLeaseIds.add(leaseId);
+  entry.lastTouch = now;
+  if (halfOpen) entry.halfOpenLeaseId = leaseId;
+  return { key, leaseId, generation: entry.generation, halfOpen };
+}
+
+function matchingEntry(lease: UpstreamHostAdmissionLease): InternalUpstreamHostHealth | null {
+  const entry = hostHealth.get(lease.key);
+  if (!entry || entry.generation !== lease.generation || !entry.activeLeaseIds.has(lease.leaseId)) {
+    return null;
+  }
+  if (lease.halfOpen && entry.halfOpenLeaseId !== lease.leaseId) return null;
+  return entry;
+}
+
+function settleLease(entry: InternalUpstreamHostHealth, lease: UpstreamHostAdmissionLease): void {
+  entry.activeLeaseIds.delete(lease.leaseId);
+  if (entry.halfOpenLeaseId === lease.leaseId) delete entry.halfOpenLeaseId;
+}
+
+/**
+ * Admit one logical request. A disabled threshold returns a null lease and has
+ * byte-for-byte compatible call-site behavior with the observational ledger.
+ */
+export function acquireUpstreamHostAdmission(
+  key: string,
+  thresholdValue: unknown,
+  now = Date.now(),
+): UpstreamHostAdmission {
+  const threshold = normalizeUpstreamHostCircuitThreshold(thresholdValue);
+  if (threshold === 0) return { kind: "admitted", lease: null };
+
+  pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now);
+  let entry = hostHealth.get(key);
+  if (
+    entry?.activeLeaseIds.size === 0
+    && entry.cooldownUntil === undefined
+    && entry.consecutiveFailures > 0
+    && now - entry.lastFailureAt > UPSTREAM_HOST_FAILURE_WINDOW_MS
+  ) {
+    hostHealth.delete(key);
+    entry = undefined;
+  }
+  if (!entry) {
+    makeRoom(now);
+    entry = newEntry(now, true);
+    hostHealth.set(key, entry);
+  } else if (!entry.circuitManaged) {
+    // Observational history predating opt-in admission must not count toward
+    // opening the circuit. Start a fresh fenced generation.
+    entry.consecutiveFailures = 0;
+    entry.lastFailureAt = 0;
+    entry.circuitManaged = true;
+    delete entry.lastFailureCode;
+    delete entry.cooldownUntil;
+    advanceGeneration(entry);
+  }
+
+  if (entry.cooldownUntil !== undefined) {
+    if (entry.cooldownUntil > now) {
+      return {
+        kind: "blocked",
+        retryAfterSeconds: Math.max(1, Math.ceil((entry.cooldownUntil - now) / 1_000)),
+      };
+    }
+    if (entry.halfOpenLeaseId !== undefined) {
+      return { kind: "blocked", retryAfterSeconds: 1 };
+    }
+    advanceGeneration(entry);
+    return { kind: "admitted", lease: issueLease(key, entry, true, now) };
+  }
+
+  return { kind: "admitted", lease: issueLease(key, entry, false, now) };
+}
+
+/** Release an admitted request without recording transport evidence. */
+export function releaseUpstreamHostAdmission(
+  lease: UpstreamHostAdmissionLease | null | undefined,
+  now = Date.now(),
+): boolean {
+  if (!lease) return false;
+  const entry = matchingEntry(lease);
+  if (!entry) return false;
+  settleLease(entry, lease);
+  entry.lastTouch = now;
+  if (entry.activeLeaseIds.size === 0 && entry.consecutiveFailures === 0) {
+    hostHealth.delete(lease.key);
+  }
+  pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now);
+  return true;
+}
+
+/**
+ * Downgrade circuit-owned state when the operator disables the circuit.
+ * Failure history remains observational, but cooldown and all in-flight lease
+ * authority are revoked so disabled-mode traffic can update the ledger normally.
+ */
+export function disableUpstreamHostCircuitForKey(key: string, now = Date.now()): boolean {
+  const entry = hostHealth.get(key);
+  if (!entry?.circuitManaged) return false;
+  entry.circuitManaged = false;
+  delete entry.cooldownUntil;
+  advanceGeneration(entry);
+  entry.lastTouch = now;
+  if (entry.consecutiveFailures === 0) hostHealth.delete(key);
+  else pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now);
+  return true;
+}
+
+/** Record one terminal logical `connect_neutral` failure. */
 export function recordUpstreamHostFailure(
   key: string,
-  opts: { code?: string; now?: number } = {},
+  opts: {
+    code?: string;
+    now?: number;
+    threshold?: unknown;
+    lease?: UpstreamHostAdmissionLease | null;
+  } = {},
 ): void {
   const now = opts.now ?? Date.now();
-  const prior = hostHealth.get(key);
-  // Prune only for a genuinely new key: updating an existing entry must never
-  // evict an unrelated one.
-  if (prior === undefined) pruneForInsert();
-  const stale = prior !== undefined && now - prior.lastFailureAt > UPSTREAM_HOST_FAILURE_WINDOW_MS;
-  const code = typeof opts.code === "string" && opts.code !== "" ? opts.code : prior?.lastFailureCode;
-  hostHealth.set(key, {
-    consecutiveFailures: stale || prior === undefined ? 1 : prior.consecutiveFailures + 1,
-    lastFailureAt: now,
-    lastTouch: now,
-    ...(code !== undefined ? { lastFailureCode: code } : {}),
-  });
+  const hasCircuitSettlement = Object.hasOwn(opts, "lease");
+  // An integrated logical request may already have settled its lease on an
+  // earlier physical response. Later retry completions carry explicit null and
+  // must not mutate a newer generation opened by another request.
+  if (hasCircuitSettlement && !opts.lease) return;
+  // Legacy callers without an admission lease continue to populate only the
+  // observational ledger. Circuit mutation is fenced to explicitly admitted
+  // logical requests so an unwired side path cannot open it accidentally.
+  const threshold = opts.lease
+    ? normalizeUpstreamHostCircuitThreshold(opts.threshold)
+    : 0;
+  let entry: InternalUpstreamHostHealth | undefined;
+  if (opts.lease) {
+    entry = matchingEntry(opts.lease) ?? undefined;
+    if (!entry || opts.lease.key !== key) return; // stale completion cannot mutate a newer generation
+    settleLease(entry, opts.lease);
+  } else {
+    entry = hostHealth.get(key);
+    if (entry?.circuitManaged) {
+      const stale = entry.activeLeaseIds.size === 0
+        && entry.cooldownUntil === undefined
+        && now - entry.lastFailureAt > UPSTREAM_HOST_FAILURE_WINDOW_MS;
+      if (!stale) return;
+      hostHealth.delete(key);
+      entry = undefined;
+    }
+  }
+  if (!entry) {
+    makeRoom(now);
+    entry = newEntry(now, opts.lease !== undefined);
+    hostHealth.set(key, entry);
+  }
+
+  const reopens = opts.lease?.halfOpen === true || entry.cooldownUntil !== undefined;
+  const stale = entry.consecutiveFailures === 0
+    || (!reopens && now - entry.lastFailureAt > UPSTREAM_HOST_FAILURE_WINDOW_MS);
+  entry.consecutiveFailures = reopens && threshold > 0
+    ? Math.max(threshold, entry.consecutiveFailures + 1)
+    : stale ? 1 : entry.consecutiveFailures + 1;
+  entry.lastFailureAt = now;
+  entry.lastTouch = now;
+  const code = typeof opts.code === "string" && opts.code !== ""
+    ? opts.code
+    : entry.lastFailureCode;
+  if (code !== undefined) entry.lastFailureCode = code;
+
+  if (threshold > 0 && (reopens || entry.consecutiveFailures >= threshold)) {
+    entry.cooldownUntil = now + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS;
+    // The failing lease was settled above. Preserve only its still-in-flight
+    // same-generation peers as one-shot reachability proofs. advanceGeneration
+    // invalidates them for every other mutation and for any later half-open generation.
+    const concurrentSuccessLeaseIds = new Set(entry.activeLeaseIds);
+    advanceGeneration(entry);
+    if (concurrentSuccessLeaseIds.size > 0) {
+      entry.cooldownSuccessLeaseIds = concurrentSuccessLeaseIds;
+    }
+  } else {
+    delete entry.cooldownUntil;
+  }
+  pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now);
 }
 
-/** Any real HTTP response from the host clears its reachability streak. */
-export function resetUpstreamHostHealth(key: string): void {
-  hostHealth.delete(key);
+/** Any real HTTP response from the admitted logical request proves reachability. */
+export function resetUpstreamHostHealth(
+  key: string,
+  lease?: UpstreamHostAdmissionLease | null,
+  now = Date.now(),
+): boolean {
+  if (lease === null) return false;
+  if (lease === undefined) {
+    const entry = hostHealth.get(key);
+    if (entry?.circuitManaged) return false;
+    return hostHealth.delete(key);
+  }
+  if (lease.key !== key) return false;
+  let entry = matchingEntry(lease);
+  if (!entry) {
+    const coolingEntry = hostHealth.get(key);
+    if (
+      coolingEntry?.cooldownUntil === undefined
+      || !coolingEntry.cooldownSuccessLeaseIds?.delete(lease.leaseId)
+    ) return false;
+    entry = coolingEntry;
+  } else {
+    settleLease(entry, lease);
+  }
+  entry.consecutiveFailures = 0;
+  entry.lastFailureAt = 0;
+  entry.lastTouch = now;
+  delete entry.lastFailureCode;
+  delete entry.cooldownUntil;
+  if (entry.activeLeaseIds.size === 0) hostHealth.delete(key);
+  pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now);
+  return true;
 }
 
 export function getUpstreamHostHealth(key: string): UpstreamHostHealthEntry | null {
-  return hostHealth.get(key) ?? null;
+  const entry = hostHealth.get(key);
+  if (!entry || entry.consecutiveFailures === 0) return null;
+  return snapshot(entry);
 }
 
 /** Test hook: clear the whole ledger. */

package/src/combos/request.ts

@@ -40,6 +40,8 @@ export function concreteComboRequestBody(
     && !Object.prototype.hasOwnProperty.call(reasoning, "effort")
   );
   if (!needsDefault) return clone;
+  // Picker availability treats an unknown ladder as a wildcard, but runtime
+  // injection stays fail-closed until this concrete target advertises support.
   if (!targetReasoningEfforts?.includes(defaultEffort)) {
     const key = `${target.provider}/${target.model}:${defaultEffort}`;
     if (!warnedUnsupportedDefaults.has(key)) {