@iislee/opencodex 2.11.0 → 2.35.0

package/src/providers/request-pacing.ts

@@ -0,0 +1,310 @@
+import type { OcxProviderConfig, RequestPacingRule } from "../types";
+import type { GenerationContext } from "../lib/state-store-sweeper";
+
+export const REQUEST_PACING_MAX_QUEUE_DEPTH = 256;
+export const REQUEST_PACING_MAX_QUEUE_AGE_MS = 60_000;
+
+let maxQueueDepth = REQUEST_PACING_MAX_QUEUE_DEPTH;
+let maxQueueAgeMs = REQUEST_PACING_MAX_QUEUE_AGE_MS;
+
+export type RequestPacingQueueOverloadReason = "queue_full" | "queue_expired";
+
+export class RequestPacingQueueOverloadError extends Error {
+  constructor(
+    public readonly providerName: string,
+    public readonly reason: RequestPacingQueueOverloadReason,
+    public readonly retryAfterSeconds: number,
+  ) {
+    super(reason === "queue_full"
+      ? `request pacing queue for provider '${providerName}' is full`
+      : `request pacing queue for provider '${providerName}' exceeded the maximum queued age`);
+    this.name = "RequestPacingQueueOverloadError";
+  }
+}
+
+export class RequestPacingProviderRemovedError extends Error {
+  constructor(public readonly providerName: string) {
+    super(`request pacing provider '${providerName}' was removed`);
+    this.name = "RequestPacingProviderRemovedError";
+  }
+}
+
+interface Waiter {
+  modelId?: string;
+  providerIntervalMs: number;
+  modelIntervalMs: number;
+  queuedAt: number;
+  signal?: AbortSignal;
+  resolve: () => void;
+  reject: (reason: unknown) => void;
+  abort?: () => void;
+}
+
+interface ProviderPacer {
+  queue: Waiter[];
+  providerNextStartAt: number;
+  modelNextStartAt: Map<string, number>;
+  timer?: unknown;
+  lastStartedAt?: number;
+  lastModelId?: string;
+}
+
+export interface RequestPacingRuntime {
+  now: () => number;
+  setTimer: (callback: () => void, delayMs: number) => unknown;
+  clearTimer: (handle: unknown) => void;
+  enqueueMicrotask: (callback: () => void) => void;
+}
+
+export interface ProviderRequestPacingStatus {
+  provider: string;
+  enabled: boolean;
+  queued: number;
+  nextSlotInMs: number;
+  lastStartedAt?: number;
+  lastModelId?: string;
+}
+
+const pacers = new Map<string, ProviderPacer>();
+const defaultRuntime: RequestPacingRuntime = {
+  now: Date.now,
+  setTimer: (callback, delayMs) => setTimeout(callback, delayMs),
+  clearTimer: handle => clearTimeout(handle as ReturnType<typeof setTimeout>),
+  enqueueMicrotask: queueMicrotask,
+};
+let runtime = defaultRuntime;
+let lastReconciledGeneration = 0;
+
+function abortReason(signal: AbortSignal): unknown {
+  return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
+}
+
+function normalizedInterval(rule: RequestPacingRule | undefined): number {
+  if (!rule) return 0;
+  const rpmInterval = typeof rule.requestsPerMinute === "number" && rule.requestsPerMinute > 0
+    ? 60_000 / rule.requestsPerMinute
+    : 0;
+  const fixedInterval = typeof rule.minIntervalMs === "number" && rule.minIntervalMs > 0
+    ? rule.minIntervalMs
+    : 0;
+  return Math.max(rpmInterval, fixedInterval);
+}
+
+export function requestPacingIntervalMs(provider: OcxProviderConfig, modelId?: string): number {
+  const policy = provider.requestPacing;
+  if (!policy?.enabled) return 0;
+  const override = modelId ? policy.models?.[modelId] : undefined;
+  return Math.max(normalizedInterval(policy), normalizedInterval(override));
+}
+
+function requestPacingIntervals(provider: OcxProviderConfig, modelId?: string): {
+  providerIntervalMs: number;
+  modelIntervalMs: number;
+} {
+  const policy = provider.requestPacing;
+  if (!policy?.enabled) return { providerIntervalMs: 0, modelIntervalMs: 0 };
+  return {
+    providerIntervalMs: normalizedInterval(policy),
+    modelIntervalMs: modelId ? normalizedInterval(policy.models?.[modelId]) : 0,
+  };
+}
+
+function waiterReadyAt(state: ProviderPacer, modelId: string | undefined): number {
+  return Math.max(
+    state.providerNextStartAt,
+    modelId ? (state.modelNextStartAt.get(modelId) ?? 0) : 0,
+  );
+}
+
+function pacingRetryAfterSeconds(state: ProviderPacer, modelId: string | undefined, now: number): number {
+  return Math.max(1, Math.ceil(Math.max(0, waiterReadyAt(state, modelId) - now) / 1000));
+}
+
+function rejectExpiredWaiters(providerName: string, state: ProviderPacer, now: number): void {
+  for (let index = state.queue.length - 1; index >= 0; index -= 1) {
+    const waiter = state.queue[index]!;
+    if (now - waiter.queuedAt < maxQueueAgeMs) continue;
+    state.queue.splice(index, 1);
+    if (waiter.abort) waiter.signal?.removeEventListener("abort", waiter.abort);
+    waiter.reject(new RequestPacingQueueOverloadError(
+      providerName,
+      "queue_expired",
+      pacingRetryAfterSeconds(state, waiter.modelId, now),
+    ));
+  }
+}
+
+function removeProviderPacer(providerName: string, state: ProviderPacer): void {
+  if (state.timer) runtime.clearTimer(state.timer);
+  state.timer = undefined;
+  pacers.delete(providerName);
+  const waiters = state.queue.splice(0);
+  const error = new RequestPacingProviderRemovedError(providerName);
+  for (const waiter of waiters) {
+    if (waiter.abort) waiter.signal?.removeEventListener("abort", waiter.abort);
+    waiter.reject(error);
+  }
+}
+
+function runQueue(providerName: string, state: ProviderPacer): void {
+  if (state.queue.length === 0) {
+    if (state.timer) runtime.clearTimer(state.timer);
+    state.timer = undefined;
+    return;
+  }
+  if (state.timer) return;
+  const now = runtime.now();
+  for (const [modelId, readyAt] of state.modelNextStartAt) {
+    if (readyAt <= now) state.modelNextStartAt.delete(modelId);
+  }
+  rejectExpiredWaiters(providerName, state, now);
+  if (state.queue.length === 0) return;
+
+  const providerReadyAt = Math.max(now, state.providerNextStartAt);
+  const waiterIndex = state.queue.findIndex(waiter => {
+    const modelReadyAt = waiter.modelId ? (state.modelNextStartAt.get(waiter.modelId) ?? 0) : 0;
+    return Math.max(providerReadyAt, modelReadyAt) <= now;
+  });
+  if (waiterIndex < 0) {
+    let earliestAt = Number.POSITIVE_INFINITY;
+    for (const waiter of state.queue) {
+      const modelReadyAt = waiter.modelId ? (state.modelNextStartAt.get(waiter.modelId) ?? 0) : 0;
+      const readyAt = Math.max(providerReadyAt, modelReadyAt);
+      const expiresAt = waiter.queuedAt + maxQueueAgeMs;
+      earliestAt = Math.min(earliestAt, readyAt, expiresAt);
+    }
+    const delayMs = Math.max(0, earliestAt - now);
+    state.timer = runtime.setTimer(() => {
+      state.timer = undefined;
+      runQueue(providerName, state);
+    }, delayMs);
+    return;
+  }
+  const waiter = state.queue[waiterIndex]!;
+  state.queue.splice(waiterIndex, 1);
+  if (waiter.abort) waiter.signal?.removeEventListener("abort", waiter.abort);
+  const startedAt = runtime.now();
+  state.lastStartedAt = startedAt;
+  state.lastModelId = waiter.modelId;
+  state.providerNextStartAt = startedAt + waiter.providerIntervalMs;
+  if (waiter.modelId && waiter.modelIntervalMs > 0) {
+    state.modelNextStartAt.set(waiter.modelId, startedAt + waiter.modelIntervalMs);
+  }
+  waiter.resolve();
+  runtime.enqueueMicrotask(() => runQueue(providerName, state));
+}
+
+export async function waitForProviderRequestSlot(
+  providerName: string,
+  provider: OcxProviderConfig,
+  modelId?: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  const intervals = requestPacingIntervals(provider, modelId);
+  if (Math.max(intervals.providerIntervalMs, intervals.modelIntervalMs) <= 0) return;
+  if (signal?.aborted) throw abortReason(signal);
+
+  const state = pacers.get(providerName) ?? {
+    queue: [], providerNextStartAt: 0, modelNextStartAt: new Map<string, number>(),
+  };
+  pacers.set(providerName, state);
+
+  // Give already-eligible or expired waiters a chance to leave before applying the
+  // admission bound to the newest request. This preserves FIFO-ish fairness while
+  // keeping the retained queue strictly bounded under burst load.
+  if (state.timer) {
+    runtime.clearTimer(state.timer);
+    state.timer = undefined;
+  }
+  runQueue(providerName, state);
+  if (state.queue.length >= maxQueueDepth) {
+    throw new RequestPacingQueueOverloadError(
+      providerName,
+      "queue_full",
+      pacingRetryAfterSeconds(state, modelId, runtime.now()),
+    );
+  }
+
+  await new Promise<void>((resolve, reject) => {
+    const waiter: Waiter = { modelId, ...intervals, queuedAt: runtime.now(), signal, resolve, reject };
+    waiter.abort = () => {
+      const index = state.queue.indexOf(waiter);
+      if (index >= 0) state.queue.splice(index, 1);
+      if (state.timer) {
+        runtime.clearTimer(state.timer);
+        state.timer = undefined;
+      }
+      reject(abortReason(signal!));
+      runQueue(providerName, state);
+    };
+    signal?.addEventListener("abort", waiter.abort, { once: true });
+    state.queue.push(waiter);
+    // Abort may race between the eager check above and listener registration.
+    if (signal?.aborted) {
+      waiter.abort();
+      return;
+    }
+    if (state.timer) {
+      runtime.clearTimer(state.timer);
+      state.timer = undefined;
+    }
+    runQueue(providerName, state);
+  });
+}
+
+export function providerRequestPacingStatus(
+  providerName: string,
+  provider: OcxProviderConfig,
+  now = runtime.now(),
+): ProviderRequestPacingStatus {
+  const state = pacers.get(providerName);
+  let nextSlotAt = state?.providerNextStartAt ?? 0;
+  if (state && state.queue.length > 0) {
+    let earliestQueuedSlotAt = Number.POSITIVE_INFINITY;
+    for (const waiter of state.queue) {
+      earliestQueuedSlotAt = Math.min(earliestQueuedSlotAt, waiterReadyAt(state, waiter.modelId));
+    }
+    if (Number.isFinite(earliestQueuedSlotAt)) nextSlotAt = earliestQueuedSlotAt;
+  }
+  return {
+    provider: providerName,
+    enabled: provider.requestPacing?.enabled === true,
+    queued: state?.queue.length ?? 0,
+    nextSlotInMs: Math.max(0, Math.ceil(nextSlotAt - now)),
+    ...(state?.lastStartedAt !== undefined ? { lastStartedAt: state.lastStartedAt } : {}),
+    ...(state?.lastModelId ? { lastModelId: state.lastModelId } : {}),
+  };
+}
+
+export function setProviderRequestPacingLimitsForTest(limits: {
+  maxQueueDepth?: number;
+  maxQueueAgeMs?: number;
+}): void {
+  if (limits.maxQueueDepth !== undefined) maxQueueDepth = limits.maxQueueDepth;
+  if (limits.maxQueueAgeMs !== undefined) maxQueueAgeMs = limits.maxQueueAgeMs;
+}
+
+export function setProviderRequestPacingRuntimeForTest(nextRuntime: RequestPacingRuntime): void {
+  runtime = nextRuntime;
+}
+
+export function reconcileProviderRequestPacing(context: GenerationContext): number {
+  if (context.generation <= lastReconciledGeneration) return 0;
+  let removed = 0;
+  for (const [providerName, state] of pacers) {
+    if (context.providerNames.has(providerName)) continue;
+    removeProviderPacer(providerName, state);
+    removed += 1;
+  }
+  lastReconciledGeneration = context.generation;
+  return removed;
+}
+
+export function resetProviderRequestPacingForTest(): void {
+  for (const state of pacers.values()) if (state.timer) runtime.clearTimer(state.timer);
+  pacers.clear();
+  maxQueueDepth = REQUEST_PACING_MAX_QUEUE_DEPTH;
+  maxQueueAgeMs = REQUEST_PACING_MAX_QUEUE_AGE_MS;
+  runtime = defaultRuntime;
+  lastReconciledGeneration = 0;
+}

package/src/providers/service-tier.ts

@@ -0,0 +1,277 @@
+import type { OcxProviderConfig } from "../types";
+import { captureWireAdapterHardPins } from "../types";
+import { isCanonicalOpenAiForwardProvider } from "./openai-tiers";
+import {
+  getProviderRegistryEntry,
+  providerMatchesRegistryTransport,
+  registryModelServiceTierCapabilityApplies,
+  type InboundWire,
+  type ModelWireDefault,
+} from "./registry";
+import {
+  cloneFastWire,
+  resolveFastPolicy,
+  resolveProviderAuthTransport,
+  type FastPolicyAuthority,
+  type ResolvedFastPolicy,
+} from "./fastwire";
+
+/** OpenAI-compatible adapters that can carry the standard `service_tier` field. */
+export const SERVICE_TIER_ADAPTERS = new Set(["openai-chat", "openai-responses"]);
+
+/** @deprecated A1 evolves this snapshot into the complete FastPolicyAuthority. */
+export type CapturedServiceTierAdapterAuthority = FastPolicyAuthority;
+
+const capturedFastPolicyAuthorities = new WeakMap<object, FastPolicyAuthority>();
+
+type ServiceTierCapabilityProvider = Pick<
+  OcxProviderConfig,
+  | "adapter"
+  | "supportsServiceTier"
+  | "modelSupportsServiceTier"
+  | "modelAdapters"
+  | "baseUrl"
+  | "authMode"
+  | "apiKeyTransport"
+  | "chatServiceTier"
+  | "fastWire"
+>;
+
+function cloneRegistryWireDefaults(
+  defaults: Readonly<Record<string, ModelWireDefault>> | undefined,
+): Readonly<Record<string, ModelWireDefault>> {
+  if (!defaults) return Object.freeze({});
+  const clone: Record<string, ModelWireDefault> = {};
+  for (const [modelId, declaration] of Object.entries(defaults)) {
+    clone[modelId.trim().toLowerCase()] = typeof declaration === "string"
+      ? declaration
+      : Object.freeze({
+          wire: declaration.wire,
+          inbound: Object.freeze([...declaration.inbound]),
+          ...(declaration.authModes
+            ? { authModes: Object.freeze([...declaration.authModes]) }
+            : {}),
+          ...(declaration.forwardCallerServiceTier !== undefined
+            ? { forwardCallerServiceTier: declaration.forwardCallerServiceTier }
+            : {}),
+        });
+  }
+  return Object.freeze(clone);
+}
+
+/**
+ * Capture every registry-owned input before an asynchronous catalog flight begins.
+ * The resolver itself is pure and never reads the live provider registry.
+ */
+function buildFastPolicyAuthority(
+  providerName: string,
+  provider: ServiceTierCapabilityProvider,
+  registryTransportMatch: boolean,
+  capabilityProvider: ServiceTierCapabilityProvider = provider,
+): FastPolicyAuthority {
+  const registry = registryTransportMatch ? getProviderRegistryEntry(providerName) : undefined;
+  const authTransport = resolveProviderAuthTransport(
+    provider.adapter,
+    provider.authMode ?? registry?.authKind ?? "key",
+    provider.apiKeyTransport,
+  );
+  const keyAuthDefaults = registry?.allowKeyAuthOverride === true
+    && (authTransport === "authorization_bearer" || authTransport === "x_api_key")
+    ? registry.keyAuthServiceTier
+    : undefined;
+  const registryModelCapabilities = registry
+    && registryModelServiceTierCapabilityApplies(registry, capabilityProvider)
+    ? registry.modelSupportsServiceTier
+    : undefined;
+  const providerCapability = capabilityProvider.supportsServiceTier
+    ?? keyAuthDefaults?.supportsServiceTier
+    ?? registry?.supportsServiceTier;
+  const authority: FastPolicyAuthority = Object.freeze({
+    providerAdapter: provider.adapter,
+    providerAuthMode: provider.authMode ?? registry?.authKind ?? "key",
+    fastWireDeclaration: cloneFastWire(
+      provider.fastWire !== undefined ? provider.fastWire : registry?.fastWire,
+      { freeze: true },
+    ),
+    ...(registry?.fastTierDescription !== undefined
+      ? { fastTierDescription: registry.fastTierDescription }
+      : {}),
+    modelWireOverrideAllowed: !isCanonicalOpenAiForwardProvider(provider as OcxProviderConfig),
+    authTransport,
+    capability: Object.freeze({
+      ...(providerCapability !== undefined ? { provider: providerCapability } : {}),
+      models: Object.freeze({
+        ...(registryModelCapabilities ?? {}),
+        ...(keyAuthDefaults?.modelSupportsServiceTier ?? {}),
+        ...(capabilityProvider.modelSupportsServiceTier ?? {}),
+      }),
+      ...(provider.chatServiceTier !== undefined
+        ? { chatServiceTier: provider.chatServiceTier }
+        : keyAuthDefaults?.chatServiceTier !== undefined
+          ? { chatServiceTier: keyAuthDefaults.chatServiceTier }
+          : {}),
+    }),
+    modelAdapters: Object.freeze({ ...(provider.modelAdapters ?? {}) }),
+    hardPins: captureWireAdapterHardPins(providerName),
+    registryWireDefaults: cloneRegistryWireDefaults(registry?.modelWireDefaults),
+  });
+  return authority;
+}
+
+export function captureFastPolicyAuthority(
+  providerName: string,
+  provider: ServiceTierCapabilityProvider,
+  registryTransportMatch: boolean,
+  capabilityProvider: ServiceTierCapabilityProvider = provider,
+): FastPolicyAuthority {
+  const authority = buildFastPolicyAuthority(
+    providerName,
+    provider,
+    registryTransportMatch,
+    capabilityProvider,
+  );
+  if (Object.isFrozen(provider)) capturedFastPolicyAuthorities.set(provider, authority);
+  return authority;
+}
+
+/** @deprecated Use captureFastPolicyAuthority. The legacy inbound argument is now snapshot data. */
+export function captureServiceTierAdapterAuthority(
+  providerName: string,
+  provider: ServiceTierCapabilityProvider,
+  registryTransportMatch: boolean,
+  _inbound: InboundWire = "responses",
+): FastPolicyAuthority {
+  return captureFastPolicyAuthority(providerName, provider, registryTransportMatch);
+}
+
+function authorityForProvider(
+  provider: ServiceTierCapabilityProvider,
+  providerName?: string,
+  capabilityProvider?: ServiceTierCapabilityProvider,
+): FastPolicyAuthority {
+  // Preserve the legacy no-name short circuit: serviceTierSupportForModel() used the
+  // provider adapter directly when no provider identity was available, so no configured
+  // override, hard pin, or registry default may participate on this path in A1.
+  if (providerName === undefined) {
+    const authority = buildFastPolicyAuthority("", provider, false, capabilityProvider ?? provider);
+    return Object.freeze({
+      ...authority,
+      modelAdapters: Object.freeze({}),
+      hardPins: Object.freeze({}),
+      registryWireDefaults: Object.freeze({}),
+    });
+  }
+  const captured = capabilityProvider === undefined && Object.isFrozen(provider)
+    ? capturedFastPolicyAuthorities.get(provider)
+    : undefined;
+  if (captured) return captured;
+  const registryTransportMatch = providerMatchesRegistryTransport(providerName, provider);
+  const authority = buildFastPolicyAuthority(
+    providerName,
+    provider,
+    registryTransportMatch,
+    capabilityProvider ?? provider,
+  );
+  // Frozen provider snapshots cannot drift, so repeated catalog/runtime projections may safely
+  // reuse the registry lookup and detached declaration maps. Mutable configs still rebuild.
+  if (Object.isFrozen(provider)) capturedFastPolicyAuthorities.set(provider, authority);
+  return authority;
+}
+
+/** Resolve the pure Fast policy for a provider/model pair. */
+export function fastPolicyForModel(
+  provider: ServiceTierCapabilityProvider,
+  modelId: string,
+  providerName?: string,
+  inbound: InboundWire = "responses",
+  capabilityProvider?: ServiceTierCapabilityProvider,
+): ResolvedFastPolicy {
+  return resolveFastPolicy(
+    authorityForProvider(provider, providerName, capabilityProvider),
+    modelId,
+    inbound,
+  );
+}
+
+/**
+ * Resolve the declared provider/model capability without applying wire availability.
+ * Kept as a public compatibility helper for callers that need the pure tri-state.
+ */
+export function supportsServiceTierForModel(
+  provider: Pick<OcxProviderConfig, "supportsServiceTier" | "modelSupportsServiceTier">,
+  modelId: string,
+): boolean | undefined {
+  const authority: FastPolicyAuthority = {
+    providerAdapter: "openai-responses",
+    fastWireDeclaration: undefined,
+    modelWireOverrideAllowed: true,
+    authTransport: "authorization_bearer",
+    capability: {
+      ...(provider.supportsServiceTier !== undefined ? { provider: provider.supportsServiceTier } : {}),
+      models: provider.modelSupportsServiceTier ?? {},
+    },
+    modelAdapters: {},
+    hardPins: {},
+    registryWireDefaults: {},
+  };
+  return resolveFastPolicy(authority, modelId).capability;
+}
+
+/** Whether a Chat route may forward an arbitrary caller tier rather than canonical Fast. */
+export function canForwardForeignServiceTierForChatModel(
+  provider: Pick<OcxProviderConfig, "supportsServiceTier" | "modelSupportsServiceTier" | "chatServiceTier">,
+  modelId: string,
+): boolean {
+  const capability = supportsServiceTierForModel(provider, modelId);
+  return capability !== false && provider.chatServiceTier === true;
+}
+
+/** Final adapter selected by the Fast policy's four-level wire resolver. */
+export function serviceTierAdapterForModel(
+  providerName: string,
+  provider: ServiceTierCapabilityProvider,
+  modelId: string,
+  inbound: InboundWire = "responses",
+): string {
+  return fastPolicyForModel(provider, modelId, providerName, inbound).adapter;
+}
+
+/** Whether the final provider/model pair can publish/send OpenAI service tiers. */
+export function canForwardServiceTierForModel(
+  provider: ServiceTierCapabilityProvider,
+  modelId: string,
+  providerName?: string,
+  inbound: InboundWire = "responses",
+): boolean {
+  return serviceTierSupportForModel(provider, modelId, providerName, inbound) === true;
+}
+
+/**
+ * Compatibility projection for catalog, routing, and fingerprint consumers. The new
+ * resolver carries richer eligibility internally while preserving the old tri-state bytes.
+ */
+export function serviceTierSupportForModel(
+  provider: ServiceTierCapabilityProvider,
+  modelId: string,
+  providerName?: string,
+  inbound: InboundWire = "responses",
+): boolean | undefined {
+  const policy = fastPolicyForModel(provider, modelId, providerName, inbound);
+  return serviceTierSupportFromPolicy(policy);
+}
+
+/** Compatibility projection shared by catalog, routing, and request logging. */
+export function serviceTierSupportFromPolicy(
+  policy: Pick<ResolvedFastPolicy, "eligibility" | "adapter" | "forwardCallerTier">,
+): boolean | undefined {
+  if (policy.eligibility === "eligible") return true;
+  if (policy.eligibility === "unclassified") {
+    // An unclassified route that cannot forward a caller tier has definitive negative
+    // evidence even when its adapter can normally serialize service_tier. This covers both
+    // Chat routes without chatServiceTier and a registry default that explicitly closes a
+    // subscription gateway. Generic unclassified Responses routes still project unknown.
+    if (!policy.forwardCallerTier) return false;
+    return undefined;
+  }
+  return false;
+}