cartethyia 2.5.9 → 2.6.0

package/src/transport/guards/error-lifecycle.ts

@@ -6,6 +6,7 @@ import { GATEWAY_SECURITY_HEADERS } from "../../security/outbound-headers";
 import type { TelemetryBatchBuffer, TelemetryEventInput } from "../../observability/telemetry-buffer";
 import type { TelemetryPayloadCapture } from "../../observability/payload-capture";
 import { captureEarlyRejectionPayload } from "./early-rejection-capture";
+import type { CartethyiaDatabase } from "../../persistence/db-handle";
 import { reasoningEffortFromIntent } from "../../providers/provider-reasoning";
 import { metrics } from "../../observability/metrics";
 import { computeTokensPerSec } from "../../observability/token-speed";
@@ -168,6 +169,7 @@ export interface AfterResponseApp {
 export function finalizeRequestTelemetry(
   state: ProxyRequestState,
   telemetryBuffer: TelemetryBatchBuffer,
+  db?: CartethyiaDatabase,
 ): void {
   // Discovery/surface routes under `/v1` (e.g. `/v1/models`) are authenticated
   // gateway routes that never dispatch upstream. They hold request state but
@@ -225,7 +227,11 @@ export function finalizeRequestTelemetry(
   // would have nothing to read. Capture the ingress body and the error the
   // client saw, keyed to the same request id as the row.
   if (requestData === undefined && state.ingressBody !== undefined) {
-    void captureEarlyRejectionPayload(state, authorization);
+    // `deps.db` so the helper resolves the tenant's capture mode through the
+    // same preference path every dispatched request uses — without it, an
+    // early rejection captured bodies with the deployment default regardless
+    // of the tenant's `none`/`metadata` choice.
+    void captureEarlyRejectionPayload(state, authorization, db);
   }
 
   telemetryBuffer.enqueue(
@@ -385,6 +391,8 @@ export function registerTelemetryLifecycle(
      * telemetry row without a payload frame rather than failing.
      */
     readonly payloadCapture?: TelemetryPayloadCapture;
+    /** Database handle for early-rejection capture-mode resolution. */
+    readonly db?: CartethyiaDatabase;
   },
 ): void {
   app.afterResponse(async ({ request }) => {
@@ -404,7 +412,7 @@ export function registerTelemetryLifecycle(
       // finalizer for requests that never reached completion (early
       // rejections). Cleanup below must still run for completed requests:
       // skipping it leaks one in-flight count per request, forever.
-      if (!state.completed) finalizeRequestTelemetry(state, deps.telemetryBuffer);
+      if (!state.completed) finalizeRequestTelemetry(state, deps.telemetryBuffer, deps.db);
     } finally {
       state.cleanup();
     }

package/src/transport/guards/guard-errors.ts

@@ -0,0 +1,23 @@
+/**
+ * Shared construction points for the guard errors that more than one guard
+ * raises. Normalisation downstream is single-seam (`error-lifecycle.ts`); the
+ * duplication this module removes was on the construction side, where four
+ * guards each hand-rolled their own code + message for the same conditions.
+ */
+import { GatewayError } from "../../protocol/gateway-error";
+
+/** Thrown when the request's client peer address cannot be established. */
+export function peerAddressUnavailableError(): GatewayError {
+  return new GatewayError("admission_unavailable", 503, "client peer address unavailable");
+}
+
+/** Thrown when the request body exceeds the endpoint's byte budget. */
+export function bodyTooLargeError(maxBytes: number): GatewayError {
+  return new GatewayError(
+    "invalid_request",
+    413,
+    `The request body is larger than this endpoint accepts (${maxBytes} bytes). Shorten the prompt or split the request.`,
+  );
+}
+
+

package/src/transport/guards/headroom-guard.ts

@@ -17,15 +17,14 @@
  * metrics are exempt so an operator can always reach the console that explains
  * the shedding.
  */
-import { totalmem } from "node:os";
 import { Elysia } from "elysia";
 import { GatewayError } from "../../protocol/gateway-error";
 import type { RedisClient } from "../../persistence/redis";
+import { deriveMemoryHeadroomBytes } from "../../config/resources";
+import type { RuntimeMemorySnapshot } from "../../observability/runtime-metrics";
 import type { ProxyRequestStateStore } from "../request/state";
 import { isProxyRequest } from "./body-policy";
 
-/** Free bytes the process must keep below its RSS before it stops accepting. */
-const DEFAULT_MIN_FREE_BYTES = 64 * 1024 * 1024;
 /** How long the shared shed marker outlives the last shed, so peers can see it. */
 const SHED_MARKER_TTL_SECONDS = 5;
 /** Fleet-visible marker: set on the first shed so every instance reports the same state. */
@@ -34,7 +33,22 @@ const SHED_RETRY_MS = 1_000;
 
 export interface HeadroomGuardDeps {
   readonly store: ProxyRequestStateStore;
-  /** Free bytes to keep; defaults to 64 MiB. A smaller box wants a larger value. */
+  /**
+  * The last sampled memory state, published once per sample by
+  * `RuntimeMetricsSampler`.
+  *
+  * Taken from the sampler rather than measured here: this runs on EVERY
+  * request, and `process.memoryUsage()` on the request path is a cost paid
+  * per request to answer a question that changes over seconds. When absent
+  * (before the first sample) the guard falls back to measuring once — an
+  * unsampled process is still a process that can be OOM-killed.
+  */
+  readonly memorySnapshot?: () => RuntimeMemorySnapshot | undefined;
+  /**
+   * Free bytes to keep below the resolved limit. Defaults to
+   * `MEMORY_HEADROOM_FRACTION` of it, so a 512 MiB container keeps ~102 MiB
+   * free rather than the flat 64 MiB a 2 GiB one would keep.
+   */
   readonly minFreeBytes?: number;
   /**
    * Optional shared client. Used only to publish a shed marker for visibility;
@@ -45,11 +59,27 @@ export interface HeadroomGuardDeps {
 }
 
 export function createHeadroomGuard(deps: HeadroomGuardDeps): Elysia {
-  const minFreeBytes = deps.minFreeBytes ?? DEFAULT_MIN_FREE_BYTES;
+  const fixedHeadroom = deps.minFreeBytes;
   return new Elysia()
     .beforeHandle(({ request }) => {
       if (!isProxyRequest(request)) return;
-      if (process.memoryUsage().rss + minFreeBytes <= totalmem()) return;
+      // Shed when RSS plus the headroom would exceed this process's LIMIT —
+      // not the host's total memory. A container capped at 512 MiB on a 16 GiB
+      // host never approached the old `os.totalmem()` threshold, so the guard
+      // never fired and the process was OOM-killed instead of shedding.
+      //
+      // The snapshot is up to one sample interval old. That is deliberate:
+      // this is the hottest path in the process, and the condition it guards
+      // develops over seconds.
+      const snapshot = deps.memorySnapshot?.();
+      const limit = snapshot?.limitBytes;
+      if (limit === undefined) {
+        // No limit discoverable and no sample yet: nothing to shed against.
+        return;
+      }
+      const rss = snapshot?.rssBytes ?? process.memoryUsage().rss;
+      const headroom = fixedHeadroom ?? deriveMemoryHeadroomBytes(limit);
+      if (rss + headroom <= limit) return;
       if (deps.redis) {
         // Fire-and-forget: a failed marker write must not change this request's
         // outcome, and awaiting it would put Redis on the shed path.
@@ -61,7 +91,11 @@ export function createHeadroomGuard(deps: HeadroomGuardDeps): Elysia {
         "overloaded",
         503,
         "The gateway is at capacity; retry shortly.",
-        { retryAfterMs: SHED_RETRY_MS, minFreeBytes, activeRequests: deps.store.activeCount() },
+        {
+          retryAfterMs: SHED_RETRY_MS,
+          minFreeBytes: headroom,
+          activeRequests: deps.store.activeCount(),
+        },
       );
     })
     .as("plugin");

package/src/transport/guards/pipeline.ts

@@ -28,6 +28,7 @@ import type { CartethyiaDatabase } from "../../persistence/db-handle";
 import type { RedisClient } from "../../persistence/redis";
 import type { TelemetryBatchBuffer } from "../../observability/telemetry-buffer";
 import type { ModelStrikeService } from "../../security/model-abuse";
+import type { RuntimeMemorySnapshot } from "../../observability/runtime-metrics";
 import {
   TelemetryPayloadCapture,
   payloadCaptureKeepsBodies,
@@ -58,6 +59,20 @@ export interface TransportPipelineContext {
    * decision is local, and Redis is only written for fleet visibility.
    */
   readonly redis?: RedisClient | undefined;
+  /**
+   * The sampled memory state the headroom guard sheds against.
+   *
+   * Supplied by the runtime sampler. When absent the guard cannot compare RSS
+   * against a limit and does not shed — so this must be wired in production,
+   * and the sampler must be initialized at boot even before its first tick.
+   */
+  readonly memorySnapshot?: () => RuntimeMemorySnapshot | undefined;
+  /**
+   * Whether invoke is switched off for a tenant. Optional: without it the
+   * gateway never refuses on the kill switch, which is the pre-change
+   * behaviour rather than a failure.
+   */
+  readonly isInvokeDisabled?: (tenantId: string) => Promise<boolean | undefined>;
 }
 
 /**
@@ -84,6 +99,7 @@ export function createTransportPipeline(context: TransportPipelineContext): Tran
     createProcessConcurrencyMiddleware(),
     createHeadroomGuard({
       store: context.stateStore,
+      ...(context.memorySnapshot === undefined ? {} : { memorySnapshot: context.memorySnapshot }),
       ...(context.redis === undefined ? {} : { redis: context.redis }),
     }),
     createDependencyReadinessMiddleware({
@@ -103,6 +119,9 @@ export function createTransportPipeline(context: TransportPipelineContext): Tran
       db: context.db,
       stateStore: context.stateStore,
       ...(context.modelStrikes ? { modelStrikes: context.modelStrikes } : {}),
+      ...(context.isInvokeDisabled
+        ? { isInvokeDisabled: context.isInvokeDisabled }
+        : {}),
     }),
     createCanonicalRequestMiddleware({
       stateStore: context.stateStore,
@@ -181,6 +200,11 @@ export function createTransportPipeline(context: TransportPipelineContext): Tran
             (resolved) => (resolved.ok ? resolved.retentionMs : payloadRetentionMs()),
           ),
         ),
+        // The early-rejection capture resolves the tenant's capture mode
+        // through the same preference path dispatched requests use, instead of
+        // capturing bodies under the deployment default regardless of the
+        // tenant's none/metadata choice.
+        db: context.db,
       });
     } else {
       registerRequestCleanup(rootApp, { stateStore: context.stateStore });

package/src/transport/guards/request-context.ts

@@ -20,6 +20,7 @@ import {
   type ModelAbuseOutcome,
   type ModelStrikeService,
 } from "../../security/model-abuse";
+import { peerAddressUnavailableError } from "./guard-errors";
 
 /**
  * Whether a thrown error is a *model* rejection the strike layer counts: the
@@ -123,7 +124,7 @@ export function createClientIdentityMiddleware(deps: {
         ? deps.resolvePeerAddress(request)
         : (server?.requestIP(request)?.address ?? null);
       if (!peer)
-        throw new GatewayError("admission_unavailable", 503, "client peer address unavailable");
+        throw peerAddressUnavailableError();
       const address = resolveClientIdentity(request, deps.trustedProxyBoundary, peer);
       const state = deps.stateStore.require(request);
       state.clientIdentity = {

package/src/transport/routing/admission.ts

@@ -69,8 +69,20 @@ function capacityRejected(): AdmissionDecision {
 
 export class InMemoryAdmissionController implements AdmissionController {
   private inflight = new Map<string, number>();
-  snapshotAccountInflight(): ReadonlyMap<string, number> {
-    return new Map(this.inflight);
+  readAccountInflight(
+    candidates: readonly RouteCandidate[],
+  ): ReadonlyMap<string, number> {
+    // Dedupe buckets: one candidate list often repeats a bucket (combo members
+    // fan out to several candidates on one account), and the caller only needs
+    // one count per bucket to make its ordering decision.
+    const snapshot = new Map<string, number>();
+    for (const candidate of candidates) {
+      const key = admissionKey(candidate);
+      if (snapshot.has(key)) continue;
+      const value = this.inflight.get(key) ?? 0;
+      if (value > 0) snapshot.set(key, value);
+    }
+    return snapshot;
   }
   async admit(candidate: RouteCandidate): Promise<AdmissionDecision> {
     // Bucket per ACCOUNT, not per provider:model. A provider with N accounts
@@ -166,17 +178,47 @@ export class RedisAdmissionController implements AdmissionController {
     await redisEvalNumber(this.redis, RELEASE_SCRIPT, 1, this.key(reservation.candidate));
   }
 
-  async snapshotAccountInflight(): Promise<ReadonlyMap<string, number>> {
-    const keys = await this.redis.keys("admission:inflight:*");
-    if (keys.length === 0) return new Map();
-    const values = await this.redis.mget(...keys);
+  async readAccountInflight(
+    candidates: readonly RouteCandidate[],
+  ): Promise<ReadonlyMap<string, number>> {
+    // Candidate-scoped exact reads replace the fleet-wide `KEYS` + unbounded
+    // `MGET` this method used to run on every `plan()` that had any bounded
+    // candidate. `KEYS` is O(keyspace) and blocks the single-threaded Redis
+    // server; on a shared box the scan was measured as the dominant planning
+    // cost. The keys here are exactly the buckets the caller is about to order
+    // candidates by, read in bounded sequential chunks.
+    //
+    // `KEYS` and `SCAN` stay banned on this path by construction: neither can
+    // answer "what are the values of these N known keys" in fewer than
+    // O(keyspace) server work, and a fleet-wide inflight snapshot has no
+    // request-path consumer left. (If one ever appears, it belongs in a
+    // maintenance task using `SCAN` with `COUNT`, never here.)
+    const buckets = [...new Set(candidates.map(admissionKey))];
+    if (buckets.length === 0) return new Map();
+    const keys = buckets.map((bucket) => `admission:inflight:${bucket}`);
+    const values: number[] = [];
+    // Bounded sequential chunks: one huge MGET would move the unbounded
+    // payload problem from the KEYS call to the MGET call. 256 keys keeps a
+    // single command payload small while the round trips stay proportional to
+    // the (deduped, candidate-bounded) request size.
+    for (let offset = 0; offset < keys.length; offset += MGET_CHUNK_SIZE) {
+      const chunk = keys.slice(offset, offset + MGET_CHUNK_SIZE);
+      const raw = (await this.redis.mget(...chunk)) as (string | null)[];
+      for (const value of raw) values.push(Number(value ?? 0));
+    }
     const snapshot = new Map<string, number>();
-    for (const [index, key] of keys.entries()) {
-      const value = Number(values[index] ?? 0);
+    for (const [index, bucket] of buckets.entries()) {
+      const value = values[index] ?? 0;
       if (Number.isFinite(value) && value > 0) {
-        snapshot.set(key.slice("admission:inflight:".length), value);
+        snapshot.set(bucket, value);
       }
+      // The Redis controller's counter keys are TTL self-healing, so a bucket
+      // absent from the reply reads as zero and is omitted — the projection
+      // both controllers share: positive-finite counts only, unprefixed names.
     }
     return snapshot;
   }
 }
+
+/** Sequential MGET chunk bound for {@link RedisAdmissionController.readAccountInflight}. */
+const MGET_CHUNK_SIZE = 256;

package/src/transport/routing/bounded-lru.ts

@@ -0,0 +1,49 @@
+// Bounded LRU map: the one shared implementation for routing-layer caches.
+
+/**
+ * A capacity-bounded LRU `Map`.
+ *
+ * `get()` refreshes recency; `set()` evicts exactly the least-recently-used
+ * entry on overflow. Insertion order is the recency order (head = LRU,
+ * tail = MRU), which `Map` maintains for free — so eviction is one
+ * `keys().next()` and no bookkeeping arrays.
+ *
+ * The routing layer previously hand-rolled this shape twice (round-robin
+ * state, capability profiles) with subtly different code. One implementation
+ * means the bound is provable once: the map's size can never exceed
+ * `capacity`, and every entry is reachable through the normal `Map` API the
+ * callers already use.
+ */
+export class BoundedLruMap<K, V> extends Map<K, V> {
+  private readonly capacity: number;
+
+  constructor(capacity: number) {
+    if (!Number.isFinite(capacity) || capacity < 1 || Math.trunc(capacity) !== capacity) {
+      throw new Error(`BoundedLruMap capacity must be a positive integer, got ${capacity}`);
+    }
+    super();
+    this.capacity = Math.trunc(capacity);
+  }
+
+  override get(key: K): V | undefined {
+    if (!super.has(key)) return undefined;
+    // Refresh recency: delete + re-insert moves the entry to the tail.
+    const value = super.get(key)!;
+    super.delete(key);
+    super.set(key, value);
+    return value;
+  }
+
+  override set(key: K, value: V): this {
+    // An existing key is re-inserted at the tail even when the value is the
+    // same object, so a read-modify-write cycle cannot leave a hot entry
+    // parked at the head one eviction away from disappearing.
+    if (super.has(key)) super.delete(key);
+    else if (super.size >= this.capacity) {
+      const oldest = super.keys().next().value;
+      if (oldest !== undefined) super.delete(oldest);
+    }
+    super.set(key, value);
+    return this;
+  }
+}

package/src/transport/routing/candidate-index.ts

@@ -0,0 +1,88 @@
+// Immutable per-snapshot candidate index for model resolution.
+import type { RouteCandidate, RouteSnapshot } from "./route-model";
+
+/** One parsed model reference: `provider/model` or a bare `model`. */
+export interface ParsedModel {
+  readonly providerId: string | null;
+  readonly modelId: string;
+}
+
+/** Parse one model reference into its provider and bare model id. */
+export function parseModelReference(model: string): ParsedModel {
+  const slash = model.indexOf("/");
+  if (slash === -1) return { providerId: null, modelId: model };
+  return { providerId: model.slice(0, slash), modelId: model.slice(slash + 1) };
+}
+
+/**
+ * An immutable, per-snapshot index of route candidates.
+ *
+ * Snapshots are immutable by contract (rebuilt wholesale on catalog change),
+ * so the index is built once per snapshot — O(C) — and reused by every plan,
+ * instead of the previous behavior of rescanning the full candidate list per
+ * resolved model inside one plan: a combo with K unique members over C
+ * candidates cost K×C `candidateMatches` calls (each re-parsing the model
+ * string), repeated on every request.
+ */
+export interface RouteCandidateIndex {
+  /** Candidates keyed by `provider_id/model_id`, snapshot order preserved. */
+  readonly qualified: ReadonlyMap<string, readonly RouteCandidate[]>;
+  /** Candidates keyed by bare `model_id`, snapshot order preserved. */
+  readonly bare: ReadonlyMap<string, readonly RouteCandidate[]>;
+}
+
+/** WeakMap cache: an evicted/rebuilt snapshot releases its index automatically. */
+const indexBySnapshot = new WeakMap<RouteSnapshot, RouteCandidateIndex>();
+
+/**
+ * Builds (once per snapshot) the lookup index for `resolveMatchingCandidates`.
+ *
+ * Each candidate is visited exactly once and filed under both its qualified and
+ * bare key, so a lookup later is one `Map.get` regardless of catalog size.
+ * Snapshot order is preserved within each bucket because `Map` iterates in
+ * insertion order and candidates are appended in one pass.
+ */
+export function routeCandidateIndex(snapshot: RouteSnapshot): RouteCandidateIndex {
+  const cached = indexBySnapshot.get(snapshot);
+  if (cached) return cached;
+  const qualified = new Map<string, RouteCandidate[]>();
+  const bare = new Map<string, RouteCandidate[]>();
+  for (const candidate of snapshot.candidates) {
+    const qualifiedKey = `${candidate.provider_id}/${candidate.model_id}`;
+    const qualifiedList = qualified.get(qualifiedKey);
+    if (qualifiedList) qualifiedList.push(candidate);
+    else qualified.set(qualifiedKey, [candidate]);
+
+    const bareList = bare.get(candidate.model_id);
+    if (bareList) bareList.push(candidate);
+    else bare.set(candidate.model_id, [candidate]);
+  }
+  const index: RouteCandidateIndex = { qualified, bare };
+  indexBySnapshot.set(snapshot, index);
+  return index;
+}
+
+/**
+ * Looks up the candidates a parsed model reference resolves to, filtered by
+ * tenant.
+ *
+ * A qualified reference (`provider/model`) reads the exact bucket and requires
+ * the provider to match; a bare reference reads every provider's bucket for
+ * that model. The tenant rule is the resolver's existing one, unchanged:
+ * a candidate is visible only when it is global (`tenant_id == null`) or
+ * belongs to the requesting tenant.
+ */
+export function lookupRouteCandidates(
+  index: RouteCandidateIndex,
+  parsedModel: ParsedModel,
+  tenantId: string | null,
+): readonly RouteCandidate[] {
+  const group =
+    parsedModel.providerId === null
+      ? index.bare.get(parsedModel.modelId)
+      : index.qualified.get(`${parsedModel.providerId}/${parsedModel.modelId}`);
+  if (group === undefined || group.length === 0) return [];
+  return group.filter(
+    (candidate) => candidate.tenant_id == null || candidate.tenant_id === tenantId,
+  );
+}

package/src/transport/routing/reservations.ts

@@ -1,25 +1,136 @@
-// In-memory reservation lease store with lazy TTL cleanup.
+// In-memory reservation lease store with expiry-ordered cleanup.
 import type { Reservation, RouteCandidate, RoutingRevision } from "./route-model";
 
+/** One heap node: when a lease expires, and which lease it names. */
+interface ExpiryNode {
+  readonly expiresAt: number;
+  readonly leaseId: string;
+}
+
 /**
- * In-memory reservation lease store with lazy TTL cleanup. The bounded sweep
- * runs before acquisition so abandoned local metadata cannot grow forever.
+ * Binary min-heap on `expiresAt`.
+ *
+ * The sweep only ever needs the *earliest* expiring reservations, so a heap
+ * answers "which nodes are expired right now" by examining heads only — the
+ * previous implementation walked the entire lease map per sweep interval.
+ * Stale nodes (a lease released before its expiry) are harmless: the sweep
+ * verifies against the map before deleting and drops the node.
+ */
+class ExpiryHeap {
+  private nodes: ExpiryNode[] = [];
+
+  get size(): number {
+    return this.nodes.length;
+  }
+
+  insert(node: ExpiryNode): void {
+    this.nodes.push(node);
+    this.bubbleUp(this.nodes.length - 1);
+  }
+
+  /** The earliest-expiring node, without removing it. */
+  peek(): ExpiryNode | undefined {
+    return this.nodes[0];
+  }
+
+  pop(): ExpiryNode | undefined {
+    const top = this.nodes[0];
+    const last = this.nodes.pop();
+    if (this.nodes.length > 0 && last !== undefined) {
+      this.nodes[0] = last;
+      this.bubbleDown(0);
+    }
+    return top;
+  }
+
+  private bubbleUp(index: number): void {
+    let current = index;
+    while (current > 0) {
+      const parent = (current - 1) >> 1;
+      if (this.nodes[current]!.expiresAt >= this.nodes[parent]!.expiresAt) break;
+      [this.nodes[current], this.nodes[parent]] = [this.nodes[parent]!, this.nodes[current]!];
+      current = parent;
+    }
+  }
+
+  private bubbleDown(index: number): void {
+    let current = index;
+    for (;;) {
+      const left = current * 2 + 1;
+      const right = left + 1;
+      let smallest = current;
+      if (left < this.nodes.length && this.nodes[left]!.expiresAt < this.nodes[smallest]!.expiresAt) {
+        smallest = left;
+      }
+      if (
+        right < this.nodes.length &&
+        this.nodes[right]!.expiresAt < this.nodes[smallest]!.expiresAt
+      ) {
+        smallest = right;
+      }
+      if (smallest === current) break;
+      [this.nodes[current], this.nodes[smallest]] = [this.nodes[smallest]!, this.nodes[current]!];
+      current = smallest;
+    }
+  }
+}
+
+/**
+ * In-memory reservation lease store with expiry-ordered cleanup.
+ *
+ * Acquisitions insert into an expiry min-heap; the bounded sweep examines only
+ * expired heads and verifies the map still holds that exact reservation before
+ * deleting (a released entry leaves a harmless stale node behind, dropped on
+ * the next sweep that reaches it or by periodic compaction). Stale nodes are
+ * compacted when they dominate the heap, so a long stream of short-lived
+ * released reservations cannot grow the heap without bound.
+ *
+ * The sweep interval stays 30 s and the lease TTL stays 30 s by default. Live
+ * reservations are not capped — capacity is admission's job, not the lease
+ * bookkeeping's. `expires_at` is NOT used to skip admission release: the
+ * admission counters protect the real limits and carry their own TTL
+ * (`INFLIGHT_TTL_SECONDS`); this map is local bookkeeping only.
  */
 export class ReservationManager {
   private leases = new Map<string, Reservation>();
+  private expiry = new ExpiryHeap();
   private lastSweepAt = 0;
   private static readonly SWEEP_INTERVAL_MS = 30_000;
 
+  constructor(private readonly now: () => number = Date.now) {}
+
   private sweepExpired(now: number): void {
     if (now - this.lastSweepAt < ReservationManager.SWEEP_INTERVAL_MS) return;
     this.lastSweepAt = now;
-    for (const [leaseId, reservation] of this.leases) {
-      if (reservation.expires_at < now) this.leases.delete(leaseId);
+    // Only expired heads are examined: once the head expires later than `now`,
+    // every remaining node does too, so the loop stops without touching them.
+    while (this.expiry.size > 0) {
+      const head = this.expiry.peek()!;
+      if (head.expiresAt >= now) break;
+      this.expiry.pop();
+      const lease = this.leases.get(head.leaseId);
+      // A released reservation is a harmless stale node: nothing to delete.
+      // (Lease identity is the unique lease id, so a reused id cannot alias —
+      // the id embeds acquisition time plus randomness.)
+      if (lease && lease.expires_at === head.expiresAt && lease.expires_at < now) {
+        this.leases.delete(head.leaseId);
+      }
+    }
+    // Periodic compaction of stale nodes (entries whose lease was released or
+    // re-keyed before expiry): rebuild the heap only when stale nodes make up
+    // the bulk of it, so the common all-live case pays nothing.
+    if (this.expiry.size > this.leases.size * 2) {
+      const live: ExpiryNode[] = [];
+      for (const [leaseId, lease] of this.leases) {
+        live.push({ expiresAt: lease.expires_at, leaseId });
+      }
+      this.expiry = new ExpiryHeap();
+      for (const node of live) this.expiry.insert(node);
     }
   }
 
   acquire(candidate: RouteCandidate, _revision: RoutingRevision, ttlMs = 30_000): Reservation {
-    const now = Date.now();
+    const now = this.now();
     this.sweepExpired(now);
     const lease_id = `${candidate.provider_id}:${candidate.model_id}:${now}:${Math.random().toString(16).slice(2)}`;
     const r: Reservation = {
@@ -29,17 +140,20 @@ export class ReservationManager {
       expires_at: now + ttlMs,
     };
     this.leases.set(lease_id, r);
+    this.expiry.insert({ expiresAt: r.expires_at, leaseId: lease_id });
     return r;
   }
 
   release(lease_id: string): void {
+    // The heap node stays until its expiry passes — the sweep treats it as
+    // stale and drops it. Cheaper than a heap search per release.
     this.leases.delete(lease_id);
   }
 
   get(lease_id: string): Reservation | undefined {
     const r = this.leases.get(lease_id);
     if (!r) return undefined;
-    if (r.expires_at < Date.now()) {
+    if (r.expires_at < this.now()) {
       this.leases.delete(lease_id);
       return undefined;
     }

package/src/transport/routing/round-robin.ts

@@ -1,5 +1,6 @@
 // Round-robin rotation state and its bounded per-key LRU.
 import type { RouteCandidate } from "./route-model";
+import { BoundedLruMap } from "./bounded-lru";
 
 export class RoundRobinState {
   private idx = 0;
@@ -22,29 +23,16 @@ export class RoundRobinState {
  * combinations would retain one entry forever. 1000 is far above the working
  * set of any real catalog, so eviction only ever targets genuinely cold keys.
  */
-const MAX_ROUND_ROBIN_ENTRIES = 1000;
+export const MAX_ROUND_ROBIN_ENTRIES = 1000;
 
 /**
- * Bounded LRU accessor for round-robin state: an existing key is refreshed to
- * the tail (most-recently-used), and inserting past the cap evicts the head
- * (least-recently-used). Keeps the hot routing working set while guaranteeing
- * the map cannot grow without bound.
+ * Creates the bounded LRU map the router keeps round-robin state in.
+ *
+ * The bound is the same 1000-entry cap the hand-rolled version enforced;
+ * `BoundedLruMap` is now the single implementation of that behavior, so the
+ * eviction rule (get refreshes recency, overflow evicts the LRU head) cannot
+ * drift between this cache and the capability-profile cache.
  */
-export function getOrCreateRoundRobin(
-  map: Map<string, RoundRobinState>,
-  key: string,
-): RoundRobinState {
-  const existing = map.get(key);
-  if (existing) {
-    map.delete(key);
-    map.set(key, existing);
-    return existing;
-  }
-  if (map.size >= MAX_ROUND_ROBIN_ENTRIES) {
-    const oldest = map.keys().next().value;
-    if (oldest !== undefined) map.delete(oldest);
-  }
-  const created = new RoundRobinState();
-  map.set(key, created);
-  return created;
+export function createRoundRobinMap(): BoundedLruMap<string, RoundRobinState> {
+  return new BoundedLruMap<string, RoundRobinState>(MAX_ROUND_ROBIN_ENTRIES);
 }