cartethyia 2.6.7 → 2.7.1

package/src/transport/request/inflight.ts

@@ -1,101 +1,10 @@
 /**
- * In-flight provider-dispatch registry — the single source of truth for the
- * live "in flight" gauge. One entry per request that has actually acquired
- * provider-dispatch leases, keyed by `requestId`, carrying the client IP
- * captured at ingress and the hard deadline the flight must not outlive.
- *
- * The registry is *owned* by the `ProxyRequestStateStore` (`state.ts`), which
- * is also the one place that evicts a request, so a flight cannot outlive its
- * state. A separate process-global counter alongside the store's
- * `liveControllers` map would be a second registry able to disagree with it: a
- * flight the store had already torn down would stay counted there forever.
- *
- * The store hands each entry an `expiresAtMs` (the request's hard deadline). A
- * periodic backstop sweep reads `overdue()` and force-releases any flight whose
- * deadline has passed, so the gauge settles even if an abort path never fires.
- *
- * Pub/sub lets the console push the live snapshot over SSE instead of polling.
- * Process-local: each gateway instance reports its own flights.
+ * The live "in flight" snapshot shape, shared by the store, the console live
+ * domain, and the dashboard. Derived from the request state store's own
+ * `liveControllers` map — the single registry that already owns a request's
+ * lifetime — so the gauge cannot disagree with what is actually running.
  */
-import { metrics } from "../../observability/metrics";
-
 export interface InFlightSnapshot {
   readonly inFlight: number;
   readonly uniqueIps: number;
 }
-
-interface Flight {
-  readonly clientIp: string;
-  /** Hard deadline after which the backstop sweep releases this flight. */
-  expiresAtMs: number;
-}
-
-export interface InFlightRegistry {
-  /** Register a flight once its dispatch leases are acquired. Idempotent per id. */
-  track(requestId: string, clientIp: string, expiresAtMs: number): void;
-  /** Release a flight. Idempotent; an unknown id is a no-op. */
-  untrack(requestId: string): void;
-  /** Re-arm a flight's hard deadline (streaming swaps the pre-stream budget). */
-  extend(requestId: string, expiresAtMs: number): void;
-  snapshot(): InFlightSnapshot;
-  count(): number;
-  /** Ids whose deadline plus the sweep grace has elapsed — the backstop's worklist. */
-  overdue(nowMs: number, graceMs: number): readonly string[];
-  subscribe(listener: (snapshot: InFlightSnapshot) => void): () => void;
-}
-
-export function createInFlightRegistry(): InFlightRegistry {
-  const flights = new Map<string, Flight>();
-  const listeners = new Set<(snapshot: InFlightSnapshot) => void>();
-
-  const snapshot = (): InFlightSnapshot => {
-    let uniqueIps = 0;
-    const seen = new Set<string>();
-    for (const flight of flights.values()) {
-      if (!seen.has(flight.clientIp)) {
-        seen.add(flight.clientIp);
-        uniqueIps += 1;
-      }
-    }
-    return { inFlight: flights.size, uniqueIps };
-  };
-
-  const publish = (): void => {
-    const current = snapshot();
-    metrics.proxy_in_flight.set(current.inFlight);
-    for (const listener of listeners) listener(current);
-  };
-
-  return {
-    track(requestId, clientIp, expiresAtMs) {
-      flights.set(requestId, { clientIp: clientIp.length > 0 ? clientIp : "unknown", expiresAtMs });
-      publish();
-    },
-    untrack(requestId) {
-      if (!flights.delete(requestId)) return;
-      publish();
-    },
-    extend(requestId, expiresAtMs) {
-      const flight = flights.get(requestId);
-      if (flight === undefined) return;
-      flight.expiresAtMs = expiresAtMs;
-    },
-    snapshot,
-    count() {
-      return flights.size;
-    },
-    overdue(nowMs, graceMs) {
-      const ids: string[] = [];
-      for (const [requestId, flight] of flights) {
-        if (nowMs >= flight.expiresAtMs + graceMs) ids.push(requestId);
-      }
-      return ids;
-    },
-    subscribe(listener) {
-      listeners.add(listener);
-      return () => {
-        listeners.delete(listener);
-      };
-    },
-  };
-}

package/src/transport/request/preparer.ts

@@ -5,14 +5,14 @@ import type { ApiKeyAdmissionService } from "../../security/admission/service";
 import type { RouteCandidate as RouteCandidate, InMemoryRouteSnapshotService, RoutePlan } from "../routing/route-model";
 import type { RouteContinuationAffinity } from "../routing/route-model";
 import { resolveAliasTarget, type RoutingEngine } from "../routing/router";
-import { projectForRoute, routeCapabilitiesFor } from "../translation/capabilities";
-import type { RequiredCapability } from "../translation/capabilities";
+import { projectForRoute, routeCapabilitiesFor } from "../../protocol/translation/capabilities";
+import type { RequiredCapability } from "../../protocol/translation/capabilities";
 import { isModelAllowed, type ResolvedApiKey } from "../../security/api-key-auth";
 import { allowsCliToolMappings } from "../../security/cli-client-fingerprint";
-import { dropIncompleteToolRounds, repairRequestToolCalls } from "../translation/tool-repair";
-import { sanitizeRequestToolIds } from "../translation/tool-id";
-import { dropCorruptAttachments } from "../translation/attachment-integrity";
-import { parseThinkingSuffix, withThinkingSuffixIntent } from "../translation/thinking";
+import { dropIncompleteToolRounds, repairRequestToolCalls } from "../../protocol/translation/tool-repair";
+import { sanitizeRequestToolIds } from "../../protocol/translation/tool-id";
+import { dropCorruptAttachments } from "../../protocol/translation/attachment-integrity";
+import { parseThinkingSuffix, withThinkingSuffixIntent } from "../../protocol/translation/thinking";
 import { nativeServicePathFor } from "../contracts";
 import { log } from "../../observability/logger";
 import { BUDDY_PROVIDER_IDS } from "../../providers/provider-metadata";

package/src/transport/request/state.ts

@@ -5,8 +5,8 @@ import type { ResolvedApiKey } from "../../security/api-key-auth";
 import type { PreparedProxyRequest } from "./preparer";
 import type { PayloadCaptureState, TelemetryPayloadCapture } from "../../observability/payload-capture";
 import type { TelemetryPayloadDepth, TelemetryPayloadMode } from "../../console/settings/contracts";
-import { createInFlightRegistry, type InFlightSnapshot } from "./inflight";
 import { log } from "../../observability/logger";
+import type { InFlightSnapshot } from "./inflight";
 import { fastPathname } from "../pathname";
 
 export interface ProxyRequestOutcome {
@@ -182,15 +182,39 @@ export interface RequestTracker {
  * shutdown coordination, which waits on tracked requests (or aborts them
  * past the grace window) instead of relying on connection close events.
  */
+/**
+ * Minimal pub/sub over a store's snapshot. Polling already re-reads
+ * `inFlightSnapshot()`, so this only pushes the same derived value to the
+ * console's SSE stream; no second registry behind it.
+ */
+function subscribeInFlightSnapshot(
+  store: { inFlightSnapshot(): InFlightSnapshot },
+  listener: (snapshot: InFlightSnapshot) => void,
+): () => void {
+  const timers = new Set<ReturnType<typeof setTimeout>>();
+  const push = (): void => listener(store.inFlightSnapshot());
+  push();
+  const handle = setInterval(push, 1_000);
+  (handle as unknown as { unref?: () => void })?.unref?.();
+  return () => {
+    clearInterval(handle);
+    for (const t of timers) clearTimeout(t);
+    timers.clear();
+  };
+}
+
 export class ProxyRequestStateStore {
   private readonly states = new WeakMap<Request, ProxyRequestState>();
   private readonly liveControllers = new Map<string, AbortController>();
   /**
-   * The in-flight gauge lives here, not in a free-standing module: this store
-   * is the one place that owns a request's lifetime and evicts it, so the
-   * gauge can never count a request the store has already torn down.
+   * The in-flight gauge is derived from the same map that owns a request's
+   * lifetime. There is no second registry: a count that disagreed with
+   * `liveControllers` was the ghost — entries that outlived the request they
+   * described. The deadline each flight last armed is kept alongside, so the
+   * backstop sweep still finds requests whose abort path never fired.
    */
-  private readonly inFlight = createInFlightRegistry();
+  private readonly inFlightDeadlines = new Map<string, number>();
+  private readonly inFlightIps = new Map<string, string>();
 
   constructor(private readonly tracker?: RequestTracker) {}
 
@@ -199,43 +223,38 @@ export class ProxyRequestStateStore {
   }
 
   inFlightSnapshot(): InFlightSnapshot {
-    return this.inFlight.snapshot();
+    let uniqueIps = 0;
+    const seen = new Set<string>();
+    for (const ip of this.inFlightIps.values()) {
+      if (!seen.has(ip)) {
+        seen.add(ip);
+        uniqueIps += 1;
+      }
+    }
+    return { inFlight: this.inFlightDeadlines.size, uniqueIps };
   }
 
   /** Live flight count for this store. */
   inFlightCount(): number {
-    return this.inFlight.count();
+    return this.inFlightDeadlines.size;
   }
 
   subscribeInFlight(listener: (snapshot: InFlightSnapshot) => void): () => void {
-    return this.inFlight.subscribe(listener);
+    return subscribeInFlightSnapshot(this, listener);
   }
 
   /**
-   * Backstop worklist: ids whose hard deadline plus `graceMs` has elapsed while
-   * still tracked. A correctly-released request never appears here — this is
-   * the safety net for an abort path that did not fire (a half-closed socket
-   * that neither pulls, cancels, nor aborts).
-   */
-  overdueInFlight(nowMs: number, graceMs: number): readonly string[] {
-    return this.inFlight.overdue(nowMs, graceMs);
-  }
-
-  /**
-   * Releases every flight past its deadline. Two tiers, because aborting the
-   * controller unwinds the fetch but only *releases* the flight if some path
-   * observes the abort:
-   *
-   * - Past `expiresAtMs + graceMs`: abort the controller, which unblocks a
-   *   paused read and fires the release in `dispatch/proxy-request`.
-   * - Past `expiresAtMs + hardGraceMs`: the abort did not release it, so
-   *   force-drop the gauge entry. The fetch is already torn down by the abort,
-   *   so leaving the entry would only mean a permanently wrong number — the
-   *   exact defect this backstop exists to end. Logged, never silent.
+   * Releases every flight past its deadline. Aborting the controller unwinds
+   * the fetch and lets the request's own `cleanup` run, which drops the
+   * flight's entry with it. A request nothing observes — a half-closed socket
+   * that neither pulls, cancels, nor aborts — never runs that cleanup, so
+   * after a further grace window the sweep drops the entry itself: the fetch
+   * is already torn down by the abort, and a permanently wrong number is the
+   * exact defect this backstop exists to end. Logged, never silent.
    */
   sweepOverdueInFlight(nowMs: number, graceMs: number, hardGraceMs: number): number {
-    const released = this.inFlight.overdue(nowMs, graceMs);
-    for (const requestId of released) {
+    for (const [requestId, deadline] of this.inFlightDeadlines) {
+      if (nowMs < deadline + graceMs) continue;
       const controller = this.liveControllers.get(requestId);
       try {
         if (controller && !controller.signal.aborted) {
@@ -246,14 +265,19 @@ export class ProxyRequestStateStore {
       } catch {
       }
     }
-    const stuck = this.inFlight.overdue(nowMs, hardGraceMs);
-    for (const requestId of stuck) {
+    let forced = 0;
+    for (const [requestId, deadline] of this.inFlightDeadlines) {
+      if (nowMs < deadline + hardGraceMs) continue;
+      this.inFlightDeadlines.delete(requestId);
+      this.inFlightIps.delete(requestId);
+      forced += 1;
+    }
+    if (forced > 0) {
       log.error(
-        `[inflight-backstop] force-releasing request ${requestId}: abort did not release it within ${hardGraceMs}ms`,
+        `[inflight-backstop] force-releasing ${forced} flight(s): abort did not release them within ${hardGraceMs}ms`,
       );
-      this.inFlight.untrack(requestId);
     }
-    return stuck.length > 0 ? stuck.length : released.length;
+    return forced;
   }
 
   abortAll(reason?: unknown): void {
@@ -325,7 +349,8 @@ export class ProxyRequestStateStore {
       startProviderFlight: () => {
         if (cleaned || providerFlightStarted) return;
         providerFlightStarted = true;
-        this.inFlight.track(requestId, state.clientIdentity?.address ?? "unknown", state.deadlineMs);
+        this.inFlightDeadlines.set(requestId, state.deadlineMs);
+        this.inFlightIps.set(requestId, state.clientIdentity?.address || "unknown");
       },
       releaseIngressBody: () => {
         state.ingressBody = undefined;
@@ -342,7 +367,7 @@ export class ProxyRequestStateStore {
         // request that legitimately extends its budget must move the flight's
         // deadline too — otherwise the sweep would force-release a healthy
         // long stream the moment the pre-stream budget elapsed.
-        this.inFlight.extend(requestId, state.deadlineMs);
+        if (providerFlightStarted) this.inFlightDeadlines.set(requestId, state.deadlineMs);
         armDeadline(ms);
       },
       cleanup: () => {
@@ -351,7 +376,10 @@ export class ProxyRequestStateStore {
         // Every step is guarded: one throwing cleanup must not skip the
         // remaining cleanups, the abort, or the map eviction (leak).
         try {
-          if (providerFlightStarted) this.inFlight.untrack(state.requestId);
+          if (providerFlightStarted) {
+            this.inFlightDeadlines.delete(state.requestId);
+            this.inFlightIps.delete(state.requestId);
+          }
         } catch {
         }
         try {

package/src/transport/request/tenant-preferences.ts

@@ -10,7 +10,7 @@
 import type { CartethyiaDatabase } from "../../persistence/db-handle";
 import type { CanonicalRequest } from "../../protocol/canonical-model";
 import type { PreparedProxyRequest } from "./preparer";
-import { normalizeThinkingConfig } from "../translation/thinking";
+import { normalizeThinkingConfig } from "../../protocol/translation/thinking";
 import { compressRequest } from "./rtk/compress-request";
 import { preferencesReaderFor } from "../../persistence/tenant-preferences";
 import { resolveUpstreamTimeoutMs } from "../../config/timeouts";

package/src/transport/routing/route-catalog.ts

@@ -741,43 +741,53 @@ class RouteCatalogRepository {
             : {}),
         };
         if (account.status === "cooldown") {
-          // Expired cooldowns read as healthy here without a fire-and-forget
-          // write: AccountHealthSweeper materializes the recovery every 30s,
-          // so the builder stays a pure read (no UPDATE racing the request).
+          // A row that still says `cooldown` stays excluded until the sweeper
+          // flips it back to `active`. An *expired* deadline used to read as
+          // healthy here (the comment below's "pure read" reasoning), which let
+          // routing redial the account in the window between the deadline
+          // elapsing and the next 30s sweep — reproducing the refusal that
+          // cooled it and logging a fresh `cooldown → cooldown` row each time
+          // (the operator's "still hit a cooled-down account" report). The
+          // sweep is the only exit from `cooldown`; until it runs the row is
+          // not proof the account can serve.
           //
           // `forceDisableCooldown` skips the exclusion entirely. The cooldown
           // is still real and still recorded — it simply stops deciding
           // routing. That is what makes the setting reversible: nothing was
           // written, so turning it off needs no cleanup, and Health still
           // reports the truth about the account.
-          if (account.cooldownUntil && account.cooldownUntil.getTime() > Date.now()) {
-            const overrideActive =
-              rowTenantId === null
-                ? anyTenantOverrides
-                : cooldownOverrideTenants.has(rowTenantId);
-            if (overrideActive) {
-              // The account is still cooling — this only stops it from being
-              // excluded. `plan()` reads `cooldown_until` to answer the
-              // all-cooling case, so it stays populated either way.
-              //
-              // Neither `health_status` nor `cooldown_kind` is set: the
-              // evaluator excludes a hard cooldown on `health_status ===
-              // "cooldown"` alone, so stamping it here would keep excluding
-              // the very accounts the override exists to reach — a spent quota
-              // or a policy block, which are the ones an operator actually
-              // wants to override.
-              candidate.cooldown_until = account.cooldownUntil.getTime();
-            } else {
-              candidate.health_status = "cooldown";
-              candidate.cooldown_kind = hardCooldownCategory(account.lastErrorCategory)
-                ? "hard"
-                : "soft";
-              // The deadline rides with the exclusion it explains. `plan()`
-              // reads it to answer the all-cooling case with the real reset
-              // instead of a bare 429, so a client backs off for the window
-              // that is actually open rather than inventing one.
-              candidate.cooldown_until = account.cooldownUntil.getTime();
-            }
+          const overrideActive =
+            rowTenantId === null
+              ? anyTenantOverrides
+              : cooldownOverrideTenants.has(rowTenantId);
+          const accountDeadline = account.cooldownUntil;
+          const unexpired =
+            accountDeadline !== null && accountDeadline.getTime() > Date.now();
+          if (overrideActive) {
+            // The account is still cooling — this only stops it from being
+            // excluded. `plan()` reads `cooldown_until` to answer the
+            // all-cooling case, so it stays populated either way.
+            //
+            // Neither `health_status` nor `cooldown_kind` is set: the
+            // evaluator excludes a hard cooldown on `health_status ===
+            // "cooldown"` alone, so stamping it here would keep excluding
+            // the very accounts the override exists to reach — a spent quota
+            // or a policy block, which are the ones an operator actually
+            // wants to override.
+            if (unexpired) candidate.cooldown_until = accountDeadline!.getTime();
+          } else {
+            candidate.health_status = "cooldown";
+            candidate.cooldown_kind = hardCooldownCategory(account.lastErrorCategory)
+              ? "hard"
+              : "soft";
+            // The deadline rides with the exclusion it explains. `plan()`
+            // reads it to answer the all-cooling case with the real reset
+            // instead of a bare 429, so a client backs off for the window
+            // that is actually open rather than inventing one. An expired
+            // deadline carries no reset — the sweep is about to recover the
+            // row, and quoting a past instant would tell the client to retry
+            // "immediately" for a wait that is already over.
+            if (unexpired) candidate.cooldown_until = accountDeadline!.getTime();
           }
         }
 

package/src/transport/routing/router.ts

@@ -21,7 +21,7 @@ import {
 import {
   candidateSupportsRequest,
   type RequiredCapability,
-} from "../translation/capabilities";
+} from "../../protocol/translation/capabilities";
 import { planAllowsCandidate } from "./plan-gate";
 import { InMemoryAdmissionController, admissionKey } from "./admission";
 import { ReservationManager } from "./reservations";
@@ -32,7 +32,7 @@ import {
   routeCandidateIndex,
 } from "./candidate-index";
 import { totalRemainingCredit, totalRemainingPercent, type ProviderQuotaWindow } from "../../providers/quota/quota-contracts";
-import { getCachedQuotaEntry, quotaLens } from "../../console/quota/cache";
+import { getCachedQuotaEntry, quotaLens } from "../../providers/quota/cache";
 import type { RedisClient } from "../../persistence/redis";
 import { sharedClaudeSlowLanes } from "../../providers/integrations/claude/claude-slow-lane";
 

package/src/transport/surface/messages/parse.ts

@@ -6,7 +6,7 @@ import { parseToolDefinition } from "../dialects";
 import { canContainToolResult } from "../../../protocol/canonical-model";
 import { MessagesLedgerError } from "./errors";
 import { finiteNumber, stringValue } from "../../../protocol/codecs";
-import { LEVEL_TO_BUDGET } from "../../translation/thinking";
+import { LEVEL_TO_BUDGET } from "../../../protocol/translation/thinking";
 
 export interface MessagesParseOptions {
   /** Explicit Claude model generation (for example `4.6` or `4.7`). */