cartethyia 2.7.4 → 2.7.5

package/src/transport/routing/candidate-index.ts

@@ -2,7 +2,7 @@
 import type { RouteCandidate, RouteSnapshot } from "./route-model";
 
 /** One parsed model reference: `provider/model` or a bare `model`. */
-export interface ParsedModel {
+interface ParsedModel {
   readonly providerId: string | null;
   readonly modelId: string;
 }
@@ -24,7 +24,7 @@ export function parseModelReference(model: string): ParsedModel {
  * candidates cost K×C `candidateMatches` calls (each re-parsing the model
  * string), repeated on every request.
  */
-export interface RouteCandidateIndex {
+interface RouteCandidateIndex {
   /** Candidates keyed by `provider_id/model_id`, snapshot order preserved. */
   readonly qualified: ReadonlyMap<string, readonly RouteCandidate[]>;
   /** Candidates keyed by bare `model_id`, snapshot order preserved. */

package/src/transport/routing/round-robin.ts

@@ -23,7 +23,7 @@ export class RoundRobinState {
  * combinations would retain one entry forever. 1000 is far above the working
  * set of any real catalog, so eviction only ever targets genuinely cold keys.
  */
-export const MAX_ROUND_ROBIN_ENTRIES = 1000;
+const MAX_ROUND_ROBIN_ENTRIES = 1000;
 
 /**
  * Creates the bounded LRU map the router keeps round-robin state in.

package/src/transport/routing/route-catalog.ts

@@ -20,12 +20,12 @@ import {
 import { DEFAULT_PROVIDER_MAX_INFLIGHT } from "../../persistence/schema";
 import { resolveTenantOverride } from "../../persistence/tenant-scope";
 import {
-  type RouteCandidate,
   type ComboDefinition,
   type PoolRoutingSetting,
   type ProviderRoutingMap,
   type RouteSnapshot,
   type SnapshotBuilder,
+  type RoutedCandidate,
 } from "./route-model";
 import { DEFAULT_PROXY_SKIP_PROVIDER_IDS } from "../../providers/provider-metadata";
 import { buildClaudeCliUserAgent } from "../../providers/operations/custom-cli-headers";
@@ -84,18 +84,6 @@ export function cliMappingSourceKeys(toolId: string, sourceModel: string): reado
  * — `RoutingEngine.plan()` filters candidates by the requesting tenant.
  */
 
-type RouteCandidateWithHealth = RouteCandidate & {
-  health_status?: "cooldown" | "model_cooldown" | "disabled";
-  cooldown_kind?: "hard" | "soft";
-  /**
-   * Epoch milliseconds at which this candidate's cooling window ends, when one
-   * is in force. Routing-internal: it is what lets `plan()` report a real reset
-   * on the all-accounts-cooling answer, and it never reaches a client verbatim —
-   * only the derived `retry-after` does.
-   */
-  cooldown_until?: number;
-};
-
 const HARD_COOLDOWN_CATEGORIES: Readonly<Record<string, true>> = {
   quota_exhausted: true,
   policy_blocked: true,
@@ -246,6 +234,7 @@ interface MergedModelRow {
   providerId: string;
   modelId: string;
   wireFamily: WireFamily;
+  dialect?: string | null;
   serviceKind: ServiceKind;
   endpointPath: string;
   modalities: unknown;
@@ -270,6 +259,7 @@ function mergeModelCatalog(
       providerId: row.providerId,
       modelId: row.modelId,
       wireFamily: row.wireFamily as WireFamily,
+      ...(row.dialect === null ? {} : { dialect: row.dialect }),
       serviceKind: row.serviceKind as ServiceKind,
       endpointPath: row.endpointPath,
       modalities: row.modalities,
@@ -309,6 +299,7 @@ const MODEL_COLUMNS = {
   providerId: models.providerId,
   modelId: models.modelId,
   wireFamily: models.wireFamily,
+  dialect: models.dialect,
   serviceKind: models.serviceKind,
   endpointPath: models.endpointPath,
   modalities: models.modalities,
@@ -678,7 +669,7 @@ class RouteCatalogRepository {
       accountsByProvider.set(account.providerId, list);
     }
 
-    const candidates: RouteCandidateWithHealth[] = [];
+    const candidates: RoutedCandidate[] = [];
     for (const model of mergedModelRows) {
       const providerTenantId = providerTenantById.get(model.providerId);
       if (providerTenantId === undefined) continue;
@@ -717,15 +708,17 @@ class RouteCatalogRepository {
               const tenantPools = resolveNetworkPools(model.providerId, tenantOwnerId);
               if (!tenantPools) continue;
               emittedPoolAccount = true;
-              const tenantCandidate: RouteCandidateWithHealth = {
+              const tenantCandidate: RoutedCandidate = {
                 provider_id: model.providerId,
                 model_id: model.modelId,
                 wire_family: model.wireFamily as WireFamily,
+                ...(model.dialect == null ? {} : { dialect: model.dialect }),
                 service_kind: model.serviceKind,
                 endpoint: model.endpointPath,
                 ...(model.outputLimit == null ? {} : { output_limit: model.outputLimit }),
                 capability_profile: capabilityProfile,
                 tenant_id: tenantOwnerId,
+                ...(model.freeTier === true ? { free_tier: true } : {}),
                 // Public credential-less candidates have no account to own them.
                 owner_tenant_id: null,
                 requires_account: false as const,
@@ -756,15 +749,17 @@ class RouteCatalogRepository {
           for (const tId of configuredPoolTenants) {
             const tenantPools = resolveNetworkPools(model.providerId, tId);
             if (tenantPools) {
-              const tenantCandidate: RouteCandidateWithHealth = {
+              const tenantCandidate: RoutedCandidate = {
                 provider_id: model.providerId,
                 model_id: model.modelId,
                 wire_family: model.wireFamily as WireFamily,
+                ...(model.dialect == null ? {} : { dialect: model.dialect }),
                 service_kind: model.serviceKind,
                 endpoint: model.endpointPath,
                 ...(model.outputLimit == null ? {} : { output_limit: model.outputLimit }),
                 capability_profile: capabilityProfile,
                 tenant_id: tId,
+                ...(model.freeTier === true ? { free_tier: true } : {}),
                 // Public credential-less candidates have no account to own them.
                 owner_tenant_id: null,
                 requires_account: false as const,
@@ -784,15 +779,17 @@ class RouteCatalogRepository {
         const networkPools = providerRequiresAccount
           ? undefined
           : resolveNetworkPools(model.providerId, providerTenantId);
-        const candidate: RouteCandidateWithHealth = {
+        const candidate: RoutedCandidate = {
           provider_id: model.providerId,
           model_id: model.modelId,
           wire_family: model.wireFamily as WireFamily,
+          ...(model.dialect == null ? {} : { dialect: model.dialect }),
           service_kind: model.serviceKind,
           endpoint: model.endpointPath,
           ...(model.outputLimit == null ? {} : { output_limit: model.outputLimit }),
           capability_profile: capabilityProfile,
           tenant_id: providerTenantId,
+          ...(model.freeTier === true ? { free_tier: true } : {}),
           // Public credential-less candidates have no account to own them.
           owner_tenant_id: null,
           ...(providerRequiresAccount
@@ -826,15 +823,17 @@ class RouteCatalogRepository {
         }
         const networkPools = resolveNetworkPools(model.providerId, rowTenantId);
         const routeUserAgent = resolveUserAgent(model.providerId, rowTenantId);
-        const candidate: RouteCandidateWithHealth = {
+        const candidate: RoutedCandidate = {
           provider_id: model.providerId,
           model_id: model.modelId,
           wire_family: model.wireFamily as WireFamily,
+          ...(model.dialect == null ? {} : { dialect: model.dialect }),
           service_kind: model.serviceKind,
           endpoint: model.endpointPath,
           ...(model.outputLimit == null ? {} : { output_limit: model.outputLimit }),
           capability_profile: capabilityProfile,
           ...(routeUserAgent === undefined ? {} : { user_agent: routeUserAgent }),
+          ...(model.freeTier === true ? { free_tier: true } : {}),
           tenant_id: rowTenantId,
           provider_account_id: account.id,
           // The account's owning tenant (null = shared pool-wide) — the bucket

package/src/transport/routing/route-model.ts

@@ -2,6 +2,8 @@
 import { GatewayError } from "../../protocol/gateway-error";
 import { type ServiceKind, type WireFamily } from "../../protocol/canonical-model";
 import type { CallerIdentity } from "../../security/access-control";
+import type { ProviderQuotaWindow } from "../../providers/quota/quota-contracts";
+import { log } from "../../observability/logger";
 
 export type { CallerIdentity };
 
@@ -14,6 +16,8 @@ export interface RouteCandidate {
   readonly provider_id: string;
   readonly model_id: string;
   readonly wire_family: WireFamily;
+  /** Explicit wire dialect (from `models.dialect`), when set. Absent = resolver fallback. */
+  readonly dialect?: string;
   /**
    * Protocol shape of the model. `llm` (the default) flows through the
    * canonical surface pipeline; a non-`llm` model is dispatched by its native
@@ -84,6 +88,12 @@ export interface RouteCandidate {
    * refused.
    */
   readonly pro_only?: boolean;
+  /**
+   * True when the provider publishes this model as free (its own directory
+   * statement, e.g. Qoder's `is_free`). A free model may be routed to any
+   * account — the credit-preference ordering skips it entirely.
+   */
+  readonly free_tier?: boolean;
   /**
    * The plan the provider last reported for `provider_account_id`.
    *
@@ -95,9 +105,44 @@ export interface RouteCandidate {
   readonly usage_floor?: number;
   readonly usage_floor_unit?: "credit" | "percent";
   /** Live quota windows (cache) the floor check evaluates. */
-  readonly quota_windows?: readonly unknown[];
+  readonly quota_windows?: readonly ProviderQuotaWindow[];
+  /** A successful Start Plan balance authoritatively lists its model grants. */
+  readonly quota_entitlements_known?: boolean;
 }
 
+/**
+ * A candidate as the routing engine actually evaluates it: the catalog fields
+ * above plus the health/cooling stamp `route-catalog.ts` projects onto it and
+ * the plan-time slow-lane marker.
+ *
+ * Declared here — not as a route-catalog-local intersection — so
+ * `EligibilityEvaluator.evaluate` can take the routed shape without a cast,
+ * and so every stamping site agrees on the same field set by construction.
+ */
+export type RoutedCandidate = RouteCandidate & {
+  /**
+   * Health marker the catalog projects from `provider_accounts.status` and
+   * unexpired `model_cooldowns` entries. Absent = no health exclusion known.
+   * Mutable by declaration: the catalog builds the base candidate first and
+   * stamps health/cooling onto it as account rows are walked.
+   */
+  health_status?: "cooldown" | "model_cooldown" | "disabled";
+  /** Cooldown class the catalog derives from the account's last error category. */
+  cooldown_kind?: "hard" | "soft";
+  /**
+   * Epoch milliseconds at which this candidate's cooling window ends, when one
+   * is in force. Routing-internal: it is what lets `plan()` report a real reset
+   * on the all-accounts-cooling answer, and it never reaches a client verbatim —
+   * only the derived `retry-after` does.
+   */
+  cooldown_until?: number;
+  /**
+   * Slow-lane stamp applied at plan time (test-supported; runtime slow-lane
+   * activity is read live via `sharedClaudeSlowLanes.isActive`).
+   */
+  lane?: "slow";
+};
+
 /**
  * Provider-routing strategies. The runtime tuple is the source: the console's
  * Elysia body schema builds its `t.Literal` union from these values and its
@@ -270,7 +315,6 @@ export type EligibilityReason =
   | "cooldown_hard"
   | "model_cooldown"
   | "quota_exhausted"
-  | "locked"
   | "disabled"
 
 export interface EligibilityDecision {
@@ -280,7 +324,7 @@ export interface EligibilityDecision {
   /**
    * Epoch milliseconds at which this candidate's cooling window ends, when the
    * exclusion that produced this decision carries one. Absent for a healthy
-   * candidate and for every reason with no deadline (`disabled`, `locked`,
+   * candidate and for every reason with no deadline (`disabled`,
    * `quota_exhausted` from spent windows or the usage floor). `plan()` takes the
    * minimum across the cooling set to report the earliest reset a client could
    * retry against.
@@ -328,7 +372,7 @@ export interface AdmissionController {
  * Translates the evaluator's internal reason codes into operator English.
  *
  * These names are the vocabulary the routing layers use with each other —
- * `model_cooldown`, `cooldown_hard`, `locked` — not something to print into the
+ * `model_cooldown`, `cooldown_hard` — not something to print into the
  * public message, where the client would read an enum instead of an
  * explanation. The codes stay in the envelope's `reasons` field for anyone who
  * wants them; the sentence carries what they mean.
@@ -342,9 +386,9 @@ const ELIGIBILITY_REASON_LABELS: Readonly<Record<string, string>> = {
   cooldown_hard: "every account is in a hard cooldown",
   cooldown: "accounts are cooling down",
   disabled: "all accounts are disabled",
-  locked: "all accounts are locked by another request",
   credit_floor_reached: "credit or quota is exhausted",
   quota_exhausted: "every account has an exhausted quota window or is below its configured usage floor",
+  capability_unsupported: "the accounts serving this model do not support the request's capabilities (e.g. images, tools, or reasoning)",
 };
 
 function explainEligibilityReasons(reasons: readonly string[]): string {
@@ -388,15 +432,6 @@ export function modelNotFoundError(requested: string, resolved: readonly string[
   );
 }
 
-export function capabilityUnsupportedError(capability: string): GatewayError {
-  return new GatewayError(
-    "capability_unsupported",
-    400,
-    `No account serving this model supports ${capability} (request {requestId}). Drop the ${capability} part of the request, or route to a model that accepts it. Know your tools — this one just isn't in the box 🧰`,
-    { capability },
-  );
-}
-
 /**
  * Every candidate in the plan refused admission: each account is already
  * serving its configured max-inflight.
@@ -439,7 +474,7 @@ export function capacityExhaustedError(context?: {
 
 /**
  * The model exists and is routed, but every account serving it is currently
- * unusable (disabled / locked). This is transient and
+ * unusable (disabled). This is transient and
  * retry-able, so it must NOT surface as `model_not_found` — a 404 tells the
  * client to fix its request when the real fix is to wait or add capacity.
  *
@@ -453,10 +488,18 @@ export function accountsUnavailableError(
   reasons: readonly string[],
 ): GatewayError {
   const distinct = [...new Set(reasons)].sort();
+  const capabilityBlocked = distinct.includes("capability_unsupported");
+  // When the blocker is a capability (the model exists and accounts exist, but
+  // none can carry the request's images/tools/reasoning), telling the client to
+  // "add an account" or wait out a coffee break is misleading — more accounts
+  // cannot conjure a capability the model itself lacks. Name the real fix.
+  const hint = capabilityBlocked
+    ? "Request a model that supports this capability (or a vision-capable model for images), or drop the unsupported part of the request. The model is the limit, not the pool 🧰"
+    : "Add an account, or wait for an operator to re-enable one. Every account is on a coffee break — try later ☕️";
   return new GatewayError(
     "accounts_unavailable",
     503,
-    `No account in the pool can serve '${requested}' right now (request {requestId}) (${explainEligibilityReasons(distinct)}). Add an account, or wait for an operator to re-enable one. Every account is on a coffee break — try later ☕️`,
+    `No account in the pool can serve '${requested}' right now (request {requestId}) (${explainEligibilityReasons(distinct)}). ${hint}`,
     { model: requested, reasons: distinct, candidate_count: reasons.length, gateway_hint: "no_healthy_account" },
   );
 }
@@ -536,42 +579,83 @@ export class InMemoryRouteSnapshotService implements RouteSnapshotService {
    * only populate the cache if no mutation happened while it ran.
    */
   private building: { revision: RoutingRevision; promise: Promise<RouteSnapshot> } | undefined;
-
-  constructor(private readonly builder: SnapshotBuilder) {
+  /**
+   * Debounced background-rebuild timer (G-02). One invalidate schedules a
+   * rebuild; a burst of invalidates within `rebuildDebounceMs` coalesces into
+   * the single latest-revision rebuild. The timer is cleared on `stop()`.
+   */
+  private rebuildTimer: ReturnType<typeof setTimeout> | undefined;
+  private stopped = false;
+
+  constructor(
+    private readonly builder: SnapshotBuilder,
+    options: { readonly rebuildDebounceMs?: number } = {},
+  ) {
+    this.rebuildDebounceMs = options.rebuildDebounceMs ?? 500;
     // Peers' mutations arrive once per process, not per instance: the handler
-    // drops this instance's copy, which is the only one there is.
+    // schedules a background rebuild, which is the only one there is.
     this.unsubscribe = subscribeRoutingInvalidation(() => {
       this.revision += 1;
-      this.snapshot = undefined;
+      this.scheduleRebuild();
     });
   }
 
-  /** Ends the peer-invalidation subscription. */
+  /** Ends the peer-invalidation subscription and clears the rebuild timer. */
   readonly unsubscribe: () => void;
 
-  async getSnapshot(): Promise<RouteSnapshot> {
-    if (this.snapshot) return this.snapshot;
-    if (this.building && this.building.revision === this.revision) return this.building.promise;
+  /** Wall-clock debounce for coalescing invalidate bursts. */
+  private readonly rebuildDebounceMs: number;
+
+  /**
+   * Schedules a background rebuild at the current revision. Debounced: while a
+   * timer is pending, further invalidates just bump the revision and the
+   * pending rebuild picks up the latest. Single-flight by construction — the
+   * rebuild itself reuses {@link building}, so concurrent callers share one
+   * promise for the same revision.
+   */
+  private scheduleRebuild(): void {
+    if (this.stopped || this.rebuildTimer !== undefined) return;
+    this.rebuildTimer = setTimeout(() => {
+      this.rebuildTimer = undefined;
+      void this.rebuildNow().catch((error: unknown) => {
+        // A background rebuild failure is a soft miss: requests served the
+        // last-completed snapshot, and the next invalidation retries.
+        log.warn("[route-snapshot] background rebuild failed", error as Error);
+      });
+    }, this.rebuildDebounceMs);
+    if (typeof this.rebuildTimer.unref === "function") this.rebuildTimer.unref();
+  }
+
+  /** Starts (or joins) the build for the current revision. */
+  private rebuildNow(): Promise<RouteSnapshot> {
     const revision = this.revision;
+    if (this.building && this.building.revision === revision) return this.building.promise;
     const promise = this.build(revision);
     this.building = { revision, promise };
-    try {
-      return await promise;
-    } finally {
+    void promise.finally(() => {
       // Only the build that is still current may clear the slot; a build
       // started after a mutation must not be dropped by an older one finishing.
       if (this.building?.promise === promise) this.building = undefined;
-    }
+    });
+    return promise;
   }
 
+  async getSnapshot(): Promise<RouteSnapshot> {
+    // G-02: serve the last-completed snapshot while a rebuild runs, so the
+    // page-open request path never pays the catalog build. Only a cold start
+    // (no snapshot at all) falls through to a synchronous build here.
+    if (this.snapshot) return this.snapshot;
+    return this.rebuildNow();
+  }
 
   async invalidate(): Promise<RoutingRevision> {
     this.revision += 1;
-    this.snapshot = undefined;
-    // Lazy: the next getSnapshot() rebuilds. Dashboard mutations only need
-    // the new revision token — they must not pay seven Postgres round-trips
-    // inside the mutating request while holding no pool pressure budget.
-    //
+    // G-02: trigger a BACKGROUND rebuild immediately instead of clearing the
+    // snapshot and making the next request pay. The rebuild is debounced so a
+    // burst of invalidates (a recovery sweep, a cascade of console writes)
+    // coalesces into one rebuild, and it never blocks the mutating request —
+    // a console write does not wait on its own rebuild here.
+    this.scheduleRebuild();
     // Broadcast, because `reusePort` runs several gateway processes and each
     // holds its own snapshot: a console write handled by one of them left the
     // others serving pre-write routing until something else invalidated them.
@@ -581,6 +665,15 @@ export class InMemoryRouteSnapshotService implements RouteSnapshotService {
     return this.revision;
   }
 
+  /** Stops background rebuilds. Test/teardown helper. */
+  stop(): void {
+    this.stopped = true;
+    if (this.rebuildTimer !== undefined) {
+      clearTimeout(this.rebuildTimer);
+      this.rebuildTimer = undefined;
+    }
+  }
+
   private async build(revision: RoutingRevision): Promise<RouteSnapshot> {
     const built = await this.builder();
     const providerRouting: ProviderRoutingMap | undefined = built.providerRouting