cartethyia 2.4.4 → 2.4.6

package/src/observability/metrics.ts

@@ -169,10 +169,11 @@ interface HistogramSeries {
 class Histogram implements HistogramMetric {
   private readonly series = new Map<string, HistogramSeries>();
   /**
-   * The label names this histogram emits, fixed by the first LABELLED
-   * observation. A Prometheus metric family must expose one consistent label set,
-   * so a later observation carrying different keys is normalized against this set
-   * rather than creating a second, incompatible series.
+   * The label names this histogram emits. Declared at registration when the
+   * caller passes them to `histogram()`, otherwise fixed by the first LABELLED
+   * observation. A Prometheus metric family must expose one consistent label
+   * set, so a later observation carrying different keys is normalized against
+   * this set rather than creating a second, incompatible series.
    */
   private labelNames: readonly string[] = [];
 
@@ -180,12 +181,18 @@ class Histogram implements HistogramMetric {
     readonly name: string,
     readonly help: string,
     readonly buckets: readonly number[],
-  ) {}
+    labelNames: readonly string[] = [],
+  ) {
+    // A declared set is stable across runs and workers, so the exposition is
+    // deterministic even before the first observation arrives. Sorting keeps
+    // the order fixed either way.
+    this.labelNames = [...labelNames].sort();
+  }
 
   observe(value: number, labels?: MetricLabels): void {
+    // Fall back to inferring the set from the first labelled observation when
+    // the metric was registered without declaring one.
     if (this.labelNames.length === 0 && labels !== undefined) {
-      // Sorting makes the exposition order deterministic, so two runs of the same
-      // code produce byte-identical output.
       this.labelNames = Object.keys(labels).sort();
     }
     const normalized = normalizeLabels(labels, this.labelNames);
@@ -271,7 +278,8 @@ export class PrometheusRegistry {
   readonly cartethyia_routing_roundrobin_entries: GaugeMetric;
   readonly cartethyia_ip_abuse_keys: GaugeMetric;
   readonly cartethyia_quota_cache_entries: GaugeMetric;
-  readonly cartethyia_proxy_dial_dns_fallback_total: CounterMetric;
+  readonly cartethyia_model_catalog_entries: GaugeMetric;
+  readonly cartethyia_proxy_dial_proxy_resolved_total: CounterMetric;
   readonly proxy_provider_adapter_load_ms: HistogramMetric;
   readonly pool_cooldown_record_failed: CounterMetric;
   readonly quota_cache_invalidate_failed: CounterMetric;
@@ -348,9 +356,13 @@ export class PrometheusRegistry {
       "cartethyia_quota_cache_entries",
       "Entries in the in-process provider quota cache",
     );
-    this.cartethyia_proxy_dial_dns_fallback_total = this.counter(
-      "cartethyia_proxy_dial_dns_fallback_total",
-      "Pool/relay-bound dials whose local DNS resolution failed and were downgraded to proxy-resolved egress",
+    this.cartethyia_model_catalog_entries = this.gauge(
+      "cartethyia_model_catalog_entries",
+      "Entries in the per-provider model catalog cache (at the cap means eviction churn)",
+    );
+    this.cartethyia_proxy_dial_proxy_resolved_total = this.counter(
+      "cartethyia_proxy_dial_proxy_resolved_total",
+      "Pool/relay-bound dials whose target is resolved by the proxy or relay rather than by the local resolver, so local DNS for the target is advisory only",
     );
     this.proxy_provider_adapter_load_ms = this.histogram(
       "proxy_provider_adapter_load_ms",
@@ -379,8 +391,23 @@ export class PrometheusRegistry {
     return metric;
   }
 
-  histogram(name: string, help: string, buckets: readonly number[]): HistogramMetric {
-    const metric = new Histogram(name, help, buckets);
+  /**
+   * Registers a histogram.
+   *
+   * `labelNames` declares the label set up front. It is optional because a
+   * histogram observed without labels still has to render, but passing it is
+   * the correct call: a declared set is stable across runs and workers, so the
+   * exposition is deterministic even before the first observation, and an
+   * observation carrying undeclared keys cannot create a second, incompatible
+   * series.
+   */
+  histogram(
+    name: string,
+    help: string,
+    buckets: readonly number[],
+    labelNames: readonly string[] = [],
+  ): HistogramMetric {
+    const metric = new Histogram(name, help, buckets, labelNames);
     this.all.push(metric);
     return metric;
   }

package/src/observability/runtime-metrics.ts

@@ -24,6 +24,12 @@ export interface RuntimeMetricSources {
   readonly ipAbuseKeys: () => number;
   /** Tracked keys in the in-process quota cache tier (0 when Redis-only paths are idle). */
   readonly quotaCacheEntries: () => number;
+
+  /**
+   * Entries in the per-provider model-catalog cache. Bounded, but a count that
+   * sits at the cap means every rebuild is evicting something it needs.
+   */
+  readonly modelCatalogEntries: () => number;
   /**
    * Live in-flight request count. Zero means the gateway is idle, so a
    * synchronous full collection is safe (no request to stall).
@@ -129,6 +135,7 @@ export class RuntimeMetricsSampler {
     metrics.cartethyia_routing_roundrobin_entries.set(roundRobin.provider, { scope: "provider" });
     metrics.cartethyia_ip_abuse_keys.set(this.sources.ipAbuseKeys());
     metrics.cartethyia_quota_cache_entries.set(this.sources.quotaCacheEntries());
+    metrics.cartethyia_model_catalog_entries.set(this.sources.modelCatalogEntries());
 
     this.checkPressure(memory.rss);
     this.checkIdleReclaim(memory.rss);

package/src/persistence/schema.ts

@@ -1,5 +1,5 @@
 /** Canonical Drizzle schema: every persistence table, enum, index, and shared type. */
-import { sql } from "drizzle-orm";
+import { desc, sql } from "drizzle-orm";
 import {
   bigint, boolean, check, customType, index, integer, jsonb, numeric, pgEnum, pgTable,
   primaryKey, text, timestamp, uniqueIndex, uuid, type AnyPgColumn,
@@ -168,6 +168,14 @@ export const providers = pgTable(
     // see `src/transport/routing/route-catalog.ts`.
     requiresAccount: boolean("requires_account").notNull().default(true),
   },
+  (table) => [
+    // `globalOrOwnedBy(providers.tenantId, …)` is the predicate behind
+    // `listModelsForTenant` and `/quota/overview`, and it is a bare
+    // `tenant_id IS NULL OR tenant_id = ?` — which no other index here can
+    // serve, so both reads scanned the whole provider list. Cheap to keep:
+    // providers are few and written rarely.
+    index("providers_tenant_id_idx").on(table.tenantId),
+  ],
 );
 
 // Upstream account credentials plus the full health state machine.
@@ -262,6 +270,12 @@ export const providerAccounts = pgTable("provider_accounts", {
     ),
     index("provider_accounts_status_cooldown_idx").on(table.status, table.cooldownUntil),
     index("provider_accounts_provider_tenant_idx").on(table.providerId, table.tenantId),
+    // The bare `tenant_id IS NULL OR tenant_id = ?` that `listAllAccounts`
+    // uses is served by neither index above: `(status, cooldown_until)` leads
+    // with the wrong column, and `(provider_id, tenant_id)` cannot serve a
+    // predicate that does not constrain `provider_id`. This one matches the
+    // hot listing path.
+    index("provider_accounts_tenant_id_idx").on(table.tenantId),
   ],
 );
 
@@ -473,7 +487,16 @@ export const healthEvents = pgTable(
   },
   (table) => [
     index("health_events_account_id_created_at_idx").on(table.accountId, table.createdAt),
-    index("health_events_account_model_created_idx").on(table.accountId, table.modelId, table.createdAt),
+    // Descending `created_at` is deliberate: health history is always read
+    // newest-first. The SQLite baseline drops the direction (see
+    // `scripts/generate/sqlite-ddl.ts` — SQLite scans an index either way),
+    // so this is a Postgres-only access-path choice, not a cross-backend
+    // disagreement.
+    index("health_events_account_model_created_idx").on(
+      table.accountId,
+      table.modelId,
+      desc(table.createdAt),
+    ),
     index("health_events_network_pool_id_created_at_idx").on(table.networkPoolId, table.createdAt),
     check(
       "health_events_entity_reference_check",
@@ -723,6 +746,21 @@ export const apiKeys = pgTable(
   },
   (table) => [
     uniqueIndex("api_keys_key_hash_idx").on(table.keyHash),
+    /**
+     * One *live* default key per tenant, enforced by the database.
+     *
+     * The baseline SQL has always created this; the schema never declared it,
+     * so the two descriptions disagreed and nothing noticed. It is the only
+     * database-level guarantee behind `isDefault`, and
+     * `test/persistence/sqlite-baseline.test.ts` pins it by name — so the
+     * schema was the incomplete side, not the migration.
+     *
+     * Partial on `revoked_at IS NULL` so a tenant can rotate its default key
+     * without the old, revoked row blocking the new one.
+     */
+    uniqueIndex("api_keys_one_default_per_tenant_uidx")
+      .on(table.tenantId)
+      .where(sql`${table.isDefault} = true AND ${table.revokedAt} IS NULL`),
     index("api_keys_tenant_id_idx").on(table.tenantId),
     index("api_keys_enabled_tenant_idx").on(table.tenantId).where(sql`${table.enabled} = true`),
     index("api_keys_parent_key_id_idx").on(table.parentKeyId),

package/src/protocol/response/codex.ts

@@ -281,9 +281,7 @@ export class CodexStreamFrameProcessor {
   // output_index routing + `output_item.added` registration.
   readonly #openItems = new Map<string, OpenItemEntry>();
   readonly #openItemsByOutputIndex = new Map<number, OpenItemEntry>();
-  // Track tool call argument accumulation, retained for diagnostics and
-  // future reconciliation of streamed arguments against authoritative ones.
-  readonly #toolCallArgs = new Map<string, string[]>();
+
   // Whitespace-loop guard: a model stuck emitting only whitespace tool-call
   // argument deltas would otherwise hang the dispatch loop forever.
   readonly #whitespaceDeltaTracking = new Map<
@@ -427,9 +425,6 @@ export class CodexStreamFrameProcessor {
           : undefined;
       const key = itemId ?? callId;
       this.#checkWhitespaceLoopGuard(key, json["delta"] as string);
-      const chunks = this.#toolCallArgs.get(key) ?? [];
-      chunks.push(json["delta"] as string);
-      this.#toolCallArgs.set(key, chunks);
       events.push({
         type: "tool_call_delta",
         sequence_number: this.#seq++,
@@ -443,14 +438,7 @@ export class CodexStreamFrameProcessor {
       typeof json["delta"] === "string"
     ) {
       const { callId, name } = this.#resolveToolIdentity(json);
-      const itemId =
-        typeof json["item_id"] === "string"
-          ? (json["item_id"] as string)
-          : undefined;
-      const key = itemId ?? callId;
-      const chunks = this.#toolCallArgs.get(key) ?? [];
-      chunks.push(json["delta"] as string);
-      this.#toolCallArgs.set(key, chunks);
+
       events.push({
         type: "tool_call_delta",
         sequence_number: this.#seq++,
@@ -505,37 +493,13 @@ export class CodexStreamFrameProcessor {
         },
       });
     } else if (t === "response.function_call_arguments.done") {
-      const rawCallId =
-        (json["call_id"] as string | undefined) ??
-        (json["item_id"] as string | undefined);
-      const itemId =
-        typeof json["item_id"] === "string"
-          ? (json["item_id"] as string)
-          : undefined;
-      let key: string | undefined;
-      if (itemId !== undefined) key = itemId;
-      else if (rawCallId !== undefined) key = decodeCodexToolCallId(rawCallId);
-      else {
-        const outputIndex = readOutputIndex(json);
-        if (outputIndex !== undefined) key = `idx_${outputIndex}`;
-      }
-      const finalArgs = (json as Record<string, unknown>)["arguments"];
-      if (typeof finalArgs === "string" && key !== undefined) {
-        this.#toolCallArgs.set(key, [finalArgs]);
-      }
+      // The arguments themselves are emitted by the delta events above; the
+      // `.done` event only marks the call complete.
+
       this.hasToolCall = true;
     } else if (t === "response.custom_tool_call_input.done") {
-      const itemId =
-        typeof json["item_id"] === "string"
-          ? (json["item_id"] as string)
-          : undefined;
-      const rawCallId = (json["call_id"] as string | undefined) ?? itemId;
-      let key: string | undefined;
-      if (itemId !== undefined) key = itemId;
-      else if (rawCallId !== undefined) key = decodeCodexToolCallId(rawCallId);
-      const finalInput = (json as Record<string, unknown>)["input"];
-      if (typeof finalInput === "string" && key !== undefined)
-        this.#toolCallArgs.set(key, [finalInput]);
+      // As above: the input arrives as deltas, so `.done` only completes it.
+
       this.hasToolCall = true;
     } else if (t === "response.output_item.done") {
       const item = json["item"];
@@ -598,43 +562,13 @@ export class CodexStreamFrameProcessor {
         }
         if (outputItem["type"] === "function_call") {
           this.hasToolCall = true;
-          const callIdRaw =
-            typeof outputItem["call_id"] === "string"
-              ? (outputItem["call_id"] as string)
-              : undefined;
-          const decodedCallId =
-            callIdRaw !== undefined
-              ? decodeCodexToolCallId(callIdRaw)
-              : undefined;
-          const authoritative =
-            typeof outputItem["arguments"] === "string"
-              ? (outputItem["arguments"] as string)
-              : undefined;
-          if (authoritative !== undefined && decodedCallId !== undefined) {
-            const key = outputItemId ?? decodedCallId;
-            this.#toolCallArgs.set(key, [authoritative]);
-          }
+
           if (outputItemId !== undefined) this.#openItems.delete(outputItemId);
           if (outputIndex !== undefined)
             this.#openItemsByOutputIndex.delete(outputIndex);
         } else if (outputItem["type"] === "custom_tool_call") {
           this.hasToolCall = true;
-          const callIdRaw =
-            typeof outputItem["call_id"] === "string"
-              ? (outputItem["call_id"] as string)
-              : undefined;
-          const decodedCallId =
-            callIdRaw !== undefined
-              ? decodeCodexToolCallId(callIdRaw)
-              : undefined;
-          const authoritative =
-            typeof outputItem["input"] === "string"
-              ? (outputItem["input"] as string)
-              : undefined;
-          if (authoritative !== undefined && decodedCallId !== undefined) {
-            const key = outputItemId ?? decodedCallId;
-            this.#toolCallArgs.set(key, [authoritative]);
-          }
+
           if (outputItemId !== undefined) this.#openItems.delete(outputItemId);
           if (outputIndex !== undefined)
             this.#openItemsByOutputIndex.delete(outputIndex);

package/src/protocol/tool-identity.ts

@@ -7,12 +7,33 @@ export interface ToolCallState {
 }
 
 /**
- * Per-state argument fragments. `ToolCallState.arguments` stays a plain
- * string field for readers, but the tracker accumulates fragments here and
- * joins once per read instead of re-copying the whole prefix on every
- * delta (which is O(n²) for large tool arguments).
+ * Per-state argument fragments and their joined snapshot.
+ *
+ * The snapshot is cached on the CHUNK ARRAY, not on the state, because the
+ * array is deliberately handed forward from a superseded state to its
+ * replacement and mutated in place — so a reader holding the older
+ * `ToolCallState` still observes the newest total. Caching per state would
+ * freeze each state at the value it had when created and silently break that
+ * invariant (pinned by test/protocol/tool-identity.test.ts).
+ *
+ * The cache is dropped on push, so N deltas cost O(N) total: without it each
+ * read re-joined the whole array, and a reader that slices the string per
+ * delta — as the Messages stream does — re-joined and re-sliced once per
+ * fragment, which is O(n²) over a large tool argument.
  */
 const argumentChunks = new WeakMap<ToolCallState, string[]>();
+const argumentJoined = new WeakMap<string[], string>();
+
+/** The accumulated arguments, joining only when a fragment has been added. */
+function joinedArguments(state: ToolCallState): string {
+  const chunks = argumentChunks.get(state);
+  if (chunks === undefined) return "";
+  const cached = argumentJoined.get(chunks);
+  if (cached !== undefined) return cached;
+  const value = chunks.join("");
+  argumentJoined.set(chunks, value);
+  return value;
+}
 
 /**
  * Single tool-call tracking home: id/index/arguments-accumulation keyed by
@@ -60,17 +81,17 @@ export function trackToolCall(
   tracker.nextIndex = Math.max(tracker.nextIndex, index + 1);
   const prior = tracker.states.get(tool.id);
   const name = tool.name ?? prior?.name;
-  // Append the fragment in O(1) amortized; the joined snapshot is produced
-  // lazily by the `arguments` getter below, so N deltas cost O(N) total
-  // instead of O(N²) prefix re-copies. The chunk array is shared forward
-  // along one call id (superseded states are never read after replacement).
+  // Append the fragment in O(1) amortized. The chunk array is shared forward
+  // along one call id, so the joined cache is keyed on the array and dropped
+  // here: the next read re-joins once and re-caches.
   const chunks =
     prior !== undefined ? (argumentChunks.get(prior) ?? [prior.arguments]) : [];
   chunks.push(tool.args ?? "");
+  argumentJoined.delete(chunks);
   const state: ToolCallState = {
     id: tool.id || prior?.id || "",
     get arguments() {
-      return chunks.join("");
+      return joinedArguments(state);
     },
     index,
   };
@@ -80,6 +101,19 @@ export function trackToolCall(
   return { index, isFirst: prior === undefined, state };
 }
 
+/**
+ * The not-yet-emitted tail of a call's arguments, from `offset`.
+ *
+ * Reads the cached joined string rather than re-joining the chunk array, so a
+ * caller that asks once per delta — as the Messages stream does — stays
+ * linear in the total argument length instead of quadratic. The `slice` itself
+ * is unavoidable: the caller needs the tail, and its own length is the output.
+ */
+export function argumentsFrom(state: ToolCallState, offset: number): string {
+  const all = joinedArguments(state);
+  return offset <= 0 ? all : all.slice(offset);
+}
+
 // ===== Emission ledger =====
 
 /**