cartethyia 2.4.5 → 2.5.0

package/src/network/pool-health-machine.ts

@@ -1,3 +1,4 @@
+import { healthEventBuffer } from "../persistence/health-event-flush";
 import { and, desc, eq, inArray, isNotNull, lte, sql } from "drizzle-orm";
 import type { CartethyiaDatabase } from "../persistence/db-handle";
 import { healthEvents, networkPools } from "../persistence/schema";
@@ -123,17 +124,23 @@ export async function recordPoolDispatchOutcome(
     ...(input.succeeded && fromStatus !== "active" ? { lastRecoveredAt: now } : {}),
   }).where(eq(networkPools.id, poolId));
   if (statusChanged || (input.succeeded && pool.consecutiveFailures > 0)) {
-    await db.insert(healthEvents).values({
-      entityKind: "pool",
+    // Off the request path when a buffer is installed: this row is
+    // append-only and read by nobody in dispatch. The pool's own status write
+    // above stays synchronous, because pool selection reads it.
+    const buffer = healthEventBuffer();
+    const row = {
+      entityKind: "pool" as const,
       networkPoolId: poolId,
       fromStatus,
       toStatus: status,
-      reason: input.succeeded ? "Proxy dispatch succeeded — pool recovered" : lastError,
-      errorCategory: input.succeeded ? null : category,
+      reason: input.succeeded ? "Proxy dispatch succeeded — pool recovered" : (lastError ?? ""),
+      errorCategory: input.succeeded ? null : (category ?? null),
       // The pool's own cooldown deadline is when the state releases, so the log
       // shows a countdown; a recovery releases immediately.
       createdAt: now,
-    });
+    };
+    if (buffer !== undefined) buffer.enqueue(row);
+    else await db.insert(healthEvents).values(row as never);
   }
   if (healthChanged) {
     publishPoolHealth({

package/src/observability/event-log.ts

@@ -176,19 +176,37 @@ export async function readTelemetryEvents(): Promise<TelemetryEventRecord[]> {
 export interface EventQuery {
   readonly tenantId?: string;
   readonly since?: Date;
+  /**
+   * Exclusive upper bound on `createdAt`, used to push a pagination cursor into
+   * the query instead of filtering after the whole window was parsed. Without
+   * it a cursor page drops `limit` — the reader cannot know where the page
+   * ends — so only the first page of a list is bounded.
+   */
+  readonly until?: Date;
   readonly apiKeyIds?: readonly string[];
   readonly accountIds?: readonly string[];
   readonly requestId?: string;
+  /**
+   * Distinct from {@link requestId}: `id` defaults to the request id when a row
+   * does not carry one of its own (`event-log.ts:68`), so a row can match by
+   * either. A lookup by `id` short-circuits like `requestId` does, which is the
+   * whole point — without it a single-event detail read parses the retention
+   * window and then scans it in JS.
+   */
+  readonly id?: string;
   readonly httpStatus?: number;
   readonly limit?: number;
 }
 
-function matches(event: TelemetryEventRecord, query: EventQuery, since: number | undefined, keys: ReadonlySet<string> | undefined, accounts: ReadonlySet<string> | undefined): boolean {
+function matches(event: TelemetryEventRecord, query: EventQuery, since: number | undefined, until: number | undefined, keys: ReadonlySet<string> | undefined, accounts: ReadonlySet<string> | undefined): boolean {
   if (query.tenantId !== undefined && event.tenantId !== query.tenantId) return false;
-  if (since !== undefined && new Date(event.createdAt).getTime() < since) return false;
+  const at = new Date(event.createdAt).getTime();
+  if (since !== undefined && at < since) return false;
+  if (until !== undefined && at > until) return false;
   if (keys && (event.apiKeyId === null || !keys.has(event.apiKeyId))) return false;
   if (accounts && (event.accountId === null || !accounts.has(event.accountId))) return false;
   if (query.requestId !== undefined && event.requestId !== query.requestId) return false;
+  if (query.id !== undefined && event.id !== query.id) return false;
   if (query.httpStatus !== undefined && event.httpStatus !== query.httpStatus) return false;
   return true;
 }
@@ -199,6 +217,7 @@ export async function queryTelemetryEvents(query: EventQuery): Promise<Telemetry
     .filter((file) => file.endsWith(".jsonl") && (query.since === undefined || eventFileHour(file) >= hourStamp(query.since)))
     .sort((a, b) => (a < b ? 1 : a > b ? -1 : 0));
   const since = query.since?.getTime();
+  const until = query.until?.getTime();
   const keys = query.apiKeyIds ? new Set(query.apiKeyIds) : undefined;
   const accounts = query.accountIds ? new Set(query.accountIds) : undefined;
   const events: TelemetryEventRecord[] = [];
@@ -207,19 +226,21 @@ export async function queryTelemetryEvents(query: EventQuery): Promise<Telemetry
   // field, and a false negative is not possible while a false positive costs
   // only the parse it would have paid anyway.
   const allow =
-    query.tenantId === undefined && query.requestId === undefined
+    query.tenantId === undefined && query.requestId === undefined && query.id === undefined
       ? undefined
       : (line: string): boolean =>
           (query.tenantId === undefined || line.includes(query.tenantId)) &&
-          (query.requestId === undefined || line.includes(query.requestId));
+          (query.requestId === undefined || line.includes(query.requestId)) &&
+          (query.id === undefined || line.includes(query.id));
+  // A single-row lookup: stop at the first (newest) match instead of parsing
+  // the whole retention window.
+  const oneRow = query.requestId !== undefined || query.id !== undefined;
   for (const file of files) {
     const matched: TelemetryEventRecord[] = [];
     for await (const event of readEventFiles([file], allow)) {
-      if (!matches(event, query, since, keys, accounts)) continue;
+      if (!matches(event, query, since, until, keys, accounts)) continue;
       matched.push(event);
-      // A requestId lookup needs one row: stop at the first (newest) match
-      // instead of parsing the whole retention window.
-      if (query.requestId !== undefined) return matched;
+      if (oneRow) return matched;
     }
     matched.sort((a, b) => (a.createdAt < b.createdAt ? 1 : a.createdAt > b.createdAt ? -1 : 0));
     events.push(...matched);

package/src/observability/metrics.ts

@@ -169,10 +169,11 @@ interface HistogramSeries {
 class Histogram implements HistogramMetric {
   private readonly series = new Map<string, HistogramSeries>();
   /**
-   * The label names this histogram emits, fixed by the first LABELLED
-   * observation. A Prometheus metric family must expose one consistent label set,
-   * so a later observation carrying different keys is normalized against this set
-   * rather than creating a second, incompatible series.
+   * The label names this histogram emits. Declared at registration when the
+   * caller passes them to `histogram()`, otherwise fixed by the first LABELLED
+   * observation. A Prometheus metric family must expose one consistent label
+   * set, so a later observation carrying different keys is normalized against
+   * this set rather than creating a second, incompatible series.
    */
   private labelNames: readonly string[] = [];
 
@@ -180,12 +181,18 @@ class Histogram implements HistogramMetric {
     readonly name: string,
     readonly help: string,
     readonly buckets: readonly number[],
-  ) {}
+    labelNames: readonly string[] = [],
+  ) {
+    // A declared set is stable across runs and workers, so the exposition is
+    // deterministic even before the first observation arrives. Sorting keeps
+    // the order fixed either way.
+    this.labelNames = [...labelNames].sort();
+  }
 
   observe(value: number, labels?: MetricLabels): void {
+    // Fall back to inferring the set from the first labelled observation when
+    // the metric was registered without declaring one.
     if (this.labelNames.length === 0 && labels !== undefined) {
-      // Sorting makes the exposition order deterministic, so two runs of the same
-      // code produce byte-identical output.
       this.labelNames = Object.keys(labels).sort();
     }
     const normalized = normalizeLabels(labels, this.labelNames);
@@ -271,7 +278,8 @@ export class PrometheusRegistry {
   readonly cartethyia_routing_roundrobin_entries: GaugeMetric;
   readonly cartethyia_ip_abuse_keys: GaugeMetric;
   readonly cartethyia_quota_cache_entries: GaugeMetric;
-  readonly cartethyia_proxy_dial_dns_fallback_total: CounterMetric;
+  readonly cartethyia_model_catalog_entries: GaugeMetric;
+  readonly cartethyia_proxy_dial_proxy_resolved_total: CounterMetric;
   readonly proxy_provider_adapter_load_ms: HistogramMetric;
   readonly pool_cooldown_record_failed: CounterMetric;
   readonly quota_cache_invalidate_failed: CounterMetric;
@@ -348,9 +356,13 @@ export class PrometheusRegistry {
       "cartethyia_quota_cache_entries",
       "Entries in the in-process provider quota cache",
     );
-    this.cartethyia_proxy_dial_dns_fallback_total = this.counter(
-      "cartethyia_proxy_dial_dns_fallback_total",
-      "Pool/relay-bound dials whose local DNS resolution failed and were downgraded to proxy-resolved egress",
+    this.cartethyia_model_catalog_entries = this.gauge(
+      "cartethyia_model_catalog_entries",
+      "Entries in the per-provider model catalog cache (at the cap means eviction churn)",
+    );
+    this.cartethyia_proxy_dial_proxy_resolved_total = this.counter(
+      "cartethyia_proxy_dial_proxy_resolved_total",
+      "Pool/relay-bound dials whose target is resolved by the proxy or relay rather than by the local resolver, so local DNS for the target is advisory only",
     );
     this.proxy_provider_adapter_load_ms = this.histogram(
       "proxy_provider_adapter_load_ms",
@@ -379,8 +391,23 @@ export class PrometheusRegistry {
     return metric;
   }
 
-  histogram(name: string, help: string, buckets: readonly number[]): HistogramMetric {
-    const metric = new Histogram(name, help, buckets);
+  /**
+   * Registers a histogram.
+   *
+   * `labelNames` declares the label set up front. It is optional because a
+   * histogram observed without labels still has to render, but passing it is
+   * the correct call: a declared set is stable across runs and workers, so the
+   * exposition is deterministic even before the first observation, and an
+   * observation carrying undeclared keys cannot create a second, incompatible
+   * series.
+   */
+  histogram(
+    name: string,
+    help: string,
+    buckets: readonly number[],
+    labelNames: readonly string[] = [],
+  ): HistogramMetric {
+    const metric = new Histogram(name, help, buckets, labelNames);
     this.all.push(metric);
     return metric;
   }