cartethyia 2.7.3 → 2.7.5

package/src/console/inferhub/normalize.ts

@@ -26,7 +26,7 @@ import type {
 /** Ask buckets are keyed to this many decimals so float noise cannot split one price into two tiers. */
 const PRICE_PRECISION = 6;
 /** Default number of live providers a capacity-weighted price must have behind it. */
-export const DEFAULT_MIN_CAPACITY = 2;
+const DEFAULT_MIN_CAPACITY = 2;
 
 const FAMILY_STATES: readonly FamilyState[] = [
   "operational",
@@ -114,7 +114,7 @@ export function groupTiers(
  * provider count would make `providersLive` wrong, and a wrong availability
  * number is worse than an absent one.
  */
-export function pricePointsToTiers(
+function pricePointsToTiers(
   points: unknown,
   official: number | null,
 ): readonly Tier[] {

package/src/console/inferhub/routes.ts

@@ -57,7 +57,7 @@ import type {
 /** Success rate below which a model is not routed to, when one is reported. */
 const MIN_SUCCESS_RATE_PCT = 80;
 
-export interface InferhubRoutesConfig {
+interface InferhubRoutesConfig {
   readonly accessResolver: (request: Request) => AccessDecision | undefined;
   /** Resolves the `config_read` key. Absent means the catalog cannot be read. */
   readonly resolveConfigKey?: () => Promise<string | undefined> | string | undefined;

package/src/console/inferhub/types.ts

@@ -34,7 +34,7 @@ export interface ModelHealth {
   readonly familyState: FamilyState | null;
 }
 
-export interface ModelCapabilities {
+interface ModelCapabilities {
   readonly contextWindow: number | null;
   readonly maxOutputTokens: number | null;
   readonly modalities: readonly string[];

package/src/console/observability/contracts.ts

@@ -173,7 +173,7 @@ export interface UsageCacheResponse {
   cacheWriteTokens: number;
   hitRate: number;
 }
-export type UsageRequestMode = "stream" | "non_stream";
+type UsageRequestMode = "stream" | "non_stream";
 export interface UsageRequestItem {
   requestId: string;
   surface?: string;
@@ -191,6 +191,10 @@ export interface UsageRequestItem {
   apiKeyId?: string;
   userAgent?: string;
   clientName?: string;
+  /** The proxy pool that carried this request, when one was used. */
+  networkPoolId?: string;
+  /** Operator-facing label of the pool; absent when the pool is gone. */
+  networkPoolLabel?: string;
   estimatedCost?: number;
   /** Provider-billed credits; absent when the upstream reported no credit field. */
   creditUsed?: number;

package/src/console/observability/live.ts

@@ -7,7 +7,7 @@
 // Every endpoint requires `dashboard:read`. The global request count is
 // process-level; pool usage and health are filtered to the caller's tenant.
 
-import { Elysia } from "elysia";
+import { Elysia, type AnyElysia } from "elysia";
 import { errorResponse, requireScope } from "../shared/errors";
 import type { ConsoleAccessResolver } from "../auth/access";
 /**
@@ -34,7 +34,7 @@ import { poolByteSnapshot } from "../../network/pool/byte-accounting";
 import { consoleSseResponse, createConsoleSseStream } from "./sse";
 import { subscribeRequestCompleted } from "../../observability/request-notify";
 
-export interface LiveConfig {
+interface LiveConfig {
   readonly accessResolver: ConsoleAccessResolver;
   readonly poolSelector?: NetworkPoolSelector;
   readonly db?: CartethyiaDatabase;
@@ -86,7 +86,7 @@ function withPoolBytes<T extends { poolId: string; currentInflight?: number }>(
   });
 }
 
-export function createLiveRoutes(config: LiveConfig): Elysia {
+export function createLiveRoutes(config: LiveConfig): AnyElysia {
   return new Elysia()
     .get("/live/requests/stream", ({ request, set }) => {
       try {
@@ -94,10 +94,9 @@ export function createLiveRoutes(config: LiveConfig): Elysia {
         requireScope(access, "dashboard:read");
         const tenantId = access?.tenantId;
         if (!tenantId) return errorResponse(new Error("Tenant required"), set, "Live operation failed");
-        set.headers["content-type"] = "text/event-stream";
-        set.headers["cache-control"] = "no-cache, no-transform";
-        set.headers["connection"] = "keep-alive";
-        set.headers["x-accel-buffering"] = "no";
+        // No `set.headers` writes here: this handler returns a `Response`, so
+        // Elysia discards `set` and `consoleSseResponse` below owns the SSE
+        // headers (content-type, cache-control, connection, buffering).
         return consoleSseResponse(
           createConsoleSseStream(request.signal, ({ send }) => {
             return subscribeRequestCompleted((notice) => {
@@ -119,22 +118,36 @@ export function createLiveRoutes(config: LiveConfig): Elysia {
     })
     .get("/live/in-flight/stream", ({ request, set }) => {
       try {
-        requireScope(config.accessResolver(request), "dashboard:read");
+        const access = requireScope(config.accessResolver(request), "dashboard:read");
+        const tenantId = access?.tenantId;
+        return consoleSseResponse(
+          createConsoleSseStream(request.signal, ({ send }) => {
+            const store = config.stateStore;
+            if (!store) {
+              send("count", EMPTY_IN_FLIGHT);
+              return () => {};
+            }
+            // The Usage page keeps one stream open for both the in-flight gauge
+            // and request-table invalidation, so this route also fans out
+            // `request_completed` events (tagged by kind) for the tenant,
+            // mirroring `/live/requests/stream`.
+            const unsubscribe = store.subscribeInFlight((snapshot) => send("count", snapshot));
+            let unsubscribeCompleted: (() => void) | undefined;
+            if (tenantId) {
+              unsubscribeCompleted = subscribeRequestCompleted((notice) => {
+                if (notice.tenantId === tenantId) send("request_completed", notice);
+              });
+            }
+            send("count", store.inFlightSnapshot());
+            return () => {
+              unsubscribe();
+              unsubscribeCompleted?.();
+            };
+          }),
+        );
       } catch (e) {
         return errorResponse(e, set, "Live operation failed");
       }
-      return consoleSseResponse(
-        createConsoleSseStream(request.signal, ({ send }) => {
-          const store = config.stateStore;
-          if (!store) {
-            send("count", EMPTY_IN_FLIGHT);
-            return () => {};
-          }
-          const unsubscribe = store.subscribeInFlight((snapshot) => send("count", snapshot));
-          send("count", store.inFlightSnapshot());
-          return unsubscribe;
-        }),
-      );
     })
     .get("/live/pools", async ({ request, set }) => {
       try {
@@ -176,5 +189,5 @@ export function createLiveRoutes(config: LiveConfig): Elysia {
           };
         }),
       );
-    }) as unknown as Elysia;
+    });
 }

package/src/console/observability/logs.ts

@@ -8,7 +8,7 @@
 // metadata and their args are redacted at the source, the same audience
 // that already reads request/response bodies on Usage.
 
-import { Elysia, t } from "elysia";
+import { Elysia, t, type AnyElysia } from "elysia";
 import { ConsoleDomainError, errorResponse, requireScope } from "../shared/errors";
 import { parseQueryLimit } from "../shared/query";
 import type { ConsoleAccessResolver } from "../auth/access";
@@ -23,7 +23,7 @@ import {
 } from "../../observability/log-ring";
 import { consoleSseResponse, createConsoleSseStream } from "./sse";
 
-export interface LogsConfig {
+interface LogsConfig {
   readonly accessResolver: ConsoleAccessResolver;
   readonly auditSink?: AuditSink;
 }
@@ -37,7 +37,7 @@ const logsQuery = t.Object({
   limit: t.Optional(t.String()),
 });
 
-export function createLogRoutes(config: LogsConfig): Elysia {
+export function createLogRoutes(config: LogsConfig): AnyElysia {
   const readLogs = (access: AccessDecision | undefined, level?: string, limitRaw?: string) => {
     requireScope(access, "dashboard:read");
     if (level !== undefined && !isLogLevel(level)) {
@@ -83,5 +83,5 @@ export function createLogRoutes(config: LogsConfig): Elysia {
           });
         }),
       );
-    }) as unknown as Elysia;
+    });
 }

package/src/console/observability/performance.ts

@@ -5,12 +5,12 @@
 // this instance's own module graph and egress), so the route is guarded by
 // `platform:admin` like the other instance-level operations.
 
-import { Elysia } from "elysia";
+import { Elysia, type AnyElysia } from "elysia";
 import { errorResponse, requireGlobalAdmin } from "../shared/errors";
 import { performanceMetricsSnapshot } from "../../observability/performance-metrics";
 import type { PerformanceConfig } from "./performance-contracts";
 
-export function createPerformanceRoutes(config: PerformanceConfig): Elysia {
+export function createPerformanceRoutes(config: PerformanceConfig): AnyElysia {
   return new Elysia()
     .get("/system/performance", ({ request, set }) => {
       try {
@@ -19,5 +19,5 @@ export function createPerformanceRoutes(config: PerformanceConfig): Elysia {
       } catch (e) {
         return errorResponse(e, set, "Performance operation failed");
       }
-    }) as unknown as Elysia;
+    });
 }

package/src/console/observability/routes.ts

@@ -1,4 +1,4 @@
-import { Elysia, t } from "elysia";
+import { Elysia, t, type AnyElysia } from "elysia";
 import { ConsoleDomainError, consoleErrorHandler } from "../shared/errors";
 import { USAGE_DIMENSIONS, type UsageDimension } from "./usage-dimensions";
 import { createObservabilityOperations, type ObservabilityConfig } from "./contracts";
@@ -18,7 +18,7 @@ const telemetryDetailQuery = t.Object({
   includePayload: t.Optional(t.String()),
   createdAt: t.Optional(t.String()),
 });
-export function createObservabilityRoutes(config: ObservabilityConfig): Elysia {
+export function createObservabilityRoutes(config: ObservabilityConfig): AnyElysia {
   const factory = createObservabilityOperations(config);
   return new Elysia()
     .error(consoleErrorHandler("Observability operation failed"))
@@ -114,5 +114,5 @@ export function createObservabilityRoutes(config: ObservabilityConfig): Elysia {
           query.createdAt,
         );
 },
-    ) as unknown as Elysia;
+    );
 }

package/src/console/observability/sse.ts

@@ -5,14 +5,14 @@
  * the shared `buildSseEventFrame` core so the two layers cannot drift.
  */
 
-export interface SseSender {
+interface SseSender {
   send(event: string, data: unknown): void;
 }
 
 /** Register endpoint-specific SSE subscriptions; return their teardown callback. */
-export type SseSetup = (sender: SseSender) => () => void;
+type SseSetup = (sender: SseSender) => () => void;
 
-import { buildSseEventFrame as formatSseFrame } from "../../protocol/sse-frame";
+import { TEXT_ENCODER, buildSseEventFrame as formatSseFrame } from "../../protocol/sse-frame";
 
 export { formatSseFrame };
 
@@ -22,38 +22,82 @@ export { formatSseFrame };
  */
 const SSE_HEARTBEAT_MS = 25_000;
 
+/**
+ * Hard cap on one stream's lifetime. A tab that is abandoned but never
+ * unloads (a backgrounded browser window, a parked laptop) would otherwise
+ * hold its subscription and timers forever; the client reconnects on close,
+ * so this only forces a refresh. 45 minutes is far longer than any live view
+ * needs in one stretch.
+ */
+const SSE_MAX_LIFETIME_MS = 45 * 60_000;
+
+/**
+ * Idle timeout: if no *data* frame is sent for this long, the stream sends a
+ * terminal `close` event and closes. Heartbeats deliberately do not count —
+ * they are there to keep proxies alive, not to prove someone is reading, so
+ * they must not defeat the abandoned-tab cutoff.
+ */
+const SSE_IDLE_TIMEOUT_MS = 5 * 60_000;
+
+/** Watchdog cadence; the two cutoffs above are checked on this tick. */
+const SSE_WATCHDOG_MS = 30_000;
+
+/** Frame sent when the server cuts the stream (idle or max lifetime). */
+function closeFrame(reason: string): Uint8Array {
+  return TEXT_ENCODER.encode(formatSseFrame("close", JSON.stringify({ reason })));
+}
+
 /**
  * Console SSE response stream with uniform JSON event encoding, heartbeat,
- * and abort/cancel cleanup. The setup callback runs synchronously at stream
- * start so the initial snapshot is enqueued before the first read.
+ * idle/max-lifetime cutoff, and abort/cancel cleanup. The setup callback runs
+ * synchronously at stream start so the initial snapshot is enqueued before the
+ * first read.
+ *
+ * The encoder is the module-level shared `TextEncoder` from the frame module:
+ * allocating one per stream (let alone per send) is unnecessary garbage for a
+ * hot path that encodes a frame every few hundred milliseconds per connection.
  */
 export function createConsoleSseStream(
   signal: AbortSignal,
   setup: SseSetup,
 ): ReadableStream<Uint8Array> {
-  const encoder = new TextEncoder();
   let cleanupSubscription: (() => void) | undefined;
   let heartbeat: ReturnType<typeof setInterval> | undefined;
+  let watchdog: ReturnType<typeof setInterval> | undefined;
   let close: (() => void) | undefined;
 
   return new ReadableStream<Uint8Array>({
     start(controller) {
       let closed = false;
+      const startedAt = Date.now();
+      let lastDataAt = startedAt;
       const sender: SseSender = {
         send(event, data) {
           if (closed) return;
           try {
-            controller.enqueue(encoder.encode(formatSseFrame(event, JSON.stringify(data))));
+            lastDataAt = Date.now();
+            controller.enqueue(TEXT_ENCODER.encode(formatSseFrame(event, JSON.stringify(data))));
           } catch {
             close?.();
           }
         },
       };
 
+      const terminate = (reason: string): void => {
+        if (closed) return;
+        try {
+          controller.enqueue(closeFrame(reason));
+        } catch {
+          // Fall through to close regardless.
+        }
+        close?.();
+      };
+
       close = () => {
         if (closed) return;
         closed = true;
         clearInterval(heartbeat);
+        clearInterval(watchdog);
         cleanupSubscription?.();
         try {
           controller.close();
@@ -70,12 +114,22 @@ export function createConsoleSseStream(
       heartbeat = setInterval(() => {
         if (closed) return;
         try {
-          controller.enqueue(encoder.encode(": ping\n\n"));
+          controller.enqueue(TEXT_ENCODER.encode(": ping\n\n"));
         } catch {
           close?.();
         }
       }, SSE_HEARTBEAT_MS);
 
+      watchdog = setInterval(() => {
+        if (closed) return;
+        const now = Date.now();
+        if (now - lastDataAt > SSE_IDLE_TIMEOUT_MS) {
+          terminate("idle");
+        } else if (now - startedAt > SSE_MAX_LIFETIME_MS) {
+          terminate("max_lifetime");
+        }
+      }, SSE_WATCHDOG_MS);
+
       signal.addEventListener("abort", close, { once: true });
     },
     cancel() {

package/src/console/observability/store.ts

@@ -7,11 +7,8 @@ import { type TelemetryEventRecord } from "../../observability/event-log";
 import {
   breakdownTelemetry,
   chartTelemetry,
-  healthTelemetry,
   queryTelemetryEvents as queryTelemetryEventPage,
-  summarizeHealthPerformance,
-  summarizeUsagePerformanceSql,
-  summarizeTelemetry,
+  summarizeUsageWithPerformance,
 } from "../../observability/event-query";
 import { clientNameFromUserAgent } from "../../observability/client-name";
 
@@ -145,6 +142,7 @@ function mapUsageRequestItem(
       : {}),
     ...(event.endpoint ? { endpoint: event.endpoint } : {}),
     ...(event.apiKeyId ? { apiKeyId: event.apiKeyId } : {}),
+    ...(event.networkPoolId ? { networkPoolId: event.networkPoolId } : {}),
     ...(event.userAgent ? { userAgent: event.userAgent } : {}),
     ...(event.userAgent ? { clientName: clientNameFromUserAgent(event.userAgent) } : {}),
     ...(event.clientIp
@@ -186,6 +184,14 @@ async function resolveProxyLabel(
 }
 const HEALTH_WINDOW_MS = 24 * 3_600_000;
 const TOP_N = 50;
+/**
+ * TTL for the usage analytics read-through (summary, chart, breakdown, totals).
+ * The dashboard polls these every 5–10s; a 30s TTL means the heavy telemetry
+ * window scans run once per 30s and the polls in between are answered from
+ * Redis (or the in-process fallback) with a `304`. Health stays on the
+ * default 10s — it feeds the live status tile.
+ */
+const USAGE_READ_THROUGH_TTL_SECONDS = 30;
 
 /**
  * Nearest-rank percentile over a sorted array — kept for the JS-side
@@ -199,7 +205,7 @@ function percentile(sorted: readonly number[], fraction: number): number {
   return Math.round(sorted[index] ?? 0);
 }
 
-export interface UsagePerformanceSummary {
+interface UsagePerformanceSummary {
   avgResolveMs: number;
   avgDurationMs: number;
   latencyP50Ms: number;
@@ -326,11 +332,10 @@ export class DrizzleObservabilityStore implements ObservabilityStore {
       // cost on the dashboard. `healthTelemetry` returns the totals and the
       // database computes the percentiles.
       const since = new Date(Date.now() - HEALTH_WINDOW_MS);
-      const totals = await healthTelemetry(this.db, tenantId, since);
-      const performance = await summarizeHealthPerformance(this.db, tenantId, since);
+      const { summary: totals, performance } = await summarizeUsageWithPerformance(this.db, tenantId, since);
       total = totals.requests;
       errors = totals.errors;
-      avg = performance.latencyAvgMs === null ? 0 : Math.round(performance.latencyAvgMs);
+      avg = Math.round(performance.avgDurationMs ?? 0);
       p95 = performance.latencyP95Ms ?? 0;
       p99 = performance.latencyP99Ms ?? 0;
       const inputTokens = totals.inputTokens;
@@ -392,6 +397,7 @@ export class DrizzleObservabilityStore implements ObservabilityStore {
   async usage(tenantId: string, period: string): Promise<CachedResult<UsageResponse>> {
     return this.readThroughWithEtag(this.usageCacheKey(tenantId, period, "usage"), () =>
       this.computeUsage(tenantId, period),
+    USAGE_READ_THROUGH_TTL_SECONDS,
     );
   }
 
@@ -408,8 +414,12 @@ export class DrizzleObservabilityStore implements ObservabilityStore {
    * configured — and returns the strong validator alongside it, so a repeated
    * poll can be answered with a `304` instead of the whole payload.
    */
-  private readThroughWithEtag<T>(key: string, compute: () => Promise<T>): Promise<CachedResult<T>> {
-    return cachedReadWithEtag(this.redis, key, compute);
+  private readThroughWithEtag<T>(
+    key: string,
+    compute: () => Promise<T>,
+    ttlSeconds?: number,
+  ): Promise<CachedResult<T>> {
+    return cachedReadWithEtag(this.redis, key, compute, ttlSeconds);
   }
 
   private async computeUsage(tenantId: string, period: string): Promise<UsageResponse> {
@@ -417,8 +427,8 @@ export class DrizzleObservabilityStore implements ObservabilityStore {
     // Totals AND the top-model/top-provider counts come from SQL: the old
     // shape parsed the whole retained window into an array to count two
     // dimensions in JavaScript.
-    const [summary, models, providers] = await Promise.all([
-      summarizeTelemetry(this.db, tenantId, since),
+    const [{ summary }, models, providers] = await Promise.all([
+      summarizeUsageWithPerformance(this.db, tenantId, since),
       breakdownTelemetry(this.db, tenantId, "model", since),
       breakdownTelemetry(this.db, tenantId, "provider", since),
     ]);
@@ -462,18 +472,18 @@ export class DrizzleObservabilityStore implements ObservabilityStore {
   async usageSummary(tenantId: string, period: string): Promise<CachedResult<UsageSummaryResponse>> {
     return this.readThroughWithEtag(this.usageCacheKey(tenantId, period, "summary"), () =>
       this.computeSummary(tenantId, period),
+    USAGE_READ_THROUGH_TTL_SECONDS,
     );
   }
 
   private async computeSummary(tenantId: string, period: string): Promise<UsageSummaryResponse> {
     const since = this.periodSince(period);
     // Totals, status breakdown, and latency percentiles all computed in SQL.
-    // This used to parse the whole retained window into JavaScript to sum six
-    // counters and sort a latency array.
-    const [summary, statuses, performance] = await Promise.all([
-      summarizeTelemetry(this.db, tenantId, since),
+    // The summary totals and performance figures come from ONE combined window
+    // scan (one percentile pass), not two independent scans.
+    const [{ summary, performance }, statuses] = await Promise.all([
+      summarizeUsageWithPerformance(this.db, tenantId, since),
       breakdownTelemetry(this.db, tenantId, "status", since),
-      summarizeUsagePerformanceSql(this.db, tenantId, since),
     ]);
     const inputTokens = summary.inputTokens;
     const cachedTokens = summary.cachedInputTokens;
@@ -503,6 +513,7 @@ export class DrizzleObservabilityStore implements ObservabilityStore {
   async usageChart(tenantId: string, period: string): Promise<CachedResult<UsageChartResponse>> {
     return this.readThroughWithEtag(this.usageCacheKey(tenantId, period, "chart"), () =>
       this.computeChart(tenantId, period),
+    USAGE_READ_THROUGH_TTL_SECONDS,
     );
   }
 
@@ -526,7 +537,7 @@ export class DrizzleObservabilityStore implements ObservabilityStore {
         input: bucket.inputTokens,
         // `cached` is the cached share of the prompt: the SQL reader sums
         // cached_input_tokens, which is the same number the old JS loop used.
-        cached: bucket.inputTokens,
+        cached: bucket.cachedInputTokens,
         output: bucket.outputTokens,
       });
     }
@@ -550,6 +561,7 @@ export class DrizzleObservabilityStore implements ObservabilityStore {
   ): Promise<CachedResult<UsageByResponse>> {
     return this.readThroughWithEtag(this.usageCacheKey(tenantId, `${dimension}:${period}`, "by"), () =>
       this.computeBy(tenantId, dimension, period),
+    USAGE_READ_THROUGH_TTL_SECONDS,
     );
   }
 
@@ -655,7 +667,7 @@ export class DrizzleObservabilityStore implements ObservabilityStore {
   }
 
   private async computeCache(tenantId: string, period: string): Promise<UsageCacheResponse> {
-    const summary = await summarizeTelemetry(this.db, tenantId, this.periodSince(period));
+    const { summary } = await summarizeUsageWithPerformance(this.db, tenantId, this.periodSince(period));
     const inputTokens = summary.inputTokens;
     const outputTokens = summary.outputTokens;
     const cachedTokens = summary.cachedInputTokens;
@@ -766,7 +778,11 @@ export class DrizzleObservabilityStore implements ObservabilityStore {
     const item = mapUsageRequestItem(event, hideClientIp);
     const keyLabel = event.apiKeyId ? keyLabels?.get(event.apiKeyId) : undefined;
     const labeled = keyLabel === undefined ? item : { ...item, apiKeyLabel: keyLabel };
-    const presented = { ...labeled, proxy };
+    const presented = {
+      ...labeled,
+      proxy,
+      ...(event.networkPoolId ? { networkPoolLabel: proxy } : {}),
+    };
     // Filter the payload table directly instead of joining `telemetry_events`
     // only to re-apply the tenant predicate: `telemetry_payloads` carries
     // `tenant_id` itself, so the join added a second table's rows to filter for

package/src/console/providers/catalog/catalog-projections.ts

@@ -23,7 +23,7 @@ import type { ProviderRecord } from "./contracts";
  * purpose: the projection should not be able to reach a loader, so the type
  * cannot drift into "call something expensive to answer a boolean".
  */
-export interface ProviderDiscoveryLookup {
+interface ProviderDiscoveryLookup {
   hasModelDiscovery(providerId: string): boolean;
 }