cartethyia 2.7.0 → 2.7.2

package/src/network/pool-cooldown-store.ts

@@ -0,0 +1,252 @@
+/**
+ * The durable mirror of the pool selector's provider cooldown state.
+ *
+ * The selector's own map is process-local, mirrored to Redis only when Redis
+ * exists — so the usual installation ran with the flags surviving just a
+ * restart, while the upstream's refusal of the egress did not. The result was
+ * the report that motivated this: no cooldown shown on the proxy page, and
+ * still-429 requests, because dispatch re-picked a spent egress it no longer
+ * remembered. `network_pools.provider_cooldowns` is that state's home between
+ * boots: dispatch flaggers write each marker, boot reads them all back.
+ *
+ * One entry per cooling provider, keyed by the lowercased provider id — the
+ * same normalization the selector applies to its in-memory key.
+ */
+import { randomUUID } from "node:crypto";
+import { eq } from "drizzle-orm";
+import type { CartethyiaDatabase } from "../persistence/db-handle";
+import { networkPools, healthEvents } from "../persistence/schema";
+import { log } from "../observability/logger";
+import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
+import { PROVIDER_BLOCK_MS, type ProxyCooldownEntry } from "./pool/selector";
+
+type PoolProviderCooldownMap = NonNullable<
+  (typeof networkPools.$inferSelect)["providerCooldowns"]
+>;
+
+function readMarkerMap(
+  db: CartethyiaDatabase,
+  poolId: string,
+): Promise<PoolProviderCooldownMap> {
+  return db
+    .select({ providerCooldowns: networkPools.providerCooldowns })
+    .from(networkPools)
+    .where(eq(networkPools.id, poolId))
+    .limit(1)
+    .then((rows) => rows[0]?.providerCooldowns ?? {});
+}
+
+function writeMarkerMap(
+  db: CartethyiaDatabase,
+  poolId: string,
+  map: PoolProviderCooldownMap,
+): Promise<unknown> {
+  return db.update(networkPools).set({ providerCooldowns: map }).where(eq(networkPools.id, poolId));
+}
+
+function entryOf(marker: Omit<ProxyCooldownEntry, "poolId" | "providerId">): {
+  until: number;
+  reason: string;
+  blocked?: boolean;
+} {
+  return marker.blocked === true
+    ? { until: marker.until, reason: marker.reason, blocked: true }
+    : { until: marker.until, reason: marker.reason };
+}
+
+/**
+ * Writes up (or replaces) one provider's marker on the pool row.
+ *
+ * Fill-in, not replace: an entry for another cooling provider must survive.
+ * Failures are logged and swallowed — the in-memory/Redis state stays
+ * conservative for the running process, and only the *next* restart would miss
+ * a flag this write failed to land.
+ */
+export async function upsertProviderCooldownMarker(
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  marker: { until: number; reason: string; blocked?: boolean },
+): Promise<void> {
+  try {
+    const key = providerId.toLowerCase();
+    const map = await readMarkerMap(db, poolId);
+    map[key] = entryOf(marker);
+    await writeMarkerMap(db, poolId, map);
+  } catch (error) {
+    log.warn("pool provider cooldown mirror write failed", { networkPoolId: poolId, providerId, error: String(error) });
+  }
+}
+
+/**
+ * Removes one provider's marker, optionally only when it is a *block*.
+ *
+ * `onlyBlocked` mirrors `clearProviderBlockedOnly`: a probe that answered 429
+ * may correct a retired claim but must not release a live cooldown, and the
+ * same rule applies to the durable copy.
+ */
+export async function dropProviderCooldownMarker(
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  onlyBlocked = false,
+): Promise<void> {
+  try {
+    const key = providerId.toLowerCase();
+    const map = await readMarkerMap(db, poolId);
+    const current = map[key];
+    if (current === undefined) return;
+    if (onlyBlocked && current.blocked !== true) return;
+    delete map[key];
+    await writeMarkerMap(db, poolId, map);
+  } catch (error) {
+    log.warn("pool provider cooldown mirror clear failed", { networkPoolId: poolId, providerId, error: String(error) });
+  }
+}
+
+/**
+ * Removes every provider's marker from one pool row (disable/replace paths).
+ */
+export async function dropAllProviderCooldownMarkers(
+  db: CartethyiaDatabase,
+  poolId: string,
+): Promise<void> {
+  try {
+    const map = await readMarkerMap(db, poolId);
+    if (Object.keys(map).length === 0) return;
+    await writeMarkerMap(db, poolId, {});
+  } catch (error) {
+    log.warn("pool provider cooldown mirror purge failed", { networkPoolId: poolId, error: String(error) });
+  }
+}
+
+/**
+ * Writes a probe-measured verdict the same way a dispatch-side event would be
+ * recorded: one mirror marker and one audit row.
+ *
+ * The probe is the only writer that can flag a cooldown from the control
+ * plane — its 429 is a fresh measurement of "this egress is out of allowance"
+ * (a cooldown marker), and its 403 is a fresh measurement of "this egress was
+ * retired" (a block marker). Without this, an operator who tested a
+ * rate-limited or retired pool saw the verdict once, refreshing lost it, and
+ * the activity panel showed nothing: the fact existed only in the remembered
+ * HTTP response.
+ *
+ * Not coalesced: a probe is operator-initiated and one row per click is the
+ * visible history the panel exists for.
+ */
+export async function recordProbeVerdict(
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  marker: { until: number; reason: string; blocked?: boolean },
+): Promise<void> {
+  await upsertProviderCooldownMarker(db, poolId, providerId, marker);
+  try {
+    await db.insert(healthEvents).values({
+      id: randomUUID(),
+      entityKind: "pool",
+      networkPoolId: poolId,
+      fromStatus: null,
+      toStatus: "cooldown",
+      reason: `provider ${providerId}: ${marker.reason}`,
+      errorCategory: marker.blocked === true ? "egress_blocked" : "rate_limit_transient",
+      createdAt: new Date(),
+    });
+  } catch (error) {
+    log.warn("pool probe event failed", { networkPoolId: poolId, providerId, error: String(error) });
+  }
+}
+
+/**
+ * Reads every live marker back into the selector at boot.
+ *
+ * Only future-dated entries are restored: an expired marker describes an
+ * allowance that refilled while the process was down, and resurrecting it
+ * would sideline a working address. Repeat 429s re-insert markers, so nothing
+ * is lost by dropping the expired ones.
+ */
+export async function restoreProviderCooldownMarkers(
+  selector: { hydrateProviderCooldown(entry: ProxyCooldownEntry): void },
+  db: CartethyiaDatabase,
+): Promise<number> {
+  const rows = await db
+    .select({ id: networkPools.id, providerCooldowns: networkPools.providerCooldowns })
+    .from(networkPools);
+  let restored = 0;
+  for (const row of rows) {
+    const map = row.providerCooldowns;
+    if (map === null || map === undefined) continue;
+    for (const [providerId, marker] of Object.entries(map)) {
+      if (Date.now() >= marker.until) continue;
+      selector.hydrateProviderCooldown({
+        poolId: row.id,
+        providerId,
+        until: marker.until,
+        reason: marker.reason,
+        ...(marker.blocked === true ? { blocked: true } : {}),
+      });
+      restored += 1;
+    }
+  }
+  return restored;
+}
+
+/** The columns the probe-verdict reconciliation reads, mirroring the row's own read path. */
+function probeEndpointOf(endpointConfig: unknown): string {
+  const raw = (endpointConfig ?? {}) as Record<string, unknown>;
+  return typeof raw["endpoint"] === "string" ? raw["endpoint"] : "";
+}
+
+/**
+ * Re-enforces the stored probe verdicts at boot (new: replay, read-ahead).
+ *
+ * A stored `blocked` verdict is a *state*, not a decoration. If the marker the
+ * probe flagged failed to persist, or the verdict predated the marker column,
+ * the egress would return to rotation after restart and get refused again —
+ * dispatch spending a real request to rediscover a refusal the row already
+ * knew. So the boot read reconciles: a fresh (endpoint still matches) `blocked`
+ * verdict for an address-keyed provider becomes a block flag, even when the
+ * mirror has nothing (or has an older, cooler entry — the retirement claim is
+ * the sharper fact and wins).
+ *
+ * `throttled` is deliberately **not** replayed into a cooldown: the cooldown
+ * the probe flagged already carries the measured window, and blind-replaying
+ * one from a verdict that predates the restart could outlive the upstream's
+ * actual window.
+ *
+ * Returns how many verdicts were replayed.
+ */
+export async function restorePoolProbeVerdicts(
+  selector: { hydrateProviderCooldown(entry: ProxyCooldownEntry): void },
+  db: CartethyiaDatabase,
+): Promise<number> {
+  const rows = await db
+    .select({
+      id: networkPools.id,
+      status: networkPools.status,
+      endpointConfig: networkPools.endpointConfig,
+      providerProbeProviderId: networkPools.providerProbeProviderId,
+      providerProbeStatus: networkPools.providerProbeStatus,
+      providerProbeReason: networkPools.providerProbeReason,
+      providerProbeEndpoint: networkPools.providerProbeEndpoint,
+    })
+    .from(networkPools);
+  let replayed = 0;
+  for (const row of rows) {
+    if (row.status !== "active") continue;
+    if (row.providerProbeStatus !== "blocked") continue;
+    if (row.providerProbeEndpoint !== probeEndpointOf(row.endpointConfig)) continue;
+    const providerId = row.providerProbeProviderId;
+    if (typeof providerId !== "string" || !providerRateLimitIsIpScoped(providerId)) continue;
+    selector.hydrateProviderCooldown({
+      poolId: row.id,
+      providerId: providerId.toLowerCase(),
+      until: Date.now() + PROVIDER_BLOCK_MS,
+      reason: row.providerProbeReason ?? "Upstream refused this egress address",
+      blocked: true,
+    });
+    replayed += 1;
+  }
+  return replayed;
+}

package/src/network/pool-health-machine.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import { healthEventBuffer } from "../persistence/health-event-flush";
 import { and, desc, eq, inArray, isNotNull, lte, sql } from "drizzle-orm";
 import type { CartethyiaDatabase } from "../persistence/db-handle";
@@ -140,7 +141,7 @@ export async function recordPoolDispatchOutcome(
       createdAt: now,
     };
     if (buffer !== undefined) buffer.enqueue(row);
-    else await db.insert(healthEvents).values(row as never);
+    else await db.insert(healthEvents).values({ id: randomUUID(), ...row } as never);
   }
   if (healthChanged) {
     publishPoolHealth({
@@ -198,6 +199,7 @@ export async function disablePoolForProxyHttpStatus(
   if (!updated[0]) return false;
 
   await db.insert(healthEvents).values({
+    id: randomUUID(),
     entityKind: "pool",
     networkPoolId: poolId,
     fromStatus: pool.status,
@@ -226,7 +228,7 @@ export async function recoverNetworkPool(db: CartethyiaDatabase, tenantId: strin
   if (!pool) return false;
   const now = new Date();
   await db.update(networkPools).set({ status: "active", consecutiveFailures: 0, cooldownUntil: null, lastRecoveredAt: now }).where(and(eq(networkPools.tenantId, tenantId), eq(networkPools.id, poolId)));
-  await db.insert(healthEvents).values({ entityKind: "pool", networkPoolId: poolId, fromStatus: pool.status, toStatus: "active", reason: "Manual operator recovery", errorCategory: null, createdAt: now });
+  await db.insert(healthEvents).values({ id: randomUUID(), entityKind: "pool", networkPoolId: poolId, fromStatus: pool.status, toStatus: "active", reason: "Manual operator recovery", errorCategory: null, createdAt: now });
   publishPoolHealth({ poolId, tenantId: pool.tenantId, status: "active", consecutiveFailures: 0 });
   return true;
 }
@@ -248,7 +250,7 @@ export async function sweepExpiredPoolCooldowns(db: CartethyiaDatabase): Promise
   const expired = await db.select({ id: networkPools.id, tenantId: networkPools.tenantId, status: networkPools.status }).from(networkPools).where(and(inArray(networkPools.status, ["cooldown"]), isNotNull(networkPools.cooldownUntil), lte(networkPools.cooldownUntil, now))).limit(POOL_COOLDOWN_RECOVERY_BATCH);
   if (expired.length === 0) return 0;
   await db.update(networkPools).set({ status: "active", consecutiveFailures: 0, cooldownUntil: null, lastRecoveredAt: now }).where(inArray(networkPools.id, expired.map((row) => row.id)));
-  await db.insert(healthEvents).values(expired.map((row) => ({ entityKind: "pool" as const, networkPoolId: row.id, fromStatus: row.status, toStatus: "active" as const, reason: "Cooldown elapsed — auto-recovered", errorCategory: null, createdAt: now })));
+  await db.insert(healthEvents).values(expired.map((row) => ({ id: randomUUID(), entityKind: "pool" as const, networkPoolId: row.id, fromStatus: row.status, toStatus: "active" as const, reason: "Cooldown elapsed — auto-recovered", errorCategory: null, createdAt: now })));
   for (const row of expired) publishPoolHealth({ poolId: row.id, tenantId: row.tenantId, status: "active", consecutiveFailures: 0 });
   return expired.length;
 }

package/src/network/pool-health.ts

@@ -1,21 +1,26 @@
+import { randomUUID } from "node:crypto";
 import type { CartethyiaDatabase } from "../persistence/db-handle";
 import { healthEvents } from "../persistence/schema";
 import type { NetworkPoolSelector } from "./pool/selector";
 import type { GatewayError } from "../protocol/gateway-error";
 import { metrics } from "../observability/metrics";
 import { log } from "../observability/logger";
+import { upsertProviderCooldownMarker } from "./pool-cooldown-store";
+import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
 
 /**
  * Durable pool rate-limit history.
  *
  * Per-(pool, provider) 429 cooldowns live in `NetworkPoolSelector` (fast,
- * volatile, consulted on every dispatch). This recorder mirrors each flag
- * into the append-only `health_events` table (`entity_kind: "pool"`) — the
- * same merged log the account recorder writes to for accounts — so
- * operators can audit which pool throttled which provider and when. It
- * never touches `network_pools.status`: a single provider's 429 must not
- * flip the whole pool to `cooldown` (that would head-of-line-block every
- * other provider on the pool).
+ * volatile, consulted on every dispatch). This recorder mirrors each flag into
+ * two durable homes: the append-only `health_events` table
+ * (`entity_kind: "pool"`, the same merged log the account recorder writes to
+ * for accounts) for audit, and the `provider_cooldowns` column on the pool row
+ * as the state boot restores — a restart must not make dispatch forget the
+ * egresses the upstream is still refusing. It never touches
+ * `network_pools.status`: a single provider's 429 must not flip the whole pool
+ * to `cooldown` (that would head-of-line-block every other provider on the
+ * pool).
  */
 
 /**
@@ -41,12 +46,14 @@ async function recordPoolCooldownEvent(
   if (last !== undefined && now - last < POOL_COOLDOWN_EVENT_WINDOW_MS) return;
   try {
     await db.insert(healthEvents).values({
+      id: randomUUID(),
       entityKind: "pool",
       networkPoolId: poolId,
       fromStatus: null,
       toStatus: "cooldown",
       reason: `provider ${providerId}: ${reason}`,
       errorCategory: "rate_limit_transient",
+      createdAt: new Date(),
     });
   } catch {
     // Non-fatal: observability writes must never fail dispatch. The
@@ -85,6 +92,12 @@ export function flagPoolBlocked(
   const reason = `Provider ${providerId} refused this egress`;
   void poolSelector
     .flagProviderBlocked(poolId, providerId, reason)
+    .then((entry) =>
+      // Persisted even when the flag was refused (a credential-keyed
+      // provider): the selector returned an inert entry, so the mirror
+      // records nothing a reader would act on.
+      upsertProviderCooldownMarker(db, poolId, providerId, entry),
+    )
     .catch((cause: unknown) => {
       metrics.pool_cooldown_record_failed.inc(1);
       log.warn("pool block flag failed", {
@@ -119,12 +132,14 @@ async function recordPoolBlockEvent(
   error: GatewayError,
 ): Promise<void> {
   await db.insert(healthEvents).values({
+    id: randomUUID(),
     entityKind: "pool",
     networkPoolId: poolId,
     fromStatus: null,
     toStatus: "cooldown",
     reason: `provider ${providerId}: ${reason}${providerCodeSuffix(error)}`,
     errorCategory: "egress_blocked",
+    createdAt: new Date(),
   });
 }
 
@@ -137,9 +152,20 @@ function providerCodeSuffix(error: GatewayError): string {
 /** Cooldown applied when an upstream 429 carries no `retryAfterMs`. */
 const DEFAULT_PROVIDER_COOLDOWN_MS = 15 * 60 * 1000;
 
+/**
+ * The fallback window for a provider-scoped egress cooldown. Upstream `Retry-After`
+ * always wins; the configured default exists because a request-counted free
+ * tier — the one provider that flags at all — stays refused for hours after
+ * the 15-minute guess this function used before.
+ */
+function providerCooldownFallbackMs(): number {
+  return Math.max(resolvePoolProviderRateLimitCooldownMs(), DEFAULT_PROVIDER_COOLDOWN_MS);
+}
+
 /**
  * Records a provider-scoped 429 cooldown: the volatile per-(pool, provider)
- * flag consulted on every dispatch, plus its durable audit row.
+ * flag consulted on every dispatch, its durable audit row, and the durable
+ * state boot restores from.
  *
  * Both paths that reach this decision — the non-streaming attempt loop and
  * the streaming failure path — call this one function, because each write
@@ -161,9 +187,10 @@ export function flagPoolCooldown(
   const retryAfterMs =
     typeof error.details.retryAfterMs === "number"
       ? error.details.retryAfterMs
-      : DEFAULT_PROVIDER_COOLDOWN_MS;
+      : providerCooldownFallbackMs();
   void poolSelector
     .flagProviderCooldown(poolId, providerId, retryAfterMs, reason)
+    .then((entry) => upsertProviderCooldownMarker(db, poolId, providerId, entry))
     .catch((cause: unknown) => {
       metrics.pool_cooldown_record_failed.inc(1);
       log.warn("pool cooldown flag failed", {

package/src/observability/event-query.ts

@@ -537,6 +537,63 @@ export async function chartTelemetry(
   }));
 }
 
+/** One day × model usage cell for a single provider — the Activity Matrix feed. */
+export interface ProviderUsageDayRow {
+  /** UTC day, `YYYY-MM-DD` (epoch-day buckets, so the same key on both backends). */
+  readonly day: string;
+  /** What the client asked for; `null` when the event carried no model. */
+  readonly model: string | null;
+  readonly requests: number;
+  readonly inputTokens: number;
+  readonly cachedInputTokens: number;
+  readonly outputTokens: number;
+  readonly costUsd: number;
+}
+
+/**
+ * Per-provider daily usage grouped by `requested_model`, day-bucketed like
+ * {@link chartTelemetry} but with a second grouping dimension (requested
+ * model) and the totals the tooltip/summary strip need. Provider rows carry
+ * no catalog foreign keys, so an unknown or deleted provider id returns empty
+ * rather than erroring.
+ */
+export async function providerUsageMatrix(
+  db: CartethyiaDatabase,
+  tenantId: string,
+  providerId: string,
+  since: Date,
+): Promise<ProviderUsageDayRow[]> {
+  // Same inlining rule as chartTelemetry: SELECT and GROUP BY must spell the
+  // identical expression or Postgres 42803-rejects the grouped reference.
+  const w = sql.raw(String(86400));
+  const dayExpression = isPostgres(db)
+    ? sql<number>`(cast(extract(epoch from ${telemetryEvents.createdAt}) as bigint) / ${w}) * ${w}`
+    : sql<number>`(cast(strftime('%s', ${telemetryEvents.createdAt}) as integer) / ${w}) * ${w}`;
+  const rows = await db
+    .select({
+      day: dayExpression,
+      model: telemetryEvents.requestedModel,
+      requests: sql<number>`count(*)`,
+      inputTokens: sql<number>`coalesce(sum(${telemetryEvents.inputTokens}), 0)`,
+      cachedInputTokens: sql<number>`coalesce(sum(${telemetryEvents.cachedInputTokens}), 0)`,
+      outputTokens: sql<number>`coalesce(sum(${telemetryEvents.outputTokens}), 0)`,
+      costUsd: sql<number>`coalesce(sum(${telemetryEvents.estimatedCostUsd}), 0)`,
+    })
+    .from(telemetryEvents)
+    .where(and(scoped(tenantId, since), eq(telemetryEvents.providerId, providerId)))
+    .groupBy(dayExpression, telemetryEvents.requestedModel)
+    .orderBy(dayExpression);
+  return rows.map((row) => ({
+    day: new Date(numeric(row.day) * 1000).toISOString().slice(0, 10),
+    model: row.model,
+    requests: numeric(row.requests),
+    inputTokens: numeric(row.inputTokens),
+    cachedInputTokens: numeric(row.cachedInputTokens),
+    outputTokens: numeric(row.outputTokens),
+    costUsd: numeric(row.costUsd),
+  }));
+}
+
 /** Overview/health totals for one tenant window. */
 export async function healthTelemetry(
   db: CartethyiaDatabase,