cartethyia 2.6.7 → 2.7.1

package/src/network/pool/pool-state.ts

@@ -28,6 +28,13 @@ export interface ProxyCooldownEntry {
 export interface PoolRotation {
   readonly key: string;
   readonly rotateCount: number;
+  /**
+   * The strategy that produced this rotation request. `round_robin` serves
+   * `rotateCount` requests per pool before advancing; `random` scatters each
+   * selection across the eligible pools in a fresh random order. Absent means
+   * the default weighted least-loaded scan.
+   */
+  readonly strategy?: "round_robin" | "random";
 }
 
 export interface PoolUsageSnapshot {

package/src/network/pool/selector.ts

@@ -10,6 +10,7 @@
 import { redisEvalNumber, type RedisClient } from "../../persistence/redis";
 import { resolveInflightTtlSeconds } from "../../config/timeouts";
 import { log } from "../../observability/logger";
+import { providerRateLimitIsIpScoped } from "../../providers/provider-metadata";
 import {
   COOLDOWN_CLEAR_SCRIPT,
   COOLDOWN_FLAG_SCRIPT,
@@ -67,6 +68,21 @@ export function effectiveConcurrency(limit: number): number {
   return limit;
 }
 
+/**
+ * In-place Fisher–Yates shuffle. The `random` pool strategy uses it to scatter
+ * each selection across the eligible pools instead of always scanning from a
+ * fixed (or cursor-driven) order, so a burst does not clump onto the first
+ * pool and burn it while siblings sit idle.
+ */
+function shuffleArray<T>(items: T[]): void {
+  for (let i = items.length - 1; i > 0; i -= 1) {
+    const j = Math.floor(Math.random() * (i + 1));
+    const swap = items[i]!;
+    items[i] = items[j]!;
+    items[j] = swap;
+  }
+}
+
 function normalizedWeight(weight: number | undefined): number {
   if (weight === undefined) return DEFAULT_PROXY_WEIGHT;
   if (!Number.isInteger(weight) || weight < 1 || weight > MAX_PROXY_WEIGHT) return 0;
@@ -257,6 +273,20 @@ export class NetworkPoolSelector {
     return `${poolId}::${providerId.toLowerCase()}`;
   }
 
+  /**
+   * Seeds one cooldown entry into the local map without touching Redis.
+   *
+   * Boot restore is the only writer: it replays the durable markers the
+   * previous process recorded (`network_pools.provider_cooldowns`, see
+   * `pool-cooldown-store.ts`) so dispatch starts remembering the egresses the
+   * upstream is still refusing. Anything whose window already passed is
+   * dropped by `restore…` before this is reached.
+   */
+  hydrateProviderCooldown(entry: ProxyCooldownEntry): void {
+    enforceCooldownLimit(this.state.cooldowns);
+    this.state.cooldowns.set(this.cooldownKey(entry.poolId, entry.providerId), entry);
+  }
+
   async flagProviderCooldown(
     poolId: string,
     providerId: string,
@@ -308,12 +338,32 @@ export class NetworkPoolSelector {
    * The entry uses the same storage as a cooldown so every reader that already
    * skips a cooling pool skips this one too; `reason` is what the console
    * renders to tell an operator-blocked proxy from a throttled one.
+   *
+   * Refuses to retire an egress for a provider whose limits belong to a
+   * credential. An address-scoped verdict only exists for providers that count
+   * requests by egress (`ipScopedRateLimit` — opencode); for everyone else a
+   * refusal belongs to the account that was dialed, so retiring the pool would
+   * sideline an address that is still serving every sibling account. Callers
+   * keep their own guard, and this one is the backstop that makes the rule
+   * impossible to route around from a new call site.
    */
   async flagProviderBlocked(
     poolId: string,
     providerId: string,
     reason = "Upstream refused this egress address",
   ): Promise<ProxyCooldownEntry> {
+    if (!providerRateLimitIsIpScoped(providerId)) {
+      log.warn("[pool-selector] refused to block an egress for a non-IP-scoped provider", {
+        networkPoolId: poolId,
+        providerId,
+      });
+      return {
+        poolId,
+        providerId: providerId.toLowerCase(),
+        until: Date.now(),
+        reason,
+      };
+    }
     const entry = await this.flagProviderCooldown(poolId, providerId, PROVIDER_BLOCK_MS, reason);
     const blocked: ProxyCooldownEntry = { ...entry, blocked: true };
     // Rewrite both copies with the flag: the cooldown write above is shared
@@ -455,6 +505,38 @@ export class NetworkPoolSelector {
     await this.clearRedisCooldown(poolId, providerId);
   }
 
+  /**
+   * Releases a *retired* egress only, leaving a cooldown intact.
+   *
+   * The two are one key, so `clearProviderCooldown` cannot be reused: it would
+   * also drop a live 429 cooldown and hand a spent address back to dispatch.
+   * This is the correction a 429 probe is entitled to make — the address is
+   * answering its own rate limiter, so it was never retired — and nothing more.
+   *
+   * No-op when the marker is a cooldown, and no-op when there is no marker.
+   */
+  async clearProviderBlockedOnly(poolId: string, providerId: string): Promise<void> {
+    const key = this.cooldownKey(poolId, providerId);
+    const local = this.state.cooldowns.get(key);
+    if (local !== undefined) {
+      if (local.blocked !== true) return;
+      this.state.cooldowns.delete(key);
+    }
+    if (!this.redis) return;
+    try {
+      const raw = await this.redis.get(proxyCooldownKey(poolId, providerId));
+      const entry = raw ? parseCooldownEntry(raw) : undefined;
+      // A missing or unparseable marker says nothing about the address, and
+      // deleting a key we cannot read would drop a cooldown this is forbidden
+      // from touching. Only a marker that names itself a block is released.
+      if (entry?.blocked !== true) return;
+      await this.clearRedisCooldown(poolId, providerId);
+    } catch {
+      // The local copy is already corrected; the Redis mirror stays until its
+      // TTL or the next write.
+    }
+  }
+
   /**
    * Removes one cooldown marker and its index entry in one atomic step.
    *
@@ -564,6 +646,26 @@ export class NetworkPoolSelector {
     if (scored.length === 0) return undefined;
 
     if (!this.redis) {
+      if (rotation?.strategy === "random") {
+        // Random scatter: shuffle the eligible pools and take the first one
+        // with a free slot, falling through like the ordered scans do.
+        shuffleArray(scored);
+        for (const candidate of scored) {
+          if (this.getInflight(candidate.poolId) >= candidate.capacity) continue;
+          const slot =
+            providerCapacity === undefined
+              ? this.acquire(candidate.poolId, candidate.capacity)
+              : this.acquireProvider(
+                  candidate.poolId,
+                  providerId,
+                  candidate.capacity,
+                  providerCapacity,
+                );
+          if (!slot.acquired) continue;
+          return { poolId: candidate.poolId, release: slot.release };
+        }
+        return undefined;
+      }
       if (rotation) {
         // Round robin: scan from the tenant cursor; a pool that is full falls
         // through to the next offset (failover). Only a successful admission
@@ -662,7 +764,13 @@ export class NetworkPoolSelector {
     // Original (pre-sort) index = absolute rotation position and
     // deterministic tie-breaking.
     const absoluteIndex = new Map(withCounts.map((candidate, index) => [candidate.poolId, index]));
-    if (rotation) {
+    if (rotation?.strategy === "random") {
+      // Random scatter: shuffle before the admit loop, which still enforces
+      // capacity and fails over to the next pool exactly like the ordered
+      // scans — but the order each request tries is fresh, so a burst does
+      // not gang up on the first pools in a fixed order.
+      shuffleArray(withCounts);
+    } else if (rotation) {
       // Round robin orders strictly by the tenant cursor; the CAS admit below
       // still enforces capacity, so a full pool falls through to the next
       // offset exactly like account failover.
@@ -715,7 +823,7 @@ export class NetworkPoolSelector {
       // to the next pool, which is the whole point of the ceiling: the request
       // moves to an egress that still has room for this provider.
       if (result !== 1) continue;
-      if (rotation) {
+      if (rotation && rotation.strategy !== "random") {
         advanceRotation(
           this.state.rotationCursors,
           rotation.key,

package/src/network/pool-cooldown-store.ts

@@ -0,0 +1,252 @@
+/**
+ * The durable mirror of the pool selector's provider cooldown state.
+ *
+ * The selector's own map is process-local, mirrored to Redis only when Redis
+ * exists — so the usual installation ran with the flags surviving just a
+ * restart, while the upstream's refusal of the egress did not. The result was
+ * the report that motivated this: no cooldown shown on the proxy page, and
+ * still-429 requests, because dispatch re-picked a spent egress it no longer
+ * remembered. `network_pools.provider_cooldowns` is that state's home between
+ * boots: dispatch flaggers write each marker, boot reads them all back.
+ *
+ * One entry per cooling provider, keyed by the lowercased provider id — the
+ * same normalization the selector applies to its in-memory key.
+ */
+import { randomUUID } from "node:crypto";
+import { eq } from "drizzle-orm";
+import type { CartethyiaDatabase } from "../persistence/db-handle";
+import { networkPools, healthEvents } from "../persistence/schema";
+import { log } from "../observability/logger";
+import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
+import { PROVIDER_BLOCK_MS, type ProxyCooldownEntry } from "./pool/selector";
+
+type PoolProviderCooldownMap = NonNullable<
+  (typeof networkPools.$inferSelect)["providerCooldowns"]
+>;
+
+function readMarkerMap(
+  db: CartethyiaDatabase,
+  poolId: string,
+): Promise<PoolProviderCooldownMap> {
+  return db
+    .select({ providerCooldowns: networkPools.providerCooldowns })
+    .from(networkPools)
+    .where(eq(networkPools.id, poolId))
+    .limit(1)
+    .then((rows) => rows[0]?.providerCooldowns ?? {});
+}
+
+function writeMarkerMap(
+  db: CartethyiaDatabase,
+  poolId: string,
+  map: PoolProviderCooldownMap,
+): Promise<unknown> {
+  return db.update(networkPools).set({ providerCooldowns: map }).where(eq(networkPools.id, poolId));
+}
+
+function entryOf(marker: Omit<ProxyCooldownEntry, "poolId" | "providerId">): {
+  until: number;
+  reason: string;
+  blocked?: boolean;
+} {
+  return marker.blocked === true
+    ? { until: marker.until, reason: marker.reason, blocked: true }
+    : { until: marker.until, reason: marker.reason };
+}
+
+/**
+ * Writes up (or replaces) one provider's marker on the pool row.
+ *
+ * Fill-in, not replace: an entry for another cooling provider must survive.
+ * Failures are logged and swallowed — the in-memory/Redis state stays
+ * conservative for the running process, and only the *next* restart would miss
+ * a flag this write failed to land.
+ */
+export async function upsertProviderCooldownMarker(
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  marker: { until: number; reason: string; blocked?: boolean },
+): Promise<void> {
+  try {
+    const key = providerId.toLowerCase();
+    const map = await readMarkerMap(db, poolId);
+    map[key] = entryOf(marker);
+    await writeMarkerMap(db, poolId, map);
+  } catch (error) {
+    log.warn("pool provider cooldown mirror write failed", { networkPoolId: poolId, providerId, error: String(error) });
+  }
+}
+
+/**
+ * Removes one provider's marker, optionally only when it is a *block*.
+ *
+ * `onlyBlocked` mirrors `clearProviderBlockedOnly`: a probe that answered 429
+ * may correct a retired claim but must not release a live cooldown, and the
+ * same rule applies to the durable copy.
+ */
+export async function dropProviderCooldownMarker(
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  onlyBlocked = false,
+): Promise<void> {
+  try {
+    const key = providerId.toLowerCase();
+    const map = await readMarkerMap(db, poolId);
+    const current = map[key];
+    if (current === undefined) return;
+    if (onlyBlocked && current.blocked !== true) return;
+    delete map[key];
+    await writeMarkerMap(db, poolId, map);
+  } catch (error) {
+    log.warn("pool provider cooldown mirror clear failed", { networkPoolId: poolId, providerId, error: String(error) });
+  }
+}
+
+/**
+ * Removes every provider's marker from one pool row (disable/replace paths).
+ */
+export async function dropAllProviderCooldownMarkers(
+  db: CartethyiaDatabase,
+  poolId: string,
+): Promise<void> {
+  try {
+    const map = await readMarkerMap(db, poolId);
+    if (Object.keys(map).length === 0) return;
+    await writeMarkerMap(db, poolId, {});
+  } catch (error) {
+    log.warn("pool provider cooldown mirror purge failed", { networkPoolId: poolId, error: String(error) });
+  }
+}
+
+/**
+ * Writes a probe-measured verdict the same way a dispatch-side event would be
+ * recorded: one mirror marker and one audit row.
+ *
+ * The probe is the only writer that can flag a cooldown from the control
+ * plane — its 429 is a fresh measurement of "this egress is out of allowance"
+ * (a cooldown marker), and its 403 is a fresh measurement of "this egress was
+ * retired" (a block marker). Without this, an operator who tested a
+ * rate-limited or retired pool saw the verdict once, refreshing lost it, and
+ * the activity panel showed nothing: the fact existed only in the remembered
+ * HTTP response.
+ *
+ * Not coalesced: a probe is operator-initiated and one row per click is the
+ * visible history the panel exists for.
+ */
+export async function recordProbeVerdict(
+  db: CartethyiaDatabase,
+  poolId: string,
+  providerId: string,
+  marker: { until: number; reason: string; blocked?: boolean },
+): Promise<void> {
+  await upsertProviderCooldownMarker(db, poolId, providerId, marker);
+  try {
+    await db.insert(healthEvents).values({
+      id: randomUUID(),
+      entityKind: "pool",
+      networkPoolId: poolId,
+      fromStatus: null,
+      toStatus: "cooldown",
+      reason: `provider ${providerId}: ${marker.reason}`,
+      errorCategory: marker.blocked === true ? "egress_blocked" : "rate_limit_transient",
+      createdAt: new Date(),
+    });
+  } catch (error) {
+    log.warn("pool probe event failed", { networkPoolId: poolId, providerId, error: String(error) });
+  }
+}
+
+/**
+ * Reads every live marker back into the selector at boot.
+ *
+ * Only future-dated entries are restored: an expired marker describes an
+ * allowance that refilled while the process was down, and resurrecting it
+ * would sideline a working address. Repeat 429s re-insert markers, so nothing
+ * is lost by dropping the expired ones.
+ */
+export async function restoreProviderCooldownMarkers(
+  selector: { hydrateProviderCooldown(entry: ProxyCooldownEntry): void },
+  db: CartethyiaDatabase,
+): Promise<number> {
+  const rows = await db
+    .select({ id: networkPools.id, providerCooldowns: networkPools.providerCooldowns })
+    .from(networkPools);
+  let restored = 0;
+  for (const row of rows) {
+    const map = row.providerCooldowns;
+    if (map === null || map === undefined) continue;
+    for (const [providerId, marker] of Object.entries(map)) {
+      if (Date.now() >= marker.until) continue;
+      selector.hydrateProviderCooldown({
+        poolId: row.id,
+        providerId,
+        until: marker.until,
+        reason: marker.reason,
+        ...(marker.blocked === true ? { blocked: true } : {}),
+      });
+      restored += 1;
+    }
+  }
+  return restored;
+}
+
+/** The columns the probe-verdict reconciliation reads, mirroring the row's own read path. */
+function probeEndpointOf(endpointConfig: unknown): string {
+  const raw = (endpointConfig ?? {}) as Record<string, unknown>;
+  return typeof raw["endpoint"] === "string" ? raw["endpoint"] : "";
+}
+
+/**
+ * Re-enforces the stored probe verdicts at boot (new: replay, read-ahead).
+ *
+ * A stored `blocked` verdict is a *state*, not a decoration. If the marker the
+ * probe flagged failed to persist, or the verdict predated the marker column,
+ * the egress would return to rotation after restart and get refused again —
+ * dispatch spending a real request to rediscover a refusal the row already
+ * knew. So the boot read reconciles: a fresh (endpoint still matches) `blocked`
+ * verdict for an address-keyed provider becomes a block flag, even when the
+ * mirror has nothing (or has an older, cooler entry — the retirement claim is
+ * the sharper fact and wins).
+ *
+ * `throttled` is deliberately **not** replayed into a cooldown: the cooldown
+ * the probe flagged already carries the measured window, and blind-replaying
+ * one from a verdict that predates the restart could outlive the upstream's
+ * actual window.
+ *
+ * Returns how many verdicts were replayed.
+ */
+export async function restorePoolProbeVerdicts(
+  selector: { hydrateProviderCooldown(entry: ProxyCooldownEntry): void },
+  db: CartethyiaDatabase,
+): Promise<number> {
+  const rows = await db
+    .select({
+      id: networkPools.id,
+      status: networkPools.status,
+      endpointConfig: networkPools.endpointConfig,
+      providerProbeProviderId: networkPools.providerProbeProviderId,
+      providerProbeStatus: networkPools.providerProbeStatus,
+      providerProbeReason: networkPools.providerProbeReason,
+      providerProbeEndpoint: networkPools.providerProbeEndpoint,
+    })
+    .from(networkPools);
+  let replayed = 0;
+  for (const row of rows) {
+    if (row.status !== "active") continue;
+    if (row.providerProbeStatus !== "blocked") continue;
+    if (row.providerProbeEndpoint !== probeEndpointOf(row.endpointConfig)) continue;
+    const providerId = row.providerProbeProviderId;
+    if (typeof providerId !== "string" || !providerRateLimitIsIpScoped(providerId)) continue;
+    selector.hydrateProviderCooldown({
+      poolId: row.id,
+      providerId: providerId.toLowerCase(),
+      until: Date.now() + PROVIDER_BLOCK_MS,
+      reason: row.providerProbeReason ?? "Upstream refused this egress address",
+      blocked: true,
+    });
+    replayed += 1;
+  }
+  return replayed;
+}

package/src/network/pool-health-machine.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import { healthEventBuffer } from "../persistence/health-event-flush";
 import { and, desc, eq, inArray, isNotNull, lte, sql } from "drizzle-orm";
 import type { CartethyiaDatabase } from "../persistence/db-handle";
@@ -140,7 +141,7 @@ export async function recordPoolDispatchOutcome(
       createdAt: now,
     };
     if (buffer !== undefined) buffer.enqueue(row);
-    else await db.insert(healthEvents).values(row as never);
+    else await db.insert(healthEvents).values({ id: randomUUID(), ...row } as never);
   }
   if (healthChanged) {
     publishPoolHealth({
@@ -198,6 +199,7 @@ export async function disablePoolForProxyHttpStatus(
   if (!updated[0]) return false;
 
   await db.insert(healthEvents).values({
+    id: randomUUID(),
     entityKind: "pool",
     networkPoolId: poolId,
     fromStatus: pool.status,
@@ -226,7 +228,7 @@ export async function recoverNetworkPool(db: CartethyiaDatabase, tenantId: strin
   if (!pool) return false;
   const now = new Date();
   await db.update(networkPools).set({ status: "active", consecutiveFailures: 0, cooldownUntil: null, lastRecoveredAt: now }).where(and(eq(networkPools.tenantId, tenantId), eq(networkPools.id, poolId)));
-  await db.insert(healthEvents).values({ entityKind: "pool", networkPoolId: poolId, fromStatus: pool.status, toStatus: "active", reason: "Manual operator recovery", errorCategory: null, createdAt: now });
+  await db.insert(healthEvents).values({ id: randomUUID(), entityKind: "pool", networkPoolId: poolId, fromStatus: pool.status, toStatus: "active", reason: "Manual operator recovery", errorCategory: null, createdAt: now });
   publishPoolHealth({ poolId, tenantId: pool.tenantId, status: "active", consecutiveFailures: 0 });
   return true;
 }
@@ -248,7 +250,7 @@ export async function sweepExpiredPoolCooldowns(db: CartethyiaDatabase): Promise
   const expired = await db.select({ id: networkPools.id, tenantId: networkPools.tenantId, status: networkPools.status }).from(networkPools).where(and(inArray(networkPools.status, ["cooldown"]), isNotNull(networkPools.cooldownUntil), lte(networkPools.cooldownUntil, now))).limit(POOL_COOLDOWN_RECOVERY_BATCH);
   if (expired.length === 0) return 0;
   await db.update(networkPools).set({ status: "active", consecutiveFailures: 0, cooldownUntil: null, lastRecoveredAt: now }).where(inArray(networkPools.id, expired.map((row) => row.id)));
-  await db.insert(healthEvents).values(expired.map((row) => ({ entityKind: "pool" as const, networkPoolId: row.id, fromStatus: row.status, toStatus: "active" as const, reason: "Cooldown elapsed — auto-recovered", errorCategory: null, createdAt: now })));
+  await db.insert(healthEvents).values(expired.map((row) => ({ id: randomUUID(), entityKind: "pool" as const, networkPoolId: row.id, fromStatus: row.status, toStatus: "active" as const, reason: "Cooldown elapsed — auto-recovered", errorCategory: null, createdAt: now })));
   for (const row of expired) publishPoolHealth({ poolId: row.id, tenantId: row.tenantId, status: "active", consecutiveFailures: 0 });
   return expired.length;
 }

package/src/network/pool-health.ts

@@ -1,21 +1,26 @@
+import { randomUUID } from "node:crypto";
 import type { CartethyiaDatabase } from "../persistence/db-handle";
 import { healthEvents } from "../persistence/schema";
 import type { NetworkPoolSelector } from "./pool/selector";
 import type { GatewayError } from "../protocol/gateway-error";
 import { metrics } from "../observability/metrics";
 import { log } from "../observability/logger";
+import { upsertProviderCooldownMarker } from "./pool-cooldown-store";
+import { resolvePoolProviderRateLimitCooldownMs } from "../config/cooldowns";
 
 /**
  * Durable pool rate-limit history.
  *
  * Per-(pool, provider) 429 cooldowns live in `NetworkPoolSelector` (fast,
- * volatile, consulted on every dispatch). This recorder mirrors each flag
- * into the append-only `health_events` table (`entity_kind: "pool"`) — the
- * same merged log the account recorder writes to for accounts — so
- * operators can audit which pool throttled which provider and when. It
- * never touches `network_pools.status`: a single provider's 429 must not
- * flip the whole pool to `cooldown` (that would head-of-line-block every
- * other provider on the pool).
+ * volatile, consulted on every dispatch). This recorder mirrors each flag into
+ * two durable homes: the append-only `health_events` table
+ * (`entity_kind: "pool"`, the same merged log the account recorder writes to
+ * for accounts) for audit, and the `provider_cooldowns` column on the pool row
+ * as the state boot restores — a restart must not make dispatch forget the
+ * egresses the upstream is still refusing. It never touches
+ * `network_pools.status`: a single provider's 429 must not flip the whole pool
+ * to `cooldown` (that would head-of-line-block every other provider on the
+ * pool).
  */
 
 /**
@@ -41,12 +46,14 @@ async function recordPoolCooldownEvent(
   if (last !== undefined && now - last < POOL_COOLDOWN_EVENT_WINDOW_MS) return;
   try {
     await db.insert(healthEvents).values({
+      id: randomUUID(),
       entityKind: "pool",
       networkPoolId: poolId,
       fromStatus: null,
       toStatus: "cooldown",
       reason: `provider ${providerId}: ${reason}`,
       errorCategory: "rate_limit_transient",
+      createdAt: new Date(),
     });
   } catch {
     // Non-fatal: observability writes must never fail dispatch. The
@@ -85,6 +92,12 @@ export function flagPoolBlocked(
   const reason = `Provider ${providerId} refused this egress`;
   void poolSelector
     .flagProviderBlocked(poolId, providerId, reason)
+    .then((entry) =>
+      // Persisted even when the flag was refused (a credential-keyed
+      // provider): the selector returned an inert entry, so the mirror
+      // records nothing a reader would act on.
+      upsertProviderCooldownMarker(db, poolId, providerId, entry),
+    )
     .catch((cause: unknown) => {
       metrics.pool_cooldown_record_failed.inc(1);
       log.warn("pool block flag failed", {
@@ -119,12 +132,14 @@ async function recordPoolBlockEvent(
   error: GatewayError,
 ): Promise<void> {
   await db.insert(healthEvents).values({
+    id: randomUUID(),
     entityKind: "pool",
     networkPoolId: poolId,
     fromStatus: null,
     toStatus: "cooldown",
     reason: `provider ${providerId}: ${reason}${providerCodeSuffix(error)}`,
     errorCategory: "egress_blocked",
+    createdAt: new Date(),
   });
 }
 
@@ -137,9 +152,20 @@ function providerCodeSuffix(error: GatewayError): string {
 /** Cooldown applied when an upstream 429 carries no `retryAfterMs`. */
 const DEFAULT_PROVIDER_COOLDOWN_MS = 15 * 60 * 1000;
 
+/**
+ * The fallback window for a provider-scoped egress cooldown. Upstream `Retry-After`
+ * always wins; the configured default exists because a request-counted free
+ * tier — the one provider that flags at all — stays refused for hours after
+ * the 15-minute guess this function used before.
+ */
+function providerCooldownFallbackMs(): number {
+  return Math.max(resolvePoolProviderRateLimitCooldownMs(), DEFAULT_PROVIDER_COOLDOWN_MS);
+}
+
 /**
  * Records a provider-scoped 429 cooldown: the volatile per-(pool, provider)
- * flag consulted on every dispatch, plus its durable audit row.
+ * flag consulted on every dispatch, its durable audit row, and the durable
+ * state boot restores from.
  *
  * Both paths that reach this decision — the non-streaming attempt loop and
  * the streaming failure path — call this one function, because each write
@@ -161,9 +187,10 @@ export function flagPoolCooldown(
   const retryAfterMs =
     typeof error.details.retryAfterMs === "number"
       ? error.details.retryAfterMs
-      : DEFAULT_PROVIDER_COOLDOWN_MS;
+      : providerCooldownFallbackMs();
   void poolSelector
     .flagProviderCooldown(poolId, providerId, retryAfterMs, reason)
+    .then((entry) => upsertProviderCooldownMarker(db, poolId, providerId, entry))
     .catch((cause: unknown) => {
       metrics.pool_cooldown_record_failed.inc(1);
       log.warn("pool cooldown flag failed", {

package/src/observability/event-query.ts

@@ -537,6 +537,63 @@ export async function chartTelemetry(
   }));
 }
 
+/** One day × model usage cell for a single provider — the Activity Matrix feed. */
+export interface ProviderUsageDayRow {
+  /** UTC day, `YYYY-MM-DD` (epoch-day buckets, so the same key on both backends). */
+  readonly day: string;
+  /** What the client asked for; `null` when the event carried no model. */
+  readonly model: string | null;
+  readonly requests: number;
+  readonly inputTokens: number;
+  readonly cachedInputTokens: number;
+  readonly outputTokens: number;
+  readonly costUsd: number;
+}
+
+/**
+ * Per-provider daily usage grouped by `requested_model`, day-bucketed like
+ * {@link chartTelemetry} but with a second grouping dimension (requested
+ * model) and the totals the tooltip/summary strip need. Provider rows carry
+ * no catalog foreign keys, so an unknown or deleted provider id returns empty
+ * rather than erroring.
+ */
+export async function providerUsageMatrix(
+  db: CartethyiaDatabase,
+  tenantId: string,
+  providerId: string,
+  since: Date,
+): Promise<ProviderUsageDayRow[]> {
+  // Same inlining rule as chartTelemetry: SELECT and GROUP BY must spell the
+  // identical expression or Postgres 42803-rejects the grouped reference.
+  const w = sql.raw(String(86400));
+  const dayExpression = isPostgres(db)
+    ? sql<number>`(cast(extract(epoch from ${telemetryEvents.createdAt}) as bigint) / ${w}) * ${w}`
+    : sql<number>`(cast(strftime('%s', ${telemetryEvents.createdAt}) as integer) / ${w}) * ${w}`;
+  const rows = await db
+    .select({
+      day: dayExpression,
+      model: telemetryEvents.requestedModel,
+      requests: sql<number>`count(*)`,
+      inputTokens: sql<number>`coalesce(sum(${telemetryEvents.inputTokens}), 0)`,
+      cachedInputTokens: sql<number>`coalesce(sum(${telemetryEvents.cachedInputTokens}), 0)`,
+      outputTokens: sql<number>`coalesce(sum(${telemetryEvents.outputTokens}), 0)`,
+      costUsd: sql<number>`coalesce(sum(${telemetryEvents.estimatedCostUsd}), 0)`,
+    })
+    .from(telemetryEvents)
+    .where(and(scoped(tenantId, since), eq(telemetryEvents.providerId, providerId)))
+    .groupBy(dayExpression, telemetryEvents.requestedModel)
+    .orderBy(dayExpression);
+  return rows.map((row) => ({
+    day: new Date(numeric(row.day) * 1000).toISOString().slice(0, 10),
+    model: row.model,
+    requests: numeric(row.requests),
+    inputTokens: numeric(row.inputTokens),
+    cachedInputTokens: numeric(row.cachedInputTokens),
+    outputTokens: numeric(row.outputTokens),
+    costUsd: numeric(row.costUsd),
+  }));
+}
+
 /** Overview/health totals for one tenant window. */
 export async function healthTelemetry(
   db: CartethyiaDatabase,