cartethyia 2.7.2 → 2.7.4

package/src/observability/event-query.ts

@@ -17,9 +17,9 @@
  *   latency arrays are never returned to JavaScript;
  * - `isGatewayError()` semantics live in one shared SQL predicate.
  */
-import { and, desc, eq, gte, inArray, isNotNull, sql } from "drizzle-orm";
+import { and, desc, eq, gte, inArray, isNotNull, or, sql, type SQL } from "drizzle-orm";
 import type { CartethyiaDatabase } from "../persistence/db-handle";
-import { telemetryEvents } from "../persistence/schema";
+import { apiKeys, telemetryEvents, telemetryUsageTotals } from "../persistence/schema";
 import type { TelemetryEventRecord } from "./event-log";
 import { gatewayErrorSql } from "./telemetry-status";
 
@@ -31,7 +31,7 @@ import { gatewayErrorSql } from "./telemetry-status";
  * totals, and the durable usage rollup cannot drift apart — an unknown status
  * must be classified identically in all three.
  */
-const IS_GATEWAY_ERROR_SQL = gatewayErrorSql(telemetryEvents.status, telemetryEvents.httpStatus);
+export const IS_GATEWAY_ERROR_SQL = gatewayErrorSql(telemetryEvents.status, telemetryEvents.httpStatus);
 
 /** One bounded page of events, newest first. */
 export interface EventPage {
@@ -52,6 +52,17 @@ export interface EventListQuery {
   readonly until?: Date;
   readonly apiKeyIds?: readonly string[];
   readonly accountIds?: readonly string[];
+  /**
+   * The share-template key whose family (the template itself plus every child
+   * key it issued) scopes the read. Rendered as an `EXISTS` subquery against
+   * `api_keys` rather than an id list: a family is minted by the tenant and
+   * can grow to whatever size the tenant drives it to, so enumerating its ids
+   * in JavaScript to feed a bind list would scale the query's wire size with
+   * the family instead of with the result. The subquery keeps the same
+   * membership semantics — template id or any child of it — for any family
+   * size.
+   */
+  readonly familyApiKeyId?: string;
   readonly requestId?: string;
   /** Several request ids at once — the backup restore's "already present" check. */
   readonly requestIds?: readonly string[];
@@ -231,6 +242,94 @@ function scoped(tenantId: string, since?: Date) {
   return since === undefined ? tenant : and(tenant, gte(telemetryEvents.createdAt, since));
 }
 
+/**
+ * Thrown when a membership filter outruns the guard below. This is a handled
+ * query error, not a protocol fault: callers surface it as a request failure
+ * instead of crashing the process with a PostgreSQL wire error that systemd
+ * would only restart into.
+ */
+export class TelemetryQueryInputError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "TelemetryQueryInputError";
+  }
+}
+
+/**
+ * The deduplicated id count a list membership filter accepts before refusing.
+ *
+ * `idMembershipFilter` binds the whole set as one array parameter, so even a
+ * six-figure set is wire-safe; this guard is not a protocol necessity but a
+ * runaway-collector tripwire — a bug elsewhere (an unbounded accumulator, a
+ * vanished scope) must fail as a query error instead of materializing an
+ * unbounded parameter. Families larger than this must use the `familyApiKeyId`
+ * filter, which never materializes ids.
+ */
+const MAX_MEMBERSHIP_IDS = 200_000;
+
+/**
+ * The chunk size for the SQLite rendering of a list membership filter.
+ *
+ * SQLite has no array parameters, so the set has to come back as individual
+ * placeholders; 500 per chunk sits far below every build's
+ * SQLITE_MAX_VARIABLE_NUMBER while keeping the OR of chunks — semantically
+ * the identical membership test — to a handful of clauses for realistic sets.
+ */
+const SQLITE_IDS_PER_CHUNK = 500;
+
+/**
+ * Membership of `column` in a caller-supplied id set, safe at any set size.
+ *
+ * Expanding the members into one bind placeholder each breaks the wire twice
+ * over: PostgreSQL caps a Bind message at 65,535 parameters, and node-postgres
+ * miscounts long placeholder lists anyway (`08P01 bind message has N parameter
+ * formats but 0 parameters` — the shape that crash-looped the dev deployment,
+ * whose share family carried ~90k minted child keys). Instead the set is
+ * deduplicated and handed to the server as a single array parameter
+ * (`= ANY(…::uuid[])`), or, on SQLite, to chunks of individual placeholders;
+ * membership semantics are unchanged in both renderings.
+ */
+function idMembershipFilter(
+  db: CartethyiaDatabase,
+  column: typeof telemetryEvents.apiKeyId | typeof telemetryEvents.accountId | typeof telemetryEvents.requestId,
+  values: readonly string[],
+  cast: "uuid[]" | "text[]",
+): SQL {
+  const unique = [...new Set(values)];
+  if (unique.length > MAX_MEMBERSHIP_IDS) {
+    throw new TelemetryQueryInputError(
+      `telemetry filter exceeds ${MAX_MEMBERSHIP_IDS} distinct ids (got ${unique.length}); scope the read by family or tenant instead`,
+    );
+  }
+  if (unique.length === 0) return sql`false`;
+  if (isPostgres(db)) {
+    return sql`${column} = ANY(${unique}::${sql.raw(cast)})`;
+  }
+  if (unique.length <= SQLITE_IDS_PER_CHUNK) return inArray(column, unique);
+  const chunks: SQL[] = [];
+  for (let start = 0; start < unique.length; start += SQLITE_IDS_PER_CHUNK) {
+    chunks.push(inArray(column, unique.slice(start, start + SQLITE_IDS_PER_CHUNK)));
+  }
+  return or(...chunks) as SQL;
+}
+
+/**
+ * Membership of `api_key_id` in the family of one share template — the
+ * template itself plus every key it issued.
+ *
+ * The primary predicate is the denormalized `family_template_key_id` column:
+ * an event is in the family when it was stamped with the template at write
+ * time. That makes the read independent of the key row surviving — a revoked
+ * or removed key keeps its history. Events written before the column existed
+ * (or whose key was hard-deleted before the backfill) fall back to an
+ * `EXISTS` against `api_keys`, so the migration window is still counted when
+ * the key row is present. The tenant predicate is carried by every read
+ * already, so a child in another tenant still cannot match.
+ */
+function familyKeyCondition(templateKeyId: string): SQL {
+  return sql`(${telemetryEvents.familyTemplateKeyId} = ${templateKeyId} or (${telemetryEvents.familyTemplateKeyId} is null and exists (select 1 from ${apiKeys} where ${apiKeys.id} = ${telemetryEvents.apiKeyId} and (${apiKeys.id} = ${templateKeyId} or ${apiKeys.parentKeyId} = ${templateKeyId}))))`;
+}
+
 /**
  * Bounded detail/list read — the only place rows are materialized.
  *
@@ -249,16 +348,19 @@ export async function queryTelemetryEvents(
     conditions.push(sql`${telemetryEvents.createdAt} < ${query.until}`);
   }
   if (query.apiKeyIds !== undefined) {
-    conditions.push(inArray(telemetryEvents.apiKeyId, [...query.apiKeyIds]));
+    conditions.push(idMembershipFilter(db, telemetryEvents.apiKeyId, query.apiKeyIds, "uuid[]"));
+  }
+  if (query.familyApiKeyId !== undefined) {
+    conditions.push(familyKeyCondition(query.familyApiKeyId));
   }
   if (query.accountIds !== undefined) {
-    conditions.push(inArray(telemetryEvents.accountId, [...query.accountIds]));
+    conditions.push(idMembershipFilter(db, telemetryEvents.accountId, query.accountIds, "uuid[]"));
   }
   if (query.requestId !== undefined) {
     conditions.push(eq(telemetryEvents.requestId, query.requestId));
   }
   if (query.requestIds !== undefined) {
-    conditions.push(inArray(telemetryEvents.requestId, [...query.requestIds]));
+    conditions.push(idMembershipFilter(db, telemetryEvents.requestId, query.requestIds, "text[]"));
   }
   if (query.id !== undefined) {
     conditions.push(eq(telemetryEvents.id, query.id));
@@ -414,23 +516,39 @@ async function latencyPercentiles(
   }
 
   // SQLite: nearest-rank percentile = the value at ceil(p * n) rows into the
-  // ordered latency column, via ORDER BY + OFFSET/LIMIT in SQL.
-  const rankAt = async (fraction: number): Promise<number | null> => {
-    const offset = Math.max(0, Math.ceil(fraction * sampled) - 1);
-    const [row] = await db
-      .select({ value: telemetryEvents.latencyMs })
-      .from(telemetryEvents)
-      .where(and(...base))
-      .orderBy(telemetryEvents.latencyMs)
-      .limit(1)
-      .offset(offset);
-    return row?.value ?? null;
+  // ordered latency column. One query computes all three ranks via a window
+  // function, instead of three separate OFFSET/LIMIT passes over the same
+  // ordered column (one per percentile).
+  const n = Math.max(1, sampled);
+  const ranks = [
+    Math.ceil(0.5 * n),
+    Math.ceil(0.95 * n),
+    Math.ceil(0.99 * n),
+  ];
+  const [row] = await db
+    .select({
+      p50: sql<number | null>`max(case when rn = ${ranks[0]} then value end)`,
+      p95: sql<number | null>`max(case when rn = ${ranks[1]} then value end)`,
+      p99: sql<number | null>`max(case when rn = ${ranks[2]} then value end)`,
+    })
+    .from(
+      sql`(
+        select ${telemetryEvents.latencyMs} as value,
+               row_number() over (order by ${telemetryEvents.latencyMs}) as rn
+        from ${telemetryEvents}
+        where ${and(...base)}
+      )`,
+    );
+  return {
+    p50: numericNullable(row?.p50 ?? null),
+    p95: numericNullable(row?.p95 ?? null),
+    p99: numericNullable(row?.p99 ?? null),
   };
-  const [p50, p95, p99] = await Promise.all([rankAt(0.5), rankAt(0.95), rankAt(0.99)]);
-  return { p50, p95, p99 };
 }
 
-/** The SQL column one breakdown dimension groups by. */
+/**
+ * The SQL column one breakdown dimension groups by.
+ */
 function dimensionColumn(dimension: string) {
   if (dimension === "model") return telemetryEvents.requestedModel;
   if (dimension === "provider") return telemetryEvents.providerId;
@@ -638,14 +756,23 @@ export async function healthTelemetry(
 export async function sumTelemetryTokens(
   db: CartethyiaDatabase,
   tenantId: string,
-  filters: { readonly apiKeyIds?: readonly string[]; readonly since: Date; readonly until?: Date },
+  filters: {
+    readonly apiKeyIds?: readonly string[];
+    /** The share-template family filter; see {@link EventListQuery.familyApiKeyId}. */
+    readonly familyKeyId?: string;
+    readonly since: Date;
+    readonly until?: Date;
+  },
 ): Promise<number> {
   const conditions = [scoped(tenantId, filters.since)];
   if (filters.until !== undefined) {
     conditions.push(sql`${telemetryEvents.createdAt} < ${filters.until}`);
   }
   if (filters.apiKeyIds !== undefined && filters.apiKeyIds.length > 0) {
-    conditions.push(inArray(telemetryEvents.apiKeyId, [...filters.apiKeyIds]));
+    conditions.push(idMembershipFilter(db, telemetryEvents.apiKeyId, filters.apiKeyIds, "uuid[]"));
+  }
+  if (filters.familyKeyId !== undefined) {
+    conditions.push(familyKeyCondition(filters.familyKeyId));
   }
   const [row] = await db
     .select({
@@ -656,6 +783,196 @@ export async function sumTelemetryTokens(
   return numeric(row?.total ?? 0);
 }
 
+/** One hour, in milliseconds — the share page's hourly bucket width. */
+const HOUR_MS = 3_600_000;
+
+/**
+ * All-time family counts from the durable rollup, not from a scan of the
+ * retained telemetry window.
+ *
+ * The share KPI cards need lifetime numbers. `telemetry_usage_totals` is
+ * charged at write time and accumulates per identity, so a family's lifetime
+ * requests/errors/tokens are a SUM over its rows (keyed by
+ * `family_template_key_id`) — sub-millisecond on a small table, versus a
+ * full-window aggregate over `telemetry_events` that reads every retained
+ * event on every tick. The rollup is the same definition as the read-side
+ * counts, so the two cannot drift.
+ */
+export interface FamilyKpiTotals {
+  readonly requests: number;
+  readonly errors: number;
+  readonly tokens: number;
+}
+
+/** Sums one share family's durable lifetime totals. */
+export async function familyKpiTotals(
+  db: CartethyiaDatabase,
+  tenantId: string,
+  familyKeyId: string,
+): Promise<FamilyKpiTotals> {
+  const [row] = await db
+    .select({
+      requests: sql<number>`coalesce(sum(${telemetryUsageTotals.requests}), 0)`,
+      errors: sql<number>`coalesce(sum(${telemetryUsageTotals.errors}), 0)`,
+      tokens: sql<number>`coalesce(sum(coalesce(${telemetryUsageTotals.inputTokens}, 0) + coalesce(${telemetryUsageTotals.outputTokens}, 0)), 0)`,
+    })
+    .from(telemetryUsageTotals)
+    .where(
+      and(
+        eq(telemetryUsageTotals.tenantId, tenantId),
+        eq(telemetryUsageTotals.familyTemplateKeyId, familyKeyId),
+      ),
+    );
+  return {
+    requests: numeric(row?.requests ?? 0),
+    errors: numeric(row?.errors ?? 0),
+    tokens: numeric(row?.tokens ?? 0),
+  };
+}
+
+/**
+ * All-time family counts: the numbers a share page's KPI card shows.
+ *
+ * These must be computed in SQL, not derived from the bounded page read — the
+ * page caps at {@link MAX_LIMIT}, so a busy family would freeze at 500
+ * requests/errors/last-hour regardless of real traffic.
+ */
+export interface FamilyUsageTotals {
+  readonly requests: number;
+  readonly errors: number;
+  readonly lastHourRequests: number;
+}
+
+/** Counts one share family's requests, errors, and last-hour traffic in SQL. */
+export async function familyUsageTotals(
+  db: CartethyiaDatabase,
+  tenantId: string,
+  familyKeyId: string,
+  now: Date,
+): Promise<FamilyUsageTotals> {
+  const lastHourStart = new Date(now.getTime() - HOUR_MS);
+  const conditions = [eq(telemetryEvents.tenantId, tenantId), familyKeyCondition(familyKeyId)];
+  const [row] = await db
+    .select({
+      requests: sql<number>`count(*)`,
+      errors: sql<number>`sum(case when ${IS_GATEWAY_ERROR_SQL} then 1 else 0 end)`,
+      lastHourRequests: sql<number>`sum(case when ${telemetryEvents.createdAt} >= ${lastHourStart} then 1 else 0 end)`,
+    })
+    .from(telemetryEvents)
+    .where(and(...conditions));
+  return {
+    requests: numeric(row?.requests ?? 0),
+    errors: numeric(row?.errors ?? 0),
+    lastHourRequests: numeric(row?.lastHourRequests ?? 0),
+  };
+}
+
+/** Counts one share family's requests in a bounded window — a range scan, not a full-window aggregate. */
+export async function familyWindowRequests(
+  db: CartethyiaDatabase,
+  tenantId: string,
+  familyKeyId: string,
+  since: Date,
+): Promise<number> {
+  const [row] = await db
+    .select({
+      requests: sql<number>`count(*)`,
+    })
+    .from(telemetryEvents)
+    .where(
+      and(
+        eq(telemetryEvents.tenantId, tenantId),
+        familyKeyCondition(familyKeyId),
+        gte(telemetryEvents.createdAt, since),
+      ),
+    );
+  return numeric(row?.requests ?? 0);
+}
+
+/**
+ * One share family's client-IP breakdown, computed in SQL.
+ *
+ * Unlike the 24-hour page read, this covers a bounded history so an address
+ * that stopped before the last day still appears — marked inactive. The
+ * `active` flag is decided in the caller from `lastSeenAt` vs a cutoff; here
+ * we only aggregate. The window is bounded (`since`, default the whole
+ * retained window) so an unbounded scan is never the default for callers
+ * that only need recent addresses.
+ */
+export async function familyIpBreakdown(
+  db: CartethyiaDatabase,
+  tenantId: string,
+  familyKeyId: string,
+  limit = 50,
+  since?: Date,
+): Promise<{ ip: string; requests: number; tokens: number; lastSeenAt: Date | null }[]> {
+  const conditions = [
+    eq(telemetryEvents.tenantId, tenantId),
+    familyKeyCondition(familyKeyId),
+    ...(since === undefined ? [] : [gte(telemetryEvents.createdAt, since)]),
+  ];
+  const rows = await db
+    .select({
+      ip: telemetryEvents.clientIp,
+      requests: sql<number>`count(*)`,
+      tokens: sql<number>`coalesce(sum(coalesce(${telemetryEvents.inputTokens}, 0) + coalesce(${telemetryEvents.outputTokens}, 0)), 0)`,
+      lastSeenAt: sql<Date>`max(${telemetryEvents.createdAt})`,
+    })
+    .from(telemetryEvents)
+    .where(and(...conditions))
+    .groupBy(telemetryEvents.clientIp)
+    // Highest volume first: the table answers "who has used the link most",
+    // not just who is active right now.
+    .orderBy(desc(sql`count(*)`))
+    .limit(limit);
+  return rows.flatMap((row) =>
+    row.ip === null ? [] : [{ ip: row.ip, requests: numeric(row.requests), tokens: numeric(row.tokens), lastSeenAt: row.lastSeenAt }],
+  );
+}
+
+/**
+ * One share family's time-series buckets, computed in SQL with the family
+ * membership filter.
+ *
+ * Mirrors {@link chartTelemetry} but scoped to a share family. The share page's
+ * 24-hour strip must not be truncated to the bounded page (500 events), or a
+ * busy family's chart undercounts every hour bucket.
+ */
+export async function familyHourlyTelemetry(
+  db: CartethyiaDatabase,
+  tenantId: string,
+  familyKeyId: string,
+  since: Date,
+  bucketSeconds: number,
+): Promise<ChartBucket[]> {
+  const width = Math.max(1, Math.trunc(bucketSeconds));
+  // Inlined (not parameterised) so Postgres sees the SELECT and GROUP BY
+  // expressions as identical — see chartTelemetry for the failure this avoids.
+  const w = sql.raw(String(width));
+  const bucketExpression = isPostgres(db)
+    ? sql<number>`(cast(extract(epoch from ${telemetryEvents.createdAt}) as bigint) / ${w}) * ${w}`
+    : sql<number>`(cast(strftime('%s', ${telemetryEvents.createdAt}) as integer) / ${w}) * ${w}`;
+  const conditions = [scoped(tenantId, since), familyKeyCondition(familyKeyId)];
+  const rows = await db
+    .select({
+      at: bucketExpression,
+      requests: sql<number>`count(*)`,
+      errors: sql<number>`sum(case when ${IS_GATEWAY_ERROR_SQL} then 1 else 0 end)`,
+      inputTokens: sql<number>`coalesce(sum(${telemetryEvents.inputTokens}), 0)`,
+      outputTokens: sql<number>`coalesce(sum(${telemetryEvents.outputTokens}), 0)`,
+    })
+    .from(telemetryEvents)
+    .where(and(...conditions))
+    .groupBy(bucketExpression);
+  return rows.map((row) => ({
+    at: numeric(row.at) * 1000,
+    requests: numeric(row.requests),
+    errors: numeric(row.errors),
+    inputTokens: numeric(row.inputTokens),
+    outputTokens: numeric(row.outputTokens),
+  }));
+}
+
 /**
  * The health card's latency and throughput figures for one window.
  *
@@ -750,3 +1067,101 @@ export async function summarizeUsagePerformanceSql(
     avgTokensPerSec: numeric(row?.avgTokensPerSec ?? 0),
   };
 }
+
+/**
+ * Combined usage totals + performance in ONE window scan.
+ *
+ * `summarizeTelemetry` and `summarizeUsagePerformanceSql` both scan the same
+ * tenant window and each additionally run the percentile path, so the Usage
+ * summary endpoint paid two scans plus two percentile passes for one response.
+ * This runs the combined SELECT once and one percentile pass, returning both
+ * shapes so the caller needs no second query.
+ */
+export async function summarizeUsageWithPerformance(
+  db: CartethyiaDatabase,
+  tenantId: string,
+  since: Date,
+): Promise<{ summary: UsageSummary; performance: UsagePerformanceTotals }> {
+  const condition = scoped(tenantId, since);
+  const [row] = await db
+    .select({
+      requests: sql<number>`count(*)`,
+      errors: sql<number>`sum(case when ${IS_GATEWAY_ERROR_SQL} then 1 else 0 end)`,
+      cancelled: sql<number>`sum(case when ${telemetryEvents.status} = 'cancelled' then 1 else 0 end)`,
+      truncated: sql<number>`sum(case when ${telemetryEvents.status} = 'truncated' then 1 else 0 end)`,
+      inputTokens: sql<number>`coalesce(sum(${telemetryEvents.inputTokens}), 0)`,
+      outputTokens: sql<number>`coalesce(sum(${telemetryEvents.outputTokens}), 0)`,
+      cachedInputTokens: sql<number>`coalesce(sum(${telemetryEvents.cachedInputTokens}), 0)`,
+      reasoningTokens: sql<number>`coalesce(sum(${telemetryEvents.reasoningTokens}), 0)`,
+      estimatedCostUsd: sql<number>`coalesce(sum(${telemetryEvents.estimatedCostUsd}), 0)`,
+      sampled: sql<number>`count(${telemetryEvents.latencyMs})`,
+      latencyAvg: sql<number>`avg(${telemetryEvents.latencyMs})`,
+      unpriced: sql<number>`sum(case when ${telemetryEvents.estimatedCostUsd} is null then 1 else 0 end)`,
+      resolveSum: sql<number>`coalesce(sum(${telemetryEvents.resolveMs}), 0)`,
+      resolveCount: sql<number>`count(${telemetryEvents.resolveMs})`,
+      avgTokensPerSec: sql<number>`coalesce(avg(${telemetryEvents.tokensPerSec}), 0)`,
+    })
+    .from(telemetryEvents)
+    .where(condition);
+
+  if (!row) {
+    const empty: UsageSummary = {
+      requests: 0,
+      errors: 0,
+      cancelled: 0,
+      truncated: 0,
+      inputTokens: 0,
+      outputTokens: 0,
+      cachedInputTokens: 0,
+      reasoningTokens: 0,
+      estimatedCostUsd: 0,
+      sampled: 0,
+      latencyMs: { avg: null, p50: null, p95: null, p99: null },
+      unpriced: 0,
+    };
+    const emptyPerformance: UsagePerformanceTotals = {
+      avgResolveMs: 0,
+      avgDurationMs: 0,
+      latencyP50Ms: 0,
+      latencyP95Ms: 0,
+      latencyP99Ms: 0,
+      latencySampleCount: 0,
+      resolveSampleCount: 0,
+      avgTokensPerSec: 0,
+    };
+    return { summary: empty, performance: emptyPerformance };
+  }
+
+  const sampled = numeric(row.sampled);
+  const percentiles = await latencyPercentiles(db, tenantId, since, sampled);
+  const resolveCount = numeric(row.resolveCount);
+  return {
+    summary: {
+      requests: numeric(row.requests),
+      errors: numeric(row.errors),
+      cancelled: numeric(row.cancelled),
+      truncated: numeric(row.truncated),
+      inputTokens: numeric(row.inputTokens),
+      outputTokens: numeric(row.outputTokens),
+      cachedInputTokens: numeric(row.cachedInputTokens),
+      reasoningTokens: numeric(row.reasoningTokens),
+      estimatedCostUsd: numeric(row.estimatedCostUsd),
+      sampled,
+      latencyMs: {
+        avg: numericNullable(row.latencyAvg),
+        ...percentiles,
+      },
+      unpriced: numeric(row.unpriced),
+    },
+    performance: {
+      avgResolveMs: resolveCount > 0 ? numeric(row.resolveSum) / resolveCount : 0,
+      avgDurationMs: numericNullable(row.latencyAvg) ?? 0,
+      latencyP50Ms: percentiles.p50 ?? 0,
+      latencyP95Ms: percentiles.p95 ?? 0,
+      latencyP99Ms: percentiles.p99 ?? 0,
+      latencySampleCount: sampled,
+      resolveSampleCount: resolveCount,
+      avgTokensPerSec: numeric(row.avgTokensPerSec),
+    },
+  };
+}

package/src/observability/telemetry-status.ts

@@ -30,7 +30,7 @@
  *   rule as well, so it needs no special case.
  * - **`503`** is capacity or availability: the gateway *correctly* refusing work
  *   it cannot do right now (`admission_unavailable`, `accounts_unavailable`,
- *   `shutting_down`, `restart_for_update`, `platform_unavailable`). Retryable,
+ *   `shutting_down` (incl. update restart), `platform_unavailable`). Retryable,
  *   self-inflicted, and an operator-capacity signal rather than a defect.
  * - **`504`** is a timeout, not a malfunction. It represents a deadline that
  *   expired: the request deadline, the stream-stall

package/src/persistence/db-pg.ts

@@ -108,6 +108,44 @@ export function poolMaxFromEnv(): number {
   return value;
 }
 
+/** Target number of idle connections the pool shrinks to (default 2). */
+export function poolIdleFloorFromEnv(): number {
+  const raw = process.env.DATABASE_POOL_IDLE_FLOOR ?? "2";
+  const value = Number(raw);
+  if (!Number.isInteger(value) || value < 0) {
+    throw new Error(`DATABASE_POOL_IDLE_FLOOR must be a non-negative integer (got "${raw}")`);
+  }
+  return value;
+}
+
+/**
+ * Periodically closes idle clients above the floor so a low-traffic process
+ * does not hold `DATABASE_POOL_MAX` backends open for nothing. node-postgres
+ * only closes an idle client after `idleTimeoutMillis` (60s here); this gives
+ * the pool a faster, explicit shrink path while still keeping the floor warm.
+ */
+export function startPoolShrinker(pool: Pool, floor: number): () => void {
+  const interval = setInterval(() => {
+    try {
+      const idle = pool.idleCount;
+      const excess = idle - floor;
+      if (excess <= 0) return;
+      // Close the excess idle clients in order; `release(true)` destroys the
+      // client instead of returning it to the pool.
+      for (let index = 0; index < excess; index += 1) {
+        pool
+          .connect()
+          .then((client) => client.release(true))
+          .catch(() => undefined);
+      }
+    } catch {
+      // The pool is best-effort; a failed shrink must never crash a request.
+    }
+  }, 30_000);
+  interval.unref();
+  return () => clearInterval(interval);
+}
+
 declare global {
   // eslint-disable-next-line no-var -- globalThis augmentation requires `var`
   var __cartethyiaPool: Pool | undefined;
@@ -147,6 +185,10 @@ export function getPool(): Pool {
   pool.on("error", (err) => {
     log.error("[postgres] pool error (idle client)", err);
   });
+  // Idle-floor shrinker: keeps the pool warm at the floor instead of parking
+  // a full `max` worth of backends. Tied to the pool's lifetime, so a hot
+  // reload that replaces the pool re-registers it.
+  startPoolShrinker(pool, poolIdleFloorFromEnv());
   globalThis.__cartethyiaPool = pool;
   return pool;
 }

package/src/persistence/db-sqlite.ts

@@ -327,7 +327,8 @@ const AUTH_KEY_COLUMNS = [
   "issued_client_ip_key", "label", "scopes", "requests_per_minute", "daily_token_limit",
   "monthly_token_limit", "lifetime_token_budget", "lifetime_tokens_consumed",
   "max_concurrent_requests", "model_access_mode", "model_list", "client_router_denylist",
-  "model_prefix", "created_at", "enabled", "revoked_at", "key_prefix", "key_encrypted",
+  "model_prefix", "created_at", "enabled", "revoked_at", "banned_at", "ban_reason", "banned_until",
+  "key_prefix", "key_encrypted",
   "notes_title", "notes_subtitle", "notes_body", "is_default", "sort_index",
 ].join(", ");
 
@@ -341,7 +342,11 @@ export interface SqliteAuthStatements {
 }
 
 export function prepareAuthStatements(sqlite: Database): SqliteAuthStatements {
-  const active = `enabled = 1 AND revoked_at IS NULL`;
+  // A key is active while not revoked and not under an active ban: a ban is
+  // active when `banned_at` is set and `banned_until` is null (permanent) or
+  // still in the future. SQLite stores timestamps as ISO text, so the
+  // comparison is lexical (ISO sorts chronologically).
+  const active = `enabled = 1 AND revoked_at IS NULL AND (banned_at IS NULL OR (banned_until IS NOT NULL AND banned_until <= datetime('now')))`;
   // `query`, not `prepare`: bun's `prepare` compiles a statement per call and
   // never finalizes one, while `query` caches the compiled statement behind an
   // LRU of its own. All three are constant text run once per authenticated