omnigateway 0.10.3 → 0.11.1

package/gateway.js

@@ -24443,13 +24443,13 @@ function prepareArtifact(input2) {
     error: error61.value
   };
   const json2 = JSON.stringify(bounded);
-  const size = encoder2.encode(json2).length;
+  const size = Buffer.byteLength(json2);
   if (size <= MAX_ARTIFACT_BYTES)
     return { artifact: bounded, json: json2 };
   const marker = omission(size);
   const omitted = omitBodies(bounded, marker);
   const omittedJson = JSON.stringify(omitted);
-  if (encoder2.encode(omittedJson).length <= MAX_ARTIFACT_BYTES) {
+  if (Buffer.byteLength(omittedJson) <= MAX_ARTIFACT_BYTES) {
     return { artifact: omitted, json: omittedJson };
   }
   const stripped = { ...omitted, error: marker };
@@ -24630,6 +24630,7 @@ var FIELD_ORDER = [
   "sizeBytes",
   "host",
   "port",
+  "dayOffsetMinutes",
   "path",
   "supervisor",
   "reason"
@@ -24926,14 +24927,14 @@ function estimateCachedInputTokens(request) {
   return cached2;
 }
 // packages/ir/src/validate.ts
-function validateRequest(req) {
+function validateRequest(req, newToolUseId = () => `tu_${crypto.randomUUID()}`) {
   const seenToolUseIds = new Set;
   const cleaned = [];
   for (const message of req.messages) {
     const content = [];
     for (const block of message.content) {
       if (block.type === "toolUse") {
-        const id = block.id.length > 0 ? block.id : `tu_${crypto.randomUUID()}`;
+        const id = block.id.length > 0 ? block.id : newToolUseId();
         seenToolUseIds.add(id);
         content.push({ ...block, id });
         continue;
@@ -24953,6 +24954,261 @@ function validateRequest(req) {
   }
   return { ...req, messages: cleaned };
 }
+// packages/store/src/sqlite/rollup.ts
+var HOUR_MS = 3600000;
+function hourOf(at) {
+  return Math.floor(at / HOUR_MS);
+}
+var DAY_MS = 86400000;
+function hostDayOffsetMinutes() {
+  return -new Date().getTimezoneOffset() || 0;
+}
+function startOfDay(at, offsetMinutes) {
+  const offsetMs = offsetMinutes * 60000;
+  return Math.floor((at + offsetMs) / DAY_MS) * DAY_MS - offsetMs;
+}
+var UPSERT = `
+  INSERT INTO usage_daily
+    (day, provider, credential_id, requested_model, resolved_model, api_key_id,
+     requests, errors, input_tokens, output_tokens, cache_read_tokens,
+     cache_write_tokens, rtk_saved_tokens, rtk_applied_requests, cost_usd, duration_ms_sum)
+  VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+  ON CONFLICT (day, provider, credential_id, requested_model, resolved_model, api_key_id)
+  DO UPDATE SET
+    requests           = requests + excluded.requests,
+    errors             = errors + excluded.errors,
+    input_tokens       = input_tokens + excluded.input_tokens,
+    output_tokens      = output_tokens + excluded.output_tokens,
+    cache_read_tokens  = cache_read_tokens + excluded.cache_read_tokens,
+    cache_write_tokens    = cache_write_tokens + excluded.cache_write_tokens,
+    rtk_saved_tokens      = rtk_saved_tokens + excluded.rtk_saved_tokens,
+    rtk_applied_requests  = rtk_applied_requests + excluded.rtk_applied_requests,
+    cost_usd              = cost_usd + excluded.cost_usd,
+    duration_ms_sum    = duration_ms_sum + excluded.duration_ms_sum`;
+function keyOf(log, dayOffsetMinutes) {
+  return [
+    startOfDay(log.at, dayOffsetMinutes),
+    log.resolvedProvider ?? "",
+    log.credentialId ?? "",
+    log.requestedModel,
+    log.resolvedModel ?? "",
+    log.apiKeyId ?? ""
+  ];
+}
+function countersOf(log) {
+  return {
+    requests: 1,
+    errors: log.status >= 400 ? 1 : 0,
+    inputTokens: log.inputTokens,
+    outputTokens: log.outputTokens,
+    cacheReadTokens: log.cacheReadTokens,
+    cacheWriteTokens: log.cacheWriteTokens,
+    rtkSavedTokens: log.rtkEstimatedTokensSaved,
+    rtkAppliedRequests: log.rtkApplied ? 1 : 0,
+    costUsd: log.costUsd,
+    durationMsSum: log.durationMs
+  };
+}
+var BACKFILL_UPSERT = `
+  INSERT INTO usage_daily
+    (day, provider, credential_id, requested_model, resolved_model, api_key_id,
+     requests, errors, input_tokens, output_tokens, cache_read_tokens,
+     cache_write_tokens, cost_usd, duration_ms_sum)
+  VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+  ON CONFLICT (day, provider, credential_id, requested_model, resolved_model, api_key_id)
+  DO UPDATE SET
+    requests           = requests + excluded.requests,
+    errors             = errors + excluded.errors,
+    input_tokens       = input_tokens + excluded.input_tokens,
+    output_tokens      = output_tokens + excluded.output_tokens,
+    cache_read_tokens  = cache_read_tokens + excluded.cache_read_tokens,
+    cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
+    cost_usd           = cost_usd + excluded.cost_usd,
+    duration_ms_sum    = duration_ms_sum + excluded.duration_ms_sum`;
+function upsertPreRtk(db, key, c) {
+  db.run(BACKFILL_UPSERT, [
+    ...key,
+    c.requests,
+    c.errors,
+    c.inputTokens,
+    c.outputTokens,
+    c.cacheReadTokens,
+    c.cacheWriteTokens,
+    c.costUsd,
+    c.durationMsSum
+  ]);
+}
+function upsert(db, key, c) {
+  db.run(UPSERT, [
+    ...key,
+    c.requests,
+    c.errors,
+    c.inputTokens,
+    c.outputTokens,
+    c.cacheReadTokens,
+    c.cacheWriteTokens,
+    c.rtkSavedTokens,
+    c.rtkAppliedRequests,
+    c.costUsd,
+    c.durationMsSum
+  ]);
+}
+function rollupLog(db, log, dayOffsetMinutes) {
+  upsert(db, keyOf(log, dayOffsetMinutes), countersOf(log));
+}
+function backfillDaily(db, dayOffsetMinutes) {
+  const groups = new Map;
+  for (const row of db.query(`SELECT at, api_key_id, requested_model, resolved_provider, resolved_model, credential_id,
+              status, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens,
+              duration_ms, cost_usd
+         FROM request_logs`).all()) {
+    const key = [
+      startOfDay(row.at, dayOffsetMinutes),
+      row.resolved_provider ?? "",
+      row.credential_id ?? "",
+      row.requested_model,
+      row.resolved_model ?? "",
+      row.api_key_id ?? ""
+    ];
+    const id = key.join("\x00");
+    const seen = groups.get(id);
+    const counters = seen?.counters ?? {
+      requests: 0,
+      errors: 0,
+      inputTokens: 0,
+      outputTokens: 0,
+      cacheReadTokens: 0,
+      cacheWriteTokens: 0,
+      rtkSavedTokens: 0,
+      rtkAppliedRequests: 0,
+      costUsd: 0,
+      durationMsSum: 0
+    };
+    counters.requests += 1;
+    counters.errors += row.status >= 400 ? 1 : 0;
+    counters.inputTokens += row.input_tokens;
+    counters.outputTokens += row.output_tokens;
+    counters.cacheReadTokens += row.cache_read_tokens;
+    counters.cacheWriteTokens += row.cache_write_tokens;
+    counters.costUsd += row.cost_usd;
+    counters.durationMsSum += row.duration_ms;
+    if (seen === undefined)
+      groups.set(id, { key, counters });
+  }
+  for (const group of groups.values())
+    upsertPreRtk(db, group.key, group.counters);
+  return groups.size;
+}
+var HOURLY_UPSERT = `
+  INSERT INTO usage_rollup
+    (api_key_id, hour, requests, input_tokens, output_tokens,
+     cache_read_tokens, cache_write_tokens, cost_usd)
+  VALUES (?,?,?,?,?,?,?,?)
+  ON CONFLICT (api_key_id, hour)
+  DO UPDATE SET
+    requests           = requests + excluded.requests,
+    input_tokens       = input_tokens + excluded.input_tokens,
+    output_tokens      = output_tokens + excluded.output_tokens,
+    cache_read_tokens  = cache_read_tokens + excluded.cache_read_tokens,
+    cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
+    cost_usd           = cost_usd + excluded.cost_usd`;
+function rollupHour(db, log) {
+  const apiKeyId = log.apiKeyId;
+  if (apiKeyId === null)
+    return;
+  db.run(HOURLY_UPSERT, [
+    apiKeyId,
+    hourOf(log.at),
+    1,
+    log.inputTokens,
+    log.outputTokens,
+    log.cacheReadTokens,
+    log.cacheWriteTokens,
+    log.costUsd
+  ]);
+}
+var REBUILD = `
+  INSERT INTO usage_rollup
+    (api_key_id, hour, requests, input_tokens, output_tokens,
+     cache_read_tokens, cache_write_tokens, cost_usd)
+  SELECT api_key_id,
+         CAST(at / 3600000 AS INTEGER),
+         COUNT(*),
+         COALESCE(SUM(input_tokens), 0),
+         COALESCE(SUM(output_tokens), 0),
+         COALESCE(SUM(cache_read_tokens), 0),
+         COALESCE(SUM(cache_write_tokens), 0),
+         COALESCE(SUM(cost_usd), 0)
+    FROM request_logs
+   WHERE state = 'done' AND api_key_id IS NOT NULL
+   GROUP BY api_key_id, CAST(at / 3600000 AS INTEGER)`;
+function rebuildRollup(db) {
+  db.run("DELETE FROM usage_rollup");
+  db.run(REBUILD);
+}
+function auditRollup(db) {
+  const row = db.query(`WITH truth AS (
+         SELECT api_key_id AS k,
+                CAST(at / 3600000 AS INTEGER) AS h,
+                COUNT(*) AS requests,
+                COALESCE(SUM(input_tokens), 0) AS input_tokens,
+                COALESCE(SUM(output_tokens), 0) AS output_tokens,
+                COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens,
+                COALESCE(SUM(cache_write_tokens), 0) AS cache_write_tokens,
+                COALESCE(SUM(cost_usd), 0) AS cost_usd
+           FROM request_logs
+          WHERE state = 'done' AND api_key_id IS NOT NULL
+          GROUP BY k, h
+       )
+       SELECT (SELECT COUNT(*) FROM truth) AS buckets,
+              (SELECT COUNT(*)
+                 FROM truth
+                 LEFT JOIN usage_rollup r ON r.api_key_id = truth.k AND r.hour = truth.h
+                WHERE r.api_key_id IS NULL
+                   OR r.requests <> truth.requests
+                   OR r.input_tokens <> truth.input_tokens
+                   OR r.output_tokens <> truth.output_tokens
+                   OR r.cache_read_tokens <> truth.cache_read_tokens
+                   OR r.cache_write_tokens <> truth.cache_write_tokens
+                   OR ABS(r.cost_usd - truth.cost_usd) > 1e-9 + ABS(truth.cost_usd) * 1e-9
+              ) AS mismatched,
+              (SELECT COUNT(*)
+                 FROM usage_rollup r
+                 LEFT JOIN truth ON r.api_key_id = truth.k AND r.hour = truth.h
+                WHERE truth.k IS NULL
+              ) AS orphans`).get();
+  const buckets = row?.buckets ?? 0;
+  const mismatched = (row?.mismatched ?? 0) + (row?.orphans ?? 0);
+  return { buckets, mismatched, ok: mismatched === 0 };
+}
+function backfillRtkUsage(db, dayOffsetMinutes) {
+  const groups = new Map;
+  for (const row of db.query(`SELECT at, api_key_id, requested_model, resolved_provider, resolved_model, credential_id,
+              rtk_applied, rtk_estimated_tokens_saved
+         FROM request_logs
+        WHERE state = 'done'`).all()) {
+    const key = [
+      startOfDay(row.at, dayOffsetMinutes),
+      row.resolved_provider ?? "",
+      row.credential_id ?? "",
+      row.requested_model,
+      row.resolved_model ?? "",
+      row.api_key_id ?? ""
+    ];
+    const id = key.join("\x00");
+    const group = groups.get(id) ?? { key, saved: 0, applied: 0 };
+    group.saved += row.rtk_estimated_tokens_saved;
+    group.applied += row.rtk_applied === 1 ? 1 : 0;
+    groups.set(id, group);
+  }
+  for (const { key, saved, applied } of groups.values()) {
+    db.run(`UPDATE usage_daily
+          SET rtk_saved_tokens = ?, rtk_applied_requests = ?
+        WHERE day = ? AND provider = ? AND credential_id = ? AND requested_model = ?
+          AND resolved_model = ? AND api_key_id = ?`, [saved, applied, ...key]);
+  }
+}
+
 // packages/store/src/postgres/db.ts
 var {SQL } = globalThis.Bun;
 
@@ -25258,11 +25514,95 @@ CREATE TRIGGER api_keys_config_version
   FOR EACH STATEMENT EXECUTE FUNCTION bump_config_version();
 `;
 
+// packages/store/src/postgres/migrations/002_health_measurements.sql
+var _002_health_measurements_default = `-- DEPLOY HAZARD \u2014 read before applying to a fleet.
+--
+-- This migration drops two columns that every gateway before it writes on
+-- the success path (\`UPSERT_HEALTH\` names \`ewma_ttft_ms\` and \`last_used_at\`).
+-- The first new replica to boot applies it under the migration lock; from that
+-- moment every OLD replica still running throws on every successful request:
+-- the upstream call has completed and been billed, the client has received its
+-- content frames, and then the connection aborts with no terminal frame. The
+-- \`RollingUpdate\` in \`k8s/deployment.yaml\` does not protect against this \u2014 the
+-- "surviving" replica is the one that breaks.
+--
+-- Required: scale to ONE replica before deploying this release, or accept that
+-- every old replica fails every successful request until it is replaced.
+-- Rollback is unsafe: an old image cannot re-add the columns, and running it
+-- against this schema fails the same way. The only way back is a snapshot
+-- restore taken before the deploy. Deliberate: kept as a drop, not
+-- expand/contract, on the operator's decision \u2014 see the write-path spec.
+--
+-- \`credential_health\` keeps decisions, not measurements.
+--
+-- \`ewma_ttft_ms\` and \`last_used_at\` were written on every successful request,
+-- and a success that changes nothing else now writes nothing. Both live in the
+-- gateway's process-local \`loadRegistry\` instead. The console's "last used"
+-- reads \`request_logs\` through \`usage.lastUsedByCredential\`, served by
+-- \`idx_request_logs_cred\`.
+ALTER TABLE credential_health DROP COLUMN ewma_ttft_ms;
+ALTER TABLE credential_health DROP COLUMN last_used_at;
+
+-- The statement-level trigger bumped \`config_version\` \u2014 one row, locked until
+-- commit, by every writer on every replica \u2014 on every health write, and every
+-- bump makes every replica rebuild its routing snapshot. Only a change to a
+-- column routing *decides* on needs that; a bare \`consecutive_failures\`
+-- increment is patched into the held snapshot from the \`healthSaved\` change.
+--
+-- The INSERT arm must stay \`FOR EACH ROW\`. The write is an upsert, and a
+-- statement-level INSERT trigger fires on an upsert whichever path each row
+-- took, which would bump on every write and make the WHEN clause below moot.
+-- Row-level \`AFTER INSERT\` fires only for rows actually inserted.
+DROP TRIGGER credential_health_config_version ON credential_health;
+
+CREATE TRIGGER credential_health_config_version_upd
+  AFTER UPDATE ON credential_health
+  FOR EACH ROW
+  WHEN (OLD.breaker_state      IS DISTINCT FROM NEW.breaker_state
+     OR OLD.rate_limited_until IS DISTINCT FROM NEW.rate_limited_until
+     OR OLD.opened_at          IS DISTINCT FROM NEW.opened_at)
+  EXECUTE FUNCTION bump_config_version();
+
+CREATE TRIGGER credential_health_config_version_ins
+  AFTER INSERT ON credential_health
+  FOR EACH ROW EXECUTE FUNCTION bump_config_version();
+
+-- DELETE needs no WHEN, so it stays statement-level as in 001: a credential
+-- with M model rows bumps once, not M times.
+CREATE TRIGGER credential_health_config_version_del
+  AFTER DELETE ON credential_health
+  FOR EACH STATEMENT EXECUTE FUNCTION bump_config_version();
+`;
+
+// packages/store/src/postgres/migrations/003_key_expiry.sql
+var _003_key_expiry_default = `-- The Postgres half of the sqlite migration of the same name. \`BIGINT\`, like
+-- every other epoch-ms column in this schema, and read back through \`numOrNull\`.
+--
+-- Nullable rather than \`NOT NULL DEFAULT\`: the column is a deadline, and every
+-- finite default is a date on which somebody's client stops working. NULL is
+-- the only value that says nothing was decided, and every reader maps it to
+-- "never expires".
+--
+-- No backfill, deliberately. Every key on every existing installation was
+-- minted on the promise that it works until revoked, so adding the column and
+-- stopping is what keeps that promise. The absence of an \`UPDATE\` here is the
+-- substance of the migration, not an omission.
+--
+-- Safe to apply under a rolling deploy, unlike migration 2: this adds a
+-- nullable column and drops nothing, so a replica running the previous image
+-- keeps serving \u2014 \`SELECT *\` gains a field its mapper ignores.
+ALTER TABLE api_keys ADD COLUMN expires_at BIGINT;
+`;
+
 // packages/store/src/postgres/db.ts
 var num = (v) => Number(v);
 var numOrNull = (v) => v === null ? null : Number(v);
 var MIGRATION_LOCK = 7140641;
-var MIGRATIONS = [{ id: 1, sql: _001_init_default }];
+var MIGRATIONS = [
+  { id: 1, sql: _001_init_default },
+  { id: 2, sql: _002_health_measurements_default },
+  { id: 3, sql: _003_key_expiry_default }
+];
 async function openPg(url2) {
   const sql = new SQL({ url: url2 });
   const conn = await sql.reserve();
@@ -25447,6 +25787,9 @@ function resolvePin(target, accounts) {
     return;
   return accounts.find((account) => servesTarget(target, account));
 }
+var keyUsable = (key, now) => key.revokedAt === null && (key.expiresAt === null || key.expiresAt > now);
+var credentialPastExpiry = (credential, now) => credential.authType === "oauth" && credential.expiresAt !== null && credential.expiresAt <= now;
+var credentialExpired = (credential, now) => credentialPastExpiry(credential, now) && !credential.hasRefreshToken;
 var NODE_GRACE_MS = 60000;
 var DEFAULT_SETTINGS = {
   weights: { tier: 10, health: 3, quota: 2, load: 2, cost: 1, latency: 1 },
@@ -25574,31 +25917,25 @@ function toHealth(r) {
     breakerState: r.breaker_state,
     consecutiveFailures: r.consecutive_failures,
     openedAt: numOrNull(r.opened_at),
-    rateLimitedUntil: numOrNull(r.rate_limited_until),
-    ewmaTtftMs: r.ewma_ttft_ms,
-    lastUsedAt: numOrNull(r.last_used_at)
+    rateLimitedUntil: numOrNull(r.rate_limited_until)
   };
 }
 var UPSERT_HEALTH = `INSERT INTO credential_health
    (credential_id, model, breaker_state, consecutive_failures, opened_at,
-    rate_limited_until, ewma_ttft_ms, last_used_at)
- VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
+    rate_limited_until)
+ VALUES ($1,$2,$3,$4,$5,$6)
  ON CONFLICT (credential_id, model) DO UPDATE SET
    breaker_state = EXCLUDED.breaker_state,
    consecutive_failures = EXCLUDED.consecutive_failures,
    opened_at = EXCLUDED.opened_at,
-   rate_limited_until = EXCLUDED.rate_limited_until,
-   ewma_ttft_ms = EXCLUDED.ewma_ttft_ms,
-   last_used_at = EXCLUDED.last_used_at`;
+   rate_limited_until = EXCLUDED.rate_limited_until`;
 var healthValues = (r) => [
   r.credentialId,
   r.model,
   r.breakerState,
   r.consecutiveFailures,
   r.openedAt,
-  r.rateLimitedUntil,
-  r.ewmaTtftMs,
-  r.lastUsedAt
+  r.rateLimitedUntil
 ];
 function createCredentialRepo(sql, key, emit) {
   const open2 = async (v) => v === null ? null : decrypt(key, v);
@@ -25791,13 +26128,6 @@ function createCredentialRepo(sql, key, emit) {
       const rows = await sql.unsafe("SELECT * FROM credential_health");
       return rows.map(toHealth);
     },
-    async saveHealth(rows) {
-      await sql.begin(async (tx) => {
-        for (const r of rows)
-          await tx.unsafe(UPSERT_HEALTH, healthValues(r));
-      });
-      emit({ type: "healthSaved", rows });
-    },
     async updateHealth(credentialId, model, apply) {
       const written = await sql.begin(async (tx) => {
         await tx.unsafe("SELECT pg_advisory_xact_lock(hashtext($1))", [`${credentialId}|${model}`]);
@@ -25939,7 +26269,8 @@ var toKey = (r, logger2) => ({
   limits: parseLimits(r.id, r.limits, logger2),
   bodyLoggingOptOut: r.body_logging_opt_out,
   createdAt: num(r.created_at),
-  revokedAt: numOrNull(r.revoked_at)
+  revokedAt: numOrNull(r.revoked_at),
+  expiresAt: numOrNull(r.expires_at)
 });
 function createKeyRepo(sql, logger2) {
   const one = async (where, arg) => {
@@ -25957,8 +26288,8 @@ function createKeyRepo(sql, logger2) {
     async create(input2) {
       const now = Date.now();
       await sql.unsafe(`INSERT INTO api_keys (id, label, prefix, hash, model_allowlist, limits,
-                               body_logging_opt_out, created_at, revoked_at)
-         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,NULL)`, [
+                               body_logging_opt_out, created_at, revoked_at, expires_at)
+         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,NULL,$9)`, [
         input2.id,
         input2.label,
         input2.prefix,
@@ -25966,7 +26297,8 @@ function createKeyRepo(sql, logger2) {
         input2.modelAllowlist === null ? null : JSON.stringify(input2.modelAllowlist),
         JSON.stringify(parseLimitConfig(input2.limits)),
         input2.bodyLoggingOptOut,
-        now
+        now,
+        input2.expiresAt
       ]);
       return { ...input2, createdAt: now, revokedAt: null };
     },
@@ -25974,8 +26306,8 @@ function createKeyRepo(sql, logger2) {
       if (row.limits === null)
         throw new Error(`api key ${row.id} has unreadable limits`);
       await sql.unsafe(`INSERT INTO api_keys (id, label, prefix, hash, model_allowlist, limits,
-                               body_logging_opt_out, created_at, revoked_at)
-         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`, [
+                               body_logging_opt_out, created_at, revoked_at, expires_at)
+         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, [
         row.id,
         row.label,
         row.prefix,
@@ -25984,7 +26316,8 @@ function createKeyRepo(sql, logger2) {
         JSON.stringify(parseLimitConfig(row.limits)),
         row.bodyLoggingOptOut,
         row.createdAt,
-        row.revokedAt
+        row.revokedAt,
+        row.expiresAt
       ]);
     },
     async setLimits(id, limits) {
@@ -25999,6 +26332,9 @@ function createKeyRepo(sql, logger2) {
         id
       ]);
     },
+    async setExpiry(id, expiresAt) {
+      await sql.unsafe("UPDATE api_keys SET expires_at = $1 WHERE id = $2", [expiresAt, id]);
+    },
     async revoke(id) {
       await sql.unsafe("UPDATE api_keys SET revoked_at = $1 WHERE id = $2", [Date.now(), id]);
     }
@@ -26297,229 +26633,6 @@ function isRtkFilterId(value) {
   return FILTER_IDS.has(value);
 }
 
-// packages/store/src/sqlite/rollup.ts
-var HOUR_MS = 3600000;
-function hourOf(at) {
-  return Math.floor(at / HOUR_MS);
-}
-function startOfLocalDay(at) {
-  const day = new Date(at);
-  day.setHours(0, 0, 0, 0);
-  return day.getTime();
-}
-var UPSERT = `
-  INSERT INTO usage_daily
-    (day, provider, credential_id, requested_model, resolved_model, api_key_id,
-     requests, errors, input_tokens, output_tokens, cache_read_tokens,
-     cache_write_tokens, rtk_saved_tokens, rtk_applied_requests, cost_usd, duration_ms_sum)
-  VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
-  ON CONFLICT (day, provider, credential_id, requested_model, resolved_model, api_key_id)
-  DO UPDATE SET
-    requests           = requests + excluded.requests,
-    errors             = errors + excluded.errors,
-    input_tokens       = input_tokens + excluded.input_tokens,
-    output_tokens      = output_tokens + excluded.output_tokens,
-    cache_read_tokens  = cache_read_tokens + excluded.cache_read_tokens,
-    cache_write_tokens    = cache_write_tokens + excluded.cache_write_tokens,
-    rtk_saved_tokens      = rtk_saved_tokens + excluded.rtk_saved_tokens,
-    rtk_applied_requests  = rtk_applied_requests + excluded.rtk_applied_requests,
-    cost_usd              = cost_usd + excluded.cost_usd,
-    duration_ms_sum    = duration_ms_sum + excluded.duration_ms_sum`;
-function keyOf(log) {
-  return [
-    startOfLocalDay(log.at),
-    log.resolvedProvider ?? "",
-    log.credentialId ?? "",
-    log.requestedModel,
-    log.resolvedModel ?? "",
-    log.apiKeyId ?? ""
-  ];
-}
-function countersOf(log) {
-  return {
-    requests: 1,
-    errors: log.status >= 400 ? 1 : 0,
-    inputTokens: log.inputTokens,
-    outputTokens: log.outputTokens,
-    cacheReadTokens: log.cacheReadTokens,
-    cacheWriteTokens: log.cacheWriteTokens,
-    rtkSavedTokens: log.rtkEstimatedTokensSaved,
-    rtkAppliedRequests: log.rtkApplied ? 1 : 0,
-    costUsd: log.costUsd,
-    durationMsSum: log.durationMs
-  };
-}
-function upsert(db, key, c) {
-  db.run(UPSERT, [
-    ...key,
-    c.requests,
-    c.errors,
-    c.inputTokens,
-    c.outputTokens,
-    c.cacheReadTokens,
-    c.cacheWriteTokens,
-    c.rtkSavedTokens,
-    c.rtkAppliedRequests,
-    c.costUsd,
-    c.durationMsSum
-  ]);
-}
-function rollupLog(db, log) {
-  upsert(db, keyOf(log), countersOf(log));
-}
-function backfillDaily(db) {
-  const groups = new Map;
-  for (const row of db.query(`SELECT at, api_key_id, requested_model, resolved_provider, resolved_model, credential_id,
-              status, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens,
-              duration_ms, cost_usd
-         FROM request_logs`).all()) {
-    const key = [
-      startOfLocalDay(row.at),
-      row.resolved_provider ?? "",
-      row.credential_id ?? "",
-      row.requested_model,
-      row.resolved_model ?? "",
-      row.api_key_id ?? ""
-    ];
-    const id = key.join("\x00");
-    const seen = groups.get(id);
-    const counters = seen?.counters ?? {
-      requests: 0,
-      errors: 0,
-      inputTokens: 0,
-      outputTokens: 0,
-      cacheReadTokens: 0,
-      cacheWriteTokens: 0,
-      rtkSavedTokens: 0,
-      rtkAppliedRequests: 0,
-      costUsd: 0,
-      durationMsSum: 0
-    };
-    counters.requests += 1;
-    counters.errors += row.status >= 400 ? 1 : 0;
-    counters.inputTokens += row.input_tokens;
-    counters.outputTokens += row.output_tokens;
-    counters.cacheReadTokens += row.cache_read_tokens;
-    counters.cacheWriteTokens += row.cache_write_tokens;
-    counters.costUsd += row.cost_usd;
-    counters.durationMsSum += row.duration_ms;
-    if (seen === undefined)
-      groups.set(id, { key, counters });
-  }
-  for (const group of groups.values())
-    upsert(db, group.key, group.counters);
-  return groups.size;
-}
-var HOURLY_UPSERT = `
-  INSERT INTO usage_rollup
-    (api_key_id, hour, requests, input_tokens, output_tokens,
-     cache_read_tokens, cache_write_tokens, cost_usd)
-  VALUES (?,?,?,?,?,?,?,?)
-  ON CONFLICT (api_key_id, hour)
-  DO UPDATE SET
-    requests           = requests + excluded.requests,
-    input_tokens       = input_tokens + excluded.input_tokens,
-    output_tokens      = output_tokens + excluded.output_tokens,
-    cache_read_tokens  = cache_read_tokens + excluded.cache_read_tokens,
-    cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
-    cost_usd           = cost_usd + excluded.cost_usd`;
-function rollupHour(db, log) {
-  const apiKeyId = log.apiKeyId;
-  if (apiKeyId === null)
-    return;
-  db.run(HOURLY_UPSERT, [
-    apiKeyId,
-    hourOf(log.at),
-    1,
-    log.inputTokens,
-    log.outputTokens,
-    log.cacheReadTokens,
-    log.cacheWriteTokens,
-    log.costUsd
-  ]);
-}
-var REBUILD = `
-  INSERT INTO usage_rollup
-    (api_key_id, hour, requests, input_tokens, output_tokens,
-     cache_read_tokens, cache_write_tokens, cost_usd)
-  SELECT api_key_id,
-         CAST(at / 3600000 AS INTEGER),
-         COUNT(*),
-         COALESCE(SUM(input_tokens), 0),
-         COALESCE(SUM(output_tokens), 0),
-         COALESCE(SUM(cache_read_tokens), 0),
-         COALESCE(SUM(cache_write_tokens), 0),
-         COALESCE(SUM(cost_usd), 0)
-    FROM request_logs
-   WHERE state = 'done' AND api_key_id IS NOT NULL
-   GROUP BY api_key_id, CAST(at / 3600000 AS INTEGER)`;
-function rebuildRollup(db) {
-  db.run("DELETE FROM usage_rollup");
-  db.run(REBUILD);
-}
-function auditRollup(db) {
-  const row = db.query(`WITH truth AS (
-         SELECT api_key_id AS k,
-                CAST(at / 3600000 AS INTEGER) AS h,
-                COUNT(*) AS requests,
-                COALESCE(SUM(input_tokens), 0) AS input_tokens,
-                COALESCE(SUM(output_tokens), 0) AS output_tokens,
-                COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens,
-                COALESCE(SUM(cache_write_tokens), 0) AS cache_write_tokens,
-                COALESCE(SUM(cost_usd), 0) AS cost_usd
-           FROM request_logs
-          WHERE state = 'done' AND api_key_id IS NOT NULL
-          GROUP BY k, h
-       )
-       SELECT (SELECT COUNT(*) FROM truth) AS buckets,
-              (SELECT COUNT(*)
-                 FROM truth
-                 LEFT JOIN usage_rollup r ON r.api_key_id = truth.k AND r.hour = truth.h
-                WHERE r.api_key_id IS NULL
-                   OR r.requests <> truth.requests
-                   OR r.input_tokens <> truth.input_tokens
-                   OR r.output_tokens <> truth.output_tokens
-                   OR r.cache_read_tokens <> truth.cache_read_tokens
-                   OR r.cache_write_tokens <> truth.cache_write_tokens
-                   OR ABS(r.cost_usd - truth.cost_usd) > 1e-9 + ABS(truth.cost_usd) * 1e-9
-              ) AS mismatched,
-              (SELECT COUNT(*)
-                 FROM usage_rollup r
-                 LEFT JOIN truth ON r.api_key_id = truth.k AND r.hour = truth.h
-                WHERE truth.k IS NULL
-              ) AS orphans`).get();
-  const buckets = row?.buckets ?? 0;
-  const mismatched = (row?.mismatched ?? 0) + (row?.orphans ?? 0);
-  return { buckets, mismatched, ok: mismatched === 0 };
-}
-function backfillRtkUsage(db) {
-  const groups = new Map;
-  for (const row of db.query(`SELECT at, api_key_id, requested_model, resolved_provider, resolved_model, credential_id,
-              rtk_applied, rtk_estimated_tokens_saved
-         FROM request_logs
-        WHERE state = 'done'`).all()) {
-    const key = [
-      startOfLocalDay(row.at),
-      row.resolved_provider ?? "",
-      row.credential_id ?? "",
-      row.requested_model,
-      row.resolved_model ?? "",
-      row.api_key_id ?? ""
-    ];
-    const id = key.join("\x00");
-    const group = groups.get(id) ?? { key, saved: 0, applied: 0 };
-    group.saved += row.rtk_estimated_tokens_saved;
-    group.applied += row.rtk_applied === 1 ? 1 : 0;
-    groups.set(id, group);
-  }
-  for (const { key, saved, applied } of groups.values()) {
-    db.run(`UPDATE usage_daily
-          SET rtk_saved_tokens = ?, rtk_applied_requests = ?
-        WHERE day = ? AND provider = ? AND credential_id = ? AND requested_model = ?
-          AND resolved_model = ? AND api_key_id = ?`, [saved, applied, ...key]);
-  }
-}
-
 // packages/store/src/postgres/rollup.ts
 var DAILY_UPSERT = `
   INSERT INTO usage_daily
@@ -26539,9 +26652,9 @@ var DAILY_UPSERT = `
     rtk_applied_requests = usage_daily.rtk_applied_requests + EXCLUDED.rtk_applied_requests,
     cost_usd             = usage_daily.cost_usd + EXCLUDED.cost_usd,
     duration_ms_sum      = usage_daily.duration_ms_sum + EXCLUDED.duration_ms_sum`;
-async function rollupLog2(conn, log) {
+async function rollupLog2(conn, log, dayOffsetMinutes) {
   await conn.unsafe(DAILY_UPSERT, [
-    startOfLocalDay(log.at),
+    startOfDay(log.at, dayOffsetMinutes),
     log.resolvedProvider ?? "",
     log.credentialId ?? "",
     log.requestedModel,
@@ -26785,7 +26898,7 @@ var sum = (row) => ({
   tokens: row === undefined ? 0 : num(row.tokens),
   costUsd: row === undefined ? 0 : row.cost_usd
 });
-function createUsageRepo(sql, nodeId) {
+function createUsageRepo(sql, nodeId, dayOffsetMinutes) {
   const complete = async (conn, log) => {
     const written = (await conn.unsafe(COMPLETE, values(log, "done", nodeId))).count;
     if (written === 0)
@@ -26793,7 +26906,7 @@ function createUsageRepo(sql, nodeId) {
     const stored = (await conn.unsafe("SELECT * FROM request_logs WHERE id = $1", [log.id]))[0];
     if (stored !== undefined) {
       const restored = toLog(stored);
-      await rollupLog2(conn, restored);
+      await rollupLog2(conn, restored, dayOffsetMinutes);
       await rollupHour2(conn, restored);
     }
   };
@@ -26824,6 +26937,10 @@ function createUsageRepo(sql, nodeId) {
       ]) : await sql.unsafe("SELECT * FROM request_logs WHERE api_key_id = $1 ORDER BY at DESC LIMIT $2", [apiKeyId, limit]);
       return rows.map(toLog);
     },
+    async lastUsedByCredential(credentialId) {
+      const rows = await sql.unsafe("SELECT MAX(at) AS at FROM request_logs WHERE credential_id = $1", [credentialId]);
+      return numOrNull(rows[0]?.at ?? null);
+    },
     async scan(cursor, limit) {
       const rows = cursor === null ? await sql.unsafe("SELECT * FROM request_logs ORDER BY at, id LIMIT $1", [
         limit
@@ -26886,7 +27003,7 @@ function createUsageRepo(sql, nodeId) {
       const daily = grain === "daily";
       const key = columnFor(grain, q.groupBy);
       const split = q.splitBy === undefined ? null : columnFor(grain, q.splitBy);
-      const since = daily ? startOfLocalDay(q.since) : q.since;
+      const since = daily ? startOfDay(q.since, dayOffsetMinutes) : q.since;
       const until = q.until ?? Number.MAX_SAFE_INTEGER;
       const timeColumn = daily ? "day" : "at";
       const scope = q.apiKeyId;
@@ -26962,6 +27079,7 @@ function redacted(url2) {
 async function createPostgresStore(opts) {
   const logger2 = opts.logger ?? noopLogger;
   const nodeId = opts.nodeId ?? crypto.randomUUID();
+  const dayOffsetMinutes = opts.dayOffsetMinutes ?? hostDayOffsetMinutes();
   const listeners = new Set;
   const emit = (change) => {
     for (const listener of listeners) {
@@ -26993,7 +27111,7 @@ async function createPostgresStore(opts) {
     credentials: createCredentialRepo(sql, opts.encryptionKey, emit),
     config: createConfigRepo(sql, emit),
     keys: createKeyRepo(sql, logger2),
-    usage: createUsageRepo(sql, nodeId),
+    usage: createUsageRepo(sql, nodeId, dayOffsetMinutes),
     bodies: createBodyRepo(sql, opts.encryptionKey),
     maintenance: createMaintenanceRepo(sql, nodeId),
     plugins: createPluginRepo(sql),
@@ -27230,22 +27348,18 @@ function toHealth2(r) {
     breakerState: r.breaker_state,
     consecutiveFailures: r.consecutive_failures,
     openedAt: r.opened_at,
-    rateLimitedUntil: r.rate_limited_until,
-    ewmaTtftMs: r.ewma_ttft_ms,
-    lastUsedAt: r.last_used_at
+    rateLimitedUntil: r.rate_limited_until
   };
 }
 var UPSERT_HEALTH2 = `INSERT INTO credential_health
    (credential_id, model, breaker_state, consecutive_failures, opened_at,
-    rate_limited_until, ewma_ttft_ms, last_used_at)
- VALUES (?,?,?,?,?,?,?,?)
+    rate_limited_until)
+ VALUES (?,?,?,?,?,?)
  ON CONFLICT (credential_id, model) DO UPDATE SET
    breaker_state = excluded.breaker_state,
    consecutive_failures = excluded.consecutive_failures,
    opened_at = excluded.opened_at,
-   rate_limited_until = excluded.rate_limited_until,
-   ewma_ttft_ms = excluded.ewma_ttft_ms,
-   last_used_at = excluded.last_used_at`;
+   rate_limited_until = excluded.rate_limited_until`;
 function createCredentialRepo2(db, key, emit = () => {}) {
   const open2 = async (v) => v === null ? null : decrypt(key, v);
   const secretsFrom = async (row) => ({
@@ -27428,22 +27542,13 @@ function createCredentialRepo2(db, key, emit = () => {}) {
     async listHealth() {
       return db.query("SELECT * FROM credential_health").all().map(toHealth2);
     },
-    async saveHealth(rows) {
-      const stmt = db.prepare(UPSERT_HEALTH2);
-      db.transaction(() => {
-        for (const r of rows) {
-          stmt.run(r.credentialId, r.model, r.breakerState, r.consecutiveFailures, r.openedAt, r.rateLimitedUntil, r.ewmaTtftMs, r.lastUsedAt);
-        }
-      })();
-      emit({ type: "healthSaved", rows });
-    },
     async updateHealth(credentialId, model, apply) {
       const current = db.prepare("SELECT * FROM credential_health WHERE credential_id = ? AND model = ?");
       const stmt = db.prepare(UPSERT_HEALTH2);
       const written = db.transaction(() => {
         const row = current.get(credentialId, model);
         const next = apply(row === null ? null : toHealth2(row));
-        stmt.run(next.credentialId, next.model, next.breakerState, next.consecutiveFailures, next.openedAt, next.rateLimitedUntil, next.ewmaTtftMs, next.lastUsedAt);
+        stmt.run(next.credentialId, next.model, next.breakerState, next.consecutiveFailures, next.openedAt, next.rateLimitedUntil);
         return next;
       })();
       emit({ type: "healthSaved", rows: [written] });
@@ -28006,6 +28111,63 @@ CREATE TABLE nodes (
 ALTER TABLE credentials ADD COLUMN token_version INTEGER NOT NULL DEFAULT 0;
 `;
 
+// packages/store/src/sqlite/migrations/013_health_measurements.sql
+var _013_health_measurements_default = `-- DEPLOY HAZARD \u2014 read before upgrading an install that is still serving.
+--
+-- This migration drops two columns that every gateway before it writes on
+-- the success path (\`UPSERT_HEALTH\` names \`ewma_ttft_ms\` and \`last_used_at\`).
+-- SQLite is single-process, so the hazard is the rollback, not the rollout: an
+-- older gateway started against this database fails every successful request
+-- after the upstream call has completed and been billed \u2014 content frames out,
+-- then an aborted connection with no terminal frame. Rolling back the binary
+-- does not roll back the schema. The only way back is a snapshot taken before
+-- the upgrade. Deliberate: kept as a drop, not expand/contract, on the
+-- operator's decision \u2014 see the write-path spec.
+--
+-- \`credential_health\` keeps decisions, not measurements.
+--
+-- \`ewma_ttft_ms\` and \`last_used_at\` were written on every successful request,
+-- and a success that changes nothing else now writes nothing. Both live in the
+-- gateway's process-local \`loadRegistry\` instead: latency is a property of
+-- *this* process's path to the provider, and the round-robin tiebreak reads
+-- the value the process holds. The console's "last used" reads
+-- \`request_logs\` through \`usage.lastUsedByCredential\`, served by
+-- \`idx_request_logs_cred\`.
+--
+-- Neither column is indexed or constrained, so \`DROP COLUMN\` is permitted.
+ALTER TABLE credential_health DROP COLUMN ewma_ttft_ms;
+ALTER TABLE credential_health DROP COLUMN last_used_at;
+`;
+
+// packages/store/src/sqlite/migrations/014_key_expiry.sql
+var _014_key_expiry_default = `-- An optional expiry on a gateway API key: an absolute epoch-ms instant after
+-- which the key is refused, spelled exactly as \`credentials.expires_at\` is.
+--
+-- Nullable rather than \`NOT NULL DEFAULT\`, which is the opposite of the choice
+-- migration 9 made for \`limits\`, and for the opposite reason. There, every
+-- possible value is a policy and \`{}\` is the one that means "no ceiling", so a
+-- default gives "unlimited" a single spelling. Here the column is a deadline,
+-- and no number means "no deadline" \u2014 every finite default is a date on which
+-- somebody's client stops working. NULL is the only value that says nothing was
+-- decided, so it is the column's own default and every reader maps it to
+-- "never expires".
+--
+-- No backfill, deliberately, and this is the whole substance of the migration.
+-- Every key on every existing installation was minted on the promise that it
+-- works until revoked; an upgrade that attaches an expiry to those rows breaks
+-- clients on a date the operator never chose, and the failure surfaces days
+-- later as an authentication error with nothing in it that names this change.
+-- Adding the column and stopping is what keeps that promise, so the absence of
+-- an \`UPDATE\` here is load-bearing rather than an omission \u2014
+-- \`packages/store/test/migrations.test.ts\` pins it.
+--
+-- No index. The column is read only after a row has already been found by
+-- \`hash\` or by \`id\`, both of which are unique, so there is no scan for an index
+-- to help; and nothing sweeps expired keys, because an expired key stays listed
+-- exactly as a revoked one does so its usage history keeps its attribution.
+ALTER TABLE api_keys ADD COLUMN expires_at INTEGER;
+`;
+
 // packages/store/src/sqlite/db.ts
 var MIGRATIONS2 = [
   { id: 1, sql: _001_init_default2 },
@@ -28019,9 +28181,16 @@ var MIGRATIONS2 = [
   { id: 9, sql: _009_key_limits_default },
   { id: 10, sql: _010_usage_rollup_default, after: rebuildRollup },
   { id: 11, sql: _011_plugin_migrations_default },
-  { id: 12, sql: _012_nodes_default }
+  { id: 12, sql: _012_nodes_default },
+  { id: 13, sql: _013_health_measurements_default },
+  { id: 14, sql: _014_key_expiry_default }
 ];
-function openDb(path) {
+function statements(sql) {
+  return sql.split(`
+`).filter((line) => !line.trimStart().startsWith("--")).join(`
+`).split(/;[ \t]*(?:\n|$)/).map((s) => s.trim()).filter((s) => s.length > 0);
+}
+function openDb(path, dayOffsetMinutes = hostDayOffsetMinutes()) {
   const db = new Database(path, { create: true });
   db.run("PRAGMA journal_mode = WAL");
   db.run("PRAGMA synchronous = NORMAL");
@@ -28033,8 +28202,9 @@ function openDb(path) {
     if (done.has(m.id))
       continue;
     db.transaction(() => {
-      db.run(m.sql);
-      m.after?.(db);
+      for (const statement of statements(m.sql))
+        db.run(statement);
+      m.after?.(db, dayOffsetMinutes);
       db.run("INSERT INTO migrations (id, applied_at) VALUES (?, ?)", [m.id, Date.now()]);
     })();
   }
@@ -28069,7 +28239,8 @@ var toKey2 = (r, logger2) => ({
   limits: parseLimits2(r.id, r.limits, logger2),
   bodyLoggingOptOut: r.body_logging_opt_out === 1,
   createdAt: r.created_at,
-  revokedAt: r.revoked_at
+  revokedAt: r.revoked_at,
+  expiresAt: r.expires_at
 });
 function createKeyRepo2(db, logger2 = noopLogger) {
   return {
@@ -28087,8 +28258,8 @@ function createKeyRepo2(db, logger2 = noopLogger) {
     async create(input2) {
       const now = Date.now();
       db.run(`INSERT INTO api_keys (id, label, prefix, hash, model_allowlist, limits,
-                               body_logging_opt_out, created_at, revoked_at)
-         VALUES (?,?,?,?,?,?,?,?,NULL)`, [
+                               body_logging_opt_out, created_at, revoked_at, expires_at)
+         VALUES (?,?,?,?,?,?,?,?,NULL,?)`, [
         input2.id,
         input2.label,
         input2.prefix,
@@ -28096,7 +28267,8 @@ function createKeyRepo2(db, logger2 = noopLogger) {
         input2.modelAllowlist === null ? null : JSON.stringify(input2.modelAllowlist),
         JSON.stringify(parseLimitConfig(input2.limits)),
         input2.bodyLoggingOptOut ? 1 : 0,
-        now
+        now,
+        input2.expiresAt
       ]);
       return { ...input2, createdAt: now, revokedAt: null };
     },
@@ -28104,8 +28276,8 @@ function createKeyRepo2(db, logger2 = noopLogger) {
       if (row.limits === null)
         throw new Error(`api key ${row.id} has unreadable limits`);
       db.run(`INSERT INTO api_keys (id, label, prefix, hash, model_allowlist, limits,
-                               body_logging_opt_out, created_at, revoked_at)
-         VALUES (?,?,?,?,?,?,?,?,?)`, [
+                               body_logging_opt_out, created_at, revoked_at, expires_at)
+         VALUES (?,?,?,?,?,?,?,?,?,?)`, [
         row.id,
         row.label,
         row.prefix,
@@ -28114,7 +28286,8 @@ function createKeyRepo2(db, logger2 = noopLogger) {
         JSON.stringify(parseLimitConfig(row.limits)),
         row.bodyLoggingOptOut ? 1 : 0,
         row.createdAt,
-        row.revokedAt
+        row.revokedAt,
+        row.expiresAt
       ]);
     },
     async setLimits(id, limits) {
@@ -28129,6 +28302,9 @@ function createKeyRepo2(db, logger2 = noopLogger) {
         id
       ]);
     },
+    async setExpiry(id, expiresAt) {
+      db.run("UPDATE api_keys SET expires_at = ? WHERE id = ?", [expiresAt, id]);
+    },
     async revoke(id) {
       db.run("UPDATE api_keys SET revoked_at = ? WHERE id = ?", [Date.now(), id]);
     }
@@ -28451,14 +28627,14 @@ var COMPLETE2 = `INSERT INTO request_logs ${COLUMNS2} VALUES ${PLACEHOLDERS2}
     rtk_estimated_tokens_saved = excluded.rtk_estimated_tokens_saved,
     rtk_filters = excluded.rtk_filters
   WHERE request_logs.state = 'pending'`;
-function createUsageRepo2(db, nodeId) {
+function createUsageRepo2(db, nodeId, dayOffsetMinutes = hostDayOffsetMinutes()) {
   const complete = db.transaction((log) => {
     if (db.run(COMPLETE2, values2(log, "done", nodeId)).changes === 0)
       return;
     const stored = db.query("SELECT * FROM request_logs WHERE id = ?").get(log.id);
     if (stored !== null) {
       const restored = toLog2(stored);
-      rollupLog(db, restored);
+      rollupLog(db, restored, dayOffsetMinutes);
       rollupHour(db, restored);
     }
   });
@@ -28508,6 +28684,10 @@ function createUsageRepo2(db, nodeId) {
       }
       return db.query("SELECT * FROM request_logs WHERE api_key_id = ? ORDER BY at DESC LIMIT ?").all(apiKeyId, limit).map(toLog2);
     },
+    async lastUsedByCredential(credentialId) {
+      const row = db.query("SELECT MAX(at) AS at FROM request_logs WHERE credential_id = ?").get(credentialId);
+      return row?.at ?? null;
+    },
     async scan(cursor, limit) {
       const rows = cursor === null ? db.query("SELECT * FROM request_logs ORDER BY at, id LIMIT ?").all(limit) : db.query(`SELECT * FROM request_logs
                   WHERE at > ? OR (at = ? AND id > ?)
@@ -28569,7 +28749,7 @@ function createUsageRepo2(db, nodeId) {
       const daily = grain === "daily";
       const key = columnFor2(grain, q.groupBy);
       const split = q.splitBy === undefined ? null : columnFor2(grain, q.splitBy);
-      const since = daily ? startOfLocalDay(q.since) : q.since;
+      const since = daily ? startOfDay(q.since, dayOffsetMinutes) : q.since;
       const until = q.until ?? Number.MAX_SAFE_INTEGER;
       const timeColumn = daily ? "day" : "at";
       const scope = q.apiKeyId;
@@ -28620,6 +28800,7 @@ function createUsageRepo2(db, nodeId) {
 async function createStore(opts) {
   const logger2 = opts.logger ?? noopLogger;
   const nodeId = opts.nodeId ?? crypto.randomUUID();
+  const dayOffsetMinutes = opts.dayOffsetMinutes ?? hostDayOffsetMinutes();
   const listeners = new Set;
   const emit = (change) => {
     for (const listener of listeners) {
@@ -28629,14 +28810,14 @@ async function createStore(opts) {
     }
   };
   const open2 = () => {
-    const db = openDb(opts.path);
+    const db = openDb(opts.path, dayOffsetMinutes);
     logger2.debug("store opened", { path: opts.path });
     return {
       db,
       credentials: createCredentialRepo2(db, opts.encryptionKey, emit),
       config: createConfigRepo2(db, emit),
       keys: createKeyRepo2(db, logger2),
-      usage: createUsageRepo2(db, nodeId),
+      usage: createUsageRepo2(db, nodeId, dayOffsetMinutes),
       bodies: createBodyRepo2(db, opts.encryptionKey, bodiesDirFor(opts.path)),
       maintenance: createMaintenanceRepo2(db, nodeId),
       plugins: createPluginRepo2(db)
@@ -28662,7 +28843,6 @@ async function createStore(opts) {
       updateSecrets: (id, secrets, expiresAt, expectedVersion) => handle.credentials.updateSecrets(id, secrets, expiresAt, expectedVersion),
       remove: (id) => handle.credentials.remove(id),
       listHealth: () => handle.credentials.listHealth(),
-      saveHealth: (rows) => handle.credentials.saveHealth(rows),
       updateHealth: (id, model, apply) => handle.credentials.updateHealth(id, model, apply),
       listQuota: () => handle.credentials.listQuota(),
       saveQuota: (rows) => handle.credentials.saveQuota(rows),
@@ -28689,6 +28869,7 @@ async function createStore(opts) {
       importRow: (row) => handle.keys.importRow(row),
       setLimits: (id, limits) => handle.keys.setLimits(id, limits),
       setModelAllowlist: (id, modelAllowlist) => handle.keys.setModelAllowlist(id, modelAllowlist),
+      setExpiry: (id, expiresAt) => handle.keys.setExpiry(id, expiresAt),
       revoke: (id) => handle.keys.revoke(id)
     },
     usage: {
@@ -28697,6 +28878,7 @@ async function createStore(opts) {
       append: (log) => handle.usage.append(log),
       sweepPending: (now) => handle.usage.sweepPending(now),
       recent: (limit, apiKeyId) => handle.usage.recent(limit, apiKeyId),
+      lastUsedByCredential: (credentialId) => handle.usage.lastUsedByCredential(credentialId),
       scan: (cursor, limit) => handle.usage.scan(cursor, limit),
       aggregate: (q) => handle.usage.aggregate(q),
       sumSince: (apiKeyId, sinceMs) => handle.usage.sumSince(apiKeyId, sinceMs),
@@ -28796,7 +28978,7 @@ function createAdminAuth(store, opts) {
   };
   const keyStillValid = async (apiKeyId) => {
     const key = await store.keys.get(apiKeyId);
-    return key !== null && key.revokedAt === null;
+    return key !== null && keyUsable(key, opts.now());
   };
   const applyPassword = async (password) => {
     if (password.length < MIN_PASSWORD_LENGTH) {
@@ -28850,7 +29032,7 @@ function createAdminAuth(store, opts) {
     },
     async loginClient(rawKey) {
       const key = await store.keys.findByHash(await hashApiKey(rawKey));
-      if (key === null || key.revokedAt !== null)
+      if (key === null || !keyUsable(key, opts.now()))
         return null;
       return issue2({ kind: "client", apiKeyId: key.id });
     },
@@ -29689,6 +29871,7 @@ function toClientLog(log) {
 var MIN_KEY_LENGTH = 16;
 var TRUTHY = new Set(["1", "true", "yes", "on"]);
 var DECIMAL_INTEGER = /^\d+$/;
+var SIGNED_INTEGER = /^-?\d+$/;
 function optionalText(value, fallback) {
   return value?.trim() || fallback;
 }
@@ -29719,6 +29902,11 @@ function loadConfig(env) {
   } else if (databaseUrl !== undefined || redisUrl !== undefined) {
     throw new Error("OMNI_DATABASE_URL and OMNI_REDIS_URL are cluster-mode settings; set OMNI_CLUSTER_MODE=true or unset them");
   }
+  const rawDayOffset = env.OMNI_DAY_OFFSET_MINUTES?.trim();
+  const parsedDayOffset = rawDayOffset === undefined ? NaN : Number(rawDayOffset);
+  const validDayOffset = rawDayOffset !== undefined && SIGNED_INTEGER.test(rawDayOffset) && Number.isInteger(parsedDayOffset) && parsedDayOffset >= -1440 && parsedDayOffset <= 1440;
+  const dayOffsetMinutes = validDayOffset ? parsedDayOffset : hostDayOffsetMinutes();
+  const dayOffsetFallbackFrom = rawDayOffset !== undefined && !validDayOffset ? rawDayOffset : null;
   const bodyLoggingAllowed = TRUTHY.has((env.OMNI_BODY_LOGGING_ALLOWED ?? "").trim().toLowerCase());
   const rawLogLevel = env.OMNI_LOG_LEVEL?.trim();
   const logLevel = parseLogLevel(rawLogLevel);
@@ -29749,6 +29937,8 @@ function loadConfig(env) {
     clusterMode,
     databaseUrl: clusterMode ? databaseUrl ?? null : null,
     redisUrl: clusterMode ? redisUrl ?? null : null,
+    dayOffsetMinutes,
+    dayOffsetFallbackFrom,
     metricsToken,
     metricsMaxSeries,
     otlpEndpoint,
@@ -36448,16 +36638,19 @@ var modelSchema = exports_external.object({
   isAlias: exports_external.boolean(),
   targets: exports_external.array(targetSchema).min(1, "a virtual model needs at least one target")
 });
+var MAX_INSTANT = 8640000000000000;
 var keyCreateSchema = exports_external.object({
   label: exports_external.string().min(1).default("api key"),
   modelAllowlist: exports_external.array(exports_external.string().min(1)).nullable().default(null),
   limits: limitConfigSchema.default({}),
-  bodyLoggingOptOut: exports_external.boolean().default(false)
+  bodyLoggingOptOut: exports_external.boolean().default(false),
+  expiresAt: exports_external.number().int().positive().max(MAX_INSTANT).nullable().default(null)
 }).strict();
 var keyLimitsSchema = exports_external.object({ limits: limitConfigSchema }).strict();
 var keyModelsSchema = exports_external.object({
   modelAllowlist: exports_external.array(exports_external.string().min(1)).nullable()
 }).strict();
+var keyExpirySchema = exports_external.object({ expiresAt: exports_external.number().int().positive().max(MAX_INSTANT).nullable() }).strict();
 var retentionSchema = exports_external.object({
   keepLatest: exports_external.number().int().min(1).max(100),
   maxAgeDays: exports_external.number().int().min(1).max(3650)
@@ -36672,10 +36865,11 @@ function eligible(input2) {
         drop("disabled");
         continue;
       }
-      if (credential.authType === "oauth" && credential.expiresAt !== null && credential.expiresAt <= now && !credential.hasRefreshToken) {
+      if (credentialExpired(credential, now)) {
         drop("expired");
         continue;
       }
+      let probe = false;
       const h = snapshot.health.get(healthKey(credential.id, target.model));
       if (h !== undefined) {
         if (h.rateLimitedUntil !== null && h.rateLimitedUntil > now) {
@@ -36688,6 +36882,9 @@ function eligible(input2) {
             drop("breaker:open");
             continue;
           }
+          probe = true;
+        } else if (h.breakerState === "halfOpen") {
+          probe = true;
         }
       }
       const spent = (snapshot.quota.get(credential.id) ?? []).find((w) => w.limit !== null && w.used >= w.limit && (w.resetsAt === null || w.resetsAt > now));
@@ -36695,7 +36892,7 @@ function eligible(input2) {
         drop(`quota:${spent.windowType}`);
         continue;
       }
-      pairs.push({ credential, target });
+      pairs.push({ credential, target, probe });
     }
     if (target.credentialId === undefined && providerSeen && !servedAny) {
       excluded.push({
@@ -36772,7 +36969,7 @@ function healthScore(h) {
   return h?.breakerState === "open" || h?.breakerState === "halfOpen" ? base * 0.5 : base;
 }
 function score(pairs, input2) {
-  const { snapshot, now, load } = input2;
+  const { snapshot, now, load, recent } = input2;
   const w = snapshot.settings.weights;
   const tiers = pairs.map((p) => p.target.tier);
   const minTier = Math.min(...tiers);
@@ -36780,9 +36977,10 @@ function score(pairs, input2) {
   const tokens2 = estimateTokens(input2.request);
   const costs = pairs.map((p) => requestCost(p.target, tokens2));
   const bestCost = bestPositive(costs);
+  const ewmaOf = (key) => recent?.get(key)?.ewmaTtftMs ?? null;
   const bestLatency = bestPositive(pairs.flatMap((p) => {
-    const h = snapshot.health.get(healthKey(p.credential.id, p.target.model));
-    return h?.ewmaTtftMs != null ? [h.ewmaTtftMs] : [];
+    const ewma = ewmaOf(healthKey(p.credential.id, p.target.model));
+    return ewma === null ? [] : [ewma];
   }));
   return pairs.map((pair, i) => {
     const key = healthKey(pair.credential.id, pair.target.model);
@@ -36794,12 +36992,14 @@ function score(pairs, input2) {
     const quota = quotaHeadroom(pair.credential, snapshot.quota.get(pair.credential.id) ?? [], now, snapshot.settings.quotaPollIntervalMs);
     const ownCost = costs[i];
     const cost = ownCost <= 0 || bestCost === null ? UNKNOWN : ratio(ownCost, bestCost);
-    const latency = h?.ewmaTtftMs == null || h.ewmaTtftMs <= 0 || bestLatency === null ? UNKNOWN : ratio(h.ewmaTtftMs, bestLatency);
+    const ewma = ewmaOf(key);
+    const latency = ewma === null || ewma <= 0 || bestLatency === null ? UNKNOWN : ratio(ewma, bestLatency);
     const reasons = { tier, health, quota, cost, latency, load: loadTerm };
     const base = tier * w.tier + health * w.health + quota * w.quota + cost * w.cost + latency * w.latency + loadTerm * w.load;
     return {
       credential: pair.credential,
       target: pair.target,
+      probe: pair.probe,
       score: base * pair.credential.weight * pair.target.weight,
       reasons
     };
@@ -36825,7 +37025,6 @@ var PENALTY = {
   ALL_CANDIDATES_FAILED: "none",
   INTERNAL: "none"
 };
-var EWMA_ALPHA = 0.3;
 var DEFAULT_RATE_LIMIT_MS = 60000;
 var QUOTA_PARK_MS = 3600000;
 var MAX_JITTER_MS = 2000;
@@ -36836,21 +37035,28 @@ function blankHealth(credentialId, model) {
     breakerState: "closed",
     consecutiveFailures: 0,
     openedAt: null,
-    rateLimitedUntil: null,
-    ewmaTtftMs: null,
-    lastUsedAt: null
+    rateLimitedUntil: null
   };
 }
-function recordSuccess(current, opts) {
-  const ewma = opts.ttftMs === null ? current.ewmaTtftMs : current.ewmaTtftMs === null ? opts.ttftMs : current.ewmaTtftMs * (1 - EWMA_ALPHA) + opts.ttftMs * EWMA_ALPHA;
+var SUCCESS_RESETS = [
+  "breakerState",
+  "consecutiveFailures",
+  "openedAt",
+  "rateLimitedUntil"
+];
+function successWouldChange(current) {
+  if (current === undefined)
+    return false;
+  const blank = blankHealth(current.credentialId, current.model);
+  return SUCCESS_RESETS.some((field) => current[field] !== blank[field]);
+}
+function recordSuccess(current) {
   return {
     ...current,
     breakerState: "closed",
     consecutiveFailures: 0,
     openedAt: null,
-    rateLimitedUntil: null,
-    ewmaTtftMs: ewma,
-    lastUsedAt: opts.now
+    rateLimitedUntil: null
   };
 }
 function recordFailure(current, opts) {
@@ -36860,7 +37066,7 @@ function recordFailure(current, opts) {
   if (penalty === "soft") {
     const base = opts.code === "QUOTA_EXHAUSTED" ? QUOTA_PARK_MS : opts.retryAfterMs ?? DEFAULT_RATE_LIMIT_MS;
     const until = opts.now + base + Math.round((opts.jitter ?? 0) * MAX_JITTER_MS);
-    return { ...current, rateLimitedUntil: until, lastUsedAt: opts.now };
+    return { ...current, rateLimitedUntil: until };
   }
   const failures = current.consecutiveFailures + 1;
   const open2 = opts.code === "AUTH" || current.breakerState === "halfOpen" || failures >= opts.settings.breakerThreshold;
@@ -36868,8 +37074,7 @@ function recordFailure(current, opts) {
     ...current,
     consecutiveFailures: failures,
     breakerState: open2 ? "open" : "closed",
-    openedAt: open2 ? opts.now : current.openedAt,
-    lastUsedAt: opts.now
+    openedAt: open2 ? opts.now : current.openedAt
   };
 }
 // packages/router/src/resolve.ts
@@ -36950,8 +37155,8 @@ function rank(input2) {
       break;
     case "roundRobin": {
       const idle = (c) => {
-        const h = input2.snapshot.health.get(healthKey(c.credential.id, c.target.model));
-        return h?.lastUsedAt == null ? Number.POSITIVE_INFINITY : input2.now - h.lastUsedAt;
+        const at = input2.recent?.get(healthKey(c.credential.id, c.target.model))?.lastReleasedAt;
+        return at === undefined ? Number.POSITIVE_INFINITY : input2.now - at;
       };
       const spent = (c) => headroom(c) < QUOTA_FLOOR ? 1 : 0;
       scored.sort((a, b) => spent(a) - spent(b) || inflight(a) - inflight(b) || idle(b) - idle(a));
@@ -37107,16 +37312,23 @@ async function createApiKeyCredential(store, input2, logger2 = noopLogger, provi
   return summarizeCredential(created);
 }
 async function credentialHealth(deps) {
-  const [health, quota, settings] = await Promise.all([
+  const [health, quota, settings, credentials] = await Promise.all([
     deps.store.credentials.listHealth(),
     deps.store.credentials.listQuota(),
-    deps.store.config.getSettings()
+    deps.store.config.getSettings(),
+    deps.store.credentials.list()
   ]);
+  const lastUsed = {};
+  for (const c of credentials) {
+    const at = await deps.store.usage.lastUsedByCredential(c.id);
+    if (at !== null)
+      lastUsed[c.id] = at;
+  }
   const burn = burnEstimates(quota, {
     now: deps.now(),
     pollIntervalMs: settings.quotaPollIntervalMs
   });
-  return { health, quota, burn };
+  return { health, quota, burn, lastUsed };
 }
 async function patchCredential(deps, id, input2) {
   const patch = parseOrThrow(credentialPatchSchema, input2);
@@ -37228,14 +37440,14 @@ function listSnapshots(deps) {
   }
   return snapshots.sort((a, b) => b.createdAt - a.createdAt);
 }
-var DAY_MS = 86400000;
+var DAY_MS2 = 86400000;
 var SNAPSHOT_HEADROOM = 1.1;
 function retentionOf(settings) {
   return { keepLatest: settings.snapshotKeepLatest, maxAgeDays: settings.snapshotMaxAgeDays };
 }
 function applyRetention(deps, policy, now) {
   const dir = snapshotsDir(deps);
-  const cutoff = now - policy.maxAgeDays * DAY_MS;
+  const cutoff = now - policy.maxAgeDays * DAY_MS2;
   const snapshots = listSnapshots(deps);
   const undo = snapshots.find((snapshot) => snapshot.reason === "preRestore")?.id;
   let removed = 0;
@@ -37599,7 +37811,8 @@ async function toSummary(store, key, now) {
     limitUsage: await readLimitUsage(store, key.id, key.limits, now),
     bodyLoggingOptOut: key.bodyLoggingOptOut,
     createdAt: key.createdAt,
-    revokedAt: key.revokedAt
+    revokedAt: key.revokedAt,
+    expiresAt: key.expiresAt
   };
 }
 async function listKeys(store, now = Date.now()) {
@@ -37622,7 +37835,8 @@ async function createKey(store, input2) {
     hash: await hashApiKey(raw),
     modelAllowlist: body2.modelAllowlist,
     limits: body2.limits,
-    bodyLoggingOptOut: body2.bodyLoggingOptOut
+    bodyLoggingOptOut: body2.bodyLoggingOptOut,
+    expiresAt: body2.expiresAt
   });
   return { id: created.id, label: created.label, prefix: created.prefix, key: raw };
 }
@@ -37642,6 +37856,14 @@ async function setKeyModels(store, id, input2, now = Date.now()) {
   await store.keys.setModelAllowlist(id, body2.modelAllowlist);
   return toSummary(store, { ...key, modelAllowlist: body2.modelAllowlist }, now);
 }
+async function setKeyExpiry(store, id, input2, now = Date.now()) {
+  const body2 = parseOrThrow(keyExpirySchema, input2);
+  const key = await store.keys.get(id);
+  if (key === null)
+    throw new GatewayError("BAD_REQUEST", "no such api key");
+  await store.keys.setExpiry(id, body2.expiresAt);
+  return toSummary(store, { ...key, expiresAt: body2.expiresAt }, now);
+}
 async function revokeKey(store, id) {
   await store.keys.revoke(id);
 }
@@ -38490,12 +38712,18 @@ async function accountQuota(deps) {
 }
 
 // packages/control/src/quota/history.ts
-var DAY_MS2 = 24 * 60 * 60 * 1000;
+var DAY_MS3 = 24 * 60 * 60 * 1000;
 var HOUR_MS3 = 3600000;
+var MAX_SAMPLES = 50000;
+var SAMPLE_QUERY_LIMIT = MAX_SAMPLES + 1;
+function pageSamples(samples) {
+  const truncated = samples.length > MAX_SAMPLES;
+  return { page: truncated ? samples.slice(0, MAX_SAMPLES) : samples, truncated };
+}
 async function retainedSpan(deps, input2, maxSpanMs) {
   const now = deps.now();
   const settings = await deps.store.config.getSettings();
-  const retained = now - settings.logRetentionDays * DAY_MS2;
+  const retained = now - settings.logRetentionDays * DAY_MS3;
   const oldest = maxSpanMs === undefined ? retained : Math.max(retained, now - maxSpanMs);
   return {
     since: Math.max(optionalNumber(input2.since, oldest), oldest),
@@ -38538,26 +38766,26 @@ async function quotaHistory(deps, input2) {
     deps.store.credentials.listQuotaSamples({
       since,
       until,
+      limit: SAMPLE_QUERY_LIMIT,
       ...credentialId === undefined ? {} : { credentialId }
     }),
     deps.store.credentials.listQuota()
   ]);
   const scoped = credentialId === undefined ? quota : quota.filter((w) => w.credentialId === credentialId);
-  return { samples, gatewayRates: await gatewayRatesFor(deps.store, scoped) };
+  const { page, truncated } = pageSamples(samples);
+  return { samples: page, truncated, gatewayRates: await gatewayRatesFor(deps.store, scoped) };
 }
 
 // packages/control/src/quota/clientHistory.ts
 var MAX_SPAN_MS = 16 * 24 * 60 * 60 * 1000;
-var MAX_SAMPLES = 50000;
 async function accountQuotaHistory(deps, input2) {
   const { since, until } = await retainedSpan(deps, input2, MAX_SPAN_MS);
   const [samples, credentials] = await Promise.all([
-    deps.store.credentials.listQuotaSamples({ since, until, limit: MAX_SAMPLES + 1 }),
+    deps.store.credentials.listQuotaSamples({ since, until, limit: SAMPLE_QUERY_LIMIT }),
     deps.store.credentials.list()
   ]);
   const byId = new Map(credentials.map((credential) => [credential.id, credential]));
-  const truncated = samples.length > MAX_SAMPLES;
-  const page = truncated ? samples.slice(0, MAX_SAMPLES) : samples;
+  const { page, truncated } = pageSamples(samples);
   const out = [];
   for (const sample of page) {
     const credential = byId.get(sample.credentialId);
@@ -53559,9 +53787,11 @@ function withReset(headroom, window2, resetAt) {
 // apps/gateway/src/dispatch/loadRegistry.ts
 var PREFIX6 = "load:";
 var SLOT_TTL_MS = 300000;
-function createLoadRegistry(coord = memoryCoord()) {
+var EWMA_ALPHA = 0.3;
+function createLoadRegistry(coord = memoryCoord(), now = Date.now) {
   const local = new Map;
   const providers = new Map;
+  const history = new Map;
   let remote = new Map;
   return {
     acquire(credentialId, model, provider) {
@@ -53571,10 +53801,15 @@ function createLoadRegistry(coord = memoryCoord()) {
         providers.set(provider, (providers.get(provider) ?? 0) + 1);
       coord.gauge.acquire(PREFIX6 + key, SLOT_TTL_MS);
       let released = false;
-      return () => {
+      return (ttftMs) => {
         if (released)
           return;
         released = true;
+        const prior = history.get(key)?.ewmaTtftMs ?? null;
+        history.set(key, {
+          lastReleasedAt: now(),
+          ewmaTtftMs: ttftMs === undefined || ttftMs === null ? prior : prior === null ? ttftMs : prior * (1 - EWMA_ALPHA) + ttftMs * EWMA_ALPHA
+        });
         const next = (local.get(key) ?? 0) - 1;
         if (next > 0)
           local.set(key, next);
@@ -53601,6 +53836,9 @@ function createLoadRegistry(coord = memoryCoord()) {
     localCounts() {
       return new Map(providers);
     },
+    recent() {
+      return history;
+    },
     async refresh() {
       const held = await coord.gauge.snapshot(PREFIX6);
       const sample = new Map;
@@ -54969,6 +55207,11 @@ function adminRoutes(deps) {
     const key = await setKeyModels(deps.store, params.id, await readJson(request2));
     changed("res:keys");
     return key;
+  }).put("/api/keys/:id/expiry", async ({ request: request2, params }) => {
+    await requireAdmin(request2, deps.admin);
+    const key = await setKeyExpiry(deps.store, params.id, await readJson(request2));
+    changed("res:keys");
+    return key;
   }).delete("/api/keys/:id", async ({ request: request2, params }) => {
     await requireAdmin(request2, deps.admin);
     await revokeKey(deps.store, params.id);
@@ -54979,6 +55222,7 @@ function adminRoutes(deps) {
     return {
       settings: await getSettings(deps.store),
       bodyLoggingAllowed: deps.bodyLoggingAllowed === true,
+      dayOffsetMinutes: deps.dayOffsetMinutes ?? hostDayOffsetMinutes(),
       viewerConfigured: await deps.admin.isViewerConfigured()
     };
   }).put("/api/settings/password", async ({ request: request2 }) => {
@@ -55113,7 +55357,10 @@ function clientRoutes(deps) {
     return { ok: true };
   }).get("/api/client/summary", async ({ request: request2 }) => {
     const apiKeyId = await requireClient(request2, deps.admin);
-    return readOwnKey(deps.store, apiKeyId, now());
+    return {
+      ...await readOwnKey(deps.store, apiKeyId, now()),
+      dayOffsetMinutes: deps.dayOffsetMinutes ?? hostDayOffsetMinutes()
+    };
   }).get("/api/client/usage", async ({ request: request2, query }) => {
     const apiKeyId = await requireClient(request2, deps.admin);
     return queryUsage({ store: deps.store, now }, {
@@ -55513,12 +55760,12 @@ function injectPonytail(request2, opts) {
 }
 
 // apps/gateway/src/auth/apiKey.ts
-async function authenticateApiKey(store, header, logger2 = noopLogger) {
+async function authenticateApiKey(store, header, now, logger2 = noopLogger) {
   const raw = extractToken(header);
   if (raw === null)
     throw new GatewayError("AUTH", "missing or malformed Authorization header");
   const key = await store.keys.findByHash(await hashApiKey(raw));
-  if (key === null || key.revokedAt !== null) {
+  if (key === null || !keyUsable(key, now)) {
     throw new GatewayError("AUTH", "invalid API key");
   }
   const limits = key.limits;
@@ -57826,6 +58073,7 @@ function priceOf(prices, usage, provider, providers = PROVIDER_DESCRIPTORS) {
 }
 
 // apps/gateway/src/dispatch/index.ts
+var PROBE_TTL_MS = 180000;
 async function dispatch(request2, deps, signal, requestId) {
   const logger2 = deps.logger ?? noopLogger;
   const startedAt = deps.now();
@@ -57919,6 +58167,7 @@ async function dispatch(request2, deps, signal, requestId) {
     now: startedAt,
     rand: deps.rand(),
     load: deps.loadRegistry.counts(),
+    recent: deps.loadRegistry.recent(),
     providers: deps.providers
   });
   if (deps.trace !== null && deps.trace !== undefined) {
@@ -57980,8 +58229,11 @@ async function dispatch(request2, deps, signal, requestId) {
   async function* run() {
     try {
       let lastError = null;
+      let attempts = 0;
       candidateLoop:
-        for (let i = 0;i < maxAttempts; i++) {
+        for (const candidate of candidates) {
+          if (attempts >= maxAttempts)
+            break;
           try {
             checkCancellation();
           } catch (error61) {
@@ -57990,22 +58242,45 @@ async function dispatch(request2, deps, signal, requestId) {
             lastError = error61 instanceof GatewayError ? error61 : new GatewayError("TIMEOUT", "request deadline exceeded");
             break;
           }
-          const candidate = candidates[i];
           const adoptable = eagerHeld && eager.credentialId === candidate.credential.id && eager.model === candidate.target.model;
           if (adoptable)
             eagerHeld = false;
           const releaseSlot = adoptable ? eager.release : deps.loadRegistry.acquire(candidate.credential.id, candidate.target.model, candidate.target.provider);
+          const successWrites = successWouldChange(snapshot.health.get(healthKey(candidate.credential.id, candidate.target.model)));
           const attemptStartedAt = deps.now();
           let attemptCode;
-          if (deps.trace !== null && deps.trace !== undefined) {
-            deps.trace.activeAttempt = addSpan(deps.trace, "dispatch.attempt", 0, attemptStartedAt - startedAt, attemptStartedAt - startedAt, {
-              attempt: i + 1,
-              provider: candidate.target.provider,
-              model: candidate.target.model,
-              credential_id: candidate.credential.id
-            });
-          }
+          let releaseProbe;
+          let probing = false;
           try {
+            if (candidate.probe) {
+              const key = `probe:${healthKey(candidate.credential.id, candidate.target.model)}`;
+              const held = await deps.coord.gauge.acquire(key, PROBE_TTL_MS);
+              releaseProbe = () => deps.coord.gauge.release(key);
+              if (held > 0) {
+                noteDegradations([`excluded:${candidate.credential.id}:breaker:probing`]);
+                logger2.debug("routing candidate excluded", {
+                  requestId,
+                  credentialId: candidate.credential.id,
+                  reason: "breaker:probing"
+                });
+                continue;
+              }
+              await persistHealth(candidate, (current) => {
+                if (current.breakerState === "closed")
+                  return current;
+                probing = true;
+                return { ...current, breakerState: "halfOpen" };
+              });
+            }
+            const i = attempts++;
+            if (deps.trace !== null && deps.trace !== undefined) {
+              deps.trace.activeAttempt = addSpan(deps.trace, "dispatch.attempt", 0, attemptStartedAt - startedAt, attemptStartedAt - startedAt, {
+                attempt: i + 1,
+                provider: candidate.target.provider,
+                model: candidate.target.model,
+                credential_id: candidate.credential.id
+              });
+            }
             log.attempts = i + 1;
             log.credentialId = candidate.credential.id;
             log.resolvedProvider = candidate.target.provider;
@@ -58028,6 +58303,7 @@ async function dispatch(request2, deps, signal, requestId) {
             log.cacheWriteTokens = 0;
             log.ttftMs = null;
             let committed = false;
+            let closedAtCommit = false;
             let authRefreshRetried = false;
             let retrySecrets;
             const attemptNow = deps.now();
@@ -58082,6 +58358,11 @@ async function dispatch(request2, deps, signal, requestId) {
                     for (const buffered of pending)
                       yield buffered;
                     pending.length = 0;
+                    if (probing) {
+                      await persistHealth(candidate, recordSuccess);
+                      probing = false;
+                      closedAtCommit = true;
+                    }
                   }
                   if (event.type === "end") {
                     terminal = true;
@@ -58139,11 +58420,8 @@ async function dispatch(request2, deps, signal, requestId) {
                 if (!terminal) {
                   throw new GatewayError("UPSTREAM", "upstream stream ended without a terminal event");
                 }
-                await persistHealth(candidate, (current) => recordSuccess(current, {
-                  settings: snapshot.settings,
-                  now: deps.now(),
-                  ttftMs: log.ttftMs
-                }));
+                if (successWrites && !closedAtCommit)
+                  await persistHealth(candidate, recordSuccess);
                 log.status = 200;
                 log.errorCode = null;
                 log.durationMs = deps.now() - startedAt;
@@ -58196,7 +58474,7 @@ async function dispatch(request2, deps, signal, requestId) {
                   ...failure.retryAfterMs === undefined ? {} : { retryAfterMs: failure.retryAfterMs },
                   jitter: deps.rand()
                 }));
-                if (!committed && RETRYABLE[failure.code] && i + 1 < maxAttempts) {
+                if (!committed && RETRYABLE[failure.code] && attempts < maxAttempts) {
                   logger2.warn("attempt failed; retrying", {
                     requestId,
                     provider: candidate.target.provider,
@@ -58236,14 +58514,16 @@ async function dispatch(request2, deps, signal, requestId) {
               }
               deps.trace.activeAttempt = null;
             }
-            releaseSlot();
+            releaseSlot(log.ttftMs);
+            releaseProbe?.();
           }
         }
-      const code = lastError?.code === "TIMEOUT" ? "TIMEOUT" : lastError !== null && !RETRYABLE[lastError.code] ? lastError.code : "ALL_CANDIDATES_FAILED";
+      const untried = attempts === 0 && lastError === null;
+      const code = lastError?.code === "TIMEOUT" ? "TIMEOUT" : lastError !== null && !RETRYABLE[lastError.code] ? lastError.code : untried ? "NO_CANDIDATES" : "ALL_CANDIDATES_FAILED";
       log.status = HTTP_STATUS[code];
       log.errorCode = code;
       log.durationMs = deps.now() - startedAt;
-      const message = lastError?.message ?? "all candidates failed";
+      const message = lastError?.message ?? (untried ? `no eligible credential for model "${safeToken(request2.model)}"` : "all candidates failed");
       reject(lastError?.provider === undefined ? new GatewayError(code, message) : new GatewayError(code, message, {
         provider: lastError.provider,
         ...lastError.gatewayAuthored ? { gatewayAuthored: true } : {}
@@ -59549,7 +59829,7 @@ async function handle(deps, rateLimiter, surface, request2) {
   try {
     let key;
     try {
-      key = await authenticateApiKey(deps.store, apiKeyHeader(request2.headers), deps.logger);
+      key = await authenticateApiKey(deps.store, apiKeyHeader(request2.headers), startedAt, deps.logger);
     } catch (error61) {
       if (error61 instanceof GatewayError && error61.code === "AUTH") {
         deps.logger.warn("authentication rejected", {
@@ -59706,17 +59986,14 @@ async function handle(deps, rateLimiter, surface, request2) {
 }
 function proxyRoutes(deps) {
   const logger2 = deps.logger ?? noopLogger;
-  const rateLimiter = deps.rateLimiter ?? new ApiKeyRateLimiter({
-    store: deps.store,
-    now: deps.now,
-    logger: logger2,
-    ...deps.coord === undefined ? {} : { coord: deps.coord }
-  });
+  const coord = deps.coord ?? memoryCoord();
+  const rateLimiter = deps.rateLimiter ?? new ApiKeyRateLimiter({ store: deps.store, now: deps.now, logger: logger2, coord });
   const dispatchDeps = {
     ...deps,
     logger: logger2,
+    coord,
     snapshots: deps.snapshots ?? createRoutingSnapshotCache(deps.store, logger2),
-    loadRegistry: deps.loadRegistry ?? createLoadRegistry(deps.coord),
+    loadRegistry: deps.loadRegistry ?? createLoadRegistry(coord),
     keepaliveMs: deps.keepaliveMs ?? KEEPALIVE_MS
   };
   return new Elysia().post("/v1/messages", ({ request: request2, server }) => {
@@ -59730,7 +60007,7 @@ function proxyRoutes(deps) {
     return handle(dispatchDeps, rateLimiter, "openai", request2);
   }).get("/v1/models", async ({ request: request2 }) => {
     try {
-      const key = await authenticateApiKey(deps.store, apiKeyHeader(request2.headers), logger2);
+      const key = await authenticateApiKey(deps.store, apiKeyHeader(request2.headers), deps.now(), logger2);
       const snapshot = await dispatchDeps.snapshots.get(deps.now());
       const models = [...snapshot.models.values()];
       const visibleModels = key.modelAllowlist === null ? models : models.filter((model) => key.modelAllowlist?.includes(model.id));
@@ -59746,7 +60023,7 @@ function proxyRoutes(deps) {
     }
   }).post("/v1/messages/count_tokens", async ({ request: request2 }) => {
     try {
-      const key = await authenticateApiKey(deps.store, apiKeyHeader(request2.headers), logger2);
+      const key = await authenticateApiKey(deps.store, apiKeyHeader(request2.headers), deps.now(), logger2);
       await rateLimiter.consume(key.id, key.limits, deps.requestId());
       const body2 = await request2.json();
       const chatRequest = parseAnthropicRequest(body2, request2.headers);
@@ -60597,7 +60874,8 @@ function sizeOf(payload) {
     return 0;
   }
 }
-function createRing(limits) {
+var DEFAULT_RING_LIMITS = { frames: 500, bytes: 2 * 1024 * 1024 };
+function createRing(limits = DEFAULT_RING_LIMITS) {
   const topics = new Map;
   const stateFor = (topic) => {
     const existing = topics.get(topic);
@@ -60660,7 +60938,7 @@ function createRing(limits) {
 }
 
 // apps/gateway/src/version.ts
-var VERSION = "0.10.3";
+var VERSION = "0.11.1";
 
 // apps/gateway/src/app.ts
 var ADMIN_SESSION_TTL_MS = 12 * 60 * 60 * 1000;
@@ -60747,7 +61025,7 @@ function createApp(deps) {
     now,
     onDetach: (id, topics) => channelsRef?.closed(id, topics)
   });
-  const ring = deps.ring ?? createRing({ frames: 500, bytes: 2 * 1024 * 1024 });
+  const ring = deps.ring ?? createRing();
   const broadcaster = deps.broadcaster ?? createBroadcaster({ registry: registry2, ring, coord, nodeId, now });
   const channels = deps.channels ?? createChannelRegistry({
     sockets: registry2,
@@ -60788,7 +61066,7 @@ function createApp(deps) {
       return new Response(null, { status: 401 });
     }
     const coordFallback = deps.coordHealthy?.() === false;
-    return new Response(telemetry.scrape(loadRegistry.localCounts?.() ?? new Map, registry2.stats(), coordFallback), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" } });
+    return new Response(telemetry.scrape(loadRegistry.localCounts?.() ?? new Map, registry2.stats(), coordFallback, deps.coordFaults?.() ?? 0), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" } });
   })).use(proxyRoutes({
     store: deps.store,
     snapshots,
@@ -60811,6 +61089,7 @@ function createApp(deps) {
     admin,
     baseUrl: deps.baseUrl,
     bodyLoggingAllowed: deps.bodyLoggingAllowed === true,
+    ...deps.dayOffsetMinutes === undefined ? {} : { dayOffsetMinutes: deps.dayOffsetMinutes },
     now,
     sessionTtlMs: ADMIN_SESSION_TTL_MS,
     logger: logger2,
@@ -60822,6 +61101,7 @@ function createApp(deps) {
     store: deps.store,
     admin,
     sessionTtlMs: ADMIN_SESSION_TTL_MS,
+    ...deps.dayOffsetMinutes === undefined ? {} : { dayOffsetMinutes: deps.dayOffsetMinutes },
     now,
     logger: logger2
   })).use(databaseRoutes({
@@ -60917,6 +61197,7 @@ var CHANNEL2 = `${NS}pubsub`;
 var LOCK_POLL_MS = 50;
 var FAULT_LOG_INTERVAL_MS = 30000;
 var CONNECT_WAIT_MS = 2000;
+var COMMAND_TIMEOUT_MS = 1000;
 var RECONNECTED2 = "coord:reconnected";
 var WINDOW_CLAIM = `
   redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', tonumber(ARGV[1]) - tonumber(ARGV[2]))
@@ -61030,10 +61311,17 @@ var SCRIPTS = [
   LEASE_ACQUIRE,
   PUBLISH_SEQUENCED
 ];
+
+class TimedOut extends Error {
+  constructor() {
+    super("command timed out");
+  }
+}
 function redisCoord(deps) {
   const logger2 = deps.logger ?? noopLogger;
   const now = deps.now ?? (() => Date.now());
   const faultInterval = deps.faultLogIntervalMs ?? FAULT_LOG_INTERVAL_MS;
+  const commandTimeout = deps.commandTimeoutMs ?? COMMAND_TIMEOUT_MS;
   const fallback = memoryCoord({ now });
   const client = new RedisClient(deps.url, {
     connectionTimeout: 2000,
@@ -61047,17 +61335,15 @@ function redisCoord(deps) {
     const colon = name.indexOf(":");
     return `${NS}gidx:${colon === -1 ? name : name.slice(0, colon + 1)}`;
   };
+  let faults = 0;
   const fault = (error61) => {
     healthy = false;
+    faults++;
     const at = now();
     if (at - lastFault < faultInterval)
       return;
     lastFault = at;
-    logger2.warn("coordinator unreachable; serving from memory", {
-      coord: "redis",
-      coordFallback: true,
-      reason: describeError(error61, "unknown")
-    });
+    logger2.warn(error61 instanceof TimedOut ? "coordinator slow; serving from memory" : "coordinator unreachable; serving from memory", { coord: "redis", coordFallback: true, reason: describeError(error61, "unknown") });
   };
   const shas = new Map;
   const loadScripts = async () => {
@@ -61085,22 +61371,39 @@ function redisCoord(deps) {
     }, (error61) => fault(error61)),
     Bun.sleep(CONNECT_WAIT_MS)
   ]);
+  let lostDebit = false;
   const reseedAfterOutage = async () => {
     const keys = await scan(client, `${key("b", "")}*`);
     if (keys.length > 0)
       await client.send("UNLINK", keys);
   };
+  const deadline = async (redis) => {
+    let timer;
+    try {
+      return await Promise.race([
+        redis(),
+        new Promise((_resolve, reject) => {
+          timer = setTimeout(() => reject(new TimedOut), commandTimeout);
+        })
+      ]);
+    } finally {
+      clearTimeout(timer);
+    }
+  };
   const attempt2 = async (redis, instead) => {
     try {
       await ready;
       if (!client.connected)
         throw new Error("not connected");
-      const out = await redis();
-      if (!healthy && lastFault !== Number.NEGATIVE_INFINITY) {
-        healthy = true;
-        reseedAfterOutage().catch(fault);
-      }
+      const out = await deadline(redis);
       healthy = true;
+      if (lostDebit) {
+        lostDebit = false;
+        reseedAfterOutage().catch((error61) => {
+          lostDebit = true;
+          fault(error61);
+        });
+      }
       return out;
     } catch (error61) {
       fault(error61);
@@ -61146,6 +61449,7 @@ function redisCoord(deps) {
   subscriber.connect().catch(fault);
   return {
     healthy: () => healthy,
+    faults: () => faults,
     close() {
       subscriber.close();
       client.close();
@@ -61192,7 +61496,10 @@ function redisCoord(deps) {
       add(name, grainMs, windowMs, at, delta2) {
         return attempt2(async () => {
           await evalScript(BUCKETS_ADD, [key("b", name)], [at, grainMs, windowMs, delta2.requests, delta2.tokens, delta2.costUsd]);
-        }, () => fallback.buckets.add(name, grainMs, windowMs, at, delta2));
+        }, () => {
+          lostDebit = true;
+          return fallback.buckets.add(name, grainMs, windowMs, at, delta2);
+        });
       },
       sum(name, grainMs, windowMs, at) {
         return attempt2(async () => {
@@ -61231,14 +61538,15 @@ function redisCoord(deps) {
       async withLock(name, ttlMs, waitMs, fn) {
         const full = key("lock", name);
         const token = crypto.randomUUID();
-        const deadline = performance.now() + waitMs;
+        const deadline2 = performance.now() + waitMs;
         for (;; ) {
           const taken = await attempt2(async () => await client.set(full, token, "PX", String(ttlMs), "NX") === "OK", async () => {
+            evalScript(MUTEX_RELEASE, [full], [token]).catch(() => {});
             throw new LockUnavailable(name);
           });
           if (taken)
             break;
-          if (performance.now() >= deadline)
+          if (performance.now() >= deadline2)
             throw new LockUnavailable(name);
           await Bun.sleep(LOCK_POLL_MS);
         }
@@ -61274,17 +61582,17 @@ function redisCoord(deps) {
       }
     },
     pubsub: {
-      publish(topic, payload) {
+      async publish(topic, payload) {
+        await listened;
         return attempt2(async () => {
-          await listened;
           if (!listening)
             throw new Error("not listening");
           await client.publish(CHANNEL2, JSON.stringify({ topic, payload }));
         }, () => fallback.pubsub.publish(topic, payload));
       },
-      publishSequenced(topic, payload, seqKey) {
+      async publishSequenced(topic, payload, seqKey) {
+        await listened;
         return attempt2(async () => {
-          await listened;
           if (!listening)
             throw new Error("not listening");
           return await evalScript(PUBLISH_SEQUENCED, [key("seq", seqKey)], [CHANNEL2, topic, payload]);
@@ -61378,13 +61686,13 @@ async function underLease(lease, name, ttlMs, fn) {
 
 // apps/gateway/src/maintenance.ts
 var SWEEP_INTERVAL_MS2 = 60 * 60 * 1000;
-var DAY_MS3 = 24 * 60 * 60 * 1000;
+var DAY_MS4 = 24 * 60 * 60 * 1000;
 var ROLLUP_RETENTION_DAYS = 400;
 async function pruneLogs(store, now) {
   const settings = await store.config.getSettings();
-  const rawHorizon = now - settings.logRetentionDays * DAY_MS3;
+  const rawHorizon = now - settings.logRetentionDays * DAY_MS4;
   const raw = await store.usage.prune(rawHorizon);
-  const daily = await store.usage.pruneDaily(now - ROLLUP_RETENTION_DAYS * DAY_MS3);
+  const daily = await store.usage.pruneDaily(now - ROLLUP_RETENTION_DAYS * DAY_MS4);
   const quotaSamples = await store.credentials.pruneQuotaSamples(rawHorizon);
   const bodies = await store.bodies.prune(rawHorizon);
   const bodiesOverCap = await store.bodies.pruneToCap();
@@ -61455,7 +61763,7 @@ async function sweep(deps) {
   let touched = 0;
   for (const credential of due(credentials, deps.now())) {
     if (!credential.hasRefreshToken) {
-      if (credential.expiresAt !== null && credential.expiresAt <= deps.now()) {
+      if (credentialExpired(credential, deps.now())) {
         await deps.store.credentials.update(credential.id, {
           enabled: false,
           disabledReason: "expiredNoRefresh",
@@ -62147,6 +62455,10 @@ var DEFINITIONS = {
     help: "Whether this process is using coordination fallback.",
     type: "gauge"
   },
+  omni_coord_faults_total: {
+    help: "Coordinator calls this process answered from memory, timeouts included.",
+    type: "counter"
+  },
   omni_metrics_series_folded_total: {
     help: "Metric series folded into api_key_id other.",
     type: "counter"
@@ -62392,13 +62704,14 @@ function createTelemetry(opts) {
       if (opts.metricsEnabled)
         registry2.add("omni_ratelimit_rejected_total", { dimension, window: window2 ?? "none" });
     },
-    scrape(localInflight, streams, coordFallback) {
+    scrape(localInflight, streams, coordFallback, coordFaults) {
       for (const [provider, value] of localInflight)
         registry2.set("omni_inflight", { provider }, value);
       registry2.set("omni_stream_connections", {}, streams.connections);
       registry2.set("omni_stream_queued", {}, streams.queued);
       registry2.set("omni_stream_dropped_total", {}, streams.dropped);
       registry2.set("omni_coord_fallback", {}, coordFallback ? 1 : 0);
+      registry2.set("omni_coord_faults_total", {}, coordFaults);
       return renderPrometheus(registry2.snapshot());
     },
     stop() {
@@ -62450,7 +62763,12 @@ async function main() {
   if (config2.logLevelFallbackFrom !== null) {
     logger2.info("invalid log level; using info", { reason: config2.logLevelFallbackFrom });
   }
-  logger2.info("omnigateway booting", { host: config2.host, port: config2.port });
+  logger2.info(config2.dayOffsetFallbackFrom === null ? "omnigateway booting" : "omnigateway booting; invalid day offset, using host offset", {
+    host: config2.host,
+    port: config2.port,
+    dayOffsetMinutes: config2.dayOffsetMinutes,
+    ...config2.dayOffsetFallbackFrom === null ? {} : { reason: config2.dayOffsetFallbackFrom }
+  });
   function dashboardDir() {
     if (config2.staticDir !== null)
       return config2.staticDir;
@@ -62467,7 +62785,8 @@ async function main() {
     ...config2.clusterMode && config2.databaseUrl !== null ? { url: config2.databaseUrl } : { path: config2.databasePath },
     encryptionKey,
     logger: logger2,
-    nodeId
+    nodeId,
+    dayOffsetMinutes: config2.dayOffsetMinutes
   });
   await store.maintenance.heartbeat(now());
   const swept = await store.usage.sweepPending(now());
@@ -62507,7 +62826,7 @@ async function main() {
     now,
     onDetach: (id, topics) => pluginChannelsRef?.closed(id, topics)
   });
-  const streamRing = createRing({ frames: 500, bytes: 2 * 1024 * 1024 });
+  const streamRing = createRing();
   const broadcaster = createBroadcaster({
     registry: streamRegistry,
     ring: streamRing,
@@ -62544,7 +62863,7 @@ async function main() {
     coord,
     nodeId,
     mode: config2.clusterMode ? "cluster" : "single",
-    ...shared === null ? {} : { coordHealthy: shared.healthy },
+    ...shared === null ? {} : { coordHealthy: shared.healthy, coordFaults: shared.faults },
     baseUrl: config2.baseUrl,
     http,
     now,
@@ -62559,6 +62878,7 @@ async function main() {
     telemetry,
     ...config2.metricsToken === null ? {} : { metricsToken: config2.metricsToken },
     bodyLoggingAllowed: config2.bodyLoggingAllowed,
+    dayOffsetMinutes: config2.dayOffsetMinutes,
     plugins: loadedPlugins.plugins.map((plugin) => ({ id: plugin.id, routes: plugin.routes })),
     pluginUi: loadedPlugins.plugins,
     ...loadedPlugins.plugins.length === 0 ? {} : { emit: (event) => pluginEvents.emitRequestCompleted(event) },