cartethyia 2.5.10 → 2.6.0

package/src/security/admission/redis-store.ts

@@ -61,6 +61,7 @@ export class RedisAdmissionCounterStore implements AdmissionCounterStore {
         request.apiKeyId,
         String(LEASE_TTL_MS),
         String(LEASE_KEY_TTL_SECONDS),
+        String(Math.max(0, request.concurrencyTolerance ?? 0)),
       );
     assertResult(result, "reserve");
   }
@@ -142,11 +143,20 @@ export class RedisAdmissionCounterStore implements AdmissionCounterStore {
    * Drops every counter for one key (revocation). Active in-flight leases
    * settle through the release script first so their concurrency slots do
    * not leak; remaining lease rows expire via TTL. Best-effort by contract.
+   *
+   * Bounded per invocation: a keyspace that has grown for months cannot turn
+   * one revocation into an unbounded walk — lease-scan pages, leases released,
+   * and counter-scan pages each stop cleanly at their budget, and the rest
+   * ages out via TTL (the fallback this contract already relies on). A later
+   * revocation or the lease sweeper picks up what a budgeted pass left.
    */
   async purge(apiKeyId: string): Promise<void> {
     try {
+      let leaseScanPages = 0;
+      let leasesReleased = 0;
       let cursor = "0";
       do {
+        if (leaseScanPages >= MAX_PURGE_LEASE_SCAN_PAGES) break;
         const [nextCursor, keys] = await this.redis.scan(
           cursor,
           "MATCH",
@@ -155,6 +165,7 @@ export class RedisAdmissionCounterStore implements AdmissionCounterStore {
           100,
         );
         cursor = String(nextCursor);
+        leaseScanPages += 1;
         const leaseKeys = keys as string[];
         if (leaseKeys.length === 0) continue;
         // One pipeline for the whole scan page instead of an HGET per lease:
@@ -172,15 +183,22 @@ export class RedisAdmissionCounterStore implements AdmissionCounterStore {
           return Array.isArray(entry) && entry[1] === apiKeyId;
         });
         for (const leaseKey of owned) {
+          if (leasesReleased >= MAX_PURGE_LEASES_RELEASED) break;
           await redisEvalNumber(this.redis, RELEASE_SCRIPT, 1, leaseKey, apiKeyId).catch(
             () => -1,
           );
+          leasesReleased += 1;
         }
+        // Releasing is the point of the lease phase; once its budget is spent
+        // there is nothing more this pass can usefully do with further pages.
+        if (leasesReleased >= MAX_PURGE_LEASES_RELEASED) break;
       } while (cursor !== "0");
+      let counterScanPages = 0;
       let counterCursor = "0";
       const patterns = [`admission:*:${apiKeyId}`, `admission:*:${apiKeyId}:*`];
       for (const pattern of patterns) {
         do {
+          if (counterScanPages >= MAX_PURGE_COUNTER_SCAN_PAGES) break;
           const [nextCursor, keys] = await this.redis.scan(
             counterCursor,
             "MATCH",
@@ -189,8 +207,10 @@ export class RedisAdmissionCounterStore implements AdmissionCounterStore {
             100,
           );
           counterCursor = String(nextCursor);
+          counterScanPages += 1;
           if ((keys as string[]).length > 0) await this.redis.del(...(keys as string[]));
         } while (counterCursor !== "0");
+        if (counterScanPages >= MAX_PURGE_COUNTER_SCAN_PAGES) break;
       }
     } catch {
       // Best-effort: revocation already succeeded; stale counters expire via TTL.
@@ -198,6 +218,16 @@ export class RedisAdmissionCounterStore implements AdmissionCounterStore {
   }
 }
 
+/**
+ * Per-invocation purge budgets. Chosen so one full-budgeted pass still costs
+ * well under a second of Redis time (hundreds of small commands), while a
+ * normal revocation — tens of leases, a handful of counters — never notices
+ * them. Mirrors the lease sweeper's own bounded-pass philosophy.
+ */
+export const MAX_PURGE_LEASE_SCAN_PAGES = 40;
+export const MAX_PURGE_LEASES_RELEASED = 100;
+export const MAX_PURGE_COUNTER_SCAN_PAGES = 40;
+
 /**
  * Releases a lease identified only by its Redis key (used by the sweeper).
  * Reads the owning api key id from the lease hash and runs RELEASE_SCRIPT,

package/src/security/admission/service.ts

@@ -35,6 +35,14 @@ export class ApiKeyAdmissionService {
     ) => Promise<number | null> | number | null = () => null,
     private readonly persistLifetimeUsage?: LifetimeUsagePersister,
     private readonly freshLifetimeConsumed?: FreshLifetimeConsumedReader,
+    /**
+     * Hidden allowance above each key's concurrency limit. Injected rather
+     * than read from the snapshot so the number is operator configuration,
+     * not something a key can carry about itself.
+     */
+    private readonly concurrencyTolerance?: (
+      tenantId: string,
+    ) => Promise<number | null> | number | null,
   ) {
     if (!store) throw new Error("ApiKeyAdmissionService requires an admission store");
   }
@@ -96,6 +104,15 @@ export class ApiKeyAdmissionService {
     const reservationId = `${apiKeyId}:${now}:${crypto.randomUUID()}`;
     const rawTenantLimit = this.tenantConcurrencyProvider(snapshot.tenant_id);
     const tenantLimit = rawTenantLimit instanceof Promise ? await rawTenantLimit : rawTenantLimit;
+    /*
+     * One number, one place. The allowance used to be overridable per key,
+     * so the tenant-wide value was only the answer for a key that had never
+     * been given one — an operator who raised it in Settings could not tell
+     * why one key still refused early, because the override lived in a modal
+     * the Settings page does not show. It is tenant-wide only now.
+     */
+    const rawTolerance = this.concurrencyTolerance?.(snapshot.tenant_id);
+    const tolerance = rawTolerance instanceof Promise ? await rawTolerance : rawTolerance;
 
     try {
       await this.store.reserve({
@@ -113,6 +130,7 @@ export class ApiKeyAdmissionService {
         lifetimeBudget: snapshot.lifetime_token_budget ?? null,
         lifetimeConsumed: snapshot.lifetime_tokens_consumed ?? 0,
         concurrencyLimit: snapshot.max_concurrent ?? null,
+        concurrencyTolerance: tolerance ?? null,
         tenantId: snapshot.tenant_id,
         tenantConcurrencyLimit: tenantLimit,
         // Fresh lifetime reader is invoked by the store only when the counter

package/src/security/api-key-auth-cache.ts

@@ -0,0 +1,96 @@
+/** Bounded process-local authorization cache; shared-cache I/O stays in the auth service. */
+export const AUTH_CACHE_TTL_MS = 3_000;
+const AUTH_CACHE_MAX_ENTRIES = 10_000;
+const AUTH_NEGATIVE_CACHE_TTL_MS = 1_500;
+const AUTH_FAILURE_COOLDOWN_MS = 1_000;
+
+interface CacheEntry<T> {
+  readonly at: number;
+  readonly value: T;
+  readonly keyId: string;
+}
+
+type Lookup<T> =
+  | { readonly kind: "positive"; readonly value: T }
+  | { readonly kind: "negative" }
+  | { readonly kind: "miss" };
+
+/** Create the process-local positive/negative authorization cache. */
+export function createApiKeyAuthorizationCache<T>() {
+  const entries = new Map<string, CacheEntry<T>>();
+  const hashesByKeyId = new Map<string, Set<string>>();
+  const negativeEntries = new Map<string, number>();
+  let failureUntil = 0;
+
+  const evictOldest = (): void => {
+    while (entries.size > AUTH_CACHE_MAX_ENTRIES) {
+      const oldest = entries.keys().next().value;
+      if (oldest === undefined) return;
+      const entry = entries.get(oldest);
+      entries.delete(oldest);
+      if (entry === undefined) continue;
+      const hashes = hashesByKeyId.get(entry.keyId);
+      if (hashes === undefined) continue;
+      hashes.delete(oldest);
+      if (hashes.size === 0) hashesByKeyId.delete(entry.keyId);
+    }
+  };
+
+  const evictNegativeOldest = (): void => {
+    while (negativeEntries.size > AUTH_CACHE_MAX_ENTRIES) {
+      const oldest = negativeEntries.keys().next().value;
+      if (oldest === undefined) return;
+      negativeEntries.delete(oldest);
+    }
+  };
+
+  return {
+    lookup(hash: string, now: number): Lookup<T> {
+      const negativeAt = negativeEntries.get(hash);
+      if (negativeAt !== undefined) {
+        if (now - negativeAt < AUTH_NEGATIVE_CACHE_TTL_MS) return { kind: "negative" };
+        negativeEntries.delete(hash);
+      }
+      const cached = entries.get(hash);
+      if (cached === undefined || now - cached.at >= AUTH_CACHE_TTL_MS) return { kind: "miss" };
+      entries.delete(hash);
+      entries.set(hash, cached);
+      return { kind: "positive", value: cached.value };
+    },
+
+    rememberPositive(hash: string, keyId: string, value: T, now: number): void {
+      entries.set(hash, { at: now, value, keyId });
+      negativeEntries.delete(hash);
+      const hashes = hashesByKeyId.get(keyId) ?? new Set<string>();
+      hashes.add(hash);
+      hashesByKeyId.set(keyId, hashes);
+      evictOldest();
+    },
+
+    rememberNegative(hash: string, now: number): void {
+      negativeEntries.set(hash, now);
+      evictNegativeOldest();
+    },
+
+    invalidate(keyId?: string): void {
+      negativeEntries.clear();
+      if (keyId === undefined) {
+        entries.clear();
+        hashesByKeyId.clear();
+        return;
+      }
+      const hashes = hashesByKeyId.get(keyId);
+      if (hashes === undefined) return;
+      for (const hash of hashes) entries.delete(hash);
+      hashesByKeyId.delete(keyId);
+    },
+
+    markStoreUnavailable(now: number): void {
+      failureUntil = now + AUTH_FAILURE_COOLDOWN_MS;
+    },
+
+    isStoreUnavailable(now: number): boolean {
+      return now < failureUntil;
+    },
+  };
+}

package/src/security/api-key-auth.ts

@@ -5,6 +5,7 @@ import { deleteShared, publishInvalidation, readShared, writeShared } from "../p
 import { DrizzleApiKeyStore } from "../persistence/stores/api-key-store";
 import { GatewayError } from "../protocol/gateway-error";
 import { hashSecret } from "./crypto";
+import { AUTH_CACHE_TTL_MS, createApiKeyAuthorizationCache } from "./api-key-auth-cache";
 import type { AccessScope } from "./access-control";
 import type { ApiKeyModelAccessMode } from "../persistence/schema";
 
@@ -23,37 +24,7 @@ import type { ApiKeyModelAccessMode } from "../persistence/schema";
  * Negative results (unknown/revoked tokens) are deliberately NOT cached, so
  * a newly minted key is usable immediately.
  */
-const AUTH_CACHE_TTL_MS = 3_000;
-const AUTH_CACHE_MAX_ENTRIES = 10_000;
-/** Negative-result TTL: unknown/revoked tokens miss the DB at most this often. */
-const AUTH_NEGATIVE_CACHE_TTL_MS = 1_500;
-
-interface AuthCacheEntry {
-  readonly at: number;
-  readonly value: ResolvedApiKey;
-  /** Owner of `value`; the reverse-index key this entry is registered under. */
-  readonly keyId: string;
-}
-
-const authCache = new Map<string, AuthCacheEntry>();
-/** Reverse index: keyId → set of token hashes that resolved to it. */
-const authCacheByKeyId = new Map<string, Set<string>>();
-/** Unknown/revoked token HMACs with their miss time; cleared on any key mutation. */
-const authNegativeCache = new Map<string, number>();
-
-/**
- * Short circuit after a credential-store failure.
- *
- * An overloaded or down database fails every lookup, and each failure costs a
- * connection attempt. Without this, a saturated backend received one query per
- * inbound request — load the operator sees as traffic, on top of the load that
- * caused the outage. For the window, lookups answer "unavailable" instead.
- *
- * Deliberately short: this is an auth path, and a brief outage must not lock
- * out valid keys for long.
- */
-const AUTH_FAILURE_COOLDOWN_MS = 1_000;
-let authFailureUntil = 0;
+const authorizationCache = createApiKeyAuthorizationCache<ResolvedApiKey>();
 
 /** Thrown when the credential store could not be reached at all. */
 export class ApiKeyStoreUnavailableError extends Error {
@@ -84,52 +55,6 @@ let pendingUsage: (apiKeyId: string) => number = () => 0;
 export function setPendingUsageSource(source: (apiKeyId: string) => number): void {
   pendingUsage = source;
 }
-/**
- * Negative-result fast path. A repeat miss for an unknown/revoked token within
- * 1.5 s costs one HMAC + one map lookup and no query. Keyed by the full HMAC: a
- * cheap pre-hash would let a crafted collision reject someone else's valid key,
- * while the HMAC key can only match its own token. Bound below beside the
- * positive cache.
- */
-
-/**
- * Drops the oldest entries until `authCache` is within budget. Every removal
- * must also unregister the hash from `authCacheByKeyId`, otherwise the reverse
- * index keeps referencing hashes that no longer exist and grows without bound
- * (one dead hash per eviction, forever).
- */
-function evictAuthCacheOldest(): void {
-  while (authCache.size > AUTH_CACHE_MAX_ENTRIES) {
-    const oldest = authCache.keys().next().value;
-    if (oldest === undefined) return;
-    const entry = authCache.get(oldest);
-    authCache.delete(oldest);
-    if (entry === undefined) continue;
-    const hashes = authCacheByKeyId.get(entry.keyId);
-    if (hashes === undefined) continue;
-    hashes.delete(oldest);
-    if (hashes.size === 0) authCacheByKeyId.delete(entry.keyId);
-  }
-}
-
-/** Registers a resolved hash in both the cache and the keyId reverse index. */
-function indexAuthCache(hash: string, entry: AuthCacheEntry): void {
-  authCache.set(hash, entry);
-  authNegativeCache.delete(hash);
-  const hashes = authCacheByKeyId.get(entry.keyId) ?? new Set<string>();
-  hashes.add(hash);
-  authCacheByKeyId.set(entry.keyId, hashes);
-}
-
-/** Bounds the negative cache; same oldest-first rule as the positive cache. */
-function evictAuthNegativeOldest(): void {
-  while (authNegativeCache.size > AUTH_CACHE_MAX_ENTRIES) {
-    const oldest = authNegativeCache.keys().next().value;
-    if (oldest === undefined) return;
-    authNegativeCache.delete(oldest);
-  }
-}
-
 /**
  * Invalidates every cached authorization entry for one API key (or the entire
  * cache when `keyId` is omitted). Called by the API-key domain after any
@@ -138,19 +63,12 @@ function evictAuthNegativeOldest(): void {
 export function invalidateApiKeyCache(keyId?: string): void {
   // A mutation may mint a token the negative cache has already rejected, so
   // negative entries must go with the positive ones.
-  authNegativeCache.clear();
+  authorizationCache.invalidate(keyId);
   if (keyId === undefined) {
-    authCache.clear();
-    authCacheByKeyId.clear();
     void deleteShared("auth");
     publishInvalidation({ namespace: "auth" });
     return;
   }
-  const hashes = authCacheByKeyId.get(keyId);
-  if (hashes) {
-    for (const hash of hashes) authCache.delete(hash);
-    authCacheByKeyId.delete(keyId);
-  }
   // Peer processes hold their own copy, and `reusePort` means the console
   // write may have landed on a different one entirely. Dropping only locally
   // leaves a revoked key valid for up to a full TTL everywhere else.
@@ -388,13 +306,9 @@ export async function resolveApiKeyAuthorization(
 ): Promise<ResolvedApiKey | undefined> {
   const now = Date.now();
   const hash = hashSecret(token);
-  const negativeAt = authNegativeCache.get(hash);
-  if (negativeAt !== undefined) {
-    if (now - negativeAt < AUTH_NEGATIVE_CACHE_TTL_MS) return undefined;
-    authNegativeCache.delete(hash);
-  }
-  const cached = authCache.get(hash);
-  if (cached && now - cached.at < AUTH_CACHE_TTL_MS) return cached.value;
+  const cached = authorizationCache.lookup(hash, now);
+  if (cached.kind === "negative") return undefined;
+  if (cached.kind === "positive") return cached.value;
   // L2: a peer process may have resolved this token already. Reachable only
   // after the L1 entry lapsed, so Redis is never consulted on a warm hit.
   //
@@ -405,8 +319,7 @@ export async function resolveApiKeyAuthorization(
     try {
       const parsed = JSON.parse(shared) as ResolvedApiKey;
       if (typeof parsed?.id === "string") {
-        indexAuthCache(hash, { at: now, value: parsed, keyId: parsed.id });
-        evictAuthCacheOldest();
+        authorizationCache.rememberPositive(hash, parsed.id, parsed, now);
         return parsed;
       }
     } catch {
@@ -415,7 +328,7 @@ export async function resolveApiKeyAuthorization(
   }
   // Answered from the cooldown before any I/O: during a store outage the
   // query is the expensive part, and it is also the part that fails.
-  if (now < authFailureUntil) throw new ApiKeyStoreUnavailableError();
+  if (authorizationCache.isStoreUnavailable(now)) throw new ApiKeyStoreUnavailableError();
   const store = new DrizzleApiKeyStore(db);
   let row: Awaited<ReturnType<typeof store.findActiveByHash>>;
   try {
@@ -423,12 +336,11 @@ export async function resolveApiKeyAuthorization(
   } catch (error) {
     // Any failure here is the store's, not the key's — a revoked key is a
     // normal empty result, so this only runs on a real fault.
-    authFailureUntil = Date.now() + AUTH_FAILURE_COOLDOWN_MS;
+    authorizationCache.markStoreUnavailable(Date.now());
     throw error;
   }
   if (!row) {
-    authNegativeCache.set(hash, now);
-    evictAuthNegativeOldest();
+    authorizationCache.rememberNegative(hash, now);
     return undefined;
   }
   // A child (shared) key consumes against its parent's quota, not just its
@@ -487,8 +399,7 @@ export async function resolveApiKeyAuthorization(
     ...(policy.modelPrefix ? { modelPrefix: policy.modelPrefix } : {}),
     ...(parent?.keyMode === "share" ? { cliMappingOwnerId: parent.id } : {}),
   };
-  indexAuthCache(hash, { at: Date.now(), value, keyId: value.id });
-  evictAuthCacheOldest();
+  authorizationCache.rememberPositive(hash, value.id, value, Date.now());
   // Shared copy for peer processes, keyed by the token hash so a lookup can
   // find it without resolving the key first. Invalidation travels by broadcast,
   // since a console mutation names a key id and cannot enumerate the hashes

package/src/security/client-router-fingerprint.ts

@@ -1,3 +1,5 @@
+import { getHeader as readRecordHeader } from "../protocol/canonical-model";
+
 // Client-router fingerprinting for inbound `/v1/*` traffic.
 //
 // Several AI-gateway products can themselves be pointed at this gateway. An
@@ -133,13 +135,7 @@ export interface ClientRouterProbe {
 function readHeader(probe: ClientRouterProbe, name: string): string | undefined {
   const headers = probe.headers;
   if (headers instanceof Headers) return headers.get(name) ?? undefined;
-  const direct = headers[name];
-  if (direct !== undefined) return direct;
-  const lowered = name.toLowerCase();
-  for (const [key, value] of Object.entries(headers)) {
-    if (key.toLowerCase() === lowered) return value;
-  }
-  return undefined;
+  return readRecordHeader(headers, name);
 }
 
 function signalMatches(signal: ClientRouterSignal, value: string): boolean {

package/src/security/ip-boundary.ts

@@ -2,6 +2,7 @@
 
 import { isIP } from "node:net";
 import type { TrustedProxyBoundary } from "../config/network";
+import { log } from "../observability/logger";
 
 function ipv4ToNumber(address: string): number | undefined {
   const parts = address.split(".");
@@ -319,9 +320,12 @@ function maybeWarnIgnoredForwardedHeaders(
   if (collapseWarned) return;
   if (!isSuspectedProxyCollapse(request, boundary, peer)) return;
   collapseWarned = true;
-  // Not `log`, to keep this module importable without the logger's transport
-  // (and usable from the boundary tests, which construct bare `Request`s).
-  console.warn(
+  // Via `log`, so this ships through the same transport as everything else and
+  // is level-filtered. It was `console.warn` to keep the module importable
+  // without the logger, but the logger pulls in only pino, the console log ring
+  // and its own sanitizer — none of which import this module — so there is no
+  // cycle and the boundary tests import it the same way production does.
+  log.warn(
     `[ip-boundary] A forwarded client-address header arrived from a loopback peer ` +
       `(${unwrapMappedIpv4(peer)}) but the trusted-proxy boundary does not trust it, so ` +
       `the client address is the loopback peer for every caller. Every caller behind this ` +

package/src/shared/coerce.ts

@@ -0,0 +1,67 @@
+/**
+ * Shared value-coercion primitives for untyped upstream JSON.
+ *
+ * These used to exist as private copies in a dozen modules, spelled
+ * `record`/`asRecord`/`recordOf`/`isPlainObject` with two different nullish
+ * sentinels. One implementation, one sentinel (`undefined`), one place to fix.
+ */
+
+/** Narrow an unknown value to a plain (non-array) object, else `undefined`. */
+export function asRecord(value: unknown): Record<string, unknown> | undefined {
+  return value !== null && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+/** Narrow to a finite number, else `undefined`. `NaN`/`Infinity` are not numbers here. */
+export function asNumber(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+/**
+ * Finite number accepting numeric strings (`"42"` → 42). Whitespace-only or
+ * non-finite results are `undefined`. This is the "upstream JSON may quote a
+ * number" contract; use `asNumber` when strings must stay rejected.
+ */
+export function asCoercedNumber(value: unknown): number | undefined {
+  if (typeof value === "number") return asNumber(value);
+  if (typeof value === "string" && value.trim().length > 0) {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : undefined;
+  }
+  return undefined;
+}
+
+/** Trim a string-ish value; empty/whitespace-only becomes `undefined`. */
+export function asTrimmedString(value: unknown): string | undefined {
+  if (typeof value !== "string") return undefined;
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : undefined;
+}
+
+/** First non-empty trimmed string among `keys` of `record`. */
+export function firstNonEmptyString(
+  record: Record<string, unknown>,
+  keys: readonly string[],
+): string | undefined {
+  for (const key of keys) {
+    const value = asTrimmedString(record[key]);
+    if (value !== undefined) return value;
+  }
+  return undefined;
+}
+
+/**
+ * `JSON.parse` that yields `undefined` instead of throwing, for untyped
+ * upstream/console JSON that may be malformed. Non-string/empty input is
+ * `undefined` without attempting a parse.
+ */
+export function tryParseJson(value: string | null | undefined): unknown {
+  if (typeof value !== "string" || value.length === 0) return undefined;
+  try {
+    return JSON.parse(value) as unknown;
+  } catch {
+    return undefined;
+  }
+}
+

package/src/shared/hash.ts

@@ -0,0 +1,6 @@
+import { createHash } from "node:crypto";
+
+/** Lowercase hex SHA-256 of a UTF-8 string, for cache-key/identity hashing. */
+export function sha256Hex(value: string): string {
+  return createHash("sha256").update(value, "utf8").digest("hex");
+}

package/src/shared/time.ts

@@ -0,0 +1,7 @@
+/**
+ * `YYYY-MM-DDTHH` stamp used as the prefix of raw telemetry/payload files
+ * and as the hour comparison key on both sides of retention.
+ */
+export function hourStamp(date: Date): string {
+  return date.toISOString().slice(0, 13).replaceAll(":", "-");
+}