cartethyia 2.5.3 → 2.5.6

package/src/security/admission/redis-store.ts

@@ -1,210 +1,211 @@
-import type { RedisClient } from "../../persistence/redis";
-import { redisEvalNumber } from "../../persistence/redis";
-import type { AdmissionCounterStore, AdmissionReserveRequest } from "./contracts";
-import { assertResult, RECONCILE_SCRIPT, RELEASE_SCRIPT, RESERVE_SCRIPT } from "./lua";
-import {
-  DAILY_COUNTER_TTL_SECONDS,
-  LEASE_KEY_TTL_SECONDS,
-  LEASE_TTL_MS,
-  MONTHLY_COUNTER_TTL_SECONDS,
-  dailyBucket,
-  monthlyBucket,
-} from "./ttl";
-
-export class RedisAdmissionCounterStore implements AdmissionCounterStore {
-  constructor(private readonly redis: RedisClient) {}
-  async reserve(request: AdmissionReserveRequest): Promise<void> {
-    const id = request.reservationId ?? `${request.apiKeyId}:${request.now}:${crypto.randomUUID()}`;
-    const lifetimeKey = `admission:lifetime:${request.apiKeyId}`;
-    const keys = [
-      `admission:rpm:${request.apiKeyId}`,
-      `admission:daily:${request.apiKeyId}:${dailyBucket(request.now)}`,
-      `admission:monthly:${request.apiKeyId}:${monthlyBucket(request.now)}`,
-      `admission:concurrent:${request.apiKeyId}`,
-      lifetimeKey,
-      `admission:lease:${id}`,
-      `admission:tenant_concurrent:${request.tenantId}`,
-    ];
-    // The lifetime seed is the one write the Lua script cannot re-issue: once
-    // the counter exists it never re-reads Postgres. If we are about to create
-    // it, prefer a fresh persisted read over the ≤3s-stale snapshot value,
-    // otherwise a snapshot that missed a recent commit freezes a low baseline
-    // for the counter's whole 35-day TTL. `EXISTS` is a single extra command
-    // and runs only for keys that actually have a lifetime budget.
-    let lifetimeSeed = request.lifetimeConsumed;
-    if (request.lifetimeBudget != null && request.freshLifetimeConsumed) {
-      const exists = await this.redis.exists(lifetimeKey).catch(() => 1);
-      if (exists === 0) {
-        const fresh = await request.freshLifetimeConsumed().catch(() => undefined);
-        if (typeof fresh === "number" && Number.isFinite(fresh) && fresh >= 0) {
-          lifetimeSeed = Math.floor(fresh);
-        }
-      }
-    }
-    const result = await redisEvalNumber(
-      this.redis,
-      RESERVE_SCRIPT,
-      keys.length,
-      ...keys,
-        String(request.now),
-        String(request.estimatedTokens),
-        request.rpmLimit == null ? "" : String(request.rpmLimit),
-        request.dailyLimit == null ? "" : String(request.dailyLimit),
-        request.monthlyLimit == null ? "" : String(request.monthlyLimit),
-        request.lifetimeBudget == null ? "" : String(request.lifetimeBudget),
-        request.concurrencyLimit == null ? "" : String(request.concurrencyLimit),
-        id,
-        String(lifetimeSeed),
-        request.tenantConcurrencyLimit == null ? "" : String(request.tenantConcurrencyLimit),
-        request.tenantId,
-        request.apiKeyId,
-        String(LEASE_TTL_MS),
-        String(LEASE_KEY_TTL_SECONDS),
-      );
-    assertResult(result, "reserve");
-  }
-  async reconcile(
-    apiKeyId: string,
-    _reserved: number,
-    actual: number,
-    reservationId?: string,
-  ): Promise<void> {
-    if (!reservationId) throw new Error("reservation id required for atomic reconcile");
-    const result = await redisEvalNumber(
-      this.redis,
-      RECONCILE_SCRIPT,
-      1,
-      `admission:lease:${reservationId}`,
-      String(actual),
-      apiKeyId,
-    );
-    assertResult(result, "reconcile");
-  }
-  async release(
-    apiKeyId: string,
-    _reserved: number,
-    reservationId?: string,
-  ): Promise<void> {
-    if (!reservationId) throw new Error("reservation id required for atomic release");
-    const result = await redisEvalNumber(
-      this.redis,
-      RELEASE_SCRIPT,
-      1,
-      `admission:lease:${reservationId}`,
-      apiKeyId,
-    );
-    assertResult(result, "release");
-  }
-
-  /**
-   * `SET NX`-seeds the current bucket's counters from already-recorded usage.
-   * NX is what makes this safe: an existing counter is the live running total,
-   * and overwriting it would erase spend. Best-effort by contract.
-   */
-  async seedBuckets(request: {
-    readonly apiKeyId: string;
-    readonly now: number;
-    readonly daily?: number;
-    readonly monthly?: number;
-  }): Promise<void> {
-    try {
-      const operations: Promise<unknown>[] = [];
-      if (request.daily !== undefined && request.daily > 0) {
-        operations.push(
-          this.redis.set(
-            `admission:daily:${request.apiKeyId}:${dailyBucket(request.now)}`,
-            Math.floor(request.daily),
-            "EX",
-            DAILY_COUNTER_TTL_SECONDS,
-            "NX",
-          ),
-        );
-      }
-      if (request.monthly !== undefined && request.monthly > 0) {
-        operations.push(
-          this.redis.set(
-            `admission:monthly:${request.apiKeyId}:${monthlyBucket(request.now)}`,
-            Math.floor(request.monthly),
-            "EX",
-            MONTHLY_COUNTER_TTL_SECONDS,
-            "NX",
-          ),
-        );
-      }
-      await Promise.all(operations);
-    } catch {
-      // Seeding is an optimisation, not a correctness gate.
-    }
-  }
-
-  /**
-   * Drops every counter for one key (revocation). Active in-flight leases
-   * settle through the release script first so their concurrency slots do
-   * not leak; remaining lease rows expire via TTL. Best-effort by contract.
-   */
-  async purge(apiKeyId: string): Promise<void> {
-    try {
-      let cursor = "0";
-      do {
-        const [nextCursor, keys] = await this.redis.scan(
-          cursor,
-          "MATCH",
-          "admission:lease:*",
-          "COUNT",
-          100,
-        );
-        cursor = String(nextCursor);
-        const leaseKeys = keys as string[];
-        if (leaseKeys.length === 0) continue;
-        // One pipeline for the whole scan page instead of an HGET per lease:
-        // the owner is a hash field, not part of the key, so the page has to be
-        // read to find this key's leases — but it can be read in one round trip.
-        // The key format stays `admission:lease:<reservationId>` (the id is not
-        // always derived from the key id) so leases written before a deploy are
-        // still found.
-        const pipeline = this.redis.pipeline();
-        for (const leaseKey of leaseKeys) pipeline.hget(leaseKey, "api_key_id");
-        const owners = await pipeline.exec().catch(() => null);
-        if (owners === null) continue;
-        const owned = leaseKeys.filter((_, index) => {
-          const entry = owners[index];
-          return Array.isArray(entry) && entry[1] === apiKeyId;
-        });
-        for (const leaseKey of owned) {
-          await redisEvalNumber(this.redis, RELEASE_SCRIPT, 1, leaseKey, apiKeyId).catch(
-            () => -1,
-          );
-        }
-      } while (cursor !== "0");
-      let counterCursor = "0";
-      const patterns = [`admission:*:${apiKeyId}`, `admission:*:${apiKeyId}:*`];
-      for (const pattern of patterns) {
-        do {
-          const [nextCursor, keys] = await this.redis.scan(
-            counterCursor,
-            "MATCH",
-            pattern,
-            "COUNT",
-            100,
-          );
-          counterCursor = String(nextCursor);
-          if ((keys as string[]).length > 0) await this.redis.del(...(keys as string[]));
-        } while (counterCursor !== "0");
-      }
-    } catch {
-      // Best-effort: revocation already succeeded; stale counters expire via TTL.
-    }
-  }
-}
-
-/**
- * Releases a lease identified only by its Redis key (used by the sweeper).
- * Reads the owning api key id from the lease hash and runs RELEASE_SCRIPT,
- * which also releases the per-tenant concurrency slot when present.
- */
-export async function releaseAdmissionLease(redis: RedisClient, leaseKey: string): Promise<void> {
-  const apiKeyId = await redis.hget(leaseKey, "api_key_id");
-  if (!apiKeyId) throw new Error("admission lease missing api_key_id");
-  const result = await redisEvalNumber(redis, RELEASE_SCRIPT, 1, leaseKey, apiKeyId);
-  assertResult(result, "release");
-}
+import type { RedisClient } from "../../persistence/redis";
+import { redisEvalNumber } from "../../persistence/redis";
+import type { AdmissionCounterStore, AdmissionReserveRequest } from "./contracts";
+import { assertResult, RECONCILE_SCRIPT, RELEASE_SCRIPT, RESERVE_SCRIPT } from "./lua";
+import {
+  DAILY_COUNTER_TTL_SECONDS,
+  LEASE_KEY_TTL_SECONDS,
+  LEASE_TTL_MS,
+  MONTHLY_COUNTER_TTL_SECONDS,
+  dailyBucket,
+  monthlyBucket,
+} from "./ttl";
+
+export class RedisAdmissionCounterStore implements AdmissionCounterStore {
+  constructor(private readonly redis: RedisClient) {}
+  async reserve(request: AdmissionReserveRequest): Promise<void> {
+    const id = request.reservationId ?? `${request.apiKeyId}:${request.now}:${crypto.randomUUID()}`;
+    const lifetimeKey = `admission:lifetime:${request.apiKeyId}`;
+    const rpmConcId = request.rpmConcurrencyKeyId ?? request.apiKeyId;
+    const keys = [
+     `admission:rpm:${rpmConcId}`,
+      `admission:daily:${request.apiKeyId}:${dailyBucket(request.now)}`,
+      `admission:monthly:${request.apiKeyId}:${monthlyBucket(request.now)}`,
+     `admission:concurrent:${rpmConcId}`,
+      lifetimeKey,
+      `admission:lease:${id}`,
+      `admission:tenant_concurrent:${request.tenantId}`,
+    ];
+    // The lifetime seed is the one write the Lua script cannot re-issue: once
+    // the counter exists it never re-reads Postgres. If we are about to create
+    // it, prefer a fresh persisted read over the ≤3s-stale snapshot value,
+    // otherwise a snapshot that missed a recent commit freezes a low baseline
+    // for the counter's whole 35-day TTL. `EXISTS` is a single extra command
+    // and runs only for keys that actually have a lifetime budget.
+    let lifetimeSeed = request.lifetimeConsumed;
+    if (request.lifetimeBudget != null && request.freshLifetimeConsumed) {
+      const exists = await this.redis.exists(lifetimeKey).catch(() => 1);
+      if (exists === 0) {
+        const fresh = await request.freshLifetimeConsumed().catch(() => undefined);
+        if (typeof fresh === "number" && Number.isFinite(fresh) && fresh >= 0) {
+          lifetimeSeed = Math.floor(fresh);
+        }
+      }
+    }
+    const result = await redisEvalNumber(
+      this.redis,
+      RESERVE_SCRIPT,
+      keys.length,
+      ...keys,
+        String(request.now),
+        String(request.estimatedTokens),
+        request.rpmLimit == null ? "" : String(request.rpmLimit),
+        request.dailyLimit == null ? "" : String(request.dailyLimit),
+        request.monthlyLimit == null ? "" : String(request.monthlyLimit),
+        request.lifetimeBudget == null ? "" : String(request.lifetimeBudget),
+        request.concurrencyLimit == null ? "" : String(request.concurrencyLimit),
+        id,
+        String(lifetimeSeed),
+        request.tenantConcurrencyLimit == null ? "" : String(request.tenantConcurrencyLimit),
+        request.tenantId,
+        request.apiKeyId,
+        String(LEASE_TTL_MS),
+        String(LEASE_KEY_TTL_SECONDS),
+      );
+    assertResult(result, "reserve");
+  }
+  async reconcile(
+    apiKeyId: string,
+    _reserved: number,
+    actual: number,
+    reservationId?: string,
+  ): Promise<void> {
+    if (!reservationId) throw new Error("reservation id required for atomic reconcile");
+    const result = await redisEvalNumber(
+      this.redis,
+      RECONCILE_SCRIPT,
+      1,
+      `admission:lease:${reservationId}`,
+      String(actual),
+      apiKeyId,
+    );
+    assertResult(result, "reconcile");
+  }
+  async release(
+    apiKeyId: string,
+    _reserved: number,
+    reservationId?: string,
+  ): Promise<void> {
+    if (!reservationId) throw new Error("reservation id required for atomic release");
+    const result = await redisEvalNumber(
+      this.redis,
+      RELEASE_SCRIPT,
+      1,
+      `admission:lease:${reservationId}`,
+      apiKeyId,
+    );
+    assertResult(result, "release");
+  }
+
+  /**
+   * `SET NX`-seeds the current bucket's counters from already-recorded usage.
+   * NX is what makes this safe: an existing counter is the live running total,
+   * and overwriting it would erase spend. Best-effort by contract.
+   */
+  async seedBuckets(request: {
+    readonly apiKeyId: string;
+    readonly now: number;
+    readonly daily?: number;
+    readonly monthly?: number;
+  }): Promise<void> {
+    try {
+      const operations: Promise<unknown>[] = [];
+      if (request.daily !== undefined && request.daily > 0) {
+        operations.push(
+          this.redis.set(
+            `admission:daily:${request.apiKeyId}:${dailyBucket(request.now)}`,
+            Math.floor(request.daily),
+            "EX",
+            DAILY_COUNTER_TTL_SECONDS,
+            "NX",
+          ),
+        );
+      }
+      if (request.monthly !== undefined && request.monthly > 0) {
+        operations.push(
+          this.redis.set(
+            `admission:monthly:${request.apiKeyId}:${monthlyBucket(request.now)}`,
+            Math.floor(request.monthly),
+            "EX",
+            MONTHLY_COUNTER_TTL_SECONDS,
+            "NX",
+          ),
+        );
+      }
+      await Promise.all(operations);
+    } catch {
+      // Seeding is an optimisation, not a correctness gate.
+    }
+  }
+
+  /**
+   * Drops every counter for one key (revocation). Active in-flight leases
+   * settle through the release script first so their concurrency slots do
+   * not leak; remaining lease rows expire via TTL. Best-effort by contract.
+   */
+  async purge(apiKeyId: string): Promise<void> {
+    try {
+      let cursor = "0";
+      do {
+        const [nextCursor, keys] = await this.redis.scan(
+          cursor,
+          "MATCH",
+          "admission:lease:*",
+          "COUNT",
+          100,
+        );
+        cursor = String(nextCursor);
+        const leaseKeys = keys as string[];
+        if (leaseKeys.length === 0) continue;
+        // One pipeline for the whole scan page instead of an HGET per lease:
+        // the owner is a hash field, not part of the key, so the page has to be
+        // read to find this key's leases — but it can be read in one round trip.
+        // The key format stays `admission:lease:<reservationId>` (the id is not
+        // always derived from the key id) so leases written before a deploy are
+        // still found.
+        const pipeline = this.redis.pipeline();
+        for (const leaseKey of leaseKeys) pipeline.hget(leaseKey, "api_key_id");
+        const owners = await pipeline.exec().catch(() => null);
+        if (owners === null) continue;
+        const owned = leaseKeys.filter((_, index) => {
+          const entry = owners[index];
+          return Array.isArray(entry) && entry[1] === apiKeyId;
+        });
+        for (const leaseKey of owned) {
+          await redisEvalNumber(this.redis, RELEASE_SCRIPT, 1, leaseKey, apiKeyId).catch(
+            () => -1,
+          );
+        }
+      } while (cursor !== "0");
+      let counterCursor = "0";
+      const patterns = [`admission:*:${apiKeyId}`, `admission:*:${apiKeyId}:*`];
+      for (const pattern of patterns) {
+        do {
+          const [nextCursor, keys] = await this.redis.scan(
+            counterCursor,
+            "MATCH",
+            pattern,
+            "COUNT",
+            100,
+          );
+          counterCursor = String(nextCursor);
+          if ((keys as string[]).length > 0) await this.redis.del(...(keys as string[]));
+        } while (counterCursor !== "0");
+      }
+    } catch {
+      // Best-effort: revocation already succeeded; stale counters expire via TTL.
+    }
+  }
+}
+
+/**
+ * Releases a lease identified only by its Redis key (used by the sweeper).
+ * Reads the owning api key id from the lease hash and runs RELEASE_SCRIPT,
+ * which also releases the per-tenant concurrency slot when present.
+ */
+export async function releaseAdmissionLease(redis: RedisClient, leaseKey: string): Promise<void> {
+  const apiKeyId = await redis.hget(leaseKey, "api_key_id");
+  if (!apiKeyId) throw new Error("admission lease missing api_key_id");
+  const result = await redisEvalNumber(redis, RELEASE_SCRIPT, 1, leaseKey, apiKeyId);
+  assertResult(result, "release");
+}

package/src/security/admission/service.ts

@@ -101,6 +101,10 @@ export class ApiKeyAdmissionService {
       await this.store.reserve({
         reservationId,
         apiKeyId,
+        // rpm and concurrency are tracked per child key so each share recipient
+        // has its own independent window and slot. Token budgets (daily/monthly/
+        // lifetime) remain on `apiKeyId` (the parent) — the shared pool.
+        ...(snapshot.api_key_id !== apiKeyId ? { rpmConcurrencyKeyId: snapshot.api_key_id } : {}),
         now,
         estimatedTokens,
         rpmLimit: snapshot.rpm ?? null,

package/src/security/api-key-auth.ts

@@ -69,7 +69,7 @@ export class ApiKeyStoreUnavailableError extends Error {
  * Injected rather than imported: this module is on the auth path and must not
  * depend on the persistence/write layer it would otherwise pull in. Set once at
  * boot (`boot/dependencies.ts`); until then it reads as zero, which is the
- * value the code has always used.
+ * value the code uses when no source is installed.
  */
 let pendingUsage: (apiKeyId: string) => number = () => 0;
 
@@ -85,11 +85,11 @@ export function setPendingUsageSource(source: (apiKeyId: string) => number): voi
   pendingUsage = source;
 }
 /**
- * Negative-result fast path. Unknown/revoked tokens previously hit the DB on
- * every request; now a repeat miss within 1.5 s costs one HMAC + one map
- * lookup and no query. Keyed by the full HMAC: a cheap pre-hash would let a
- * crafted collision reject someone else's valid key, while the HMAC key can
- * only match its own token. Bound below beside the positive cache.
+ * Negative-result fast path. A repeat miss for an unknown/revoked token within
+ * 1.5 s costs one HMAC + one map lookup and no query. Keyed by the full HMAC: a
+ * cheap pre-hash would let a crafted collision reject someone else's valid key,
+ * while the HMAC key can only match its own token. Bound below beside the
+ * positive cache.
  */
 
 /**
@@ -136,8 +136,8 @@ function evictAuthNegativeOldest(): void {
  * mutation that changes a key's scopes, limits, or revoked state.
  */
 export function invalidateApiKeyCache(keyId?: string): void {
-  // A mutation may mint the previously-unknown token, so negative entries
-  // must go with the positive ones.
+  // A mutation may mint a token the negative cache has already rejected, so
+  // negative entries must go with the positive ones.
   authNegativeCache.clear();
   if (keyId === undefined) {
     authCache.clear();
@@ -314,13 +314,23 @@ export interface ResolvedApiKey {
  * surface it hit. Unknown, revoked and expired stay deliberately
  * indistinguishable — distinguishing them would let a caller confirm that a
  * guessed token once existed — so there is no variant carrying a reason.
- *
- * The rotate instruction is unconditional for the same reason. The key that
- * fails is usually the one seeded at first boot, which cannot be deleted and
- * only rotates, and an operator hitting this on a personal key loses nothing by
- * being told the same thing.
  */
+const INVALID_KEY_MESSAGE = "Invalid API key. Check the key you are sending, or create a new one in API Keys and send it as: Authorization: Bearer <key>.";
+
 export function unknownApiKeyError(): GatewayError {
+  return new GatewayError("authentication_failed", 401, INVALID_KEY_MESSAGE);
+}
+
+/**
+ * The 401 for the public model surface specifically.
+ *
+ * Model Studio is where the seeded default key is normally used, and that key
+ * cannot be deleted — it only rotates. Naming rotation there is actionable. The
+ * same sentence on a proxy request was misleading instead: a revoked or mistyped
+ * personal key was answered with an instruction that could not apply to it,
+ * sending the operator to rotate a key they were never using.
+ */
+export function modelsListApiKeyError(): GatewayError {
   return new GatewayError(
     "authentication_failed",
     401,
@@ -430,9 +440,9 @@ export async function resolveApiKeyAuthorization(
     row.parentKeyId !== null ? await store.findActiveById(row.parentKeyId) : undefined;
   // Share children authenticate as themselves but inherit the template's live
   if (row.parentKeyId !== null && parent === undefined) return undefined;
-  // policy. Copy-on-issue alone went stale the moment an operator tightened a
-  // limit or blocked a client router on the parent — existing recipients kept
-  // the old row values forever. Reading the parent here makes denylist /
+  // policy. Copy-on-issue alone goes stale the moment an operator tightens a
+  // limit or blocks a client router on the parent — existing recipients would
+  // keep the issued row values forever. Reading the parent here makes denylist /
   // one-time / recurring edits apply on the next cache miss.
   const policy = parent ?? row;
   // Only a `share` parent can have children, and children are the only thing
@@ -488,7 +498,7 @@ export async function resolveApiKeyAuthorization(
   // That figure moves constantly, and the module already documents the snapshot
   // as advisory on that axis (spending is enforced by the admission counter),
   // so a peer serving it stale for up to the TTL is the same tradeoff the
-  // in-process cache has always made.
+  // in-process cache makes.
   void writeShared("auth", hash, JSON.stringify(value), AUTH_CACHE_TTL_MS);
   return value;
 }