@ultimat3/http 27.10.3 → 27.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/http",
-  "version": "27.10.3",
+  "version": "27.11.0",
   "description": "Owned request lifecycle over Bun.serve: router, ordered pipeline, problem+json errors",
   "license": "MIT",
   "type": "module",
@@ -35,9 +35,9 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "27.10.3",
-    "@ultimat3/i18n": "27.10.3",
-    "@ultimat3/schema": "27.10.3",
-    "@ultimat3/time": "27.10.3"
+    "@ultimat3/core": "27.11.0",
+    "@ultimat3/i18n": "27.11.0",
+    "@ultimat3/schema": "27.11.0",
+    "@ultimat3/time": "27.11.0"
   }
 }

package/src/rate-limit-degrade.ts

@@ -0,0 +1,75 @@
+// What the shared rate-limit store does when the DATABASE is down: decide from this process's
+// memory instead, and say so once per window. Before, `take` rethrew `X_DB_UNAVAILABLE` and every
+// request a web pod served — a prerendered `GET /` included — answered 500 for the whole blip.
+//
+// WHY degrade rather than refuse: a per-process bucket still limits. The alternative was a total
+// outage of every route to protect a counter. Every method degrades, auth routes included.
+//
+// WHAT the weakening is, exactly: the fallback is a SEPARATE budget, not a copy of the shared one.
+// It cannot be seeded — the database it would be seeded from is the thing that is down — so each
+// pod's fallback starts full. Behind N replicas a key may spend N × the declared capacity during
+// the outage, ON TOP of whatever it had already spent in Postgres before it: a key that had just
+// emptied its shared bucket gets a fresh one per pod at the transition. When the database answers
+// again, decisions return to the shared row, which never saw the degraded takes. Weaker, bounded by
+// the outage and logged — never no limit. `reset(key)` clears both, so a key the fallback exhausted
+// is not still denied in the next outage after a reset that succeeded.
+//
+// WHY only connection-class failures: a store that answers WRONGLY (bad SQL, a missing table, an
+// executor bug) is not a store that is down, and silently switching it to memory would hide that
+// forever. Those still refuse, exactly as before.
+
+import { isUltimateError, logger } from '@ultimat3/core';
+import type { RateLimitStore } from './rate-limit';
+import { memoryRateLimitStore } from './rate-limit';
+
+/** Codes `@ultimat3/db` raises when no connection could run the statement at all. */
+const UNREACHABLE_CODES: ReadonlySet<string> = new Set(['X_DB_UNAVAILABLE', 'X_DB_POOL_EXHAUSTED']);
+
+/**
+ * SQLSTATEs that mean "this connection, right now", carried on `meta.sqlState` of an
+ * `X_DB_STATEMENT_FAILED`: class 08 (connection exception), `57P01`–`57P03` (the server is
+ * shutting down, crashed, or not yet accepting — a failover), `53300` (too many connections).
+ */
+const isConnectionSqlState = (state: string): boolean =>
+  state.startsWith('08') || ['57P01', '57P02', '57P03', '53300'].includes(state);
+
+/** Read by the store's `catch`: true only for a failure that says the database is unreachable. */
+export function isRateLimitStoreUnreachable(error: unknown): boolean {
+  if (!isUltimateError(error)) return false;
+  if (UNREACHABLE_CODES.has(error.code)) return true;
+  const state = error.meta?.['sqlState'];
+  return typeof state === 'string' && isConnectionSqlState(state);
+}
+
+/** How often a still-degraded store says so again: a long outage stays visible, a blip is one line. */
+export const DEGRADED_WARN_EVERY_MS = 60_000;
+
+export interface DegradedFallback {
+  /** The per-process store a degraded decision is taken from — one per shared store. */
+  readonly store: RateLimitStore;
+  /** Log the degradation, at most once per `DEGRADED_WARN_EVERY_MS` of the caller's clock. */
+  note(error: unknown, nowMs: number): void;
+}
+
+/**
+ * One fallback per shared store: an empty memory store costs a `Map` until the first degraded
+ * decision. The warn is throttled on the CALLER's clock — the one every take is measured against —
+ * so a frozen test clock and a real one both get one line per window, never one per request.
+ */
+export function degradedFallback(): DegradedFallback {
+  const store = memoryRateLimitStore();
+  let lastWarnMs: number | undefined;
+  return {
+    store,
+    note(error, nowMs) {
+      if (lastWarnMs !== undefined && nowMs - lastWarnMs < DEGRADED_WARN_EVERY_MS) return;
+      lastWarnMs = nowMs;
+      logger.warn('http.rate_limit.degraded', {
+        code: isUltimateError(error) ? error.code : 'unknown',
+        cause:
+          'the shared rate-limit store could not reach the database, so this pod is counting limits in its own memory until it can — each limit is per pod, not per fleet, meanwhile',
+        fix: 'x doctor --json   # names the database binding this pod cannot reach',
+      });
+    },
+  };
+}

package/src/rate-limit-postgres.ts

@@ -6,6 +6,7 @@
 import type { PgExecutor } from '@ultimat3/core';
 import type { RateLimitDecision, RateLimitScope, RateLimitStore } from './rate-limit';
 import { rateLimitDecision } from './rate-limit';
+import { degradedFallback, isRateLimitStoreUnreachable } from './rate-limit-degrade';
 import { rateLimitStoreUnavailable } from './rate-limit-errors';
 import { rateLimitPeek, refilledTokens } from './rate-limit-peek';
 
@@ -164,18 +165,32 @@ export function postgresRateLimitStore(
   options: PostgresRateLimitStoreOptions,
 ): PostgresRateLimitStore {
   const exec = options.executor;
+  // Where a decision is taken while the database is unreachable (`rate-limit-degrade.ts`). Still
+  // `scope: 'shared'` below: the store IS the fleet's, and the degraded window is logged, bounded
+  // by the outage, and a weaker limit rather than none.
+  const fallback = degradedFallback();
+  const degrade = <T>(error: unknown, nowMs: number, decide: () => Promise<T>): Promise<T> => {
+    if (!isRateLimitStoreUnreachable(error)) throw error;
+    fallback.note(error, nowMs);
+    return decide();
+  };
 
   return {
     scope: 'shared',
 
     async take(key, bucket, cost, nowMs): Promise<RateLimitDecision> {
-      const rows = await exec.query<TakeRow>(SQL_RATE_LIMIT_TAKE, [
-        key,
-        bucket.capacity,
-        bucket.refillPerSecond,
-        cost,
-        Math.floor(nowMs),
-      ]);
+      let rows: readonly TakeRow[];
+      try {
+        rows = await exec.query<TakeRow>(SQL_RATE_LIMIT_TAKE, [
+          key,
+          bucket.capacity,
+          bucket.refillPerSecond,
+          cost,
+          Math.floor(nowMs),
+        ]);
+      } catch (error) {
+        return degrade(error, nowMs, () => fallback.store.take(key, bucket, cost, nowMs));
+      }
       const row = rows[0];
       // An upsert with `returning` answers exactly one row, so none means the executor is not
       // running the statement it was handed. Refusing loudly beats inventing a decision: the
@@ -185,7 +200,13 @@ export function postgresRateLimitStore(
     },
 
     async peek(key, bucket, nowMs) {
-      const row = (await exec.query<PeekRow>(SQL_RATE_LIMIT_PEEK, [key]))[0];
+      let rows: readonly PeekRow[];
+      try {
+        rows = await exec.query<PeekRow>(SQL_RATE_LIMIT_PEEK, [key]);
+      } catch (error) {
+        return degrade(error, nowMs, () => fallback.store.peek(key, bucket, nowMs));
+      }
+      const row = rows[0];
       const tokens =
         row === undefined
           ? bucket.capacity
@@ -194,6 +215,10 @@ export function postgresRateLimitStore(
     },
 
     async reset(key): Promise<void> {
+      // Both places a decision for this key can come from: a key exhausted in the fallback during
+      // one outage would otherwise still be denied in the NEXT one, after a reset that succeeded.
+      // The fallback first, so a reset the database refuses still clears what this pod holds.
+      await fallback.store.reset(key);
       await exec.query(SQL_RATE_LIMIT_RESET, [key]);
     },