cartethyia 2.5.9 → 2.6.0

package/src/transport/dispatch/streaming/thinking-loop.ts

@@ -0,0 +1,300 @@
+/**
+ * Thinking-loop guard for streamed reasoning.
+ *
+ * A model that falls into a degenerate reasoning loop re-emits the same
+ * paragraph — byte-identical or with cosmetic wording drift — until it burns
+ * the whole output budget without ever answering. The stream never goes silent,
+ * so the only protection this codebase had (`streamStallTimeoutMs`) cannot fire:
+ * it watches for *absence* of bytes, and a runaway emits them continuously.
+ *
+ * This is a narrow port of the donor `ThinkingLoopDetector`
+ * (`Public/oh-my-pi/packages/ai/src/utils/thinking-loop.ts`). Ported:
+ * 1. **Exact suffix cycles** — a byte-identical unit repeated back-to-back.
+ *    Model-independent and calibration-free.
+ * 2. **Near-duplicate segments** — paragraphs that normalize to the same
+ *    word-trigram fingerprint.
+ *
+ * Deliberately NOT ported: the donor's progress-lexicon novelty detector (its
+ * constants were calibrated against a 536k-block corpus we do not have) and its
+ * Gemini-specific summary-header runaway. Both would be uncalibrated guesses
+ * here and are better added later, with their corpora, if the shapes are
+ * actually observed.
+ *
+ * The detector is fed one reasoning delta at a time and returns a
+ * human-readable reason the first time a loop shape is recognized; every later
+ * call returns the same reason so a caller that does not stop stays harmless.
+ * Ported constants are copied verbatim — they are calibration values, not
+ * design choices.
+ */
+
+/** Stable lead phrase of the guard's reason; exported for tests and for the
+ *  terminal event's `provider_stop_reason`. */
+export const THINKING_LOOP_MARKER = "Thinking loop detected";
+
+/** Rolling tail retained for exact suffix-cycle detection. */
+const EXACT_TAIL_WINDOW = 4096;
+/** Longest exact cycle length considered. */
+const EXACT_MAX_UNIT = 1024;
+/** New characters between scans. Large deltas are scanned immediately. */
+const EXACT_CHECK_STRIDE = 128;
+/** Short cycles need four repeats covering at least this many characters. */
+const EXACT_SHORT_MAX_UNIT = 60;
+const EXACT_SHORT_MIN_REPEATED_CHARS = 180;
+/** Long cycles need at least three repeats covering at least this many chars. */
+const EXACT_LONG_MIN_REPEATED_CHARS = 1024;
+/** Repeated characters before a punctuation-only run counts as a loop. Matches
+ *  {@link EXACT_LONG_MIN_REPEATED_CHARS}: a kilobyte of one punctuation cycle
+ *  back-to-back is degenerate, while the short separators real output produces
+ *  (table rules, dividers) never reach it. */
+const EXACT_PUNCTUATION_MIN_REPEATED_CHARS = EXACT_LONG_MIN_REPEATED_CHARS;
+
+/** Char cap for an unterminated segment; forces a flush so a wall-of-text loop
+ *  (no blank lines / headings) still segments. */
+const SEGMENT_CHAR_CAP = 700;
+/** Normalized-length floor below which a segment is ignored (too short to be a
+ *  meaningful paragraph; bare headings must not trip detection). */
+const SEGMENT_MIN_NORM_CHARS = 60;
+/** How many recent substantial segments are kept for similarity comparison. */
+const SEGMENT_WINDOW = 16;
+/** Word-trigram Jaccard at/above which two segments count as near-duplicates. */
+const SEGMENT_SIMILARITY = 0.8;
+/** Substantial segments required before detection may fire (warm-up). */
+const SEGMENT_MIN_COUNT = 8;
+/** Near-duplicate cluster size (current + matches) that trips the loop. */
+const SEGMENT_MIN_CLUSTER = 4;
+
+/** A whole line shaped like code rather than prose: a fence marker, an indented
+ *  line that is not a nested list item, or a line ending in a block, statement,
+ *  or markup delimiter. Classified per line rather than by fence state because
+ *  thought summaries routinely end inside an unclosed fence; tracking fences
+ *  would hide every later prose paragraph from the heuristics. */
+const CODE_LINE = /^(?:[ \t]*```.*|(?: {2,}|\t)(?![ \t]*(?:[-*+]|\d+[.)])[ \t]).*|.*[{}[\](;,>][ \t]*)$/gm;
+
+export interface ThinkingLoopConfig {
+  readonly enabled?: boolean;
+}
+
+/**
+ * Stateful detector fed the streamed reasoning deltas of one stream. `push`
+ * returns a human-readable reason the first time a loop shape is recognized and
+ * the same reason on every later call, so a caller that keeps feeding after the
+ * first hit cannot double-report.
+ */
+export class ThinkingLoopDetector {
+  /** Rolling char tail for exact suffix-cycle detection. */
+  #tail = "";
+  /** Total characters received when the exact detector last scanned. */
+  #exactScannedAt = 0;
+  /** Pending reasoning text not yet split into completed segments. */
+  #pending = "";
+  /** Fingerprints of the most recent substantial segments (≤ SEGMENT_WINDOW). */
+  #window: Set<string>[] = [];
+  /** Count of substantial segments seen so far (warm-up gate). */
+  #count = 0;
+  /** The reason produced by the first hit; latched for idempotent re-reporting. */
+  #reason: string | undefined;
+
+  constructor(private readonly config: ThinkingLoopConfig = {}) {}
+
+  /** Whether a loop has already been detected. */
+  get detected(): boolean {
+    return this.#reason !== undefined;
+  }
+
+  /** Feeds one reasoning delta. Returns the reason on and after the first hit. */
+  push(delta: string): string | undefined {
+    if (this.#reason !== undefined) return this.#reason;
+    if (this.config.enabled === false) return undefined;
+    if (delta.length === 0) return undefined;
+
+    // 1. Exact suffix cycles. Scan at a bounded cadence rather than doing
+    // quadratic work for every token-sized delta.
+    this.#tail += delta;
+    if (this.#tail.length > EXACT_TAIL_WINDOW) this.#tail = this.#tail.slice(-EXACT_TAIL_WINDOW);
+    this.#exactScannedAt += delta.length;
+    if (this.#exactScannedAt >= EXACT_CHECK_STRIDE || delta.length >= EXACT_CHECK_STRIDE) {
+      this.#exactScannedAt = 0;
+      const exact = detectExactSuffixCycle(this.#tail);
+      if (exact) {
+        const [unit, times] = exact;
+        this.#reason = `repeated an exact ${unit.length}-character cycle ${times}× back-to-back`;
+        return this.#reason;
+      }
+    }
+
+    // 2. Near-duplicate paragraph loop. Append, then drain completed segments.
+    this.#pending += delta;
+    for (;;) {
+      const boundary = /\n\s*\n/.exec(this.#pending);
+      let raw: string;
+      if (boundary) {
+        raw = this.#pending.slice(0, boundary.index);
+        this.#pending = this.#pending.slice(boundary.index + boundary[0].length);
+      } else if (this.#pending.length > SEGMENT_CHAR_CAP) {
+        // No boundary yet but the segment is runaway-long: force a flush.
+        raw = this.#pending.slice(0, SEGMENT_CHAR_CAP);
+        this.#pending = this.#pending.slice(SEGMENT_CHAR_CAP);
+      } else {
+        return undefined;
+      }
+      // An over-long segment is chunked so each piece stays comparable.
+      for (let rest = raw; rest.length > 0; ) {
+        const chunk = rest.length > SEGMENT_CHAR_CAP ? rest.slice(0, SEGMENT_CHAR_CAP) : rest;
+        rest = rest.slice(chunk.length);
+        const hit = this.#consumeSegment(chunk);
+        if (hit) {
+          this.#reason = hit;
+          return this.#reason;
+        }
+      }
+    }
+  }
+
+  /** Process the buffered trailing paragraph (one with no blank-line / heading
+   *  terminator). Called when the reasoning block ends so the final segment —
+   *  which may be the one that completes a duplicate cluster — is not dropped. */
+  flush(): string | undefined {
+    if (this.#reason !== undefined) return this.#reason;
+    if (this.config.enabled === false) return undefined;
+    // A stream can end before the next cadence boundary: force one final exact
+    // check even when nothing is pending.
+    const exact = detectExactSuffixCycle(this.#tail);
+    if (exact) {
+      const [unit, times] = exact;
+      this.#reason = `repeated an exact ${unit.length}-character cycle ${times}× back-to-back`;
+      return this.#reason;
+    }
+    if (this.#pending.length === 0) return undefined;
+    let rest = this.#pending;
+    this.#pending = "";
+    while (rest.length > 0) {
+      const chunk = rest.length > SEGMENT_CHAR_CAP ? rest.slice(0, SEGMENT_CHAR_CAP) : rest;
+      rest = rest.slice(chunk.length);
+      const hit = this.#consumeSegment(chunk);
+      if (hit) {
+        this.#reason = hit;
+        return this.#reason;
+      }
+    }
+    return undefined;
+  }
+
+  #consumeSegment(raw: string): string | undefined {
+    // Reasoning-summarizer titles ("**Maintaining Momentum**", "## Heading")
+    // are per-thought formatting, not chain-of-thought; their ever-changing
+    // wording would otherwise mask a loop by inflating novelty. Strip them and
+    // code lines before analysis (a title- or code-only segment then falls below
+    // the length gate).
+    const segment = raw
+      .replace(CODE_LINE, "")
+      .replace(/^[ \t]*#{1,6}[ \t].*$/gm, "")
+      .replace(/^[ \t]*\*{2,3}.+?\*{2,3}[ \t]*$/gm, "");
+    const normalized = normalizeSegment(segment);
+    if (normalized.length < SEGMENT_MIN_NORM_CHARS) return undefined;
+
+    // Near-duplicate trigram cluster: the same paragraph reused with cosmetic
+    // wording drift (high word-trigram overlap).
+    const fingerprint = trigramShingles(normalized);
+    let cluster = 1;
+    for (const prev of this.#window) {
+      if (jaccard(fingerprint, prev) >= SEGMENT_SIMILARITY) cluster++;
+    }
+
+    this.#window.push(fingerprint);
+    if (this.#window.length > SEGMENT_WINDOW) this.#window.shift();
+    this.#count++;
+
+    if (this.#count >= SEGMENT_MIN_COUNT && cluster >= SEGMENT_MIN_CLUSTER) {
+      return `${cluster} near-identical segments within the last ${SEGMENT_WINDOW}`;
+    }
+    return undefined;
+  }
+}
+
+/** Reused Z-array scratch; scans are synchronous, so one buffer suffices. */
+const exactZ = new Uint16Array(EXACT_MAX_UNIT + 1);
+
+/**
+ * Detect an exact cycle at the text suffix. A Z-array over the reversed tail
+ * finds every possible suffix period in linear time without substring churn.
+ * The reversal is virtual (indexed from the end) and only `z[1..maxUnit]` is
+ * computed — the period check reads nothing past it, and each `z[i]` reads
+ * only `z[i - left]` with `i - left < i`, which is already filled. Short
+ * cycles retain the original 180-character/four-repeat sensitivity; long
+ * cycles require at least three repeats and 1024 repeated characters.
+ *
+ * A unit carrying a letter or an emoji matches at those thresholds. A unit with
+ * neither, such as the `?!?!` run a stuck model emits, matches only once it has
+ * repeated for {@link EXACT_PUNCTUATION_MIN_REPEATED_CHARS} characters, and never
+ * when it holds only digits and whitespace: padding, indentation and zero-filled
+ * hexdumps or literals are ordinary output at any length.
+ */
+function detectExactSuffixCycle(text: string): [unit: string, count: number] | undefined {
+  const n = text.length;
+  if (n < EXACT_SHORT_MIN_REPEATED_CHARS) return undefined;
+  const last = n - 1;
+  const maxUnit = Math.min(EXACT_MAX_UNIT, Math.floor(n / 3));
+  const z = exactZ;
+  let left = 0;
+  let right = 0;
+  for (let i = 1; i <= maxUnit; i++) {
+    let zi = i <= right ? Math.min(right - i + 1, z[i - left]!) : 0;
+    // `z[i - left]` is always filled: `i - left < i` and `left >= 0`.
+    while (i + zi < n && text.charCodeAt(last - zi) === text.charCodeAt(last - i - zi)) zi++;
+    z[i] = zi;
+    if (i + zi - 1 > right) {
+      left = i;
+      right = i + zi - 1;
+    }
+  }
+
+  for (let len = 2; len <= maxUnit; len++) {
+    // Every `z[len]` in this range was written by the loop above.
+    const count = 1 + Math.floor(z[len]! / len);
+    const minCount = len <= EXACT_SHORT_MAX_UNIT ? 4 : 3;
+    if (count < minCount) continue;
+    const unit = text.slice(-len);
+    if (/\p{L}|\p{Extended_Pictographic}/u.test(unit)) {
+      const minChars = len <= EXACT_SHORT_MAX_UNIT ? EXACT_SHORT_MIN_REPEATED_CHARS : EXACT_LONG_MIN_REPEATED_CHARS;
+      if (len * count >= minChars) return [unit, count];
+      continue;
+    }
+    if (/^[\p{N}\s]*$/u.test(unit)) continue;
+    if (len * count >= EXACT_PUNCTUATION_MIN_REPEATED_CHARS) return [unit, count];
+  }
+  return undefined;
+}
+
+/** Lowercase and tokenize prose plus code/path payloads, dropping pure numbers. */
+function normalizeSegment(segment: string): string {
+  return segment
+    .toLowerCase()
+    .replace(/`([^`]*)`/g, " $1 ")
+    .replace(/[^a-z0-9]+/g, " ")
+    .split(/\s+/)
+    .filter((token) => /[a-z]/.test(token))
+    .join(" ")
+    .trim();
+}
+
+/** Word-trigram shingle set of a normalized segment. */
+function trigramShingles(normalized: string): Set<string> {
+  const words = normalized.split(" ").filter(Boolean);
+  if (words.length < 3) return new Set(words.length > 0 ? [words.join(" ")] : []);
+  const shingles = new Set<string>();
+  for (let i = 0; i + 3 <= words.length; i++) {
+    shingles.add(`${words[i]} ${words[i + 1]} ${words[i + 2]}`);
+  }
+  return shingles;
+}
+
+function jaccard(a: Set<string>, b: Set<string>): number {
+  if (a.size === 0 || b.size === 0) return 0;
+  const [small, large] = a.size < b.size ? [a, b] : [b, a];
+  let intersection = 0;
+  for (const x of small) {
+    if (large.has(x)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}

package/src/transport/failure-policy.ts

@@ -582,6 +582,8 @@ const CONTEXT_LENGTH_CODES: ReadonlySet<string> = new Set([
   "context_length_exceeded",
   "context_too_large",
   "model_context_window_exceeded",
+  // InferHub's spelling for the same condition.
+  "context_too_long",
 ]);
 
 /**
@@ -733,6 +735,14 @@ const QUOTA_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "usage_not_included",
   "subscription:free-usage-exhausted",
   "resource_exhausted",
+  // InferHub publishes its own vocabulary in `error.code`. The two 403s below
+  // are deliberate absences: `account_blocked` means the account was blocked
+  // (an auth/entitlement verdict, not a spend limit) and
+  // `model_not_allowed_for_key` is a per-key allowlist refusal — both are
+  // classified separately rather than cooled as quota.
+  "rate_limited",
+  "providers_out_of_quota",
+  "providers_at_capacity",
 ]);
 const CAPACITY_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "overloaded_error",
@@ -740,6 +750,9 @@ const CAPACITY_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "model_at_capacity",
   "server_is_overloaded",
   "model_capacity",
+  // InferHub: no live provider could take the request right now.
+  "providers_busy",
+  "providers_overloaded",
 ]);
 const AUTH_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "authentication_error",
@@ -749,6 +762,12 @@ const AUTH_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "unauthenticated",
   "permission_denied",
   "permissiondenied",
+  // InferHub: the account was blocked, and a key whose scope or allowlist does
+  // not cover the request. `credentialEvidence` is what cools the account
+  // rather than the provider.
+  "account_blocked",
+  "insufficient_key_scope",
+  "model_not_allowed_for_key",
 ]);
 const PLATFORM_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "server_error",
@@ -895,6 +914,14 @@ export function classifyUpstreamError(
     };
   if (normalized === "model_not_found")
     return { code: "model_not_found", status: 404, origin: "upstream" };
+  // InferHub: no live publisher serves the requested model at all. That is a
+  // property of the model/upstream pair, not of the account or a spend limit,
+  // so it is not retried as quota and does not cool the account — the next
+  // attempt should pick a different route.
+  if (normalized === "no_provider")
+    return { code: "model_not_found", status: 404, origin: "upstream" };
+  if (normalized === "content_rejected")
+    return { code: "policy_rejected", status: 403, origin: "upstream" };
   if (
     normalized === "not_found_error" ||
     normalized === "resource_not_found" ||

package/src/transport/guards/abuse-guard.ts

@@ -1,12 +1,12 @@
 // IP-abuse protection guard: counts every `/v1/*` attempt against the per-IP
 // ceiling via the root `request` hook.
 import { Elysia } from "elysia";
-import { GatewayError } from "../../protocol/gateway-error";
 import type { TrustedProxyBoundary } from "../../config/network";
 import { resolveClientIdentity } from "../../security/ip-boundary";
 import type { ProxyRequestStateStore } from "../request/state";
 import { fastPathname } from "../pathname";
 import type { IpAbuseProtectionService } from "../../security/abuse";
+import { peerAddressUnavailableError } from "./guard-errors";
 
 /**
  * Counts every `/v1/*` attempt against the per-IP ceiling, including the ones
@@ -66,7 +66,7 @@ export function createIpAbuseProtectionMiddleware(deps: {
       ? deps.resolvePeerAddress(request)
       : (server?.requestIP(request)?.address ?? null);
     if (!peer)
-      throw new GatewayError("admission_unavailable", 503, "client peer address unavailable");
+      throw peerAddressUnavailableError();
     const address = resolveClientIdentity(request, deps.trustedProxyBoundary, peer);
     const state = deps.stateStore.get(request);
     try {

package/src/transport/guards/auth-guard.ts

@@ -28,6 +28,12 @@ export function createApiKeyAuthenticationMiddleware(deps: {
    * console error, which is exactly the noise this layer exists to stop.
    */
   readonly modelStrikes?: Pick<ModelStrikeService, "check">;
+  /**
+   * Tenant invoke kill switch. Checked after the key resolves, because the
+   * tenant is a property of the key: the same gateway serves several tenants
+   * and one switching itself off must not stop the others.
+   */
+  readonly isInvokeDisabled?: (tenantId: string) => Promise<boolean | undefined>;
 }): Elysia {
   // Single policy for every `/v1/*` gateway route: authenticated, scoped to
   // `routing:invoke`, and tenant-bound (the snapshot's tenant is the only
@@ -73,6 +79,17 @@ export function createApiKeyAuthenticationMiddleware(deps: {
       });
       if (!decision.scopes.includes("routing:invoke"))
         throw new GatewayError("invalid_request", 403, "This API key is not allowed to invoke routing. Grant it the routing:invoke scope, or use a key that has it.");
+      // Kill switch: refuse before any upstream is touched. Distinct from the
+      // scope check above, which is a property of the key — this is the tenant
+      // having stopped all traffic, and it applies to every key the tenant owns.
+      // 503 rather than 403: nothing about the request is invalid, and the
+      // caller should retry later instead of "fixing" a request that is fine.
+      if ((await deps.isInvokeDisabled?.(authorization.tenantId)) === true)
+        throw new GatewayError(
+          "platform_unavailable",
+          503,
+          "This gateway has invoke switched off. Nothing upstream is being dialed while the switch is off; turn it back on from the dashboard overview to resume routing.",
+        );
       // A key may refuse a downstream router it has been resold to. This is a
       // policy check on the resolved key, so it belongs with the other
       // authorization decisions rather than in routing: the request is refused

package/src/transport/guards/body-policy.ts

@@ -5,6 +5,7 @@ import { assertBoundedJsonDepth } from "../../protocol/codecs";
 import type { ProxyRequestStateStore } from "../request/state";
 import { fastPathname } from "../pathname";
 import { NATIVE_SERVICE_PATHS, TOKEN_COUNT_PATH } from "../contracts";
+import { bodyTooLargeError } from "./guard-errors";
 
 /** Whether this request is a gateway (`/v1/*`) request. */
 export function isProxyRequest(request: Request): boolean {
@@ -87,7 +88,7 @@ export async function readIngressBody(
   const declared = request.headers.get("content-length");
   const parsed = declared === null ? undefined : Number(declared);
   if (declared !== null && (!/^\d+$/.test(declared) || parsed === undefined || parsed > maxBytes))
-    throw new GatewayError("invalid_request", 413, `The request body is larger than this endpoint accepts (${maxBytes} bytes). Shorten the prompt or split the request.`);
+    throw bodyTooLargeError(maxBytes);
 
   const reader = request.body?.getReader();
   if (!reader) throw new GatewayError("invalid_request", 400, "The request body could not be read. Resend it with a JSON body.");
@@ -104,7 +105,7 @@ export async function readIngressBody(
       totalBytes += chunk.byteLength;
       if (totalBytes > maxBytes) {
         await reader.cancel().catch(() => undefined);
-        throw new GatewayError("invalid_request", 413, `The request body is larger than this endpoint accepts (${maxBytes} bytes). Shorten the prompt or split the request.`);
+        throw bodyTooLargeError(maxBytes);
       }
       chunks.push(chunk);
     }

package/src/transport/guards/concurrency-guard.ts

@@ -6,23 +6,97 @@
  * their own cap still open a hundred upstream calls at once, and the event
  * loop, the database pool, and the upstream all feel that as one burst.
  *
- * This counts requests in flight in this process and refuses past the cap
- * with 503 and a three-second retry hint. It is deliberately not a queue:
- * queueing would hold the connection open and make the overload look like
- * latency. A client that retries after the hint lands on a process that has
- * drained.
+ * This admits requests into this process and refuses past the cap with 503 and
+ * a retry hint. It is deliberately not a queue: queueing would hold the
+ * connection open and make the overload look like latency. A client that
+ * retries after the hint lands on a process that has drained.
+ *
+ * The cap is now DERIVED — see `config/resources.ts` — rather than a constant,
+ * because a constant cap admits the same concurrency on a 512 MiB container as
+ * on a 4 GiB one, and only the latter can actually hold it.
  */
 import { Elysia } from "elysia";
 import { GatewayError } from "../../protocol/gateway-error";
+import { resolveProcessConcurrency } from "../../config/resources";
+import { resolveMemoryLimitBytes } from "../../observability/runtime-metrics";
 import { fastPathname } from "../pathname";
 
 /** In-flight `/v1` requests one process will carry before refusing. */
-export const PROCESS_CONCURRENCY_CAP = 64;
+export const PROCESS_CONCURRENCY_CAP = resolveProcessConcurrency(
+  resolveMemoryLimitBytes() ?? 512 * 1024 * 1024,
+);
 const PROCESS_CONCURRENCY_RETRY_MS = 3_000;
 
-let inFlight = 0;
+/**
+ * A bounded set of in-flight tokens.
+ *
+ * Owned by the caller rather than held in a module global: the previous shape
+ * was one `let inFlight` for the whole module, so two middleware instances
+ * (tests, or a second app in one process) shared and corrupted a single count.
+ */
+export interface ProcessConcurrencyController {
+  /**
+   * Takes a token, or returns `undefined` when the process is full.
+   *
+   * The returned release function is idempotent — released twice still frees
+   * exactly one token — because a request can reach more than one completion
+   * path (an abort racing a normal finish).
+   */
+  tryAcquire(): (() => void) | undefined;
+  /** Tokens currently held. */
+  activeCount(): number;
+  /** The effective cap. */
+  limit(): number;
+}
+
+export function createProcessConcurrencyController(
+  limit: number = PROCESS_CONCURRENCY_CAP,
+): ProcessConcurrencyController {
+  const effective = Math.min(64, Math.max(1, Math.trunc(limit)));
+  let active = 0;
+  return {
+    tryAcquire() {
+      if (active >= effective) return undefined;
+      active += 1;
+      let released = false;
+      return () => {
+        if (released) return;
+        released = true;
+        if (active > 0) active -= 1;
+      };
+    },
+    activeCount() {
+      return active;
+    },
+    limit() {
+      return effective;
+    },
+  };
+}
+
+/**
+ * The controller the middleware below uses when none is injected.
+ *
+ * Exported so the metrics sampler can publish its live count: in-flight
+ * concurrency is the number that explains both a latency rise and a shed, and
+ * a gauge nobody sets is worse than no gauge.
+ */
+export const defaultProcessConcurrencyController = createProcessConcurrencyController();
 
-export function createProcessConcurrencyMiddleware(): Elysia {
+/**
+ * The middleware.
+ *
+ * The token is keyed by the Request object in a `WeakMap` and released only if
+ * one was actually taken. The previous code decremented a module-global count
+ * in `afterResponse` for every `/v1` request — including the ones this guard
+ * had just REJECTED, which never incremented it. Under sustained overload that
+ * drove the count down spuriously and let more requests through than the cap
+ * allows, which is the opposite of what a cap is for.
+ */
+export function createProcessConcurrencyMiddleware(
+  controller: ProcessConcurrencyController = defaultProcessConcurrencyController,
+): Elysia {
+  const tokens = new WeakMap<Request, () => void>();
   return new Elysia()
     .beforeHandle(({ request }) => {
       const path = fastPathname(request.url);
@@ -30,7 +104,8 @@ export function createProcessConcurrencyMiddleware(): Elysia {
       // still answer while the process is saturated, or an operator cannot
       // see why it is refusing.
       if (!path.startsWith("/v1/")) return;
-      if (inFlight >= PROCESS_CONCURRENCY_CAP) {
+      const release = controller.tryAcquire();
+      if (release === undefined) {
         throw new GatewayError(
           "platform_unavailable",
           503,
@@ -38,12 +113,13 @@ export function createProcessConcurrencyMiddleware(): Elysia {
           { retryAfterMs: PROCESS_CONCURRENCY_RETRY_MS },
         );
       }
-      inFlight += 1;
+      tokens.set(request, release);
     })
     .afterResponse(({ request }: { request: Request }) => {
-      const path = fastPathname(request.url);
-      if (!path.startsWith("/v1/")) return;
-      if (inFlight > 0) inFlight -= 1;
+      const release = tokens.get(request);
+      if (release === undefined) return;
+      tokens.delete(request);
+      release();
     })
     .as("plugin");
 }

package/src/transport/guards/early-rejection-capture.ts

@@ -12,9 +12,15 @@
  * error envelope the client received.
  */
 import type { PayloadCaptureState } from "../../observability/payload-capture";
+import { resolvePayloadCapture } from "../../observability/payload-capture";
+import type { CartethyiaDatabase } from "../../persistence/db-handle";
 /** The capture surface this helper needs; matches the real capture's method. */
 export interface EarlyRejectionCaptureSink {
-  capture(input: EarlyRejectionCapture): Promise<unknown>;
+  capture(input: EarlyRejectionCapture): Promise<{
+    id: string;
+    expiresAt: Date;
+    payloadCaptureState?: string;
+  }>;
 }
 
 /** One early-rejection payload frame. */
@@ -34,6 +40,15 @@ export interface EarlyRejectionState {
   readonly ingressBody?: unknown;
   readonly clientResponseText?: string;
   readonly payloadCapture?: EarlyRejectionCaptureSink;
+  /**
+   * The request's memoized capture preference, when the request already
+   * resolved one. Early rejections rarely have, so the helper resolves the
+   * tenant preference itself through the same `resolvePayloadCapture` path
+   * every other capture decision uses — the bypass this field exists to end
+   * was an early rejection capturing bodies with the deployment's full cap
+   * while the tenant had configured `none` or `metadata`.
+   */
+  payloadCapturePreference?: unknown;
   /**
    * Written by this helper so telemetry can record whether the rejection's body
    * was captured. Mutable because the reason is an outcome, not an input.
@@ -51,12 +66,36 @@ export interface EarlyRejectionState {
 export async function captureEarlyRejectionPayload(
   state: EarlyRejectionState,
   authorization: { readonly tenantId: string },
+  db?: CartethyiaDatabase,
 ): Promise<void> {
   const capture = state.payloadCapture;
   if (capture === undefined) return;
   if (state.ingressBody === undefined) return;
   try {
-    await capture.capture({
+    // The same preference path a dispatched request resolves (memoized on the
+    // state when the request got far enough to have one). `none` captures
+    // nothing at all; `metadata` records stage presence only, without the
+    // ingress body; only `full` keeps bodies — and at the tenant's own
+    // retention, never the deployment default by accident.
+    const preference =
+      state.payloadCapturePreference !== undefined
+        ? (state.payloadCapturePreference as { ok: boolean })
+        : db !== undefined
+          ? await resolvePayloadCapture(db, authorization.tenantId)
+          : { ok: false };
+    const mode = preference.ok === true ? (preference as { mode: string }).mode : "none";
+    if (mode === "none") {
+      state.payloadCaptureState = "skipped_mode_none";
+      return;
+    }
+    if (mode === "metadata") {
+      // Stage presence only: the ingress body is exactly what `metadata`
+      // mode declines to retain. A frame with no body is pointless, so the
+      // state records the mode outcome and nothing is written.
+      state.payloadCaptureState = "captured_metadata";
+      return;
+    }
+    const result = await capture.capture({
       tenantId: authorization.tenantId,
       requestBody: state.ingressBody,
       // No provider exchange: the request never reached one.
@@ -66,7 +105,8 @@ export async function captureEarlyRejectionPayload(
       tenantOptIn: true,
       requestId: state.requestId,
     });
-    state.payloadCaptureState = "captured";
+    state.payloadCaptureState =
+      (result.payloadCaptureState as PayloadCaptureState | undefined) ?? "captured";
   } catch {
     // A capture failure is not a request failure; the telemetry row stands.
     state.payloadCaptureState = "failed_write";