cartethyia 2.5.10 → 2.6.0

package/src/transport/dispatch/streaming/encoder.ts

@@ -10,8 +10,11 @@
  * client-visible tracking → encode + enqueue) is preserved exactly.
  */
 import type { CanonicalEvent, SourceSurface, UsageRecord } from "../../../protocol/canonical-model";
+import { canonicalTerminal } from "../../../protocol/codecs";
+import { reasoningTextOf } from "../../../protocol/reasoning-text";
 import { MimoThinkSplitter } from "../../../providers/integrations/xiaomi-mimo/think-stream";
 import { createDispatchStreamEncoder, type DispatchStreamEncoder } from "../stream-bridge";
+import { THINKING_LOOP_MARKER, ThinkingLoopDetector } from "./thinking-loop";
 
 /** The terminal event shape every stream settles (or fails to settle) on. */
 export type StreamTerminalEvent = Extract<CanonicalEvent, { type: "terminal" }>;
@@ -78,20 +81,41 @@ export class StreamEventPipeline {
   // One decoder per stream: `new TextDecoder()` per chunk would allocate
   // on every upstream event for the telemetry transcript copy.
   private readonly clientResponseDecoder = new TextDecoder();
+  /**
+   * Resolved tenant capture mode. `none`/`metadata` gate the diagnostics the
+   * tenant will never be able to see: the JSON-serialized event transcript
+   * copy and the client-response accumulation are skipped entirely, so a
+   * privacy-lean tenant's streams cost neither the per-event `JSON.stringify`
+   * nor the up-to-512 KiB buffered client-response copy. The fields the
+   * completion path always needs (terminal, usage, timing, reported model,
+   * event count, client-visible flag) are recorded regardless — the Responses
+   * prelude retry depends on the flag, and the request row depends on the rest.
+   */
+  private readonly diagnosticsCaptureMode: "none" | "metadata" | "full";
   private readonly streamEncoder: DispatchStreamEncoder;
   // One splitter per stream: a thinking tag can straddle two provider chunks,
   // and the text/idle bookkeeping below must see the split result, not the raw
   // tagged chunk.
   private readonly thinkSplitter = new MimoThinkSplitter();
+  /**
+   * Content-level loop guard. A model that degenerates into a repeated
+   * chain-of-thought keeps sending bytes, so the stall timeout — which fires
+   * only on silence — never trips and the runaway burns the whole output
+   * budget. This detector watches the reasoning text itself and terminates the
+   * stream early when a loop shape is recognized.
+   */
+  private readonly loopDetector = new ThinkingLoopDetector();
   private lastSequenceNumber = -1;
 
   constructor(
     surface: SourceSurface,
     encoderOptions: Record<string, unknown>,
     recoverableResponsesPrelude: boolean,
+    diagnostics: { captureMode: "none" | "metadata" | "full" } = { captureMode: "full" },
   ) {
     this.streamEncoder = createDispatchStreamEncoder(surface, encoderOptions);
     this.recoverableResponsesPrelude = recoverableResponsesPrelude;
+    this.diagnosticsCaptureMode = diagnostics.captureMode;
   }
 
   /** Number of canonical events processed, including those omitted from diagnostics. */
@@ -104,6 +128,13 @@ export class StreamEventPipeline {
     return this.clientVisibleEvent;
   }
 
+  /** Whether the loop guard has terminated this stream early. The pull loop
+   *  stops consuming the upstream once it has: continuing would keep billing
+   *  and emitting a runaway the client was already told is over. */
+  get loopDetected(): boolean {
+    return this.loopDetector.detected;
+  }
+
   /** The client-facing response transcript (decoded SSE/JSON) — distinct from the provider-side canonical events. */
   get clientResponseText(): string {
     const text = this.clientResponseChunks.join("");
@@ -114,6 +145,9 @@ export class StreamEventPipeline {
   get telemetryEventTranscript():
     | readonly (CanonicalEvent & { timestamp: number })[]
     | TruncatedEventTranscript {
+    // Gated modes recorded nothing, so the transcript is the plain empty
+    // array — no truncated-marker object for content that was never kept.
+    if (this.diagnosticsCaptureMode !== "full") return [];
     const tail = this.eventTranscriptTail.slice(this.eventTranscriptTailStart).map((entry) => entry.event);
     const events = [...this.eventTranscriptHead, ...tail];
     const omittedEvents = this.totalEventCount - events.length;
@@ -140,6 +174,9 @@ export class StreamEventPipeline {
 
   private recordTelemetryEvent(event: CanonicalEvent & { timestamp: number }): void {
     this.totalEventCount += 1;
+    // Gated diagnostics: the transcript is only ever read by the terminal
+    // capture path in `full` mode, so `none`/`metadata` pay only the count.
+    if (this.diagnosticsCaptureMode !== "full") return;
     // Single serialization per event: the byte size is stored with the entry
     // so eviction subtracts without re-serializing the removed event.
     const eventBytes = Buffer.byteLength(JSON.stringify(event), "utf8");
@@ -179,6 +216,9 @@ export class StreamEventPipeline {
   }
 
   private appendClientResponse(bytes: Uint8Array): void {
+    // Gated diagnostics: the client-response copy feeds only the terminal
+    // capture's client panel, which exists only in `full` mode.
+    if (this.diagnosticsCaptureMode !== "full") return;
     if (this.clientResponseTruncated) return;
     const chunk = this.clientResponseDecoder.decode(bytes, { stream: true });
     if (this.clientResponseLength + chunk.length > CLIENT_RESPONSE_CAP) {
@@ -213,6 +253,12 @@ export class StreamEventPipeline {
     event: CanonicalEvent,
     controller: ReadableStreamDefaultController<Uint8Array>,
   ): boolean {
+    // Once the guard has terminated the stream, the surface encoders reject
+    // any further event — content *and* a second terminal both throw
+    // (`canonical event arrived after terminal`). The pull loop stops on
+    // `loopDetected`, so this only matters for an event already in flight;
+    // dropping it is the alternative to throwing mid-stream.
+    if (this.loopDetector.detected) return false;
     // The one place every surface shares. A provider that writes its reasoning
     // into the answer as `<think>…</think>` had that text forwarded verbatim:
     // the chat wire happened to split it in its own decoder, so chat looked
@@ -222,11 +268,12 @@ export class StreamEventPipeline {
       const split = this.thinkSplitter.push(event.content.text);
       let enqueued = false;
       if (split.reasoning.length > 0) {
-        enqueued =
-          this.enqueueOne(
-            { ...event, content: { kind: "reasoning", payload: null, summary: split.reasoning } },
-            controller,
-          ) || enqueued;
+        const reasoningEvent: CanonicalEvent = {
+          ...event,
+          content: { kind: "reasoning", payload: null, summary: split.reasoning },
+        };
+        enqueued = this.enqueueOne(reasoningEvent, controller) || enqueued;
+        if (this.guardReasoning(split.reasoning, controller)) return enqueued;
       }
       if (split.text.length > 0) {
         enqueued =
@@ -235,6 +282,14 @@ export class StreamEventPipeline {
       }
       return enqueued;
     }
+    // Reasoning a provider already classified: feed it to the guard too, so a
+    // loop is caught whether the reasoning arrived pre-classified or was
+    // recovered from a `<think>` span in the answer.
+    if (event.type === "content_delta" && event.content.kind === "reasoning") {
+      const enqueued = this.enqueueOne(event, controller);
+      this.guardReasoning(reasoningTextOf(event.content) ?? "", controller);
+      return enqueued;
+    }
     // Text held back for a tag that never completed must be released *before*
     // the terminal: a content event after it is rejected by the surface
     // encoders, and the tail of the answer would be lost instead.
@@ -264,6 +319,57 @@ export class StreamEventPipeline {
     return this.enqueueOne(event, controller);
   }
 
+  /**
+   * Feeds one reasoning chunk to the loop guard and, on the first hit,
+   * terminates the stream early. Returns whether it terminated.
+   *
+   * The terminal is a real canonical terminal rather than an error frame: the
+   * client already received valid reasoning, and the runaway is cut off
+   * mid-generation — which is exactly what `stop_reason: "length"` means on
+   * every surface (`length` / `max_tokens`). `provider_stop_reason` carries the
+   * marker so the cut is distinguishable from hitting the real token cap.
+   */
+  private guardReasoning(
+    text: string,
+    controller: ReadableStreamDefaultController<Uint8Array>,
+  ): boolean {
+    const reason = text.length > 0 ? this.loopDetector.push(text) : undefined;
+    if (reason === undefined) return false;
+    // Flush before the terminal: content after it is rejected by the surface
+    // encoders, and the held-back text would be lost.
+    const rest = this.thinkSplitter.flush();
+    if (rest.reasoning.length > 0) {
+      this.enqueueOne(
+        {
+          type: "content_delta",
+          sequence_number: this.lastSequenceNumber + 1,
+          content: { kind: "reasoning", payload: null, summary: rest.reasoning },
+        },
+        controller,
+      );
+    }
+    if (rest.text.length > 0) {
+      this.enqueueOne(
+        {
+          type: "content_delta",
+          sequence_number: this.lastSequenceNumber + 1,
+          content: { kind: "text", text: rest.text },
+        },
+        controller,
+      );
+    }
+    this.enqueueOne(
+      canonicalTerminal({
+        sequenceNumber: this.lastSequenceNumber + 1,
+        state: "complete",
+        stopReason: "length",
+        providerStopReason: THINKING_LOOP_MARKER,
+      }),
+      controller,
+    );
+    return true;
+  }
+
   private enqueueOne(
     event: CanonicalEvent,
     controller: ReadableStreamDefaultController<Uint8Array>,

package/src/transport/dispatch/streaming/stream.ts

@@ -43,12 +43,13 @@ import type { AdmissionLease } from "../../../security/admission/contracts";
 import type { NetworkPoolSelector } from "../../../network/pool/selector";
 import type { TelemetryBatchBuffer } from "../../../observability/telemetry-buffer";
 import type { ProviderExchangeCapture } from "../attempt-finalize";
+import { captureProviderExchange } from "../attempt-finalize";
 import {
-  captureProviderExchange,
   providerCaptureMaxBytes,
   resolveCaptureDepthForRequest,
+  resolveCaptureModeForRequest,
   shouldCaptureProviderBodyForRequest,
-} from "../attempt-finalize";
+} from "../capture-policy";
 import { resolveTenantImageClampMaxBytes } from "../../request/tenant-preferences";
 import { proxySuccessHeaders, buildUpstreamDispatchContext } from "../upstream";
 import type { AttemptLeaseOwnership } from "../leases";
@@ -179,6 +180,14 @@ export async function dispatchStreamingAttempt(input: StreamingDispatchInput): P
     prepared.authorization.snapshot.tenant_id,
     state,
   );
+  // Resolved once, before the pipeline exists: the pipeline's diagnostics are
+  // gated on it from construction, so `none`/`metadata` tenants never pay the
+  // transcript serialization or the client-response copy for any event.
+  const captureMode = await resolveCaptureModeForRequest(
+    deps.db,
+    prepared.authorization.snapshot.tenant_id,
+    state,
+  );
   const wrappedOutboundFetch = outboundFetch
     ? captureProviderExchange(
         outboundFetch,
@@ -276,6 +285,7 @@ export async function dispatchStreamingAttempt(input: StreamingDispatchInput): P
         ? { response_id: `msg-${crypto.randomUUID()}`, model: candidate.model_id }
         : streamOptions,
     recoverableResponsesPrelude,
+    { captureMode },
   );
   const finalizer = new StreamFinalizer(
     {
@@ -400,6 +410,13 @@ export async function dispatchStreamingAttempt(input: StreamingDispatchInput): P
             }
             sawUpstreamActivity = true;
             if (pipeline.enqueueEvent(result.value, controller)) return;
+            // The guard terminated the stream early: stop consuming. Pulling
+            // on would keep the runaway upstream alive and bill tokens the
+            // client was already told are not coming.
+            if (pipeline.loopDetected) {
+              await finalizer.finalizeStream(controller);
+              return;
+            }
           }
         } catch (err) {
           finalizer.clearStallWatchdog();

package/src/transport/dispatch/streaming/thinking-loop.ts

@@ -0,0 +1,300 @@
+/**
+ * Thinking-loop guard for streamed reasoning.
+ *
+ * A model that falls into a degenerate reasoning loop re-emits the same
+ * paragraph — byte-identical or with cosmetic wording drift — until it burns
+ * the whole output budget without ever answering. The stream never goes silent,
+ * so the only protection this codebase had (`streamStallTimeoutMs`) cannot fire:
+ * it watches for *absence* of bytes, and a runaway emits them continuously.
+ *
+ * This is a narrow port of the donor `ThinkingLoopDetector`
+ * (`Public/oh-my-pi/packages/ai/src/utils/thinking-loop.ts`). Ported:
+ * 1. **Exact suffix cycles** — a byte-identical unit repeated back-to-back.
+ *    Model-independent and calibration-free.
+ * 2. **Near-duplicate segments** — paragraphs that normalize to the same
+ *    word-trigram fingerprint.
+ *
+ * Deliberately NOT ported: the donor's progress-lexicon novelty detector (its
+ * constants were calibrated against a 536k-block corpus we do not have) and its
+ * Gemini-specific summary-header runaway. Both would be uncalibrated guesses
+ * here and are better added later, with their corpora, if the shapes are
+ * actually observed.
+ *
+ * The detector is fed one reasoning delta at a time and returns a
+ * human-readable reason the first time a loop shape is recognized; every later
+ * call returns the same reason so a caller that does not stop stays harmless.
+ * Ported constants are copied verbatim — they are calibration values, not
+ * design choices.
+ */
+
+/** Stable lead phrase of the guard's reason; exported for tests and for the
+ *  terminal event's `provider_stop_reason`. */
+export const THINKING_LOOP_MARKER = "Thinking loop detected";
+
+/** Rolling tail retained for exact suffix-cycle detection. */
+const EXACT_TAIL_WINDOW = 4096;
+/** Longest exact cycle length considered. */
+const EXACT_MAX_UNIT = 1024;
+/** New characters between scans. Large deltas are scanned immediately. */
+const EXACT_CHECK_STRIDE = 128;
+/** Short cycles need four repeats covering at least this many characters. */
+const EXACT_SHORT_MAX_UNIT = 60;
+const EXACT_SHORT_MIN_REPEATED_CHARS = 180;
+/** Long cycles need at least three repeats covering at least this many chars. */
+const EXACT_LONG_MIN_REPEATED_CHARS = 1024;
+/** Repeated characters before a punctuation-only run counts as a loop. Matches
+ *  {@link EXACT_LONG_MIN_REPEATED_CHARS}: a kilobyte of one punctuation cycle
+ *  back-to-back is degenerate, while the short separators real output produces
+ *  (table rules, dividers) never reach it. */
+const EXACT_PUNCTUATION_MIN_REPEATED_CHARS = EXACT_LONG_MIN_REPEATED_CHARS;
+
+/** Char cap for an unterminated segment; forces a flush so a wall-of-text loop
+ *  (no blank lines / headings) still segments. */
+const SEGMENT_CHAR_CAP = 700;
+/** Normalized-length floor below which a segment is ignored (too short to be a
+ *  meaningful paragraph; bare headings must not trip detection). */
+const SEGMENT_MIN_NORM_CHARS = 60;
+/** How many recent substantial segments are kept for similarity comparison. */
+const SEGMENT_WINDOW = 16;
+/** Word-trigram Jaccard at/above which two segments count as near-duplicates. */
+const SEGMENT_SIMILARITY = 0.8;
+/** Substantial segments required before detection may fire (warm-up). */
+const SEGMENT_MIN_COUNT = 8;
+/** Near-duplicate cluster size (current + matches) that trips the loop. */
+const SEGMENT_MIN_CLUSTER = 4;
+
+/** A whole line shaped like code rather than prose: a fence marker, an indented
+ *  line that is not a nested list item, or a line ending in a block, statement,
+ *  or markup delimiter. Classified per line rather than by fence state because
+ *  thought summaries routinely end inside an unclosed fence; tracking fences
+ *  would hide every later prose paragraph from the heuristics. */
+const CODE_LINE = /^(?:[ \t]*```.*|(?: {2,}|\t)(?![ \t]*(?:[-*+]|\d+[.)])[ \t]).*|.*[{}[\](;,>][ \t]*)$/gm;
+
+export interface ThinkingLoopConfig {
+  readonly enabled?: boolean;
+}
+
+/**
+ * Stateful detector fed the streamed reasoning deltas of one stream. `push`
+ * returns a human-readable reason the first time a loop shape is recognized and
+ * the same reason on every later call, so a caller that keeps feeding after the
+ * first hit cannot double-report.
+ */
+export class ThinkingLoopDetector {
+  /** Rolling char tail for exact suffix-cycle detection. */
+  #tail = "";
+  /** Total characters received when the exact detector last scanned. */
+  #exactScannedAt = 0;
+  /** Pending reasoning text not yet split into completed segments. */
+  #pending = "";
+  /** Fingerprints of the most recent substantial segments (≤ SEGMENT_WINDOW). */
+  #window: Set<string>[] = [];
+  /** Count of substantial segments seen so far (warm-up gate). */
+  #count = 0;
+  /** The reason produced by the first hit; latched for idempotent re-reporting. */
+  #reason: string | undefined;
+
+  constructor(private readonly config: ThinkingLoopConfig = {}) {}
+
+  /** Whether a loop has already been detected. */
+  get detected(): boolean {
+    return this.#reason !== undefined;
+  }
+
+  /** Feeds one reasoning delta. Returns the reason on and after the first hit. */
+  push(delta: string): string | undefined {
+    if (this.#reason !== undefined) return this.#reason;
+    if (this.config.enabled === false) return undefined;
+    if (delta.length === 0) return undefined;
+
+    // 1. Exact suffix cycles. Scan at a bounded cadence rather than doing
+    // quadratic work for every token-sized delta.
+    this.#tail += delta;
+    if (this.#tail.length > EXACT_TAIL_WINDOW) this.#tail = this.#tail.slice(-EXACT_TAIL_WINDOW);
+    this.#exactScannedAt += delta.length;
+    if (this.#exactScannedAt >= EXACT_CHECK_STRIDE || delta.length >= EXACT_CHECK_STRIDE) {
+      this.#exactScannedAt = 0;
+      const exact = detectExactSuffixCycle(this.#tail);
+      if (exact) {
+        const [unit, times] = exact;
+        this.#reason = `repeated an exact ${unit.length}-character cycle ${times}× back-to-back`;
+        return this.#reason;
+      }
+    }
+
+    // 2. Near-duplicate paragraph loop. Append, then drain completed segments.
+    this.#pending += delta;
+    for (;;) {
+      const boundary = /\n\s*\n/.exec(this.#pending);
+      let raw: string;
+      if (boundary) {
+        raw = this.#pending.slice(0, boundary.index);
+        this.#pending = this.#pending.slice(boundary.index + boundary[0].length);
+      } else if (this.#pending.length > SEGMENT_CHAR_CAP) {
+        // No boundary yet but the segment is runaway-long: force a flush.
+        raw = this.#pending.slice(0, SEGMENT_CHAR_CAP);
+        this.#pending = this.#pending.slice(SEGMENT_CHAR_CAP);
+      } else {
+        return undefined;
+      }
+      // An over-long segment is chunked so each piece stays comparable.
+      for (let rest = raw; rest.length > 0; ) {
+        const chunk = rest.length > SEGMENT_CHAR_CAP ? rest.slice(0, SEGMENT_CHAR_CAP) : rest;
+        rest = rest.slice(chunk.length);
+        const hit = this.#consumeSegment(chunk);
+        if (hit) {
+          this.#reason = hit;
+          return this.#reason;
+        }
+      }
+    }
+  }
+
+  /** Process the buffered trailing paragraph (one with no blank-line / heading
+   *  terminator). Called when the reasoning block ends so the final segment —
+   *  which may be the one that completes a duplicate cluster — is not dropped. */
+  flush(): string | undefined {
+    if (this.#reason !== undefined) return this.#reason;
+    if (this.config.enabled === false) return undefined;
+    // A stream can end before the next cadence boundary: force one final exact
+    // check even when nothing is pending.
+    const exact = detectExactSuffixCycle(this.#tail);
+    if (exact) {
+      const [unit, times] = exact;
+      this.#reason = `repeated an exact ${unit.length}-character cycle ${times}× back-to-back`;
+      return this.#reason;
+    }
+    if (this.#pending.length === 0) return undefined;
+    let rest = this.#pending;
+    this.#pending = "";
+    while (rest.length > 0) {
+      const chunk = rest.length > SEGMENT_CHAR_CAP ? rest.slice(0, SEGMENT_CHAR_CAP) : rest;
+      rest = rest.slice(chunk.length);
+      const hit = this.#consumeSegment(chunk);
+      if (hit) {
+        this.#reason = hit;
+        return this.#reason;
+      }
+    }
+    return undefined;
+  }
+
+  #consumeSegment(raw: string): string | undefined {
+    // Reasoning-summarizer titles ("**Maintaining Momentum**", "## Heading")
+    // are per-thought formatting, not chain-of-thought; their ever-changing
+    // wording would otherwise mask a loop by inflating novelty. Strip them and
+    // code lines before analysis (a title- or code-only segment then falls below
+    // the length gate).
+    const segment = raw
+      .replace(CODE_LINE, "")
+      .replace(/^[ \t]*#{1,6}[ \t].*$/gm, "")
+      .replace(/^[ \t]*\*{2,3}.+?\*{2,3}[ \t]*$/gm, "");
+    const normalized = normalizeSegment(segment);
+    if (normalized.length < SEGMENT_MIN_NORM_CHARS) return undefined;
+
+    // Near-duplicate trigram cluster: the same paragraph reused with cosmetic
+    // wording drift (high word-trigram overlap).
+    const fingerprint = trigramShingles(normalized);
+    let cluster = 1;
+    for (const prev of this.#window) {
+      if (jaccard(fingerprint, prev) >= SEGMENT_SIMILARITY) cluster++;
+    }
+
+    this.#window.push(fingerprint);
+    if (this.#window.length > SEGMENT_WINDOW) this.#window.shift();
+    this.#count++;
+
+    if (this.#count >= SEGMENT_MIN_COUNT && cluster >= SEGMENT_MIN_CLUSTER) {
+      return `${cluster} near-identical segments within the last ${SEGMENT_WINDOW}`;
+    }
+    return undefined;
+  }
+}
+
+/** Reused Z-array scratch; scans are synchronous, so one buffer suffices. */
+const exactZ = new Uint16Array(EXACT_MAX_UNIT + 1);
+
+/**
+ * Detect an exact cycle at the text suffix. A Z-array over the reversed tail
+ * finds every possible suffix period in linear time without substring churn.
+ * The reversal is virtual (indexed from the end) and only `z[1..maxUnit]` is
+ * computed — the period check reads nothing past it, and each `z[i]` reads
+ * only `z[i - left]` with `i - left < i`, which is already filled. Short
+ * cycles retain the original 180-character/four-repeat sensitivity; long
+ * cycles require at least three repeats and 1024 repeated characters.
+ *
+ * A unit carrying a letter or an emoji matches at those thresholds. A unit with
+ * neither, such as the `?!?!` run a stuck model emits, matches only once it has
+ * repeated for {@link EXACT_PUNCTUATION_MIN_REPEATED_CHARS} characters, and never
+ * when it holds only digits and whitespace: padding, indentation and zero-filled
+ * hexdumps or literals are ordinary output at any length.
+ */
+function detectExactSuffixCycle(text: string): [unit: string, count: number] | undefined {
+  const n = text.length;
+  if (n < EXACT_SHORT_MIN_REPEATED_CHARS) return undefined;
+  const last = n - 1;
+  const maxUnit = Math.min(EXACT_MAX_UNIT, Math.floor(n / 3));
+  const z = exactZ;
+  let left = 0;
+  let right = 0;
+  for (let i = 1; i <= maxUnit; i++) {
+    let zi = i <= right ? Math.min(right - i + 1, z[i - left]!) : 0;
+    // `z[i - left]` is always filled: `i - left < i` and `left >= 0`.
+    while (i + zi < n && text.charCodeAt(last - zi) === text.charCodeAt(last - i - zi)) zi++;
+    z[i] = zi;
+    if (i + zi - 1 > right) {
+      left = i;
+      right = i + zi - 1;
+    }
+  }
+
+  for (let len = 2; len <= maxUnit; len++) {
+    // Every `z[len]` in this range was written by the loop above.
+    const count = 1 + Math.floor(z[len]! / len);
+    const minCount = len <= EXACT_SHORT_MAX_UNIT ? 4 : 3;
+    if (count < minCount) continue;
+    const unit = text.slice(-len);
+    if (/\p{L}|\p{Extended_Pictographic}/u.test(unit)) {
+      const minChars = len <= EXACT_SHORT_MAX_UNIT ? EXACT_SHORT_MIN_REPEATED_CHARS : EXACT_LONG_MIN_REPEATED_CHARS;
+      if (len * count >= minChars) return [unit, count];
+      continue;
+    }
+    if (/^[\p{N}\s]*$/u.test(unit)) continue;
+    if (len * count >= EXACT_PUNCTUATION_MIN_REPEATED_CHARS) return [unit, count];
+  }
+  return undefined;
+}
+
+/** Lowercase and tokenize prose plus code/path payloads, dropping pure numbers. */
+function normalizeSegment(segment: string): string {
+  return segment
+    .toLowerCase()
+    .replace(/`([^`]*)`/g, " $1 ")
+    .replace(/[^a-z0-9]+/g, " ")
+    .split(/\s+/)
+    .filter((token) => /[a-z]/.test(token))
+    .join(" ")
+    .trim();
+}
+
+/** Word-trigram shingle set of a normalized segment. */
+function trigramShingles(normalized: string): Set<string> {
+  const words = normalized.split(" ").filter(Boolean);
+  if (words.length < 3) return new Set(words.length > 0 ? [words.join(" ")] : []);
+  const shingles = new Set<string>();
+  for (let i = 0; i + 3 <= words.length; i++) {
+    shingles.add(`${words[i]} ${words[i + 1]} ${words[i + 2]}`);
+  }
+  return shingles;
+}
+
+function jaccard(a: Set<string>, b: Set<string>): number {
+  if (a.size === 0 || b.size === 0) return 0;
+  const [small, large] = a.size < b.size ? [a, b] : [b, a];
+  let intersection = 0;
+  for (const x of small) {
+    if (large.has(x)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}

package/src/transport/failure-policy.ts

@@ -582,6 +582,8 @@ const CONTEXT_LENGTH_CODES: ReadonlySet<string> = new Set([
   "context_length_exceeded",
   "context_too_large",
   "model_context_window_exceeded",
+  // InferHub's spelling for the same condition.
+  "context_too_long",
 ]);
 
 /**
@@ -733,6 +735,14 @@ const QUOTA_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "usage_not_included",
   "subscription:free-usage-exhausted",
   "resource_exhausted",
+  // InferHub publishes its own vocabulary in `error.code`. The two 403s below
+  // are deliberate absences: `account_blocked` means the account was blocked
+  // (an auth/entitlement verdict, not a spend limit) and
+  // `model_not_allowed_for_key` is a per-key allowlist refusal — both are
+  // classified separately rather than cooled as quota.
+  "rate_limited",
+  "providers_out_of_quota",
+  "providers_at_capacity",
 ]);
 const CAPACITY_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "overloaded_error",
@@ -740,6 +750,9 @@ const CAPACITY_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "model_at_capacity",
   "server_is_overloaded",
   "model_capacity",
+  // InferHub: no live provider could take the request right now.
+  "providers_busy",
+  "providers_overloaded",
 ]);
 const AUTH_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "authentication_error",
@@ -749,6 +762,12 @@ const AUTH_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "unauthenticated",
   "permission_denied",
   "permissiondenied",
+  // InferHub: the account was blocked, and a key whose scope or allowlist does
+  // not cover the request. `credentialEvidence` is what cools the account
+  // rather than the provider.
+  "account_blocked",
+  "insufficient_key_scope",
+  "model_not_allowed_for_key",
 ]);
 const PLATFORM_ERROR_IDENTIFIERS: ReadonlySet<string> = new Set([
   "server_error",
@@ -895,6 +914,14 @@ export function classifyUpstreamError(
     };
   if (normalized === "model_not_found")
     return { code: "model_not_found", status: 404, origin: "upstream" };
+  // InferHub: no live publisher serves the requested model at all. That is a
+  // property of the model/upstream pair, not of the account or a spend limit,
+  // so it is not retried as quota and does not cool the account — the next
+  // attempt should pick a different route.
+  if (normalized === "no_provider")
+    return { code: "model_not_found", status: 404, origin: "upstream" };
+  if (normalized === "content_rejected")
+    return { code: "policy_rejected", status: 403, origin: "upstream" };
   if (
     normalized === "not_found_error" ||
     normalized === "resource_not_found" ||

package/src/transport/guards/abuse-guard.ts

@@ -1,12 +1,12 @@
 // IP-abuse protection guard: counts every `/v1/*` attempt against the per-IP
 // ceiling via the root `request` hook.
 import { Elysia } from "elysia";
-import { GatewayError } from "../../protocol/gateway-error";
 import type { TrustedProxyBoundary } from "../../config/network";
 import { resolveClientIdentity } from "../../security/ip-boundary";
 import type { ProxyRequestStateStore } from "../request/state";
 import { fastPathname } from "../pathname";
 import type { IpAbuseProtectionService } from "../../security/abuse";
+import { peerAddressUnavailableError } from "./guard-errors";
 
 /**
  * Counts every `/v1/*` attempt against the per-IP ceiling, including the ones
@@ -66,7 +66,7 @@ export function createIpAbuseProtectionMiddleware(deps: {
       ? deps.resolvePeerAddress(request)
       : (server?.requestIP(request)?.address ?? null);
     if (!peer)
-      throw new GatewayError("admission_unavailable", 503, "client peer address unavailable");
+      throw peerAddressUnavailableError();
     const address = resolveClientIdentity(request, deps.trustedProxyBoundary, peer);
     const state = deps.stateStore.get(request);
     try {

package/src/transport/guards/auth-guard.ts

@@ -28,6 +28,12 @@ export function createApiKeyAuthenticationMiddleware(deps: {
    * console error, which is exactly the noise this layer exists to stop.
    */
   readonly modelStrikes?: Pick<ModelStrikeService, "check">;
+  /**
+   * Tenant invoke kill switch. Checked after the key resolves, because the
+   * tenant is a property of the key: the same gateway serves several tenants
+   * and one switching itself off must not stop the others.
+   */
+  readonly isInvokeDisabled?: (tenantId: string) => Promise<boolean | undefined>;
 }): Elysia {
   // Single policy for every `/v1/*` gateway route: authenticated, scoped to
   // `routing:invoke`, and tenant-bound (the snapshot's tenant is the only
@@ -73,6 +79,17 @@ export function createApiKeyAuthenticationMiddleware(deps: {
       });
       if (!decision.scopes.includes("routing:invoke"))
         throw new GatewayError("invalid_request", 403, "This API key is not allowed to invoke routing. Grant it the routing:invoke scope, or use a key that has it.");
+      // Kill switch: refuse before any upstream is touched. Distinct from the
+      // scope check above, which is a property of the key — this is the tenant
+      // having stopped all traffic, and it applies to every key the tenant owns.
+      // 503 rather than 403: nothing about the request is invalid, and the
+      // caller should retry later instead of "fixing" a request that is fine.
+      if ((await deps.isInvokeDisabled?.(authorization.tenantId)) === true)
+        throw new GatewayError(
+          "platform_unavailable",
+          503,
+          "This gateway has invoke switched off. Nothing upstream is being dialed while the switch is off; turn it back on from the dashboard overview to resume routing.",
+        );
       // A key may refuse a downstream router it has been resold to. This is a
       // policy check on the resolved key, so it belongs with the other
       // authorization decisions rather than in routing: the request is refused