opencode-plugboard 0.13.0 → 0.15.0

package/README.md

@@ -30,7 +30,7 @@ export PLUGBOARD_KEY_MY_APP=choose-a-long-random-string
 {
   "plugins": [
     {
-      "package": "opencode-plugboard@0.13.0",
+      "package": "opencode-plugboard@0.15.0",
       "options": {
         "keys": { "my-app": "{env:PLUGBOARD_KEY_MY_APP}" },
         "models": ["anthropic/claude-sonnet-4-6", "github-copilot/*", "opencode-go/glm-5.3-flash"],
@@ -78,7 +78,7 @@ Plugboard is early (0.x) and narrow on purpose.
 - **V2 only.** OpenCode V2 has no OpenAI endpoint, and the V1 plugins that had one stopped working when the plugin API changed. V1 is not planned.
 - **Two formats:** Chat Completions and Responses. `/v1/responses` is a stateless subset, see [docs/responses.md](https://github.com/jjjermiah/opencode-plugboard/blob/main/docs/responses.md). Anthropic Messages and Gemini front-ends are on the [roadmap](#roadmap).
 - **Not planned:** embeddings, audio, image generation, `logprobs`.
-- **No concurrency limit per app** yet ([#54](https://github.com/jjjermiah/opencode-plugboard/issues/54), [#86](https://github.com/jjjermiah/opencode-plugboard/issues/86)).
+- **Not a distributed gateway.** Limits, the response cache and metrics live in one OpenCode server process; two servers don't share them.
 - **Not a gateway with its own provider keys.** It only reuses what OpenCode is logged in to.
 
 How it differs from a standalone gateway:

package/index.ts

@@ -23,8 +23,9 @@ import { Effect } from "effect"
 import { parseOptions } from "./src/config.js"
 import { errorMessage, logError } from "./src/log.js"
 import { withFallbacks } from "./src/fallback.js"
+import { withLimits } from "./src/limits.js"
 import { createOpenAiRoute } from "./src/openai/route.js"
-import { ensureBackend, ensurePump, loadServingCatalog, processMetrics, registerGeneration, sessionStore } from "./src/runtime.js"
+import { ensureBackend, ensurePump, loadServingCatalog, processLimits, processMetrics, registerGeneration, sessionStore, withResponseCache } from "./src/runtime.js"
 import { createSessionCaller } from "./src/session/caller.js"
 import { installSessionHooks } from "./src/session/context.js"
 import { createHistoryPort, createSessionPort, loadCatalog } from "./src/session/host.js"
@@ -65,14 +66,26 @@ export default Plugin.define({
         startPump()
         return loadServingCatalog(ownCatalog)
       }
+      const limits = options.limits
       const handle = createOpenAiRoute({
         options,
+        // Caps are outermost, so one request takes one slot even when it falls back (src/limits.ts).
         // Aliases written as a list retry the next target before the first output (src/fallback.ts).
-        caller: withFallbacks(createSessionCaller(callerDeps), {
-          options,
-          loadCatalog: servingCatalog,
-          onFallback: options.metrics ? (app) => processMetrics().fallback(app) : undefined,
-        }),
+        // Limits are outermost (one slot per request, refused calls never reach anything); the response
+        // cache sits outside the fallbacks, so a hit never reaches a provider (src/limits.ts, src/cache.ts).
+        caller: withLimits(
+          withResponseCache(
+            withFallbacks(createSessionCaller(callerDeps), {
+              options,
+              loadCatalog: servingCatalog,
+              onFallback: options.metrics ? (app) => processMetrics().fallback(app) : undefined,
+            }),
+            options,
+          ),
+          () => limits,
+          processLimits(),
+          { onLimited: options.metrics ? (app, reason) => processMetrics().limited(app, reason) : undefined },
+        ),
         loadCatalog: servingCatalog,
       })
       const registration = registerGeneration({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-plugboard",
-  "version": "0.13.0",
+  "version": "0.15.0",
   "type": "module",
   "license": "MIT",
   "description": "Use the ChatGPT, Claude, Copilot and OpenCode Go logins in your OpenCode server from any app. An OpenCode V2 plugin serving OpenAI-compatible Chat Completions, Responses and model endpoints.",
@@ -25,8 +25,12 @@
     "release": "bun .github/scripts/release.ts"
   },
   "dependencies": {
+    "@prometheus-io/client": "0.16.1",
     "escape-html": "1.0.3",
     "ipaddr.js": "2.5.0",
+    "lru-cache": "11.5.3",
+    "p-queue": "9.3.5",
+    "pino": "10.4.0",
     "safe-stable-stringify": "2.5.0"
   },
   "devDependencies": {

package/src/anthropic/route.ts

@@ -27,6 +27,7 @@ import {
 } from "../http.js"
 import { reportServed, type ServedInfo } from "../fallback.js"
 import { resolveModel } from "../models.js"
+import { reportCache } from "../cache.js"
 import { reportPolicy } from "../policy.js"
 import { dropRejected } from "../openai/sampling.js"
 import type { RouteHost } from "../openai/route.js"
@@ -125,6 +126,7 @@ export async function serveMessages(host: RouteHost, call: MessagesCall): Promis
         },
         onServed: reportServed(response, info, (served) => void (exchange.model = served.model)),
         onPolicy: reportPolicy(response),
+        onCache: reportCache(response),
       }),
       scope,
     )

package/src/cache.ts

@@ -0,0 +1,165 @@
+/**
+ * Opt-in exact-match response cache, as a `ModelCaller` wrapper.
+ *
+ * Pollers and bots repeat identical prompts and pay quota every time. With `cache` set, a
+ * completed answer is kept in memory, and the same request from the same app within the TTL is
+ * answered from it without creating a session or touching the provider.
+ *
+ * What is cached: a turn that streamed to the end and finished `stop` or `length`, with some text
+ * or reasoning and no tool call and no error. Everything else passes through untouched: tool calls
+ * need the caller's tool results, structured output arrives as a forced tool call (so it is not
+ * cached), and errors and aborted turns are never answers. A hit replays what the miss sent: the
+ * `PlugEvent` sequence (same-kind deltas coalesced), the `onServed` and `onPolicy` notes, and a
+ * `finish` with zero usage, because the call spent no tokens and `plugboard_tokens_total` must not
+ * count it. No history record is written on a hit.
+ *
+ * The key is a SHA-256 of the app and the resolved request: provider, model id, variant, system,
+ * messages, tools, tool choice and sampling parameters. The `model` string the caller sent (an
+ * alias, say) is not part of it, so two names for one model share entries. Apps never share
+ * entries. The store is process-wide (`src/runtime.ts`), so it survives plugin generations, is
+ * reset when the cache option or the `apps` policy changes, and is lost on restart.
+ *
+ * Policy that the key does not see (a system prompt part, a tool strip, a model scope) is applied
+ * after the key is built. That is safe because the store is reset whenever `apps` changes, so an
+ * entry always comes from the same policy as the request that replays it.
+ *
+ * Fan-out (`n` above 1, prompt arrays) calls the caller once per choice with the same request, so
+ * the fan-out sets `noCache`: every choice must be a fresh sample.
+ */
+
+import { createHash } from "node:crypto"
+import type { ServerResponse } from "node:http"
+import { LRUCache } from "lru-cache"
+import stableStringify from "safe-stable-stringify"
+import type { CallContext, ModelCaller, PlugEvent, PlugFinish, PlugRequest, PolicyNote, ServedModel } from "./contract.js"
+
+/** The `cache` plugin option after defaults and validation (see `parseOptions` in `src/config.ts`). */
+export interface CacheConfig {
+  /** How long an entry is served after it was stored. */
+  readonly ttlSeconds: number
+  /** Most entries kept; the least recently used goes first. */
+  readonly maxEntries: number
+  /** Most bytes kept (the entries as JSON); the least recently used goes first. */
+  readonly maxBytes: number
+  /** Apps that use the cache. Absent: every app. */
+  readonly apps?: ReadonlyArray<string>
+}
+
+/** What a miss produced, replayed by a hit. */
+export interface Entry {
+  readonly events: ReadonlyArray<PlugEvent>
+  readonly served?: ServedModel
+  readonly policy?: ReadonlyArray<PolicyNote>
+}
+
+export interface CacheStore {
+  get(key: string): Entry | undefined
+  set(key: string, entry: Entry): void
+  size(): number
+}
+
+export interface CacheLimits {
+  readonly ttlMs: number
+  readonly maxEntries: number
+  readonly maxBytes: number
+}
+
+/** Fixed per-entry overhead added to the payload size: the key, the entry object and the array. */
+const ENTRY_OVERHEAD = 128
+
+export function createCacheStore(limits: CacheLimits): CacheStore {
+  const lru = new LRUCache<string, Entry>({
+    max: limits.maxEntries,
+    maxSize: limits.maxBytes,
+    sizeCalculation: (entry) => Buffer.byteLength(JSON.stringify(entry)) + ENTRY_OVERHEAD,
+    ttl: limits.ttlMs,
+  })
+  return {
+    get: (key) => lru.get(key),
+    set: (key, entry) => void lru.set(key, entry),
+    size: () => lru.size,
+  }
+}
+
+/** The cache key of one call: the app and the resolved request, object keys sorted. */
+export function cacheKey(app: string, request: PlugRequest): string {
+  const { ref } = request
+  const canonical = stableStringify([app, ref.providerID, ref.id, ref.variant ?? null, request.system, request.messages, request.tools, request.toolChoice ?? null, request.generation])
+  return createHash("sha256").update(canonical ?? "").digest("hex")
+}
+
+/** Called once per call the cache applies to. */
+export type CacheCounter = (app: string, result: "hit" | "miss") => void
+
+/** The response header that says whether the answer came from the cache. */
+export const CACHE_HEADER = "x-plugboard-cache"
+
+/**
+ * The `CallContext.onCache` of a route: `x-plugboard-cache: hit|miss` while the headers are still
+ * open. The first writer wins, so several choices of one request report once.
+ */
+export function reportCache(response: ServerResponse): (result: "hit" | "miss") => void {
+  return (result) => {
+    if (response.headersSent || response.destroyed || response.hasHeader(CACHE_HEADER)) return
+    response.setHeader(CACHE_HEADER, result)
+  }
+}
+
+const CACHEABLE_FINISH: ReadonlySet<PlugFinish> = new Set<PlugFinish>(["stop", "length"])
+
+/** Appends a delta to the list, merging it into the previous event when that is the same kind. */
+function append(events: PlugEvent[], event: { readonly type: "text-delta" | "reasoning-delta"; readonly text: string }): void {
+  const last = events.at(-1)
+  if (last && last.type === event.type) events[events.length - 1] = { type: event.type, text: (last as { text: string }).text + event.text }
+  else events.push({ type: event.type, text: event.text })
+}
+
+/** Wraps `inner`: hits are replayed, misses stream through and are stored when they qualify. */
+export function cachedCaller(inner: ModelCaller, config: CacheConfig, store: CacheStore, count?: CacheCounter): ModelCaller {
+  const apps = config.apps ? new Set(config.apps) : undefined
+  return {
+    async *stream(request: PlugRequest, context: CallContext): AsyncIterable<PlugEvent> {
+      if (context.noCache || (apps && !apps.has(context.app))) return yield* inner.stream(request, context)
+      const key = cacheKey(context.app, request)
+      const hit = store.get(key)
+      if (hit) {
+        count?.(context.app, "hit")
+        context.onCache?.("hit")
+        if (hit.served) context.onServed?.(hit.served)
+        if (hit.policy) context.onPolicy?.(hit.policy)
+        for (const event of hit.events) {
+          if (context.signal.aborted) return
+          yield event.type === "finish" ? { type: "finish", reason: event.reason, usage: { inputTokens: 0, outputTokens: 0 } } : event
+        }
+        return
+      }
+      count?.(context.app, "miss")
+      context.onCache?.("miss")
+      // What the route is told before the first event is kept for a later hit; the route still gets it.
+      let served: ServedModel | undefined
+      let policy: ReadonlyArray<PolicyNote> | undefined
+      const forward: CallContext = {
+        ...context,
+        onServed: (value) => {
+          served ??= value
+          context.onServed?.(value)
+        },
+        onPolicy: (notes) => {
+          policy ??= [...notes]
+          context.onPolicy?.(notes)
+        },
+      }
+      const recorded: PlugEvent[] = []
+      let storable = true
+      for await (const event of inner.stream(request, forward)) {
+        if (event.type === "text-delta" || event.type === "reasoning-delta") append(recorded, event)
+        else if (event.type === "finish") {
+          // Stored before the event is handed on, so a consumer that stops reading at `finish` still fills the cache.
+          if (storable && CACHEABLE_FINISH.has(event.reason) && recorded.length > 0 && !context.signal.aborted)
+            store.set(key, { events: [...recorded, { type: "finish", reason: event.reason }], ...(served ? { served } : {}), ...(policy?.length ? { policy } : {}) })
+        } else storable = false
+        yield event
+      }
+    },
+  }
+}

package/src/config.ts

@@ -1,11 +1,15 @@
 /** Plugin options: type, defaults, validation, caller authentication and the loopback check. */
 
 import { createHash, timingSafeEqual } from "node:crypto"
+import type { CacheConfig } from "./cache.js"
 import type { PlugError } from "./contract.js"
 import { allowedBy, parseModelRef } from "./models.js"
 import { parseAllowEntry, type RemoteImages } from "./session/media.js"
 import type { HistoryLimits } from "./session/prune.js"
 
+/** How `/v1/models` spells a model id: the canonical `provider/model#variant`, a UI-safe slug, or both. Resolution accepts either. */
+export type ModelIds = "canonical" | "slugs" | "both"
+
 /** Plugin options, after defaults and validation (see `parseOptions`). */
 export interface PlugboardOptions {
   readonly listen: { readonly host: string; readonly port: number }
@@ -42,6 +46,12 @@ export interface PlugboardOptions {
   readonly logFormat: "text" | "json"
   /** Serve `GET /metrics` (Prometheus text). Off by default; same authentication as `/v1`. */
   readonly metrics: boolean
+  /** Ids listed by `/v1/models`: canonical, slugs (`provider--model__variant`) or both. See `toSlug` in `src/models.ts`. */
+  readonly modelIds: ModelIds
+  /** Concurrency caps and requests per minute per app (see `src/limits.ts`). Absent: unlimited. */
+  readonly limits?: Limits
+  /** Opt-in exact-match response cache (see `src/cache.ts`). Absent: off. */
+  readonly cache?: CacheConfig
 }
 
 export type SystemMode = "prepend" | "append" | "replace"
@@ -61,6 +71,24 @@ export interface AppPolicy {
   readonly maxTools?: number
 }
 
+/** One set of caps. A field that is not set is unlimited. */
+export interface LimitCaps {
+  /** Model calls in flight at once; each call holds one slot. */
+  readonly maxConcurrent?: number
+  /** Model calls started per minute (a fixed one-minute window). */
+  readonly requestsPerMinute?: number
+}
+
+/**
+ * Option `limits`. `maxConcurrent` caps all apps together; `perApp` caps every app; `apps.<name>`
+ * overrides `perApp` field by field for that app.
+ */
+export interface Limits {
+  readonly maxConcurrent?: number
+  readonly perApp?: LimitCaps
+  readonly apps?: Readonly<Record<string, LimitCaps>>
+}
+
 export const APP_NAME_PATTERN = /^[a-z0-9-]+$/
 export const DEFAULT_APP = "default"
 const DEFAULTS = {
@@ -71,8 +99,16 @@ const DEFAULTS = {
   requestTimeoutMs: 600_000,
 } as const
 
-const KNOWN = new Set(["listen", "keys", "apps", "models", "aliases", "maxTools", "maxN", "requestTimeoutMs", "remoteImages", "history", "dropUnsupportedParams", "logFormat", "metrics"])
+const KNOWN = new Set(["listen", "keys", "apps", "models", "aliases", "maxTools", "maxN", "requestTimeoutMs", "remoteImages", "history", "dropUnsupportedParams", "logFormat", "metrics", "modelIds", "limits", "cache"])
+const LIMITS_KNOWN = new Set(["maxConcurrent", "perApp", "apps"])
+const LIMIT_CAP_KNOWN = new Set(["maxConcurrent", "requestsPerMinute"])
+/** Largest cap accepted: far above any provider's limit, and keeps the error messages short. */
+const MAX_LIMIT = 1_000_000
 const HISTORY_KNOWN = new Set(["maxSessions", "maxAgeDays"])
+const CACHE_KNOWN = new Set(["ttlSeconds", "maxEntries", "maxBytes", "apps"])
+const CACHE_DEFAULTS = { maxEntries: 1000, maxBytes: 64 * 1024 * 1024 } as const
+/** One year: a longer TTL is a typo, and keeps the millisecond conversion a safe integer. */
+const MAX_CACHE_TTL_SECONDS = 31_536_000
 /** About 100 years: keeps `maxAgeDays * 86_400_000` a safe integer. */
 const MAX_AGE_DAYS = 36_500
 const MODEL_PATTERN = /^[^/#*\s]+\/(\*|[^#*\s]+(#[^#*\s]+)?)$/
@@ -139,6 +175,7 @@ function mergedKeys(keys: unknown, apps: unknown): Array<[string, unknown]> {
 
 const APP_KNOWN = new Set(["key", "models", "system", "systemMode", "tools", "maxTools"])
 const SYSTEM_MODES: ReadonlySet<string> = new Set(["prepend", "append", "replace"])
+const MODEL_IDS: ReadonlySet<string> = new Set(["canonical", "slugs", "both"])
 
 /** Validates `apps` (the `key` field was merged into `keys` already) into per-app policies. */
 function parseApps(
@@ -185,11 +222,62 @@ function parseApps(
   return Object.fromEntries(entries)
 }
 
+/** Validates one set of caps (`limits.perApp`, or one `limits.apps` entry): each cap is a positive integer. */
+function parseCaps(raw: unknown, at: string): LimitCaps {
+  if (!isRecord(raw)) throw new Error(`plugboard option \`${at}\` must be an object`)
+  for (const key of Object.keys(raw))
+    if (!LIMIT_CAP_KNOWN.has(key)) throw new Error(`Unknown plugboard option \`${at}.${key}\` (known: ${[...LIMIT_CAP_KNOWN].join(", ")})`)
+  return {
+    ...(raw.maxConcurrent === undefined ? {} : { maxConcurrent: positiveInt(raw.maxConcurrent, `${at}.maxConcurrent`, 0, MAX_LIMIT) }),
+    ...(raw.requestsPerMinute === undefined ? {} : { requestsPerMinute: positiveInt(raw.requestsPerMinute, `${at}.requestsPerMinute`, 0, MAX_LIMIT) }),
+  }
+}
+
+/** Validates `limits`. An `apps` entry must name a configured app: a key in `keys` (or `default`, only when there are no keys). */
+function parseLimits(raw: unknown, keys: Readonly<Record<string, string>>): Limits | undefined {
+  if (raw === undefined) return undefined
+  if (!isRecord(raw)) throw new Error("plugboard option `limits` must be an object")
+  for (const key of Object.keys(raw))
+    if (!LIMITS_KNOWN.has(key)) throw new Error(`Unknown plugboard option \`limits.${key}\` (known: ${[...LIMITS_KNOWN].join(", ")})`)
+  const appsRaw = raw.apps
+  if (appsRaw !== undefined && !isRecord(appsRaw)) throw new Error("plugboard option `limits.apps` must be an object of app name -> caps")
+  const apps: Record<string, LimitCaps> = {}
+  for (const [app, value] of Object.entries(appsRaw ?? {})) {
+    const at = `limits.apps.${app}`
+    const configured = app === DEFAULT_APP ? Object.keys(keys).length === 0 : Object.hasOwn(keys, app)
+    if (!configured) throw new Error(`plugboard option \`${at}\` must name a configured app (a key in \`keys\`; \`${DEFAULT_APP}\` only without keys)`)
+    apps[app] = parseCaps(value, at)
+  }
+  return {
+    ...(raw.maxConcurrent === undefined ? {} : { maxConcurrent: positiveInt(raw.maxConcurrent, "limits.maxConcurrent", 0, MAX_LIMIT) }),
+    ...(raw.perApp === undefined ? {} : { perApp: parseCaps(raw.perApp, "limits.perApp") }),
+    ...(appsRaw === undefined ? {} : { apps }),
+  }
+}
+
 /** The policy configured for `app` (`default` for the keyless app), if any. Own-property lookup: an app name never reaches the prototype. */
 export function appPolicy(options: Pick<PlugboardOptions, "apps">, app: string): AppPolicy | undefined {
   return Object.hasOwn(options.apps, app) ? options.apps[app] : undefined
 }
 
+function parseCache(value: unknown, keys: Readonly<Record<string, string>>): CacheConfig | undefined {
+  if (value === undefined) return undefined
+  if (!isRecord(value)) throw new Error("plugboard option `cache` must be an object with `ttlSeconds`")
+  for (const key of Object.keys(value))
+    if (!CACHE_KNOWN.has(key)) throw new Error(`Unknown plugboard option \`cache.${key}\` (known: ${[...CACHE_KNOWN].join(", ")})`)
+  if (value.ttlSeconds === undefined) throw new Error("plugboard option `cache.ttlSeconds` is required (how long an answer is served, in seconds)")
+  const ttlSeconds = positiveInt(value.ttlSeconds, "cache.ttlSeconds", 0, MAX_CACHE_TTL_SECONDS)
+  const maxEntries = positiveInt(value.maxEntries, "cache.maxEntries", CACHE_DEFAULTS.maxEntries)
+  const maxBytes = positiveInt(value.maxBytes, "cache.maxBytes", CACHE_DEFAULTS.maxBytes)
+  if (value.apps === undefined) return { ttlSeconds, maxEntries, maxBytes }
+  if (!Array.isArray(value.apps) || value.apps.length === 0) throw new Error("plugboard option `cache.apps` must be a non-empty array of app names")
+  // A name that can never authenticate would silently leave the cache off for the app the user meant.
+  const known = Object.keys(keys).length === 0 ? [DEFAULT_APP] : Object.keys(keys)
+  for (const app of value.apps)
+    if (typeof app !== "string" || !known.includes(app)) throw new Error(`plugboard \`cache.apps\` entry \`${String(app)}\` is not an app (known: ${known.join(", ")})`)
+  return { ttlSeconds, maxEntries, maxBytes, apps: [...new Set(value.apps as string[])] }
+}
+
 /** Validates raw plugin options and applies defaults. Throws `Error` with an actionable message. */
 export function parseOptions(raw: unknown): PlugboardOptions {
   if (raw !== undefined && raw !== null && !isRecord(raw)) throw new Error("plugboard options must be an object")
@@ -266,6 +354,8 @@ export function parseOptions(raw: unknown): PlugboardOptions {
   if (input.logFormat !== undefined && input.logFormat !== "text" && input.logFormat !== "json")
     throw new Error("plugboard option `logFormat` must be `text` or `json`")
   if (input.metrics !== undefined && typeof input.metrics !== "boolean") throw new Error("plugboard option `metrics` must be a boolean")
+  if (input.modelIds !== undefined && (typeof input.modelIds !== "string" || !MODEL_IDS.has(input.modelIds)))
+    throw new Error("plugboard option `modelIds` must be `canonical`, `slugs` or `both`")
 
   const historyInput = input.history
   if (historyInput !== undefined && !isRecord(historyInput)) throw new Error("plugboard option `history` must be an object")
@@ -274,6 +364,9 @@ export function parseOptions(raw: unknown): PlugboardOptions {
   const maxSessions = historyInput?.maxSessions === undefined ? undefined : positiveInt(historyInput.maxSessions, "history.maxSessions", 0)
   const maxAgeDays = historyInput?.maxAgeDays === undefined ? undefined : positiveInt(historyInput.maxAgeDays, "history.maxAgeDays", 0, MAX_AGE_DAYS)
 
+  const limits = parseLimits(input.limits, keys)
+  const cache = parseCache(input.cache, keys)
+
   return {
     listen: { host, port: port ?? DEFAULTS.port },
     keys,
@@ -289,6 +382,9 @@ export function parseOptions(raw: unknown): PlugboardOptions {
     dropUnsupportedParams: input.dropUnsupportedParams !== false,
     logFormat: input.logFormat === "json" ? "json" : "text",
     metrics: input.metrics === true,
+    modelIds: (input.modelIds as ModelIds | undefined) ?? "canonical",
+    ...(limits === undefined ? {} : { limits }),
+    ...(cache ? { cache } : {}),
   }
 }
 

package/src/contract.ts

@@ -153,6 +153,8 @@ export interface PlugError {
   readonly message: string
   readonly code?: string
   readonly param?: string
+  /** Seconds to wait before a retry, for a `Retry-After` header. Set by the limiter (`src/limits.ts`); no route reads it yet. */
+  readonly retryAfterSeconds?: number
 }
 
 /** The one discriminator for `PlugError` values returned in place of a result. */
@@ -180,6 +182,10 @@ export interface CallContext {
   readonly onServed?: (served: ServedModel) => void
   /** Called once, before the first event, with each change the app's policy made that the caller sent for. Front-ends report them in the `x-plugboard-policy` header. */
   readonly onPolicy?: (notes: ReadonlyArray<PolicyNote>) => void
+  /** Called once, before the first event, when the response cache applies to the call: `hit` replays a stored answer (no session, no record), `miss` runs the model. Not called when the cache is off, skipped for the app, or `noCache` is set. */
+  readonly onCache?: (result: "hit" | "miss") => void
+  /** Bypass the response cache for this call. Fan-out sets it: each choice of `n` above 1 repeats one request and must be a fresh sample. */
+  readonly noCache?: boolean
 }
 
 /** The target of an alias that answered: `provider/model[#variant]`, and the targets tried before it. */

package/src/gemini/route.ts

@@ -19,6 +19,7 @@ import { isPlugError, type PlugError, type PlugRequest } from "../contract.js"
 import { reportServed, type ServedInfo } from "../fallback.js"
 import { boundEvents, checkJsonContentType, readJsonBody, sendJson, sseWriter, type RequestScope } from "../http.js"
 import { listAvailableModels, resolveModel } from "../models.js"
+import { reportCache } from "../cache.js"
 import { reportPolicy } from "../policy.js"
 import type { RouteHost } from "../openai/route.js"
 import { dropRejected } from "../openai/sampling.js"
@@ -141,6 +142,7 @@ export async function serveGemini(host: RouteHost, { match, request, url, app, r
       },
       onServed: reportServed(response, info, (served) => void (exchange.model = served.model)),
       onPolicy: reportPolicy(response),
+      onCache: reportCache(response),
     }),
     scope,
   )

package/src/limits.ts

@@ -0,0 +1,144 @@
+/**
+ * Concurrency caps and requests per minute per app (option `limits`): a `ModelCaller` wrapper
+ * that sits outside the fallback chain, so one request holds one slot however many targets it tries.
+ *
+ * Before the inner caller runs, a call takes a slot in each concurrency cap that applies to it: the
+ * global `maxConcurrent`, and its app's own cap (`apps.<name>` over `perApp`). Each cap is a
+ * `p-queue` whose concurrency is the cap, kept on the process runtime so it survives a plugin
+ * generation. Nothing waits in a queue: a call that finds a cap full, or its app past its requests
+ * per minute, is answered at once with one `error` event (429) and never reaches the model. Every
+ * check runs before any slot or window count is taken, so a refused call costs nothing.
+ *
+ * A slot is freed when the inner stream ends, throws, is closed by the consumer, or the request
+ * signal aborts (the abort frees it even if the inner stream ignores the signal). p-queue counts a
+ * slot free one microtask after its job resolves, so a call made in that same tick can still be
+ * refused; requests arrive from the I/O loop, so this does not happen in practice. The requests per
+ * minute limit is a fixed window of 60 s per app, counting the calls admitted in it.
+ *
+ * A call whose app has no cap that applies goes straight to the wrapped caller.
+ */
+
+import PQueue from "p-queue"
+import type { LimitCaps, Limits } from "./config.js"
+import type { CallContext, ModelCaller, PlugError, PlugEvent, PlugRequest } from "./contract.js"
+import { log } from "./log.js"
+
+/** The process-wide queues and windows. Held on the runtime (`src/runtime.ts`), not on a generation. */
+export interface LimitState {
+  /** One queue per capped scope: `*` for the global cap, an app name for that app's cap. Its concurrency is the cap. */
+  readonly queues: Map<string, PQueue>
+  /** The requests-per-minute window of each app: its start (epoch ms) and the calls admitted in it. */
+  readonly windows: Map<string, { start: number; count: number }>
+}
+
+export const createLimitState = (): LimitState => ({ queues: new Map(), windows: new Map() })
+
+export type LimitReason = "concurrency" | "rate"
+
+export interface LimitOptions {
+  /** Clock in epoch ms. Tests inject one; the default is `Date.now`. */
+  readonly now?: () => number
+  /** Called once for each refused call (metrics). */
+  readonly onLimited?: (app: string, reason: LimitReason) => void
+}
+
+const WINDOW_MS = 60_000
+/** Queue key of the global cap. Not an app name: names match `[a-z0-9-]+`. */
+const GLOBAL = "*"
+
+/** The caps that apply to `app`: `perApp`, with the app's `apps` entry overriding it field by field. */
+export function capsFor(limits: Limits, app: string): LimitCaps {
+  const own = limits.apps && Object.hasOwn(limits.apps, app) ? limits.apps[app] : undefined
+  return { ...limits.perApp, ...own }
+}
+
+const isCapped = (limits: Limits, app: string): boolean => {
+  const caps = capsFor(limits, app)
+  return limits.maxConcurrent !== undefined || caps.maxConcurrent !== undefined || caps.requestsPerMinute !== undefined
+}
+
+/** The caller's events, admitted under the configured caps; see the file comment. `getLimits` is read per call. */
+export function withLimits(inner: ModelCaller, getLimits: () => Limits | undefined, state: LimitState, options: LimitOptions = {}): ModelCaller {
+  const now = options.now ?? Date.now
+  return {
+    stream(request, context) {
+      const limits = getLimits()
+      if (!limits || !isCapped(limits, context.app)) return inner.stream(request, context)
+      return admit(inner, limits, state, now, options.onLimited, request, context)
+    },
+  }
+}
+
+async function* admit(
+  inner: ModelCaller,
+  limits: Limits,
+  state: LimitState,
+  now: () => number,
+  onLimited: LimitOptions["onLimited"],
+  request: PlugRequest,
+  context: CallContext,
+): AsyncGenerator<PlugEvent> {
+  const app = context.app
+  const caps = capsFor(limits, app)
+  const scopes: PQueue[] = []
+  if (limits.maxConcurrent !== undefined) scopes.push(queueFor(state, GLOBAL, limits.maxConcurrent))
+  if (caps.maxConcurrent !== undefined) scopes.push(queueFor(state, app, caps.maxConcurrent))
+
+  // Decide everything before taking anything: a refused call holds no slot and spends no window count.
+  const at = now()
+  const current = state.windows.get(app)
+  const live = current && at - current.start < WINDOW_MS ? current : undefined
+  let refusal: { readonly reason: LimitReason; readonly retryAfterSeconds: number } | undefined
+  if (scopes.some((queue) => queue.pending + queue.size >= queue.concurrency)) refusal = { reason: "concurrency", retryAfterSeconds: 1 }
+  else if (caps.requestsPerMinute !== undefined && live && live.count >= caps.requestsPerMinute)
+    refusal = { reason: "rate", retryAfterSeconds: Math.max(1, Math.ceil((live.start + WINDOW_MS - at) / 1000)) }
+  if (refusal) {
+    onLimited?.(app, refusal.reason)
+    log("request.limited", { app, reason: refusal.reason, retryAfterSeconds: refusal.retryAfterSeconds })
+    yield { type: "error", error: limitedError(refusal.reason, refusal.retryAfterSeconds) }
+    return
+  }
+
+  if (caps.requestsPerMinute !== undefined) {
+    if (live) live.count++
+    else state.windows.set(app, { start: at, count: 1 })
+  }
+  const frees = scopes.map(take)
+  let freed = false
+  const free = () => {
+    if (freed) return
+    freed = true
+    for (const release of frees) release()
+  }
+
+  const signal = context.signal
+  signal.addEventListener("abort", free, { once: true })
+  try {
+    for await (const event of inner.stream(request, context)) yield event
+  } finally {
+    signal.removeEventListener("abort", free)
+    free()
+  }
+}
+
+/** The queue of one capped scope, created on first use, with its concurrency set to the cap in force. */
+function queueFor(state: LimitState, key: string, cap: number): PQueue {
+  let queue = state.queues.get(key)
+  if (!queue) state.queues.set(key, (queue = new PQueue({ concurrency: cap })))
+  queue.concurrency = cap
+  return queue
+}
+
+/** Takes a slot in a queue that has room (checked by the caller). The returned function frees it. */
+function take(queue: PQueue): () => void {
+  let done = () => {}
+  // The job starts at once when a slot is free, so `done` is set before `add` returns.
+  void queue.add(() => new Promise<void>((resolve) => (done = resolve)))
+  return () => done()
+}
+
+/** A 429 with a neutral kind (`rate_limit`, the closest `PlugErrorKind`) and a code the front-ends pass through. */
+const limitedError = (reason: LimitReason, retryAfterSeconds: number): PlugError =>
+  reason === "concurrency"
+    ? { status: 429, kind: "rate_limit", code: "too_many_requests", message: "Too many concurrent requests", retryAfterSeconds }
+    : { status: 429, kind: "rate_limit", code: "rate_limit_exceeded", message: "Too many requests per minute", retryAfterSeconds }