opencode-plugboard 0.14.0 → 0.15.0

package/README.md

@@ -30,7 +30,7 @@ export PLUGBOARD_KEY_MY_APP=choose-a-long-random-string
 {
   "plugins": [
     {
-      "package": "opencode-plugboard@0.14.0",
+      "package": "opencode-plugboard@0.15.0",
       "options": {
         "keys": { "my-app": "{env:PLUGBOARD_KEY_MY_APP}" },
         "models": ["anthropic/claude-sonnet-4-6", "github-copilot/*", "opencode-go/glm-5.3-flash"],
@@ -78,7 +78,7 @@ Plugboard is early (0.x) and narrow on purpose.
 - **V2 only.** OpenCode V2 has no OpenAI endpoint, and the V1 plugins that had one stopped working when the plugin API changed. V1 is not planned.
 - **Two formats:** Chat Completions and Responses. `/v1/responses` is a stateless subset, see [docs/responses.md](https://github.com/jjjermiah/opencode-plugboard/blob/main/docs/responses.md). Anthropic Messages and Gemini front-ends are on the [roadmap](#roadmap).
 - **Not planned:** embeddings, audio, image generation, `logprobs`.
-- **No concurrency limit per app** yet ([#54](https://github.com/jjjermiah/opencode-plugboard/issues/54), [#86](https://github.com/jjjermiah/opencode-plugboard/issues/86)).
+- **Not a distributed gateway.** Limits, the response cache and metrics live in one OpenCode server process; two servers don't share them.
 - **Not a gateway with its own provider keys.** It only reuses what OpenCode is logged in to.
 
 How it differs from a standalone gateway:

package/index.ts

@@ -23,8 +23,9 @@ import { Effect } from "effect"
 import { parseOptions } from "./src/config.js"
 import { errorMessage, logError } from "./src/log.js"
 import { withFallbacks } from "./src/fallback.js"
+import { withLimits } from "./src/limits.js"
 import { createOpenAiRoute } from "./src/openai/route.js"
-import { ensureBackend, ensurePump, loadServingCatalog, processMetrics, registerGeneration, sessionStore } from "./src/runtime.js"
+import { ensureBackend, ensurePump, loadServingCatalog, processLimits, processMetrics, registerGeneration, sessionStore, withResponseCache } from "./src/runtime.js"
 import { createSessionCaller } from "./src/session/caller.js"
 import { installSessionHooks } from "./src/session/context.js"
 import { createHistoryPort, createSessionPort, loadCatalog } from "./src/session/host.js"
@@ -65,14 +66,26 @@ export default Plugin.define({
         startPump()
         return loadServingCatalog(ownCatalog)
       }
+      const limits = options.limits
       const handle = createOpenAiRoute({
         options,
+        // Caps are outermost, so one request takes one slot even when it falls back (src/limits.ts).
         // Aliases written as a list retry the next target before the first output (src/fallback.ts).
-        caller: withFallbacks(createSessionCaller(callerDeps), {
-          options,
-          loadCatalog: servingCatalog,
-          onFallback: options.metrics ? (app) => processMetrics().fallback(app) : undefined,
-        }),
+        // Limits are outermost (one slot per request, refused calls never reach anything); the response
+        // cache sits outside the fallbacks, so a hit never reaches a provider (src/limits.ts, src/cache.ts).
+        caller: withLimits(
+          withResponseCache(
+            withFallbacks(createSessionCaller(callerDeps), {
+              options,
+              loadCatalog: servingCatalog,
+              onFallback: options.metrics ? (app) => processMetrics().fallback(app) : undefined,
+            }),
+            options,
+          ),
+          () => limits,
+          processLimits(),
+          { onLimited: options.metrics ? (app, reason) => processMetrics().limited(app, reason) : undefined },
+        ),
         loadCatalog: servingCatalog,
       })
       const registration = registerGeneration({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-plugboard",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "type": "module",
   "license": "MIT",
   "description": "Use the ChatGPT, Claude, Copilot and OpenCode Go logins in your OpenCode server from any app. An OpenCode V2 plugin serving OpenAI-compatible Chat Completions, Responses and model endpoints.",
@@ -28,6 +28,8 @@
     "@prometheus-io/client": "0.16.1",
     "escape-html": "1.0.3",
     "ipaddr.js": "2.5.0",
+    "lru-cache": "11.5.3",
+    "p-queue": "9.3.5",
     "pino": "10.4.0",
     "safe-stable-stringify": "2.5.0"
   },

package/src/anthropic/route.ts

@@ -27,6 +27,7 @@ import {
 } from "../http.js"
 import { reportServed, type ServedInfo } from "../fallback.js"
 import { resolveModel } from "../models.js"
+import { reportCache } from "../cache.js"
 import { reportPolicy } from "../policy.js"
 import { dropRejected } from "../openai/sampling.js"
 import type { RouteHost } from "../openai/route.js"
@@ -125,6 +126,7 @@ export async function serveMessages(host: RouteHost, call: MessagesCall): Promis
         },
         onServed: reportServed(response, info, (served) => void (exchange.model = served.model)),
         onPolicy: reportPolicy(response),
+        onCache: reportCache(response),
       }),
       scope,
     )

package/src/cache.ts

@@ -0,0 +1,165 @@
+/**
+ * Opt-in exact-match response cache, as a `ModelCaller` wrapper.
+ *
+ * Pollers and bots repeat identical prompts and pay quota every time. With `cache` set, a
+ * completed answer is kept in memory, and the same request from the same app within the TTL is
+ * answered from it without creating a session or touching the provider.
+ *
+ * What is cached: a turn that streamed to the end and finished `stop` or `length`, with some text
+ * or reasoning and no tool call and no error. Everything else passes through untouched: tool calls
+ * need the caller's tool results, structured output arrives as a forced tool call (so it is not
+ * cached), and errors and aborted turns are never answers. A hit replays what the miss sent: the
+ * `PlugEvent` sequence (same-kind deltas coalesced), the `onServed` and `onPolicy` notes, and a
+ * `finish` with zero usage, because the call spent no tokens and `plugboard_tokens_total` must not
+ * count it. No history record is written on a hit.
+ *
+ * The key is a SHA-256 of the app and the resolved request: provider, model id, variant, system,
+ * messages, tools, tool choice and sampling parameters. The `model` string the caller sent (an
+ * alias, say) is not part of it, so two names for one model share entries. Apps never share
+ * entries. The store is process-wide (`src/runtime.ts`), so it survives plugin generations, is
+ * reset when the cache option or the `apps` policy changes, and is lost on restart.
+ *
+ * Policy that the key does not see (a system prompt part, a tool strip, a model scope) is applied
+ * after the key is built. That is safe because the store is reset whenever `apps` changes, so an
+ * entry always comes from the same policy as the request that replays it.
+ *
+ * Fan-out (`n` above 1, prompt arrays) calls the caller once per choice with the same request, so
+ * the fan-out sets `noCache`: every choice must be a fresh sample.
+ */
+
+import { createHash } from "node:crypto"
+import type { ServerResponse } from "node:http"
+import { LRUCache } from "lru-cache"
+import stableStringify from "safe-stable-stringify"
+import type { CallContext, ModelCaller, PlugEvent, PlugFinish, PlugRequest, PolicyNote, ServedModel } from "./contract.js"
+
+/** The `cache` plugin option after defaults and validation (see `parseOptions` in `src/config.ts`). */
+export interface CacheConfig {
+  /** How long an entry is served after it was stored. */
+  readonly ttlSeconds: number
+  /** Most entries kept; the least recently used goes first. */
+  readonly maxEntries: number
+  /** Most bytes kept (the entries as JSON); the least recently used goes first. */
+  readonly maxBytes: number
+  /** Apps that use the cache. Absent: every app. */
+  readonly apps?: ReadonlyArray<string>
+}
+
+/** What a miss produced, replayed by a hit. */
+export interface Entry {
+  readonly events: ReadonlyArray<PlugEvent>
+  readonly served?: ServedModel
+  readonly policy?: ReadonlyArray<PolicyNote>
+}
+
+export interface CacheStore {
+  get(key: string): Entry | undefined
+  set(key: string, entry: Entry): void
+  size(): number
+}
+
+export interface CacheLimits {
+  readonly ttlMs: number
+  readonly maxEntries: number
+  readonly maxBytes: number
+}
+
+/** Fixed per-entry overhead added to the payload size: the key, the entry object and the array. */
+const ENTRY_OVERHEAD = 128
+
+export function createCacheStore(limits: CacheLimits): CacheStore {
+  const lru = new LRUCache<string, Entry>({
+    max: limits.maxEntries,
+    maxSize: limits.maxBytes,
+    sizeCalculation: (entry) => Buffer.byteLength(JSON.stringify(entry)) + ENTRY_OVERHEAD,
+    ttl: limits.ttlMs,
+  })
+  return {
+    get: (key) => lru.get(key),
+    set: (key, entry) => void lru.set(key, entry),
+    size: () => lru.size,
+  }
+}
+
+/** The cache key of one call: the app and the resolved request, object keys sorted. */
+export function cacheKey(app: string, request: PlugRequest): string {
+  const { ref } = request
+  const canonical = stableStringify([app, ref.providerID, ref.id, ref.variant ?? null, request.system, request.messages, request.tools, request.toolChoice ?? null, request.generation])
+  return createHash("sha256").update(canonical ?? "").digest("hex")
+}
+
+/** Called once per call the cache applies to. */
+export type CacheCounter = (app: string, result: "hit" | "miss") => void
+
+/** The response header that says whether the answer came from the cache. */
+export const CACHE_HEADER = "x-plugboard-cache"
+
+/**
+ * The `CallContext.onCache` of a route: `x-plugboard-cache: hit|miss` while the headers are still
+ * open. The first writer wins, so several choices of one request report once.
+ */
+export function reportCache(response: ServerResponse): (result: "hit" | "miss") => void {
+  return (result) => {
+    if (response.headersSent || response.destroyed || response.hasHeader(CACHE_HEADER)) return
+    response.setHeader(CACHE_HEADER, result)
+  }
+}
+
+const CACHEABLE_FINISH: ReadonlySet<PlugFinish> = new Set<PlugFinish>(["stop", "length"])
+
+/** Appends a delta to the list, merging it into the previous event when that is the same kind. */
+function append(events: PlugEvent[], event: { readonly type: "text-delta" | "reasoning-delta"; readonly text: string }): void {
+  const last = events.at(-1)
+  if (last && last.type === event.type) events[events.length - 1] = { type: event.type, text: (last as { text: string }).text + event.text }
+  else events.push({ type: event.type, text: event.text })
+}
+
+/** Wraps `inner`: hits are replayed, misses stream through and are stored when they qualify. */
+export function cachedCaller(inner: ModelCaller, config: CacheConfig, store: CacheStore, count?: CacheCounter): ModelCaller {
+  const apps = config.apps ? new Set(config.apps) : undefined
+  return {
+    async *stream(request: PlugRequest, context: CallContext): AsyncIterable<PlugEvent> {
+      if (context.noCache || (apps && !apps.has(context.app))) return yield* inner.stream(request, context)
+      const key = cacheKey(context.app, request)
+      const hit = store.get(key)
+      if (hit) {
+        count?.(context.app, "hit")
+        context.onCache?.("hit")
+        if (hit.served) context.onServed?.(hit.served)
+        if (hit.policy) context.onPolicy?.(hit.policy)
+        for (const event of hit.events) {
+          if (context.signal.aborted) return
+          yield event.type === "finish" ? { type: "finish", reason: event.reason, usage: { inputTokens: 0, outputTokens: 0 } } : event
+        }
+        return
+      }
+      count?.(context.app, "miss")
+      context.onCache?.("miss")
+      // What the route is told before the first event is kept for a later hit; the route still gets it.
+      let served: ServedModel | undefined
+      let policy: ReadonlyArray<PolicyNote> | undefined
+      const forward: CallContext = {
+        ...context,
+        onServed: (value) => {
+          served ??= value
+          context.onServed?.(value)
+        },
+        onPolicy: (notes) => {
+          policy ??= [...notes]
+          context.onPolicy?.(notes)
+        },
+      }
+      const recorded: PlugEvent[] = []
+      let storable = true
+      for await (const event of inner.stream(request, forward)) {
+        if (event.type === "text-delta" || event.type === "reasoning-delta") append(recorded, event)
+        else if (event.type === "finish") {
+          // Stored before the event is handed on, so a consumer that stops reading at `finish` still fills the cache.
+          if (storable && CACHEABLE_FINISH.has(event.reason) && recorded.length > 0 && !context.signal.aborted)
+            store.set(key, { events: [...recorded, { type: "finish", reason: event.reason }], ...(served ? { served } : {}), ...(policy?.length ? { policy } : {}) })
+        } else storable = false
+        yield event
+      }
+    },
+  }
+}

package/src/config.ts

@@ -1,11 +1,15 @@
 /** Plugin options: type, defaults, validation, caller authentication and the loopback check. */
 
 import { createHash, timingSafeEqual } from "node:crypto"
+import type { CacheConfig } from "./cache.js"
 import type { PlugError } from "./contract.js"
 import { allowedBy, parseModelRef } from "./models.js"
 import { parseAllowEntry, type RemoteImages } from "./session/media.js"
 import type { HistoryLimits } from "./session/prune.js"
 
+/** How `/v1/models` spells a model id: the canonical `provider/model#variant`, a UI-safe slug, or both. Resolution accepts either. */
+export type ModelIds = "canonical" | "slugs" | "both"
+
 /** Plugin options, after defaults and validation (see `parseOptions`). */
 export interface PlugboardOptions {
   readonly listen: { readonly host: string; readonly port: number }
@@ -42,6 +46,12 @@ export interface PlugboardOptions {
   readonly logFormat: "text" | "json"
   /** Serve `GET /metrics` (Prometheus text). Off by default; same authentication as `/v1`. */
   readonly metrics: boolean
+  /** Ids listed by `/v1/models`: canonical, slugs (`provider--model__variant`) or both. See `toSlug` in `src/models.ts`. */
+  readonly modelIds: ModelIds
+  /** Concurrency caps and requests per minute per app (see `src/limits.ts`). Absent: unlimited. */
+  readonly limits?: Limits
+  /** Opt-in exact-match response cache (see `src/cache.ts`). Absent: off. */
+  readonly cache?: CacheConfig
 }
 
 export type SystemMode = "prepend" | "append" | "replace"
@@ -61,6 +71,24 @@ export interface AppPolicy {
   readonly maxTools?: number
 }
 
+/** One set of caps. A field that is not set is unlimited. */
+export interface LimitCaps {
+  /** Model calls in flight at once; each call holds one slot. */
+  readonly maxConcurrent?: number
+  /** Model calls started per minute (a fixed one-minute window). */
+  readonly requestsPerMinute?: number
+}
+
+/**
+ * Option `limits`. `maxConcurrent` caps all apps together; `perApp` caps every app; `apps.<name>`
+ * overrides `perApp` field by field for that app.
+ */
+export interface Limits {
+  readonly maxConcurrent?: number
+  readonly perApp?: LimitCaps
+  readonly apps?: Readonly<Record<string, LimitCaps>>
+}
+
 export const APP_NAME_PATTERN = /^[a-z0-9-]+$/
 export const DEFAULT_APP = "default"
 const DEFAULTS = {
@@ -71,8 +99,16 @@ const DEFAULTS = {
   requestTimeoutMs: 600_000,
 } as const
 
-const KNOWN = new Set(["listen", "keys", "apps", "models", "aliases", "maxTools", "maxN", "requestTimeoutMs", "remoteImages", "history", "dropUnsupportedParams", "logFormat", "metrics"])
+const KNOWN = new Set(["listen", "keys", "apps", "models", "aliases", "maxTools", "maxN", "requestTimeoutMs", "remoteImages", "history", "dropUnsupportedParams", "logFormat", "metrics", "modelIds", "limits", "cache"])
+const LIMITS_KNOWN = new Set(["maxConcurrent", "perApp", "apps"])
+const LIMIT_CAP_KNOWN = new Set(["maxConcurrent", "requestsPerMinute"])
+/** Largest cap accepted: far above any provider's limit, and keeps the error messages short. */
+const MAX_LIMIT = 1_000_000
 const HISTORY_KNOWN = new Set(["maxSessions", "maxAgeDays"])
+const CACHE_KNOWN = new Set(["ttlSeconds", "maxEntries", "maxBytes", "apps"])
+const CACHE_DEFAULTS = { maxEntries: 1000, maxBytes: 64 * 1024 * 1024 } as const
+/** One year: a longer TTL is a typo, and keeps the millisecond conversion a safe integer. */
+const MAX_CACHE_TTL_SECONDS = 31_536_000
 /** About 100 years: keeps `maxAgeDays * 86_400_000` a safe integer. */
 const MAX_AGE_DAYS = 36_500
 const MODEL_PATTERN = /^[^/#*\s]+\/(\*|[^#*\s]+(#[^#*\s]+)?)$/
@@ -139,6 +175,7 @@ function mergedKeys(keys: unknown, apps: unknown): Array<[string, unknown]> {
 
 const APP_KNOWN = new Set(["key", "models", "system", "systemMode", "tools", "maxTools"])
 const SYSTEM_MODES: ReadonlySet<string> = new Set(["prepend", "append", "replace"])
+const MODEL_IDS: ReadonlySet<string> = new Set(["canonical", "slugs", "both"])
 
 /** Validates `apps` (the `key` field was merged into `keys` already) into per-app policies. */
 function parseApps(
@@ -185,11 +222,62 @@ function parseApps(
   return Object.fromEntries(entries)
 }
 
+/** Validates one set of caps (`limits.perApp`, or one `limits.apps` entry): each cap is a positive integer. */
+function parseCaps(raw: unknown, at: string): LimitCaps {
+  if (!isRecord(raw)) throw new Error(`plugboard option \`${at}\` must be an object`)
+  for (const key of Object.keys(raw))
+    if (!LIMIT_CAP_KNOWN.has(key)) throw new Error(`Unknown plugboard option \`${at}.${key}\` (known: ${[...LIMIT_CAP_KNOWN].join(", ")})`)
+  return {
+    ...(raw.maxConcurrent === undefined ? {} : { maxConcurrent: positiveInt(raw.maxConcurrent, `${at}.maxConcurrent`, 0, MAX_LIMIT) }),
+    ...(raw.requestsPerMinute === undefined ? {} : { requestsPerMinute: positiveInt(raw.requestsPerMinute, `${at}.requestsPerMinute`, 0, MAX_LIMIT) }),
+  }
+}
+
+/** Validates `limits`. An `apps` entry must name a configured app: a key in `keys` (or `default`, only when there are no keys). */
+function parseLimits(raw: unknown, keys: Readonly<Record<string, string>>): Limits | undefined {
+  if (raw === undefined) return undefined
+  if (!isRecord(raw)) throw new Error("plugboard option `limits` must be an object")
+  for (const key of Object.keys(raw))
+    if (!LIMITS_KNOWN.has(key)) throw new Error(`Unknown plugboard option \`limits.${key}\` (known: ${[...LIMITS_KNOWN].join(", ")})`)
+  const appsRaw = raw.apps
+  if (appsRaw !== undefined && !isRecord(appsRaw)) throw new Error("plugboard option `limits.apps` must be an object of app name -> caps")
+  const apps: Record<string, LimitCaps> = {}
+  for (const [app, value] of Object.entries(appsRaw ?? {})) {
+    const at = `limits.apps.${app}`
+    const configured = app === DEFAULT_APP ? Object.keys(keys).length === 0 : Object.hasOwn(keys, app)
+    if (!configured) throw new Error(`plugboard option \`${at}\` must name a configured app (a key in \`keys\`; \`${DEFAULT_APP}\` only without keys)`)
+    apps[app] = parseCaps(value, at)
+  }
+  return {
+    ...(raw.maxConcurrent === undefined ? {} : { maxConcurrent: positiveInt(raw.maxConcurrent, "limits.maxConcurrent", 0, MAX_LIMIT) }),
+    ...(raw.perApp === undefined ? {} : { perApp: parseCaps(raw.perApp, "limits.perApp") }),
+    ...(appsRaw === undefined ? {} : { apps }),
+  }
+}
+
 /** The policy configured for `app` (`default` for the keyless app), if any. Own-property lookup: an app name never reaches the prototype. */
 export function appPolicy(options: Pick<PlugboardOptions, "apps">, app: string): AppPolicy | undefined {
   return Object.hasOwn(options.apps, app) ? options.apps[app] : undefined
 }
 
+function parseCache(value: unknown, keys: Readonly<Record<string, string>>): CacheConfig | undefined {
+  if (value === undefined) return undefined
+  if (!isRecord(value)) throw new Error("plugboard option `cache` must be an object with `ttlSeconds`")
+  for (const key of Object.keys(value))
+    if (!CACHE_KNOWN.has(key)) throw new Error(`Unknown plugboard option \`cache.${key}\` (known: ${[...CACHE_KNOWN].join(", ")})`)
+  if (value.ttlSeconds === undefined) throw new Error("plugboard option `cache.ttlSeconds` is required (how long an answer is served, in seconds)")
+  const ttlSeconds = positiveInt(value.ttlSeconds, "cache.ttlSeconds", 0, MAX_CACHE_TTL_SECONDS)
+  const maxEntries = positiveInt(value.maxEntries, "cache.maxEntries", CACHE_DEFAULTS.maxEntries)
+  const maxBytes = positiveInt(value.maxBytes, "cache.maxBytes", CACHE_DEFAULTS.maxBytes)
+  if (value.apps === undefined) return { ttlSeconds, maxEntries, maxBytes }
+  if (!Array.isArray(value.apps) || value.apps.length === 0) throw new Error("plugboard option `cache.apps` must be a non-empty array of app names")
+  // A name that can never authenticate would silently leave the cache off for the app the user meant.
+  const known = Object.keys(keys).length === 0 ? [DEFAULT_APP] : Object.keys(keys)
+  for (const app of value.apps)
+    if (typeof app !== "string" || !known.includes(app)) throw new Error(`plugboard \`cache.apps\` entry \`${String(app)}\` is not an app (known: ${known.join(", ")})`)
+  return { ttlSeconds, maxEntries, maxBytes, apps: [...new Set(value.apps as string[])] }
+}
+
 /** Validates raw plugin options and applies defaults. Throws `Error` with an actionable message. */
 export function parseOptions(raw: unknown): PlugboardOptions {
   if (raw !== undefined && raw !== null && !isRecord(raw)) throw new Error("plugboard options must be an object")
@@ -266,6 +354,8 @@ export function parseOptions(raw: unknown): PlugboardOptions {
   if (input.logFormat !== undefined && input.logFormat !== "text" && input.logFormat !== "json")
     throw new Error("plugboard option `logFormat` must be `text` or `json`")
   if (input.metrics !== undefined && typeof input.metrics !== "boolean") throw new Error("plugboard option `metrics` must be a boolean")
+  if (input.modelIds !== undefined && (typeof input.modelIds !== "string" || !MODEL_IDS.has(input.modelIds)))
+    throw new Error("plugboard option `modelIds` must be `canonical`, `slugs` or `both`")
 
   const historyInput = input.history
   if (historyInput !== undefined && !isRecord(historyInput)) throw new Error("plugboard option `history` must be an object")
@@ -274,6 +364,9 @@ export function parseOptions(raw: unknown): PlugboardOptions {
   const maxSessions = historyInput?.maxSessions === undefined ? undefined : positiveInt(historyInput.maxSessions, "history.maxSessions", 0)
   const maxAgeDays = historyInput?.maxAgeDays === undefined ? undefined : positiveInt(historyInput.maxAgeDays, "history.maxAgeDays", 0, MAX_AGE_DAYS)
 
+  const limits = parseLimits(input.limits, keys)
+  const cache = parseCache(input.cache, keys)
+
   return {
     listen: { host, port: port ?? DEFAULTS.port },
     keys,
@@ -289,6 +382,9 @@ export function parseOptions(raw: unknown): PlugboardOptions {
     dropUnsupportedParams: input.dropUnsupportedParams !== false,
     logFormat: input.logFormat === "json" ? "json" : "text",
     metrics: input.metrics === true,
+    modelIds: (input.modelIds as ModelIds | undefined) ?? "canonical",
+    ...(limits === undefined ? {} : { limits }),
+    ...(cache ? { cache } : {}),
   }
 }
 

package/src/contract.ts

@@ -153,6 +153,8 @@ export interface PlugError {
   readonly message: string
   readonly code?: string
   readonly param?: string
+  /** Seconds to wait before a retry, for a `Retry-After` header. Set by the limiter (`src/limits.ts`); no route reads it yet. */
+  readonly retryAfterSeconds?: number
 }
 
 /** The one discriminator for `PlugError` values returned in place of a result. */
@@ -180,6 +182,10 @@ export interface CallContext {
   readonly onServed?: (served: ServedModel) => void
   /** Called once, before the first event, with each change the app's policy made that the caller sent for. Front-ends report them in the `x-plugboard-policy` header. */
   readonly onPolicy?: (notes: ReadonlyArray<PolicyNote>) => void
+  /** Called once, before the first event, when the response cache applies to the call: `hit` replays a stored answer (no session, no record), `miss` runs the model. Not called when the cache is off, skipped for the app, or `noCache` is set. */
+  readonly onCache?: (result: "hit" | "miss") => void
+  /** Bypass the response cache for this call. Fan-out sets it: each choice of `n` above 1 repeats one request and must be a fresh sample. */
+  readonly noCache?: boolean
 }
 
 /** The target of an alias that answered: `provider/model[#variant]`, and the targets tried before it. */

package/src/gemini/route.ts

@@ -19,6 +19,7 @@ import { isPlugError, type PlugError, type PlugRequest } from "../contract.js"
 import { reportServed, type ServedInfo } from "../fallback.js"
 import { boundEvents, checkJsonContentType, readJsonBody, sendJson, sseWriter, type RequestScope } from "../http.js"
 import { listAvailableModels, resolveModel } from "../models.js"
+import { reportCache } from "../cache.js"
 import { reportPolicy } from "../policy.js"
 import type { RouteHost } from "../openai/route.js"
 import { dropRejected } from "../openai/sampling.js"
@@ -141,6 +142,7 @@ export async function serveGemini(host: RouteHost, { match, request, url, app, r
       },
       onServed: reportServed(response, info, (served) => void (exchange.model = served.model)),
       onPolicy: reportPolicy(response),
+      onCache: reportCache(response),
     }),
     scope,
   )

package/src/limits.ts

@@ -0,0 +1,144 @@
+/**
+ * Concurrency caps and requests per minute per app (option `limits`): a `ModelCaller` wrapper
+ * that sits outside the fallback chain, so one request holds one slot however many targets it tries.
+ *
+ * Before the inner caller runs, a call takes a slot in each concurrency cap that applies to it: the
+ * global `maxConcurrent`, and its app's own cap (`apps.<name>` over `perApp`). Each cap is a
+ * `p-queue` whose concurrency is the cap, kept on the process runtime so it survives a plugin
+ * generation. Nothing waits in a queue: a call that finds a cap full, or its app past its requests
+ * per minute, is answered at once with one `error` event (429) and never reaches the model. Every
+ * check runs before any slot or window count is taken, so a refused call costs nothing.
+ *
+ * A slot is freed when the inner stream ends, throws, is closed by the consumer, or the request
+ * signal aborts (the abort frees it even if the inner stream ignores the signal). p-queue counts a
+ * slot free one microtask after its job resolves, so a call made in that same tick can still be
+ * refused; requests arrive from the I/O loop, so this does not happen in practice. The requests per
+ * minute limit is a fixed window of 60 s per app, counting the calls admitted in it.
+ *
+ * A call whose app has no cap that applies goes straight to the wrapped caller.
+ */
+
+import PQueue from "p-queue"
+import type { LimitCaps, Limits } from "./config.js"
+import type { CallContext, ModelCaller, PlugError, PlugEvent, PlugRequest } from "./contract.js"
+import { log } from "./log.js"
+
+/** The process-wide queues and windows. Held on the runtime (`src/runtime.ts`), not on a generation. */
+export interface LimitState {
+  /** One queue per capped scope: `*` for the global cap, an app name for that app's cap. Its concurrency is the cap. */
+  readonly queues: Map<string, PQueue>
+  /** The requests-per-minute window of each app: its start (epoch ms) and the calls admitted in it. */
+  readonly windows: Map<string, { start: number; count: number }>
+}
+
+export const createLimitState = (): LimitState => ({ queues: new Map(), windows: new Map() })
+
+export type LimitReason = "concurrency" | "rate"
+
+export interface LimitOptions {
+  /** Clock in epoch ms. Tests inject one; the default is `Date.now`. */
+  readonly now?: () => number
+  /** Called once for each refused call (metrics). */
+  readonly onLimited?: (app: string, reason: LimitReason) => void
+}
+
+const WINDOW_MS = 60_000
+/** Queue key of the global cap. Not an app name: names match `[a-z0-9-]+`. */
+const GLOBAL = "*"
+
+/** The caps that apply to `app`: `perApp`, with the app's `apps` entry overriding it field by field. */
+export function capsFor(limits: Limits, app: string): LimitCaps {
+  const own = limits.apps && Object.hasOwn(limits.apps, app) ? limits.apps[app] : undefined
+  return { ...limits.perApp, ...own }
+}
+
+const isCapped = (limits: Limits, app: string): boolean => {
+  const caps = capsFor(limits, app)
+  return limits.maxConcurrent !== undefined || caps.maxConcurrent !== undefined || caps.requestsPerMinute !== undefined
+}
+
+/** The caller's events, admitted under the configured caps; see the file comment. `getLimits` is read per call. */
+export function withLimits(inner: ModelCaller, getLimits: () => Limits | undefined, state: LimitState, options: LimitOptions = {}): ModelCaller {
+  const now = options.now ?? Date.now
+  return {
+    stream(request, context) {
+      const limits = getLimits()
+      if (!limits || !isCapped(limits, context.app)) return inner.stream(request, context)
+      return admit(inner, limits, state, now, options.onLimited, request, context)
+    },
+  }
+}
+
+async function* admit(
+  inner: ModelCaller,
+  limits: Limits,
+  state: LimitState,
+  now: () => number,
+  onLimited: LimitOptions["onLimited"],
+  request: PlugRequest,
+  context: CallContext,
+): AsyncGenerator<PlugEvent> {
+  const app = context.app
+  const caps = capsFor(limits, app)
+  const scopes: PQueue[] = []
+  if (limits.maxConcurrent !== undefined) scopes.push(queueFor(state, GLOBAL, limits.maxConcurrent))
+  if (caps.maxConcurrent !== undefined) scopes.push(queueFor(state, app, caps.maxConcurrent))
+
+  // Decide everything before taking anything: a refused call holds no slot and spends no window count.
+  const at = now()
+  const current = state.windows.get(app)
+  const live = current && at - current.start < WINDOW_MS ? current : undefined
+  let refusal: { readonly reason: LimitReason; readonly retryAfterSeconds: number } | undefined
+  if (scopes.some((queue) => queue.pending + queue.size >= queue.concurrency)) refusal = { reason: "concurrency", retryAfterSeconds: 1 }
+  else if (caps.requestsPerMinute !== undefined && live && live.count >= caps.requestsPerMinute)
+    refusal = { reason: "rate", retryAfterSeconds: Math.max(1, Math.ceil((live.start + WINDOW_MS - at) / 1000)) }
+  if (refusal) {
+    onLimited?.(app, refusal.reason)
+    log("request.limited", { app, reason: refusal.reason, retryAfterSeconds: refusal.retryAfterSeconds })
+    yield { type: "error", error: limitedError(refusal.reason, refusal.retryAfterSeconds) }
+    return
+  }
+
+  if (caps.requestsPerMinute !== undefined) {
+    if (live) live.count++
+    else state.windows.set(app, { start: at, count: 1 })
+  }
+  const frees = scopes.map(take)
+  let freed = false
+  const free = () => {
+    if (freed) return
+    freed = true
+    for (const release of frees) release()
+  }
+
+  const signal = context.signal
+  signal.addEventListener("abort", free, { once: true })
+  try {
+    for await (const event of inner.stream(request, context)) yield event
+  } finally {
+    signal.removeEventListener("abort", free)
+    free()
+  }
+}
+
+/** The queue of one capped scope, created on first use, with its concurrency set to the cap in force. */
+function queueFor(state: LimitState, key: string, cap: number): PQueue {
+  let queue = state.queues.get(key)
+  if (!queue) state.queues.set(key, (queue = new PQueue({ concurrency: cap })))
+  queue.concurrency = cap
+  return queue
+}
+
+/** Takes a slot in a queue that has room (checked by the caller). The returned function frees it. */
+function take(queue: PQueue): () => void {
+  let done = () => {}
+  // The job starts at once when a slot is free, so `done` is set before `add` returns.
+  void queue.add(() => new Promise<void>((resolve) => (done = resolve)))
+  return () => done()
+}
+
+/** A 429 with a neutral kind (`rate_limit`, the closest `PlugErrorKind`) and a code the front-ends pass through. */
+const limitedError = (reason: LimitReason, retryAfterSeconds: number): PlugError =>
+  reason === "concurrency"
+    ? { status: 429, kind: "rate_limit", code: "too_many_requests", message: "Too many concurrent requests", retryAfterSeconds }
+    : { status: 429, kind: "rate_limit", code: "rate_limit_exceeded", message: "Too many requests per minute", retryAfterSeconds }

package/src/metrics.ts

@@ -62,6 +62,10 @@ export interface Metrics {
   dropped(params: ReadonlyArray<string>): void
   /** A model call failed before any output and was retried on the next target of its alias. */
   fallback(app: string): void
+  /** A call was refused with 429 by a concurrency cap (`concurrency`) or a requests-per-minute limit (`rate`). */
+  limited(app: string, reason: "concurrency" | "rate"): void
+  /** A call the response cache applied to: a stored answer replayed (`hit`) or the model run (`miss`). */
+  cache(app: string, result: "hit" | "miss"): void
   /** One history sweep ended. */
   sweep(result: "ok" | "failed" | "timeout", removed: number): void
   /** The exposition text. Called from a request that is itself in flight, which is not counted. */
@@ -104,6 +108,18 @@ export function createMetrics(): Metrics {
     labelNames: ["app"],
     registers: [registry],
   })
+  const limited = new Counter({
+    name: "plugboard_limited_total",
+    help: "Model calls refused with 429 by a concurrency cap or a requests-per-minute limit, by app and reason.",
+    labelNames: ["app", "reason"],
+    registers: [registry],
+  })
+  const cache = new Counter({
+    name: "plugboard_cache_requests_total",
+    help: "Model calls the response cache applied to, by app and result (hit or miss). Absent unless the `cache` option is set.",
+    labelNames: ["app", "result"],
+    registers: [registry],
+  })
   const sweeps = new Counter({ name: "plugboard_history_sweeps_total", help: "History sweeps that ended, by result.", labelNames: ["result"], registers: [registry] })
   const removed = new Counter({ name: "plugboard_history_sessions_removed_total", help: "Request sessions removed by history sweeps.", registers: [registry] })
   const generations = new Gauge({ name: "plugboard_generations", help: "Accepted plugin generations (one per OpenCode Location) serving requests.", registers: [registry] })
@@ -142,6 +158,8 @@ export function createMetrics(): Metrics {
       for (const param of params) count(dropped, { param })
     },
     fallback: (app) => count(fallbacks, { app }),
+    limited: (app, reason) => count(limited, { app, reason }),
+    cache: (app, result) => count(cache, { app, result }),
     sweep(result, amount) {
       count(sweeps, { result })
       if (amount > 0 && Number.isFinite(amount)) removed.inc(amount)

package/src/models.ts

@@ -20,6 +20,33 @@ export function refToString(ref: ModelRef): string {
   return `${ref.providerID}/${ref.id}${ref.variant ? `#${ref.variant}` : ""}`
 }
 
+/**
+ * A UI-safe spelling of a canonical id: `/` becomes `--` and `#` becomes `__`, so some chat UIs that mangle
+ * `/` and `#` keep it intact. `anthropic/claude-haiku-4-5#high` -> `anthropic--claude-haiku-4-5__high`.
+ */
+export function toSlug(id: string): string {
+  return id.replaceAll("/", "--").replaceAll("#", "__")
+}
+
+/**
+ * The canonical id a slug stands for, or undefined when `slug` is not one. The first `--` ends the provider, the
+ * first `__` starts the variant; a `--` inside the model id is read back as `/`. Ids where that does not round
+ * trip are not listed as slugs (see `listAvailableModels`), so they are never offered and never resolved this way.
+ */
+export function fromSlug(slug: string): string | undefined {
+  if (typeof slug !== "string" || /[\s/#*]/.test(slug)) return undefined
+  const sep = slug.indexOf("--")
+  if (sep <= 0) return undefined
+  const providerID = slug.slice(0, sep)
+  const rest = slug.slice(sep + 2)
+  const at = rest.indexOf("__")
+  const model = (at === -1 ? rest : rest.slice(0, at)).replaceAll("--", "/")
+  const variant = at === -1 ? undefined : rest.slice(at + 2)
+  if (variant !== undefined && (!variant || variant.includes("__"))) return undefined
+  const id = refToString({ providerID, id: model, ...(variant === undefined ? {} : { variant }) })
+  return parseModelRef(id) ? id : undefined
+}
+
 /** `provider/*` any model of the provider; `provider/model` any variant; `provider/model#variant` exactly. */
 export function allowedBy(models: ReadonlyArray<string>, ref: ModelRef): boolean {
   return models.some((pattern) => {
@@ -80,7 +107,8 @@ export function resolveModel(
     code: "model_not_found",
     param: "model",
   }
-  let ref = parseModelRef(own(options.aliases, model) ?? model)
+  // Aliases keep priority over slugs; a slug and a canonical id both resolve, whatever `modelIds` lists.
+  let ref = parseModelRef(own(options.aliases, model) ?? fromSlug(model) ?? model)
   if (!ref) return notAvailable
   if (!ref.variant && reasoningEffort && find(catalog, { ...ref, variant: reasoningEffort }))
     ref = { ...ref, variant: reasoningEffort }
@@ -96,7 +124,9 @@ export interface ListedModel {
 
 /**
  * The allowlist intersected with the catalog, plus aliases whose target qualifies. Ids are unique except an alias may repeat one.
- * With `app`, the list is narrowed to that app's `models` (see `inAppScope`).
+ * Each model is listed as its canonical id, its slug, or both, per `options.modelIds`; a model whose slug does not
+ * round trip (see `fromSlug`) is left out in `slugs` mode. Aliases are listed as they are.
+ * With `app`, the list is narrowed to that app's `models` (see `inAppScope`), judged on the canonical id.
  */
 export function listAvailableModels(options: PlugboardOptions, catalog: ReadonlyArray<CatalogModel>, app?: string): ListedModel[] {
   const policy = app === undefined ? undefined : appPolicy(options, app)
@@ -110,7 +140,9 @@ export function listAvailableModels(options: PlugboardOptions, catalog: Readonly
       const id = refToString(ref)
       if (seen.has(id) || !allowed(options, ref) || !inAppScope(policy, id, ref)) continue
       seen.add(id)
-      data.push(entry(id, ref, model))
+      if (options.modelIds !== "slugs") data.push(entry(id, ref, model))
+      const slug = toSlug(id)
+      if (options.modelIds !== "canonical" && fromSlug(slug) === id) data.push(entry(slug, ref, model))
     }
   }
   for (const [alias, target] of Object.entries(options.aliases)) {
@@ -121,6 +153,22 @@ export function listAvailableModels(options: PlugboardOptions, catalog: Readonly
   return data
 }
 
+/** Capabilities `GET /v1/models?capability=` can filter on: the flags the catalog reports (`CatalogModel.capabilities`). */
+export type Capability = "vision" | "tools"
+
+const CAPABILITY_FLAG: Readonly<Record<Capability, "image" | "tools">> = { vision: "image", tools: "tools" }
+
+/** The capability a query value names, or undefined for anything else (`json` has no catalog flag, so it is not one). */
+export function parseCapability(value: string): Capability | undefined {
+  return Object.hasOwn(CAPABILITY_FLAG, value) ? (value as Capability) : undefined
+}
+
+/** The listed models whose catalog entry sets that capability flag to true. Order is kept. */
+export function filterByCapability(models: ReadonlyArray<ListedModel>, capability: Capability): ListedModel[] {
+  const flag = CAPABILITY_FLAG[capability]
+  return models.filter(({ model }) => model.capabilities?.[flag] === true)
+}
+
 /**
  * The targets an alias falls back to, in order: the fallbacks that are allowed and enabled on the server.
  * Empty for a plain model id or an alias without fallbacks. The primary target is `resolveModel`'s answer.

package/src/openai/fanout.ts

@@ -23,6 +23,7 @@
 import type { ServerResponse } from "node:http"
 import type { PlugError, PlugEvent, PlugRequest, PlugUsage } from "../contract.js"
 import { reportServed, type ServedInfo } from "../fallback.js"
+import { reportCache } from "../cache.js"
 import { reportPolicy } from "../policy.js"
 import { RequestAborted, boundEvents, sendJson, sseWriter, type RequestScope } from "../http.js"
 import { chatUsageChunk, collectCompletion, encodeError, streamCompletion, type OpenAiExchange, type StreamChoice, type StreamResult } from "./response.js"
@@ -168,6 +169,8 @@ export async function serveChoices(host: Pick<RouteHost, "caller">, { app, calls
     if (calls.length === 1) codec.setModel(s.model)
   })
   const policy = reportPolicy(response)
+  // Each choice is a fresh sample: the response cache never serves or stores fan-out calls.
+  const cache = reportCache(response)
   const open = (index: number) =>
     boundEvents(
       host.caller.stream(calls[index]!, {
@@ -180,6 +183,8 @@ export async function serveChoices(host: Pick<RouteHost, "caller">, { app, calls
         },
         onServed: served,
         onPolicy: policy,
+        onCache: cache,
+        noCache: calls.length > 1,
       }),
       gscope,
     )

package/src/openai/route.ts

@@ -42,6 +42,7 @@ import { geminiAuthorization, matchGeminiRoute, serveGemini } from "../gemini/ro
 import { errorMessage, log, logError, withRequestId } from "../log.js"
 import { NO_APP, outcomeLabel, type Metrics } from "../metrics.js"
 import { listAvailableModels, resolveModel } from "../models.js"
+import { reportCache } from "../cache.js"
 import { reportPolicy } from "../policy.js"
 import { isMessagesPath, serveMessages } from "../anthropic/route.js"
 import { serveCompletions } from "../completions/route.js"
@@ -251,6 +252,7 @@ export function createOpenAiRoute(routeHost: RouteHost) {
         // An alias with fallbacks: report which target answered, and echo it as the response model.
         onServed: reportServed(response, info, (served) => void (exchange.model = served.model)),
         onPolicy: reportPolicy(response),
+        onCache: reportCache(response),
       }),
       scope,
     )

package/src/responses/route.ts

@@ -13,6 +13,7 @@ import { isPlugError, type PlugError, type PlugRequest } from "../contract.js"
 import { reportServed, type ServedInfo } from "../fallback.js"
 import { boundEvents, sendJson, sseWriter, type RequestScope } from "../http.js"
 import { resolveModel } from "../models.js"
+import { reportCache } from "../cache.js"
 import { reportPolicy } from "../policy.js"
 import { encodeError } from "../openai/response.js"
 import type { RouteHost } from "../openai/route.js"
@@ -54,6 +55,7 @@ export async function serveResponses(host: RouteHost, { app, body, response, sco
       },
       onServed: reportServed(response, info, (served) => void (exchange.model = served.model)),
       onPolicy: reportPolicy(response),
+      onCache: reportCache(response),
     }),
     scope,
   )

package/src/runtime.ts

@@ -24,7 +24,9 @@
  *     limits (src/session/prune.ts). Both start from request handling (the store's `register`);
  *  - the metrics registry (src/metrics.ts) and the runtime start time, so counters and uptime
  *     outlive any one generation. `processMetrics` / `processStatus` read them for `/metrics`,
- *     `/v1/status` and the discovery pages.
+ *     `/v1/status` and the discovery pages;
+ *   - the request limiter's queues and rate windows (src/limits.ts), so a cap holds across
+ *     generations. `processLimits` reads them.
  *
  * Registration is explicit: `registerGeneration` returns whether the generation was accepted. A
  * rejected one (different `listen`, options that differ from the live ones, foreign runtime) never
@@ -48,8 +50,11 @@ import { setTimeout as delay } from "node:timers/promises"
 import { Cause, Effect, Stream } from "effect"
 import stableStringify from "safe-stable-stringify"
 import { raceAbort } from "./abort.js"
-import { isPlugError, type CatalogModel, type PlugError } from "./contract.js"
+import { cachedCaller, createCacheStore, type CacheStore } from "./cache.js"
+import type { PlugboardOptions } from "./config.js"
+import { isPlugError, type CatalogModel, type ModelCaller, type PlugError } from "./contract.js"
 import { errorMessage, log, logError, outsideRequest, setLogFormat } from "./log.js"
+import { createLimitState, type LimitState } from "./limits.js"
 import { createMetrics, type Metrics, type RuntimeSnapshot } from "./metrics.js"
 import { ensureAppDir } from "./session/history.js"
 import type { HistoryPort } from "./session/host.js"
@@ -107,8 +112,8 @@ const BACKEND_POLL_MS = 25
 const BIND_RETRY_MS = 15_000
 const BIND_REMINDER_MS = 5 * 60_000
 
-/** 8: `Metrics.render` is async and the registry is @prometheus-io/client's. 7: the options fingerprint comes from safe-stable-stringify, so it must not be compared with an older one. 6: the listener retries a failed bind (new fields on `Listener`). 5: adds the metrics registry and the start time. 4: the history sweep state and the session store's register hook. 3: confirmed directories and rejected generations. */
-const VERSION = 8
+/** 10: the response cache store (`src/cache.ts`) lives here too. 9: the request limiter state (`src/limits.ts`) lives here, so the concurrency caps and rate windows survive a generation. 8: `Metrics.render` is async and the registry is @prometheus-io/client's. 7: the options fingerprint comes from safe-stable-stringify, so it must not be compared with an older one. 6: the listener retries a failed bind (new fields on `Listener`). 5: adds the metrics registry and the start time. 4: the history sweep state and the session store's register hook. 3: confirmed directories and rejected generations. */
+const VERSION = 10
 const KEY = Symbol.for("opencode-plugboard.runtime")
 
 interface Listener {
@@ -158,8 +163,12 @@ interface Runtime {
   readonly sweep: { nextAt: number; running: boolean }
   /** Process-wide counters, shared by every generation (see src/metrics.ts). */
   readonly metrics: Metrics
+  /** Concurrency queues and requests-per-minute windows (see src/limits.ts). */
+  readonly limits: LimitState
   /** When this runtime was created, epoch ms. */
   readonly startedAt: number
+  /** The response cache store (src/cache.ts) and the fingerprint of the config and `apps` policy it was built for. */
+  cache?: { readonly fingerprint: string; readonly store: CacheStore }
   listener?: Listener
   /** Present while a fiber is subscribed to the server event bus. */
   pump?: Pump
@@ -171,6 +180,7 @@ const REJECTED: Registration = { accepted: false, dispose() {}, confirmed: () =>
 const detachedSessions = createSessionStore()
 const detachedMetrics = createMetrics()
 const detachedStartedAt = Date.now()
+const detachedLimits = createLimitState()
 let foreignVersionLogged = false
 
 /** The process runtime, created on first use; undefined when a different version already owns the process. */
@@ -186,6 +196,7 @@ function runtime(): Runtime | undefined {
       rejected: new Map(),
       sweep: { nextAt: 0, running: false },
       metrics: createMetrics(),
+      limits: createLimitState(),
       startedAt: Date.now(),
     }
     slot[KEY] = created
@@ -261,6 +272,42 @@ export const sessionStore = (): SessionStore => runtime()?.sessions ?? detachedS
 /** The metrics registry shared by every generation in the process (a detached one under a foreign-version runtime). */
 export const processMetrics = (): Metrics => runtime()?.metrics ?? detachedMetrics
 
+/** The request limiter's state shared by every generation in the process (detached under a foreign-version runtime). */
+export const processLimits = (): LimitState => runtime()?.limits ?? detachedLimits
+
+/**
+ * Wraps `caller` with the response cache when option `cache` is set (a no-op without it). The store
+ * lives on the runtime, so it survives plugin generations. It is replaced when the cache option or
+ * any policy input (`apps`, `models`, `aliases`, `fallbacks`) differs from the one it was built for:
+ * the key does not cover policy, and a hit skips the session caller's scope checks. The store is
+ * resolved on the first call, not when the wrapper is built: a rejected generation is built too but
+ * never serves, and must not replace the live store. Under a foreign-version runtime the store is
+ * private to the wrapper.
+ */
+export function withResponseCache(caller: ModelCaller, options: PlugboardOptions): ModelCaller {
+  const config = options.cache
+  if (!config) return caller
+  let wrapped: ModelCaller | undefined
+  const build = (): ModelCaller => {
+    const rt = runtime()
+    const fingerprint = canonicalJson({
+      cache: config,
+      apps: options.apps,
+      models: options.models,
+      aliases: options.aliases,
+      fallbacks: options.fallbacks,
+    })
+    let held = rt?.cache
+    if (!held || held.fingerprint !== fingerprint) {
+      held = { fingerprint, store: createCacheStore({ ttlMs: config.ttlSeconds * 1000, maxEntries: config.maxEntries, maxBytes: config.maxBytes }) }
+      if (rt) rt.cache = held
+    }
+    const count = options.metrics ? (app: string, result: "hit" | "miss") => processMetrics().cache(app, result) : undefined
+    return cachedCaller(caller, config, held.store, count)
+  }
+  return { stream: (request, context) => (wrapped ??= build()).stream(request, context) }
+}
+
 /** What `/v1/status` and the discovery pages report about the process, besides the model catalog. */
 export interface RuntimeStatus extends RuntimeSnapshot {
   readonly runtimeVersion: number