opencode-plugboard 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.ts +15 -8
- package/package.json +5 -1
- package/src/anthropic/response.ts +2 -2
- package/src/anthropic/route.ts +3 -1
- package/src/completions/response.ts +3 -2
- package/src/completions/route.ts +2 -1
- package/src/config.ts +22 -4
- package/src/contract.ts +13 -0
- package/src/discovery.ts +4 -1
- package/src/fallback.ts +167 -0
- package/src/gemini/response.ts +2 -2
- package/src/gemini/route.ts +3 -1
- package/src/metrics.ts +5 -0
- package/src/models.ts +20 -4
- package/src/openai/fanout.ts +11 -2
- package/src/openai/response.ts +2 -2
- package/src/openai/route.ts +5 -2
- package/src/responses/response.ts +2 -2
- package/src/responses/route.ts +3 -1
package/README.md
CHANGED
|
@@ -30,7 +30,7 @@ export PLUGBOARD_KEY_MY_APP=choose-a-long-random-string
|
|
|
30
30
|
{
|
|
31
31
|
"plugins": [
|
|
32
32
|
{
|
|
33
|
-
"package": "opencode-plugboard@0.
|
|
33
|
+
"package": "opencode-plugboard@0.12.0",
|
|
34
34
|
"options": {
|
|
35
35
|
"keys": { "my-app": "{env:PLUGBOARD_KEY_MY_APP}" },
|
|
36
36
|
"models": ["anthropic/claude-sonnet-4-6", "github-copilot/*", "opencode-go/glm-5.3-flash"],
|
package/index.ts
CHANGED
|
@@ -22,8 +22,9 @@ import { Plugin } from "@opencode/plugin/effect"
|
|
|
22
22
|
import { Effect } from "effect"
|
|
23
23
|
import { parseOptions } from "./src/config.js"
|
|
24
24
|
import { errorMessage, logError } from "./src/log.js"
|
|
25
|
+
import { withFallbacks } from "./src/fallback.js"
|
|
25
26
|
import { createOpenAiRoute } from "./src/openai/route.js"
|
|
26
|
-
import { ensureBackend, ensurePump, loadServingCatalog, registerGeneration, sessionStore } from "./src/runtime.js"
|
|
27
|
+
import { ensureBackend, ensurePump, loadServingCatalog, processMetrics, registerGeneration, sessionStore } from "./src/runtime.js"
|
|
27
28
|
import { createSessionCaller } from "./src/session/caller.js"
|
|
28
29
|
import { installSessionHooks } from "./src/session/context.js"
|
|
29
30
|
import { createHistoryPort, createSessionPort, loadCatalog } from "./src/session/host.js"
|
|
@@ -57,15 +58,21 @@ export default Plugin.define({
|
|
|
57
58
|
// The readiness gate the caller awaits before creating a session (fails closed: 503 `backend_unavailable`).
|
|
58
59
|
ensureBackend: (app: string, signal: AbortSignal) => ensureBackend(app, signal, bootLocation),
|
|
59
60
|
}
|
|
61
|
+
// The catalog comes from the preferred accepted generation (a data-root Location when there is
|
|
62
|
+
// one), not from whichever Location registered last; this generation's own is the fallback.
|
|
63
|
+
const servingCatalog = async () => {
|
|
64
|
+
startPump()
|
|
65
|
+
return loadServingCatalog(ownCatalog)
|
|
66
|
+
}
|
|
60
67
|
const handle = createOpenAiRoute({
|
|
61
68
|
options,
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
+
// Aliases written as a list retry the next target before the first output (src/fallback.ts).
|
|
70
|
+
caller: withFallbacks(createSessionCaller(callerDeps), {
|
|
71
|
+
options,
|
|
72
|
+
loadCatalog: servingCatalog,
|
|
73
|
+
onFallback: options.metrics ? (app) => processMetrics().fallback(app) : undefined,
|
|
74
|
+
}),
|
|
75
|
+
loadCatalog: servingCatalog,
|
|
69
76
|
})
|
|
70
77
|
const registration = registerGeneration({
|
|
71
78
|
listen: options.listen,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-plugboard",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"description": "Use the ChatGPT, Claude, Copilot and OpenCode Go logins in your OpenCode server from any app. An OpenCode V2 plugin serving OpenAI-compatible Chat Completions, Responses and model endpoints.",
|
|
@@ -30,11 +30,15 @@
|
|
|
30
30
|
"safe-stable-stringify": "2.5.0"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
|
+
"@commitlint/cli": "21.2.3",
|
|
34
|
+
"@commitlint/config-conventional": "21.2.3",
|
|
33
35
|
"@opencode/plugin": "2.0.26",
|
|
34
36
|
"@types/bun": "1.4.2",
|
|
35
37
|
"@types/escape-html": "^1.0.4",
|
|
36
38
|
"@types/node": "^26.6.2",
|
|
39
|
+
"conventional-commits-parser": "7.1.3",
|
|
37
40
|
"effect": "4.0.0-rc.112",
|
|
41
|
+
"lefthook": "2.2.1",
|
|
38
42
|
"typescript": "^5.9.3"
|
|
39
43
|
},
|
|
40
44
|
"peerDependencies": {
|
|
@@ -12,8 +12,8 @@ type ToolCall = Extract<PlugEvent, { type: "tool-call" }>
|
|
|
12
12
|
|
|
13
13
|
/** What the encoder needs besides the model events: Messages presentation state. The request decoder produces it. */
|
|
14
14
|
export interface MessagesExchange {
|
|
15
|
-
/** The model string the client sent; echoed in the message object. */
|
|
16
|
-
|
|
15
|
+
/** The model string the client sent; echoed in the message object. A request that used an alias with fallbacks echoes the target that answered (`CallContext.onServed`). */
|
|
16
|
+
model: string
|
|
17
17
|
readonly stream: boolean
|
|
18
18
|
/** `thinking` was enabled or adaptive: reasoning text is returned as thinking blocks. */
|
|
19
19
|
readonly thinking: boolean
|
package/src/anthropic/route.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
sseWriter,
|
|
26
26
|
type RequestScope,
|
|
27
27
|
} from "../http.js"
|
|
28
|
+
import { reportServed, type ServedInfo } from "../fallback.js"
|
|
28
29
|
import { resolveModel } from "../models.js"
|
|
29
30
|
import { dropRejected } from "../openai/sampling.js"
|
|
30
31
|
import type { RouteHost } from "../openai/route.js"
|
|
@@ -45,7 +46,7 @@ export interface MessagesCall {
|
|
|
45
46
|
readonly response: ServerResponse
|
|
46
47
|
readonly scope: RequestScope
|
|
47
48
|
/** The route's log fields: the route name, the app, the resolved model, dropped parameters, and the outcome of a stream that failed after it committed. */
|
|
48
|
-
readonly info: { route: string; app?: string; model?: string; failed?: string; dropped?: string }
|
|
49
|
+
readonly info: ServedInfo & { route: string; app?: string; model?: string; failed?: string; dropped?: string }
|
|
49
50
|
}
|
|
50
51
|
|
|
51
52
|
const header = (request: IncomingMessage, name: string): string | undefined => {
|
|
@@ -121,6 +122,7 @@ export async function serveMessages(host: RouteHost, call: MessagesCall): Promis
|
|
|
121
122
|
if (response.headersSent || response.destroyed || !/^[\x21-\x7e]+$/.test(recordID)) return
|
|
122
123
|
response.setHeader(SESSION_HEADER, recordID)
|
|
123
124
|
},
|
|
125
|
+
onServed: reportServed(response, info, (served) => void (exchange.model = served.model)),
|
|
124
126
|
}),
|
|
125
127
|
scope,
|
|
126
128
|
)
|
|
@@ -11,8 +11,8 @@ import { encodeError, type StreamChoice, type StreamResult } from "../openai/res
|
|
|
11
11
|
|
|
12
12
|
/** What the encoders need besides the model events; the request decoder produces it, the model call never sees it. */
|
|
13
13
|
export interface CompletionsExchange {
|
|
14
|
-
/** The model string the client sent; echoed in every response object. */
|
|
15
|
-
|
|
14
|
+
/** The model string the client sent; echoed in every response object. A request that used an alias with fallbacks echoes the target that answered (`CallContext.onServed`). */
|
|
15
|
+
model: string
|
|
16
16
|
readonly stream: boolean
|
|
17
17
|
/** `stream_options.include_usage`: a final chunk with `choices: []` and the summed usage. */
|
|
18
18
|
readonly includeUsage: boolean
|
|
@@ -144,4 +144,5 @@ export const completionsCodec = (exchange: CompletionsExchange): ChoiceCodec =>
|
|
|
144
144
|
choices: [],
|
|
145
145
|
usage: encodeUsage(usage),
|
|
146
146
|
}),
|
|
147
|
+
setModel: (model) => void (exchange.model = model),
|
|
147
148
|
})
|
package/src/completions/route.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import type { ServerResponse } from "node:http"
|
|
13
13
|
import { isPlugError, type PlugError, type PlugRequest } from "../contract.js"
|
|
14
|
+
import type { ServedInfo } from "../fallback.js"
|
|
14
15
|
import { sendJson, type RequestScope } from "../http.js"
|
|
15
16
|
import { resolveModel } from "../models.js"
|
|
16
17
|
import { serveChoices } from "../openai/fanout.js"
|
|
@@ -28,7 +29,7 @@ export interface CompletionsCall {
|
|
|
28
29
|
readonly response: ServerResponse
|
|
29
30
|
readonly scope: RequestScope
|
|
30
31
|
/** The route's log fields: the resolved model, the sampling parameters dropped, and the outcome of a stream that failed after it committed. */
|
|
31
|
-
readonly info: { model?: string; failed?: string; dropped?: string }
|
|
32
|
+
readonly info: ServedInfo & { model?: string; failed?: string; dropped?: string }
|
|
32
33
|
}
|
|
33
34
|
|
|
34
35
|
export async function serveCompletions(host: RouteHost, { app, body, response, scope, info }: CompletionsCall): Promise<void> {
|
package/src/config.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { createHash, timingSafeEqual } from "node:crypto"
|
|
4
4
|
import type { PlugError } from "./contract.js"
|
|
5
|
+
import { allowedBy, parseModelRef } from "./models.js"
|
|
5
6
|
import { parseAllowEntry, type RemoteImages } from "./session/media.js"
|
|
6
7
|
import type { HistoryLimits } from "./session/prune.js"
|
|
7
8
|
|
|
@@ -14,6 +15,12 @@ export interface PlugboardOptions {
|
|
|
14
15
|
readonly models: ReadonlyArray<string>
|
|
15
16
|
/** Alias -> `provider/model[#variant]`. Targets must also pass the allowlist. */
|
|
16
17
|
readonly aliases: Readonly<Record<string, string>>
|
|
18
|
+
/**
|
|
19
|
+
* Alias -> the targets after the first, in order, for an alias written as an array. `aliases` holds the
|
|
20
|
+
* first target, so everything that resolves an alias keeps working; the model caller retries the rest
|
|
21
|
+
* (`src/fallback.ts`). Absent for an alias with one target. Every target passes the allowlist.
|
|
22
|
+
*/
|
|
23
|
+
readonly fallbacks: Readonly<Record<string, ReadonlyArray<string>>>
|
|
17
24
|
readonly maxTools: number
|
|
18
25
|
/** Most model calls one request may fan out to: `n` for chat completions, prompts x `n` for legacy completions. Each is a full call and a kept session. */
|
|
19
26
|
readonly maxN: number
|
|
@@ -137,18 +144,28 @@ export function parseOptions(raw: unknown): PlugboardOptions {
|
|
|
137
144
|
throw new Error(`plugboard model \`${String(pattern)}\` must be \`provider/model\`, \`provider/model#variant\` or \`provider/*\``)
|
|
138
145
|
|
|
139
146
|
const aliasEntries: Array<[string, string]> = []
|
|
140
|
-
|
|
147
|
+
const fallbackEntries: Array<[string, string[]]> = []
|
|
148
|
+
if (input.aliases !== undefined && !isRecord(input.aliases)) throw new Error("plugboard option `aliases` must be an object of alias -> model or alias -> [model, ...]")
|
|
141
149
|
for (const [alias, target] of Object.entries(input.aliases ?? {})) {
|
|
142
150
|
if (alias.length === 0 || /[\s]/.test(alias)) throw new Error(`plugboard alias \`${alias}\` must be a non-empty string without whitespace`)
|
|
143
151
|
// Plain-object lookups downstream would hit the prototype accessor instead of the alias.
|
|
144
152
|
if (alias === "__proto__") throw new Error("plugboard alias `__proto__` is not allowed")
|
|
145
|
-
|
|
146
|
-
|
|
153
|
+
const chain = Array.isArray(target) ? target : [target]
|
|
154
|
+
if (chain.length === 0) throw new Error(`plugboard alias \`${alias}\` must list at least one target`)
|
|
155
|
+
for (const entry of chain) {
|
|
156
|
+
if (typeof entry !== "string" || !ALIAS_TARGET.test(entry))
|
|
157
|
+
throw new Error(`plugboard alias \`${alias}\` must target \`provider/model\` or \`provider/model#variant\` (or a list of them)`)
|
|
158
|
+
// A list is new, so it is strict: a target the allowlist would refuse can never serve, so fail at load.
|
|
159
|
+
if (Array.isArray(target) && !allowedBy(models as string[], parseModelRef(entry)!))
|
|
160
|
+
throw new Error(`plugboard alias \`${alias}\` target \`${entry}\` is not allowed by the \`models\` option`)
|
|
161
|
+
}
|
|
147
162
|
if (CANONICAL_SHAPE.test(alias))
|
|
148
163
|
throw new Error(`plugboard alias \`${alias}\` is written like a \`provider/model\` id; aliases must not contain \`/\` (use a short name such as \`fast\`)`)
|
|
149
|
-
aliasEntries.push([alias,
|
|
164
|
+
aliasEntries.push([alias, chain[0] as string])
|
|
165
|
+
if (chain.length > 1) fallbackEntries.push([alias, chain.slice(1) as string[]])
|
|
150
166
|
}
|
|
151
167
|
const aliases: Record<string, string> = Object.fromEntries(aliasEntries)
|
|
168
|
+
const fallbacks: Record<string, string[]> = Object.fromEntries(fallbackEntries)
|
|
152
169
|
|
|
153
170
|
const remoteImages = parseRemoteImages(input.remoteImages)
|
|
154
171
|
if (input.dropUnsupportedParams !== undefined && typeof input.dropUnsupportedParams !== "boolean")
|
|
@@ -170,6 +187,7 @@ export function parseOptions(raw: unknown): PlugboardOptions {
|
|
|
170
187
|
keys,
|
|
171
188
|
models: models as string[],
|
|
172
189
|
aliases,
|
|
190
|
+
fallbacks,
|
|
173
191
|
maxTools: positiveInt(input.maxTools, "maxTools", DEFAULTS.maxTools, 1024),
|
|
174
192
|
maxN: positiveInt(input.maxN, "maxN", DEFAULTS.maxN, 64),
|
|
175
193
|
requestTimeoutMs: positiveInt(input.requestTimeoutMs, "requestTimeoutMs", DEFAULTS.requestTimeoutMs, MAX_TIMER_MS),
|
package/src/contract.ts
CHANGED
|
@@ -168,6 +168,19 @@ export interface CallContext {
|
|
|
168
168
|
readonly signal: AbortSignal
|
|
169
169
|
/** Called once, before the first event, with the id of the history record this call writes (if any). */
|
|
170
170
|
readonly onRecord?: (recordID: string) => void
|
|
171
|
+
/**
|
|
172
|
+
* Called once, before the first event, only for a request whose model is an alias with fallbacks
|
|
173
|
+
* (`src/fallback.ts`): which target answered and how many earlier targets failed first. A front-end
|
|
174
|
+
* uses it for `x-plugboard-model` and the `model` it echoes. A call that fails on every target never
|
|
175
|
+
* reports it.
|
|
176
|
+
*/
|
|
177
|
+
readonly onServed?: (served: ServedModel) => void
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** The target of an alias that answered: `provider/model[#variant]`, and the targets tried before it. */
|
|
181
|
+
export interface ServedModel {
|
|
182
|
+
readonly model: string
|
|
183
|
+
readonly fallbacks: number
|
|
171
184
|
}
|
|
172
185
|
|
|
173
186
|
/** The seam between the core and a backend (the session backend now, `ctx.llm.stream` possibly later). */
|
package/src/discovery.ts
CHANGED
|
@@ -94,6 +94,8 @@ export interface Discovery {
|
|
|
94
94
|
readonly state: ModelsState
|
|
95
95
|
readonly allowlist: ReadonlyArray<string>
|
|
96
96
|
readonly aliases: Readonly<Record<string, string>>
|
|
97
|
+
/** Alias -> the targets tried after its first one, for aliases written as a list. Absent when there are none. */
|
|
98
|
+
readonly fallbacks?: Readonly<Record<string, ReadonlyArray<string>>>
|
|
97
99
|
/** Ids `GET /v1/models` lists (allowlist intersected with the catalog, plus aliases). */
|
|
98
100
|
readonly available?: number
|
|
99
101
|
readonly hint?: string
|
|
@@ -171,6 +173,7 @@ export function buildDiscovery({ options, app, catalog }: DiscoveryInput): Disco
|
|
|
171
173
|
state,
|
|
172
174
|
allowlist: options.models,
|
|
173
175
|
aliases: options.aliases,
|
|
176
|
+
...(Object.keys(options.fallbacks).length > 0 ? { fallbacks: options.fallbacks } : {}),
|
|
174
177
|
...(catalog ? { available: listed.length } : {}),
|
|
175
178
|
...(modelsHint ? { hint: modelsHint } : {}),
|
|
176
179
|
},
|
|
@@ -202,7 +205,7 @@ export function renderDiscoveryHtml(doc: Discovery): string {
|
|
|
202
205
|
if (models.available !== undefined) sections.push(`<p>${escapeHtml(models.available)} model ids available.</p>`)
|
|
203
206
|
sections.push(`<h3>Allowlist</h3>${models.allowlist.length > 0 ? list(models.allowlist) : "<p>Empty.</p>"}`)
|
|
204
207
|
const aliases = Object.entries(models.aliases)
|
|
205
|
-
if (aliases.length > 0) sections.push(`<h3>Aliases</h3>${list(aliases.map(([alias, target]) =>
|
|
208
|
+
if (aliases.length > 0) sections.push(`<h3>Aliases</h3>${list(aliases.map(([alias, target]) => [alias, target, ...(models.fallbacks?.[alias] ?? [])].join(" -> ")))}`)
|
|
206
209
|
}
|
|
207
210
|
if (doc.suggestedModels) {
|
|
208
211
|
sections.push(`<h2>Suggested <code>models</code> option</h2>${pre(`"models": ${JSON.stringify(doc.suggestedModels)}`)}`)
|
package/src/fallback.ts
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ordered model fallbacks: a `ModelCaller` wrapper, so every front-end gets them.
|
|
3
|
+
*
|
|
4
|
+
* An alias written as a list (`"fast": ["a/x", "b/y"]`) is a chain. The route resolves the first
|
|
5
|
+
* target as usual; this wrapper calls it, looks ahead to the first event only, and when that event
|
|
6
|
+
* is a failure of a retryable class (or the call throws, or ends with nothing) it calls the next
|
|
7
|
+
* target in a fresh call, so each attempt is its own session. Once any event has been handed on
|
|
8
|
+
* nothing is retried: a partly streamed answer is never restarted. Attempts are capped at the chain
|
|
9
|
+
* length, share the request's signal (so the deadline and a disconnect end the chain) and nothing is
|
|
10
|
+
* buffered beyond that first event.
|
|
11
|
+
*
|
|
12
|
+
* A call whose model is not an alias with fallbacks goes straight to the wrapped caller.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import type { ServerResponse } from "node:http"
|
|
16
|
+
import type { PlugboardOptions } from "./config.js"
|
|
17
|
+
import type { CallContext, CatalogModel, ModelCaller, ModelRef, PlugError, PlugEvent, PlugRequest, ServedModel } from "./contract.js"
|
|
18
|
+
import { log } from "./log.js"
|
|
19
|
+
import { fallbackRefs, refToString } from "./models.js"
|
|
20
|
+
import { dropRejected } from "./openai/sampling.js"
|
|
21
|
+
|
|
22
|
+
export interface FallbackDeps {
|
|
23
|
+
readonly options: PlugboardOptions
|
|
24
|
+
/** The serving Location's enabled models; read only after a first failure, to skip targets that are not available. */
|
|
25
|
+
loadCatalog(): Promise<ReadonlyArray<CatalogModel>>
|
|
26
|
+
/** A call is about to be retried on the next target (metrics). */
|
|
27
|
+
readonly onFallback?: (app: string) => void
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Failures that say something about this model or provider, not about the request: another target
|
|
32
|
+
* may well answer. Client errors (invalid request, context length, content filter) and plugboard's
|
|
33
|
+
* own failures (backend not ready, bad preparation) are not retried. `provider_auth_failed` is: a
|
|
34
|
+
* provider that lost its sign-in is as down as one that is rate limiting.
|
|
35
|
+
*/
|
|
36
|
+
const RETRYABLE = new Set([
|
|
37
|
+
"rate_limited",
|
|
38
|
+
"quota_exceeded",
|
|
39
|
+
"provider_unavailable",
|
|
40
|
+
"provider_timeout",
|
|
41
|
+
"provider_auth_failed",
|
|
42
|
+
"model_run_failed",
|
|
43
|
+
"prompt_failed",
|
|
44
|
+
"backend_failed",
|
|
45
|
+
])
|
|
46
|
+
|
|
47
|
+
export const MODEL_HEADER = "x-plugboard-model"
|
|
48
|
+
export const FALLBACKS_HEADER = "x-plugboard-fallbacks"
|
|
49
|
+
|
|
50
|
+
/** The route's log fields a served model fills in. */
|
|
51
|
+
export interface ServedInfo {
|
|
52
|
+
served?: string
|
|
53
|
+
fallbacks?: number
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The `CallContext.onServed` of a route: `x-plugboard-model` (and `x-plugboard-fallbacks` when targets
|
|
58
|
+
* failed first) while the headers are still open, and the log fields. The first call wins, so several
|
|
59
|
+
* choices of one request report the first answer. `then` runs for that first call (a single-call route
|
|
60
|
+
* echoes the served model in the body there).
|
|
61
|
+
*/
|
|
62
|
+
export function reportServed(response: ServerResponse, info: ServedInfo, then?: (served: ServedModel) => void): (served: ServedModel) => void {
|
|
63
|
+
return (served) => {
|
|
64
|
+
if (info.served !== undefined) return
|
|
65
|
+
info.served = served.model
|
|
66
|
+
info.fallbacks = served.fallbacks
|
|
67
|
+
if (!response.headersSent && !response.destroyed) {
|
|
68
|
+
response.setHeader(MODEL_HEADER, served.model)
|
|
69
|
+
if (served.fallbacks > 0) response.setHeader(FALLBACKS_HEADER, String(served.fallbacks))
|
|
70
|
+
}
|
|
71
|
+
then?.(served)
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export const isRetryable = (error: PlugError): boolean => error.code !== undefined && RETRYABLE.has(error.code)
|
|
76
|
+
|
|
77
|
+
/** The caller's events with a chain of fallbacks behind the request's model; see the file comment. */
|
|
78
|
+
export function withFallbacks(inner: ModelCaller, deps: FallbackDeps): ModelCaller {
|
|
79
|
+
return {
|
|
80
|
+
stream(request, context) {
|
|
81
|
+
if (!Object.hasOwn(deps.options.fallbacks, request.model)) return inner.stream(request, context)
|
|
82
|
+
return chain(inner, deps, request, context)
|
|
83
|
+
},
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function* chain(inner: ModelCaller, deps: FallbackDeps, request: PlugRequest, context: CallContext): AsyncGenerator<PlugEvent> {
|
|
88
|
+
let rest: ModelRef[] | undefined
|
|
89
|
+
let ref = request.ref
|
|
90
|
+
let attempt = 0
|
|
91
|
+
for (;;) {
|
|
92
|
+
// The history id of this attempt: handed on only if the attempt is the one that answers (or the last).
|
|
93
|
+
let recordID: string | undefined
|
|
94
|
+
const next = attempt === 0 ? request : retarget(deps.options, request, ref)
|
|
95
|
+
const iterator = inner.stream(next, { ...context, onRecord: (id) => void (recordID = id) })[Symbol.asyncIterator]()
|
|
96
|
+
|
|
97
|
+
let first: IteratorResult<PlugEvent> | undefined
|
|
98
|
+
let thrown: { readonly error: unknown } | undefined
|
|
99
|
+
try {
|
|
100
|
+
first = await iterator.next()
|
|
101
|
+
} catch (error) {
|
|
102
|
+
thrown = { error }
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Why this attempt failed before any output, or undefined when it answered.
|
|
106
|
+
let reason: string | undefined
|
|
107
|
+
if (thrown) reason = "caller_error"
|
|
108
|
+
else if (first!.done) reason = "no_response"
|
|
109
|
+
else if (first!.value.type === "error" && isRetryable(first!.value.error)) reason = first!.value.error.code
|
|
110
|
+
if (context.signal.aborted) reason = undefined
|
|
111
|
+
|
|
112
|
+
if (reason !== undefined) {
|
|
113
|
+
rest ??= await resolveRest(deps, request)
|
|
114
|
+
const target = rest[attempt]
|
|
115
|
+
if (target) {
|
|
116
|
+
if (!thrown) await close(iterator)
|
|
117
|
+
log("model.fallback", { app: context.app, from: refToString(ref), to: refToString(target), reason, attempt: attempt + 1 })
|
|
118
|
+
deps.onFallback?.(context.app)
|
|
119
|
+
ref = target
|
|
120
|
+
attempt++
|
|
121
|
+
continue
|
|
122
|
+
}
|
|
123
|
+
log("model.fallback_exhausted", { app: context.app, attempts: attempt + 1, reason })
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Answered, ran out of targets, or the request ended: this attempt's outcome is the response.
|
|
127
|
+
if (recordID !== undefined) report(context, recordID)
|
|
128
|
+
if (thrown) throw thrown.error
|
|
129
|
+
if (first!.done) return
|
|
130
|
+
// An error is not an answer: a call that fails (non-retryable, or on its last target) reports no served model.
|
|
131
|
+
if (reason === undefined && first!.value.type !== "error") context.onServed?.({ model: refToString(ref), fallbacks: attempt })
|
|
132
|
+
try {
|
|
133
|
+
yield first!.value
|
|
134
|
+
for (let step = await iterator.next(); !step.done; step = await iterator.next()) yield step.value
|
|
135
|
+
} finally {
|
|
136
|
+
await close(iterator)
|
|
137
|
+
}
|
|
138
|
+
return
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** The request for a later target: its ref, and the sampling parameters that target is known to reject dropped. */
|
|
143
|
+
function retarget(options: PlugboardOptions, request: PlugRequest, ref: ModelRef): PlugRequest {
|
|
144
|
+
return { ...request, ref, ...(options.dropUnsupportedParams ? { generation: dropRejected(ref, request.generation).generation } : {}) }
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** The fallbacks that are allowed and enabled. A catalog that cannot be loaded leaves none: the failure stands. */
|
|
148
|
+
async function resolveRest(deps: FallbackDeps, request: PlugRequest): Promise<ModelRef[]> {
|
|
149
|
+
try {
|
|
150
|
+
return fallbackRefs(deps.options, request.model, await deps.loadCatalog())
|
|
151
|
+
} catch {
|
|
152
|
+
return []
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const report = (context: CallContext, recordID: string): void => {
|
|
157
|
+
try {
|
|
158
|
+
context.onRecord?.(recordID)
|
|
159
|
+
} catch {}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Ends an attempt's stream so its cleanup runs; it must never fail the request. */
|
|
163
|
+
async function close(iterator: AsyncIterator<PlugEvent>): Promise<void> {
|
|
164
|
+
try {
|
|
165
|
+
await iterator.return?.()
|
|
166
|
+
} catch {}
|
|
167
|
+
}
|
package/src/gemini/response.ts
CHANGED
|
@@ -15,8 +15,8 @@ type ToolCall = Extract<PlugEvent, { type: "tool-call" }>
|
|
|
15
15
|
|
|
16
16
|
/** What the encoder needs besides the model events: Gemini presentation state. The request decoder produces it. */
|
|
17
17
|
export interface GeminiExchange {
|
|
18
|
-
/** The model string from the URL; echoed as `modelVersion`. */
|
|
19
|
-
|
|
18
|
+
/** The model string from the URL; echoed as `modelVersion`. A request that used an alias with fallbacks echoes the target that answered (`CallContext.onServed`). */
|
|
19
|
+
model: string
|
|
20
20
|
readonly stream: boolean
|
|
21
21
|
/** Name of the forced tool that carries `responseSchema` output; its input becomes the text of the answer. */
|
|
22
22
|
readonly structuredTool?: string
|
package/src/gemini/route.ts
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
import type { IncomingMessage, ServerResponse } from "node:http"
|
|
18
18
|
import { isPlugError, type PlugError, type PlugRequest } from "../contract.js"
|
|
19
|
+
import { reportServed, type ServedInfo } from "../fallback.js"
|
|
19
20
|
import { boundEvents, checkJsonContentType, readJsonBody, sendJson, sseWriter, type RequestScope } from "../http.js"
|
|
20
21
|
import { listAvailableModels, resolveModel } from "../models.js"
|
|
21
22
|
import type { RouteHost } from "../openai/route.js"
|
|
@@ -88,7 +89,7 @@ export interface GeminiCall {
|
|
|
88
89
|
readonly response: ServerResponse
|
|
89
90
|
readonly scope: RequestScope
|
|
90
91
|
/** The route's log fields: the resolved model, and the outcome of a stream that failed after it committed. */
|
|
91
|
-
readonly info: { model?: string; failed?: string; dropped?: string }
|
|
92
|
+
readonly info: ServedInfo & { model?: string; failed?: string; dropped?: string }
|
|
92
93
|
}
|
|
93
94
|
|
|
94
95
|
export async function serveGemini(host: RouteHost, { match, request, url, app, response, scope, info }: GeminiCall): Promise<void> {
|
|
@@ -137,6 +138,7 @@ export async function serveGemini(host: RouteHost, { match, request, url, app, r
|
|
|
137
138
|
if (response.headersSent || response.destroyed || !/^[\x21-\x7e]+$/.test(recordID)) return
|
|
138
139
|
response.setHeader(SESSION_HEADER, recordID)
|
|
139
140
|
},
|
|
141
|
+
onServed: reportServed(response, info, (served) => void (exchange.model = served.model)),
|
|
140
142
|
}),
|
|
141
143
|
scope,
|
|
142
144
|
)
|
package/src/metrics.ts
CHANGED
|
@@ -112,6 +112,8 @@ export interface Metrics {
|
|
|
112
112
|
tokens(app: string, usage: PlugUsage): void
|
|
113
113
|
/** Sampling parameters dropped for one request. */
|
|
114
114
|
dropped(params: ReadonlyArray<string>): void
|
|
115
|
+
/** A model call failed before any output and was retried on the next target of its alias. */
|
|
116
|
+
fallback(app: string): void
|
|
115
117
|
/** One history sweep ended. */
|
|
116
118
|
sweep(result: "ok" | "failed" | "timeout", removed: number): void
|
|
117
119
|
/** The exposition text. Called from a request that is itself in flight, which is not counted. */
|
|
@@ -128,6 +130,7 @@ export function createMetrics(): Metrics {
|
|
|
128
130
|
["app", "kind"],
|
|
129
131
|
)
|
|
130
132
|
const dropped = new Counter("plugboard_dropped_params_total", "Sampling parameters dropped because the target model rejects them, by parameter.", ["param"])
|
|
133
|
+
const fallbacks = new Counter("plugboard_fallbacks_total", "Model calls retried on the next target of an alias after a failure before any output, by app.", ["app"])
|
|
131
134
|
const sweeps = new Counter("plugboard_history_sweeps_total", "History sweeps that ended, by result.", ["result"])
|
|
132
135
|
const removed = new Counter("plugboard_history_sessions_removed_total", "Request sessions removed by history sweeps.", [])
|
|
133
136
|
|
|
@@ -151,6 +154,7 @@ export function createMetrics(): Metrics {
|
|
|
151
154
|
dropped(params) {
|
|
152
155
|
for (const param of params) dropped.add([param])
|
|
153
156
|
},
|
|
157
|
+
fallback: (app) => fallbacks.add([app]),
|
|
154
158
|
sweep(result, count) {
|
|
155
159
|
sweeps.add([result])
|
|
156
160
|
removed.add([], count)
|
|
@@ -163,6 +167,7 @@ export function createMetrics(): Metrics {
|
|
|
163
167
|
...gauge("plugboard_requests_in_flight", "Requests being handled right now, not counting this scrape.", Math.max(0, inFlight - 1)),
|
|
164
168
|
...tokens.render(),
|
|
165
169
|
...dropped.render(),
|
|
170
|
+
...fallbacks.render(),
|
|
166
171
|
...sweeps.render(),
|
|
167
172
|
...removed.render(),
|
|
168
173
|
...gauge("plugboard_generations", "Accepted plugin generations (one per OpenCode Location) serving requests.", snapshot.generations),
|
package/src/models.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { PlugboardOptions } from "./config.js"
|
|
|
4
4
|
import type { CatalogModel, ModelRef, PlugError } from "./contract.js"
|
|
5
5
|
|
|
6
6
|
/** `provider/model[#variant]`. The model id may contain slashes (`openrouter/vendor/model`). */
|
|
7
|
-
function parseModelRef(s: string): ModelRef | undefined {
|
|
7
|
+
export function parseModelRef(s: string): ModelRef | undefined {
|
|
8
8
|
if (typeof s !== "string") return undefined
|
|
9
9
|
const slash = s.indexOf("/")
|
|
10
10
|
if (slash <= 0) return undefined
|
|
@@ -16,19 +16,21 @@ function parseModelRef(s: string): ModelRef | undefined {
|
|
|
16
16
|
return { providerID: s.slice(0, slash), id, ...(variant ? { variant } : {}) }
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
function refToString(ref: ModelRef): string {
|
|
19
|
+
export function refToString(ref: ModelRef): string {
|
|
20
20
|
return `${ref.providerID}/${ref.id}${ref.variant ? `#${ref.variant}` : ""}`
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
/** `provider/*` any model of the provider; `provider/model` any variant; `provider/model#variant` exactly. */
|
|
24
|
-
function
|
|
25
|
-
return
|
|
24
|
+
export function allowedBy(models: ReadonlyArray<string>, ref: ModelRef): boolean {
|
|
25
|
+
return models.some((pattern) => {
|
|
26
26
|
if (pattern.endsWith("/*")) return pattern.slice(0, -2) === ref.providerID
|
|
27
27
|
const p = parseModelRef(pattern)
|
|
28
28
|
return p !== undefined && p.providerID === ref.providerID && p.id === ref.id && (p.variant === undefined || p.variant === ref.variant)
|
|
29
29
|
})
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
const allowed = (options: PlugboardOptions, ref: ModelRef): boolean => allowedBy(options.models, ref)
|
|
33
|
+
|
|
32
34
|
function find(catalog: ReadonlyArray<CatalogModel>, ref: ModelRef): CatalogModel | undefined {
|
|
33
35
|
const model = catalog.find((m) => m.providerID === ref.providerID && m.id === ref.id)
|
|
34
36
|
return model && (ref.variant === undefined || model.variants.includes(ref.variant)) ? model : undefined
|
|
@@ -98,3 +100,17 @@ export function listAvailableModels(options: PlugboardOptions, catalog: Readonly
|
|
|
98
100
|
}
|
|
99
101
|
return data
|
|
100
102
|
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* The targets an alias falls back to, in order: the fallbacks that are allowed and enabled on the server.
|
|
106
|
+
* Empty for a plain model id or an alias without fallbacks. The primary target is `resolveModel`'s answer.
|
|
107
|
+
*/
|
|
108
|
+
export function fallbackRefs(options: PlugboardOptions, alias: string, catalog: ReadonlyArray<CatalogModel>): ModelRef[] {
|
|
109
|
+
const targets = Object.hasOwn(options.fallbacks, alias) ? options.fallbacks[alias]! : []
|
|
110
|
+
const refs: ModelRef[] = []
|
|
111
|
+
for (const target of targets) {
|
|
112
|
+
const ref = parseModelRef(target)
|
|
113
|
+
if (ref && allowed(options, ref) && find(catalog, ref)) refs.push(ref)
|
|
114
|
+
}
|
|
115
|
+
return refs
|
|
116
|
+
}
|
package/src/openai/fanout.ts
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
|
|
23
23
|
import type { ServerResponse } from "node:http"
|
|
24
24
|
import type { PlugError, PlugEvent, PlugRequest, PlugUsage } from "../contract.js"
|
|
25
|
+
import { reportServed, type ServedInfo } from "../fallback.js"
|
|
25
26
|
import { RequestAborted, boundEvents, sendJson, sseWriter, type RequestScope } from "../http.js"
|
|
26
27
|
import { chatUsageChunk, collectCompletion, encodeError, streamCompletion, type OpenAiExchange, type StreamChoice, type StreamResult } from "./response.js"
|
|
27
28
|
import type { RouteHost } from "./route.js"
|
|
@@ -40,6 +41,8 @@ export interface ChoiceCodec {
|
|
|
40
41
|
streamChoice(events: AsyncIterable<PlugEvent>, write: (chunk: string) => void, choice: StreamChoice): Promise<StreamResult>
|
|
41
42
|
/** The final chunk with the summed usage (`include_usage`). */
|
|
42
43
|
usageChunk(choice: Omit<StreamChoice, "index">, usage: PlugUsage): object
|
|
44
|
+
/** Echo the served model of an alias with fallbacks as the response model. Used for a single call only: with several choices the body keeps the requested model. */
|
|
45
|
+
setModel(model: string): void
|
|
43
46
|
}
|
|
44
47
|
|
|
45
48
|
/** The Chat Completions encoder as a `ChoiceCodec`. */
|
|
@@ -50,6 +53,7 @@ export const chatCodec = (exchange: OpenAiExchange): ChoiceCodec => ({
|
|
|
50
53
|
collect: (events) => collectCompletion(events, exchange),
|
|
51
54
|
streamChoice: (events, write, choice) => streamCompletion(events, exchange, write, choice),
|
|
52
55
|
usageChunk: (choice, usage) => chatUsageChunk(exchange.model, choice.id, choice.created, usage),
|
|
56
|
+
setModel: (model) => void (exchange.model = model),
|
|
53
57
|
})
|
|
54
58
|
|
|
55
59
|
export interface ChoicesCall {
|
|
@@ -58,8 +62,8 @@ export interface ChoicesCall {
|
|
|
58
62
|
readonly calls: ReadonlyArray<PlugRequest>
|
|
59
63
|
readonly response: ServerResponse
|
|
60
64
|
readonly scope: RequestScope
|
|
61
|
-
/** The route's log fields: the outcome of a stream that failed after it committed. */
|
|
62
|
-
readonly info: { failed?: string }
|
|
65
|
+
/** The route's log fields: the outcome of a stream that failed after it committed, and the first served model. */
|
|
66
|
+
readonly info: ServedInfo & { failed?: string }
|
|
63
67
|
readonly codec: ChoiceCodec
|
|
64
68
|
}
|
|
65
69
|
|
|
@@ -158,6 +162,10 @@ export async function serveChoices(host: Pick<RouteHost, "caller">, { app, calls
|
|
|
158
162
|
const gscope = groupScope(scope, AbortSignal.any([scope.signal, group.signal]))
|
|
159
163
|
/** History record ids as the callers report them, in choice order, for the response header (comma separated). */
|
|
160
164
|
const records: Array<string | undefined> = []
|
|
165
|
+
// Headers: the first choice to be served wins. Body model: only a single call echoes the served model.
|
|
166
|
+
const served = reportServed(response, info, (s) => {
|
|
167
|
+
if (calls.length === 1) codec.setModel(s.model)
|
|
168
|
+
})
|
|
161
169
|
const open = (index: number) =>
|
|
162
170
|
boundEvents(
|
|
163
171
|
host.caller.stream(calls[index]!, {
|
|
@@ -168,6 +176,7 @@ export async function serveChoices(host: Pick<RouteHost, "caller">, { app, calls
|
|
|
168
176
|
records[index] = recordID
|
|
169
177
|
response.setHeader(SESSION_HEADER, records.filter((id) => id !== undefined).join(","))
|
|
170
178
|
},
|
|
179
|
+
onServed: served,
|
|
171
180
|
}),
|
|
172
181
|
gscope,
|
|
173
182
|
)
|
package/src/openai/response.ts
CHANGED
|
@@ -14,8 +14,8 @@ type ToolCall = Extract<PlugEvent, { type: "tool-call" }>
|
|
|
14
14
|
* presentation state. The request decoder produces it; the model call never sees it.
|
|
15
15
|
*/
|
|
16
16
|
export interface OpenAiExchange {
|
|
17
|
-
/** The model string the client sent; echoed in every response object. */
|
|
18
|
-
|
|
17
|
+
/** The model string the client sent; echoed in every response object. A request that used an alias with fallbacks echoes the target that answered (`CallContext.onServed`). */
|
|
18
|
+
model: string
|
|
19
19
|
readonly stream: boolean
|
|
20
20
|
/** `stream_options.include_usage`: the stream carries a usage field per chunk and a final usage chunk. */
|
|
21
21
|
readonly includeUsage: boolean
|
package/src/openai/route.ts
CHANGED
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
sseWriter,
|
|
37
37
|
type RequestScope,
|
|
38
38
|
} from "../http.js"
|
|
39
|
+
import { reportServed, type ServedInfo } from "../fallback.js"
|
|
39
40
|
import { encodeGeminiError } from "../gemini/response.js"
|
|
40
41
|
import { geminiAuthorization, matchGeminiRoute, serveGemini } from "../gemini/route.js"
|
|
41
42
|
import { errorMessage, log, logError, withRequestId } from "../log.js"
|
|
@@ -96,7 +97,7 @@ const decodeSegment = (raw: string): string => {
|
|
|
96
97
|
}
|
|
97
98
|
|
|
98
99
|
/** What a request log line may carry: ids and outcomes only. */
|
|
99
|
-
interface RequestInfo {
|
|
100
|
+
interface RequestInfo extends ServedInfo {
|
|
100
101
|
route: string
|
|
101
102
|
app?: string
|
|
102
103
|
model?: string
|
|
@@ -246,6 +247,8 @@ export function createOpenAiRoute(routeHost: RouteHost) {
|
|
|
246
247
|
if (response.headersSent || response.destroyed || !/^[\x21-\x7e]+$/.test(recordID)) return
|
|
247
248
|
response.setHeader(SESSION_HEADER, recordID)
|
|
248
249
|
},
|
|
250
|
+
// An alias with fallbacks: report which target answered, and echo it as the response model.
|
|
251
|
+
onServed: reportServed(response, info, (served) => void (exchange.model = served.model)),
|
|
249
252
|
}),
|
|
250
253
|
scope,
|
|
251
254
|
)
|
|
@@ -328,7 +331,7 @@ export function createOpenAiRoute(routeHost: RouteHost) {
|
|
|
328
331
|
metrics?.requestFinished({ app: info.app ?? NO_APP, route: info.route, status: response.statusCode, outcome: outcomeLabel(outcome, response.statusCode), ms })
|
|
329
332
|
// Probes and scrapes that succeed are not worth a line each.
|
|
330
333
|
if ((info.route !== "GET /healthz" && info.route !== "GET /metrics") || outcome !== "ok") {
|
|
331
|
-
const fields = { id, route: info.route, app: info.app, model: info.model, dropped: info.dropped, status: response.statusCode, ms, outcome, error }
|
|
334
|
+
const fields = { id, route: info.route, app: info.app, model: info.model, served: info.served, fallbacks: info.fallbacks, dropped: info.dropped, status: response.statusCode, ms, outcome, error }
|
|
332
335
|
if (outcome === "error" || response.statusCode >= 500) logError("request.failed", fields)
|
|
333
336
|
else log("request", fields)
|
|
334
337
|
}
|
|
@@ -12,8 +12,8 @@ type ToolCall = Extract<PlugEvent, { type: "tool-call" }>
|
|
|
12
12
|
|
|
13
13
|
/** What the encoder needs besides the model events: Responses presentation state. The request decoder produces it. */
|
|
14
14
|
export interface ResponsesExchange {
|
|
15
|
-
/** The model string the client sent; echoed in every response object. */
|
|
16
|
-
|
|
15
|
+
/** The model string the client sent; echoed in every response object. A request that used an alias with fallbacks echoes the target that answered (`CallContext.onServed`). */
|
|
16
|
+
model: string
|
|
17
17
|
readonly stream: boolean
|
|
18
18
|
/** Request fields echoed into the response object (instructions, tools, temperature, ...). */
|
|
19
19
|
readonly envelope: Readonly<Record<string, unknown>>
|
package/src/responses/route.ts
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
import type { ServerResponse } from "node:http"
|
|
12
12
|
import { isPlugError, type PlugError, type PlugRequest } from "../contract.js"
|
|
13
|
+
import { reportServed, type ServedInfo } from "../fallback.js"
|
|
13
14
|
import { boundEvents, sendJson, sseWriter, type RequestScope } from "../http.js"
|
|
14
15
|
import { resolveModel } from "../models.js"
|
|
15
16
|
import { encodeError } from "../openai/response.js"
|
|
@@ -25,7 +26,7 @@ export interface ResponsesCall {
|
|
|
25
26
|
readonly response: ServerResponse
|
|
26
27
|
readonly scope: RequestScope
|
|
27
28
|
/** The route's log fields: the resolved model, and the outcome of a stream that failed after it committed. */
|
|
28
|
-
readonly info: { model?: string; failed?: string }
|
|
29
|
+
readonly info: ServedInfo & { model?: string; failed?: string }
|
|
29
30
|
}
|
|
30
31
|
|
|
31
32
|
export async function serveResponses(host: RouteHost, { app, body, response, scope, info }: ResponsesCall): Promise<void> {
|
|
@@ -50,6 +51,7 @@ export async function serveResponses(host: RouteHost, { app, body, response, sco
|
|
|
50
51
|
if (response.headersSent || response.destroyed || !/^[\x21-\x7e]+$/.test(recordID)) return
|
|
51
52
|
response.setHeader(SESSION_HEADER, recordID)
|
|
52
53
|
},
|
|
54
|
+
onServed: reportServed(response, info, (served) => void (exchange.model = served.model)),
|
|
53
55
|
}),
|
|
54
56
|
scope,
|
|
55
57
|
)
|