dsh-cline-pass 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +1 -1
- package/README.md +1 -1
- package/lib/adapter.js +368 -23
- package/lib/catalog.js +13 -5
- package/lib/client.js +1125 -190
- package/lib/cline.js +170 -5
- package/lib/compat.js +154 -0
- package/lib/engine.js +153 -28
- package/lib/index.js +184 -28
- package/lib/panel.js +168 -85
- package/lib/protocol.js +189 -23
- package/lib/store.js +17 -2
- package/lib/tools.js +155 -14
- package/package.json +3 -2
package/README.en.md
CHANGED
|
@@ -68,7 +68,7 @@ To override defaults, add a `cline-pass` section to your profile's `$DSH_HOME/pr
|
|
|
68
68
|
apiKeyEnv: CLINE_PASS_BACKUP_KEY
|
|
69
69
|
```
|
|
70
70
|
|
|
71
|
-
Common settings include `baseURL`, `knownModels`, `models`, `perModel`, `exposeCatalog`, and `historyLimit`. The default gateway is `https://api.cline.bot/api/v1`.
|
|
71
|
+
Common settings include `baseURL`, `knownModels`, `models`, `perModel`, `exposeCatalog`, and `historyLimit`. The default gateway is `https://api.cline.bot/api/v1`. `maxConcurrentRequests` caps how many streams each account keeps open to the gateway at once (default 16); further requests wait for a free slot, and `0` lifts the cap.
|
|
72
72
|
|
|
73
73
|
## Upstream channels
|
|
74
74
|
|
package/README.md
CHANGED
|
@@ -68,7 +68,7 @@ cline_pass_accounts action=mode mode=roundrobin
|
|
|
68
68
|
apiKeyEnv: CLINE_PASS_BACKUP_KEY
|
|
69
69
|
```
|
|
70
70
|
|
|
71
|
-
常用设置包括 `baseURL`、`knownModels`、`models`、`perModel`、`exposeCatalog` 和 `historyLimit`。默认网关地址为 `https://api.cline.bot/api/v1
|
|
71
|
+
常用设置包括 `baseURL`、`knownModels`、`models`、`perModel`、`exposeCatalog` 和 `historyLimit`。默认网关地址为 `https://api.cline.bot/api/v1`。`maxConcurrentRequests` 限制每个账号同时发往网关的流数,默认 16,超出的请求排队等待空位,设为 `0` 表示不限。
|
|
72
72
|
|
|
73
73
|
## 上游渠道
|
|
74
74
|
|
package/lib/adapter.js
CHANGED
|
@@ -30,12 +30,23 @@ import {
|
|
|
30
30
|
errorText,
|
|
31
31
|
injectPrefs,
|
|
32
32
|
parseRouting,
|
|
33
|
+
pinAdherence,
|
|
34
|
+
pinWarnings,
|
|
33
35
|
unwrapEnvelope,
|
|
34
36
|
} from './protocol.js'
|
|
35
37
|
import { parseServerSentEvents } from './engine.js'
|
|
36
38
|
|
|
37
39
|
/** Default maximum idle interval while a stream read is outstanding. */
|
|
38
40
|
export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 300000
|
|
41
|
+
/**
|
|
42
|
+
* Default number of streams one account keeps open to the gateway at once.
|
|
43
|
+
*
|
|
44
|
+
* Twice dsh 0.1.7's default `maxActiveSubagents` (8): a main agent with a full
|
|
45
|
+
* set of subagents, plus a title or compaction call, never queues, while a
|
|
46
|
+
* runaway fan-out still cannot open an unbounded number of upstream streams
|
|
47
|
+
* (CWE-770). `0` lifts the cap.
|
|
48
|
+
*/
|
|
49
|
+
export const DEFAULT_MAX_CONCURRENT_REQUESTS = 16
|
|
39
50
|
/**
|
|
40
51
|
* Conservative context capacity for a model the catalog does not describe.
|
|
41
52
|
* Shipped models carry their published window, so this only bounds the
|
|
@@ -56,6 +67,84 @@ export const DEFAULT_MAX_TOKENS = 32000
|
|
|
56
67
|
* cap. These match the values the shipped adapters use.
|
|
57
68
|
*/
|
|
58
69
|
export const DEFAULT_REQUEST_IMAGE_POLICY = { maxPixels: 4194304, maxBytes: 1048576 }
|
|
70
|
+
/**
|
|
71
|
+
* Lead-in text for the user message that carries images recovered from tool
|
|
72
|
+
* results, matching the wording the shipped DeepSeek adapter uses.
|
|
73
|
+
*/
|
|
74
|
+
const TOOL_RESULT_IMAGE_TEXT = 'Attached image(s) from tool result:'
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Compute aspect-preserving integer dimensions within a total-pixel budget.
|
|
78
|
+
*
|
|
79
|
+
* Local copy of the harness's own projection: it is pure geometry, and this
|
|
80
|
+
* adapter must not add a hard dependency on `@deepseek-ai/dsh-attachment` for
|
|
81
|
+
* one helper. Mirrors `requestImageDimensions`, including the inward rounding
|
|
82
|
+
* and the "never enlarge a small image" rule.
|
|
83
|
+
*
|
|
84
|
+
* @param width - positive source width.
|
|
85
|
+
* @param height - positive source height.
|
|
86
|
+
* @param maxPixels - positive width-times-height cap.
|
|
87
|
+
* @returns projected dimensions.
|
|
88
|
+
*/
|
|
89
|
+
export function projectImageDimensions(width, height, maxPixels) {
|
|
90
|
+
const safeWidth = Number.isSafeInteger(width) && width > 0 ? width : 1
|
|
91
|
+
const safeHeight = Number.isSafeInteger(height) && height > 0 ? height : 1
|
|
92
|
+
const safeBudget = Number.isSafeInteger(maxPixels) && maxPixels > 0 ? maxPixels : DEFAULT_REQUEST_IMAGE_POLICY.maxPixels
|
|
93
|
+
const scale = Math.min(1, Math.sqrt(safeBudget / (safeWidth * safeHeight)))
|
|
94
|
+
if (scale === 1) return { width: safeWidth, height: safeHeight }
|
|
95
|
+
if (safeWidth >= safeHeight) {
|
|
96
|
+
let projectedWidth = Math.max(1, Math.floor(safeWidth * scale))
|
|
97
|
+
let projectedHeight = Math.max(1, Math.round((projectedWidth * safeHeight) / safeWidth))
|
|
98
|
+
while (projectedWidth * projectedHeight > safeBudget && projectedWidth > 1) {
|
|
99
|
+
projectedWidth -= 1
|
|
100
|
+
projectedHeight = Math.max(1, Math.round((projectedWidth * safeHeight) / safeWidth))
|
|
101
|
+
}
|
|
102
|
+
return { width: projectedWidth, height: projectedHeight }
|
|
103
|
+
}
|
|
104
|
+
let projectedHeight = Math.max(1, Math.floor(safeHeight * scale))
|
|
105
|
+
let projectedWidth = Math.max(1, Math.round((projectedHeight * safeWidth) / safeHeight))
|
|
106
|
+
while (projectedWidth * projectedHeight > safeBudget && projectedHeight > 1) {
|
|
107
|
+
projectedHeight -= 1
|
|
108
|
+
projectedWidth = Math.max(1, Math.round((projectedHeight * safeWidth) / safeHeight))
|
|
109
|
+
}
|
|
110
|
+
return { width: projectedWidth, height: projectedHeight }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Build the target object one attachment must be materialized at.
|
|
115
|
+
*
|
|
116
|
+
* The harness changed this contract between releases and the two shapes are
|
|
117
|
+
* mutually exclusive, not additive:
|
|
118
|
+
*
|
|
119
|
+
* - 0.1.2 - 0.1.5 validate `{maxPixels, maxBytes}` and project the dimensions
|
|
120
|
+
* themselves;
|
|
121
|
+
* - 0.1.6+ validate `{width, height, maxBytes}` and take the dimensions as
|
|
122
|
+
* given.
|
|
123
|
+
*
|
|
124
|
+
* A caller that sends only one shape fails on the other host — the newer one
|
|
125
|
+
* rejects a missing `width` with `Image request width must be a positive
|
|
126
|
+
* integer.`, which surfaces to the user as a failed turn on every image. Each
|
|
127
|
+
* validator reads only the fields it knows and ignores the rest, so one target
|
|
128
|
+
* carrying both the projected dimensions and the pixel budget satisfies either
|
|
129
|
+
* host. The dimensions are projected here with the same geometry the older
|
|
130
|
+
* host would have applied, so the rendered result is identical on both.
|
|
131
|
+
*
|
|
132
|
+
* @param ref - the durable attachment reference, which carries its own size.
|
|
133
|
+
* @param policy - the configured pixel and byte budget.
|
|
134
|
+
* @returns the request target for `readImageRequest`.
|
|
135
|
+
* @throws {LlmError} UNSUPPORTED_CONTENT when the reference carries no usable size.
|
|
136
|
+
*/
|
|
137
|
+
export function requestImageTarget(ref, policy) {
|
|
138
|
+
const maxPixels = policy?.maxPixels ?? DEFAULT_REQUEST_IMAGE_POLICY.maxPixels
|
|
139
|
+
const maxBytes = policy?.maxBytes ?? DEFAULT_REQUEST_IMAGE_POLICY.maxBytes
|
|
140
|
+
const width = ref?.width
|
|
141
|
+
const height = ref?.height
|
|
142
|
+
if (!Number.isSafeInteger(width) || width <= 0 || !Number.isSafeInteger(height) || height <= 0) {
|
|
143
|
+
throw new LlmError('The Cline Pass adapter received an image reference without usable dimensions.', 'UNSUPPORTED_CONTENT')
|
|
144
|
+
}
|
|
145
|
+
const projected = projectImageDimensions(width, height, maxPixels)
|
|
146
|
+
return { width: projected.width, height: projected.height, maxPixels, maxBytes }
|
|
147
|
+
}
|
|
59
148
|
|
|
60
149
|
const STREAM_IDLE_TIMEOUT_CODE = 'CLINE_PASS_STREAM_IDLE_TIMEOUT'
|
|
61
150
|
|
|
@@ -107,7 +196,7 @@ export async function prepareRequestImages(messages, attachments, policy, signal
|
|
|
107
196
|
throw new LlmError('The Cline Pass adapter needs the durable attachment service to send image input.', 'UNSUPPORTED_CONTENT')
|
|
108
197
|
}
|
|
109
198
|
const ordered = [...refs.values()]
|
|
110
|
-
const prepared = await Promise.all(ordered.map((ref) => attachments.readImageRequest(ref, policy, signal)))
|
|
199
|
+
const prepared = await Promise.all(ordered.map((ref) => attachments.readImageRequest(ref, requestImageTarget(ref, policy), signal)))
|
|
111
200
|
const versions = new Map()
|
|
112
201
|
for (const [index, ref] of ordered.entries()) versions.set(ref.attachmentId, prepared[index])
|
|
113
202
|
return versions
|
|
@@ -135,21 +224,55 @@ function serializeAssistant(message) {
|
|
|
135
224
|
}
|
|
136
225
|
}
|
|
137
226
|
|
|
227
|
+
/**
|
|
228
|
+
* Resolve one image block into its OpenAI wire part.
|
|
229
|
+
*
|
|
230
|
+
* @throws {LlmError} UNSUPPORTED_CONTENT when its request bytes were not prepared.
|
|
231
|
+
*/
|
|
232
|
+
function imagePart(block, images) {
|
|
233
|
+
const version = images?.get(block.attachment?.attachmentId)
|
|
234
|
+
if (version === undefined) {
|
|
235
|
+
throw new LlmError('The Cline Pass adapter received an image it could not resolve into request bytes.', 'UNSUPPORTED_CONTENT')
|
|
236
|
+
}
|
|
237
|
+
return { type: 'image_url', image_url: { url: `data:${version.mediaType};base64,${toBase64(version.data)}` } }
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Recursively collect image parts from blocks, including nested tool results. */
|
|
241
|
+
function collectImageParts(blocks, images, target = []) {
|
|
242
|
+
for (const block of blocks ?? []) {
|
|
243
|
+
if (block.type === 'image') target.push(imagePart(block, images))
|
|
244
|
+
else if (block.type === 'tool-result') collectImageParts(block.content, images, target)
|
|
245
|
+
}
|
|
246
|
+
return target
|
|
247
|
+
}
|
|
248
|
+
|
|
138
249
|
/**
|
|
139
250
|
* Convert the harness conversation to wire messages.
|
|
140
251
|
*
|
|
141
252
|
* Tool results become standalone `{ role: 'tool' }` messages. A user message
|
|
142
253
|
* carrying an image switches to the OpenAI content-part array; a text-only one
|
|
143
254
|
* keeps the plain-string form.
|
|
255
|
+
*
|
|
256
|
+
* An image nested in a tool result (read_image, a screenshot tool) has no
|
|
257
|
+
* OpenAI `tool` wire spelling, so it rides a following user message instead —
|
|
258
|
+
* the same shape the shipped adapters use.
|
|
144
259
|
*/
|
|
145
260
|
function serializeMessages(messages, images) {
|
|
146
261
|
const wire = []
|
|
262
|
+
let pendingToolImages = []
|
|
263
|
+
const flushToolImages = () => {
|
|
264
|
+
if (pendingToolImages.length === 0) return
|
|
265
|
+
wire.push({ role: 'user', content: [{ type: 'text', text: TOOL_RESULT_IMAGE_TEXT }, ...pendingToolImages] })
|
|
266
|
+
pendingToolImages = []
|
|
267
|
+
}
|
|
147
268
|
for (const message of messages) {
|
|
148
269
|
if (message.role === 'system') {
|
|
270
|
+
flushToolImages()
|
|
149
271
|
wire.push({ role: 'system', content: flattenText(message.content) })
|
|
150
272
|
continue
|
|
151
273
|
}
|
|
152
274
|
if (message.role === 'assistant') {
|
|
275
|
+
flushToolImages()
|
|
153
276
|
wire.push(serializeAssistant(message))
|
|
154
277
|
continue
|
|
155
278
|
}
|
|
@@ -157,26 +280,24 @@ function serializeMessages(messages, images) {
|
|
|
157
280
|
const parts = []
|
|
158
281
|
for (const block of message.content ?? []) {
|
|
159
282
|
if (block.type === 'text' && block.text.length > 0) parts.push({ type: 'text', text: block.text })
|
|
160
|
-
else if (block.type === 'image')
|
|
161
|
-
const version = images?.get(block.attachment?.attachmentId)
|
|
162
|
-
if (version === undefined) {
|
|
163
|
-
throw new LlmError('The Cline Pass adapter received an image it could not resolve into request bytes.', 'UNSUPPORTED_CONTENT')
|
|
164
|
-
}
|
|
165
|
-
parts.push({ type: 'image_url', image_url: { url: `data:${version.mediaType};base64,${toBase64(version.data)}` } })
|
|
166
|
-
}
|
|
283
|
+
else if (block.type === 'image') parts.push(imagePart(block, images))
|
|
167
284
|
}
|
|
168
285
|
const hasImage = parts.some((part) => part.type === 'image_url')
|
|
169
286
|
const text = parts.filter((part) => part.type === 'text').map((part) => part.text).join('')
|
|
170
|
-
if (hasImage
|
|
171
|
-
|
|
287
|
+
if (hasImage || text.length > 0 || toolResults.length === 0) {
|
|
288
|
+
flushToolImages()
|
|
289
|
+
wire.push({ role: 'user', content: hasImage ? parts : text })
|
|
290
|
+
}
|
|
172
291
|
for (const result of toolResults) {
|
|
173
292
|
wire.push({
|
|
174
293
|
role: 'tool',
|
|
175
294
|
tool_call_id: result.toolCallId,
|
|
176
295
|
content: flattenText(result.content) || '(no output)',
|
|
177
296
|
})
|
|
297
|
+
collectImageParts(result.content, images, pendingToolImages)
|
|
178
298
|
}
|
|
179
299
|
}
|
|
300
|
+
flushToolImages()
|
|
180
301
|
return wire
|
|
181
302
|
}
|
|
182
303
|
|
|
@@ -303,6 +424,31 @@ export function streamErrorCode(message, status) {
|
|
|
303
424
|
return 'UPSTREAM'
|
|
304
425
|
}
|
|
305
426
|
|
|
427
|
+
/**
|
|
428
|
+
* Note the moment the first byte of a response body arrives.
|
|
429
|
+
*
|
|
430
|
+
* A pass-through stream: it reports once, on the first chunk, and forwards every
|
|
431
|
+
* chunk unchanged. A keep-alive comment or an early frame reaches this before it
|
|
432
|
+
* reaches the SSE parser, so the recorded moment is the gateway's first sign of
|
|
433
|
+
* life rather than the caller's first content.
|
|
434
|
+
*
|
|
435
|
+
* @param body - the response body stream.
|
|
436
|
+
* @param onFirst - called once, when the first chunk is seen.
|
|
437
|
+
* @returns a stream equivalent to `body`.
|
|
438
|
+
*/
|
|
439
|
+
function tapFirstByte(body, onFirst) {
|
|
440
|
+
let seen = false
|
|
441
|
+
return body.pipeThrough(new TransformStream({
|
|
442
|
+
transform(chunk, controller) {
|
|
443
|
+
if (!seen) {
|
|
444
|
+
seen = true
|
|
445
|
+
onFirst()
|
|
446
|
+
}
|
|
447
|
+
controller.enqueue(chunk)
|
|
448
|
+
},
|
|
449
|
+
}))
|
|
450
|
+
}
|
|
451
|
+
|
|
306
452
|
/**
|
|
307
453
|
* Translate decoded SSE payloads into harness stream chunks.
|
|
308
454
|
*
|
|
@@ -421,6 +567,81 @@ async function* tapRouting(payloads, observed) {
|
|
|
421
567
|
}
|
|
422
568
|
}
|
|
423
569
|
|
|
570
|
+
//#region upstream concurrency
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* Per-account slots for streams open to the gateway.
|
|
574
|
+
*
|
|
575
|
+
* A slot is held for a whole attempt — from before `fetch` until the response
|
|
576
|
+
* body is closed — because the stream, not the wait for headers, is what keeps
|
|
577
|
+
* a socket and the account's concurrency busy. Accounts are counted apart, so a
|
|
578
|
+
* stalled account cannot hold up requests routed to a healthy one; absorbing
|
|
579
|
+
* exactly that is what the pool is for.
|
|
580
|
+
*
|
|
581
|
+
* Waiters are served first in, first out, and a freed slot is handed to the
|
|
582
|
+
* next waiter and counted in the same step, so a request arriving in between
|
|
583
|
+
* cannot take it as well. A waiter whose signal aborts leaves the queue at
|
|
584
|
+
* once; a held slot is also freed when its signal aborts, so a consumer that
|
|
585
|
+
* stops reading without closing the stream does not keep it.
|
|
586
|
+
*
|
|
587
|
+
* @returns `{ acquire(key, limit, signal) }`, resolving to a release function
|
|
588
|
+
* that is safe to call more than once. A `limit` below 1 means no limit.
|
|
589
|
+
*/
|
|
590
|
+
export function createUpstreamSlots() {
|
|
591
|
+
const pools = new Map()
|
|
592
|
+
const admit = (pool) => {
|
|
593
|
+
while (pool.waiting.length > 0 && pool.active < pool.limit) {
|
|
594
|
+
pool.active += 1
|
|
595
|
+
pool.waiting.shift()()
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
const hold = (pool, signal) => {
|
|
599
|
+
let held = true
|
|
600
|
+
const release = () => {
|
|
601
|
+
if (!held) return
|
|
602
|
+
held = false
|
|
603
|
+
signal?.removeEventListener('abort', release)
|
|
604
|
+
pool.active -= 1
|
|
605
|
+
admit(pool)
|
|
606
|
+
}
|
|
607
|
+
signal?.addEventListener('abort', release, { once: true })
|
|
608
|
+
return release
|
|
609
|
+
}
|
|
610
|
+
return {
|
|
611
|
+
async acquire(key, limit, signal) {
|
|
612
|
+
signal?.throwIfAborted()
|
|
613
|
+
let pool = pools.get(key)
|
|
614
|
+
if (pool === undefined) {
|
|
615
|
+
pool = { active: 0, limit: Infinity, waiting: [] }
|
|
616
|
+
pools.set(key, pool)
|
|
617
|
+
}
|
|
618
|
+
// Read on every call, so a changed setting reaches the next request; a
|
|
619
|
+
// raised limit admits the queue before this caller joins it.
|
|
620
|
+
pool.limit = Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : Infinity
|
|
621
|
+
admit(pool)
|
|
622
|
+
if (pool.waiting.length === 0 && pool.active < pool.limit) {
|
|
623
|
+
pool.active += 1
|
|
624
|
+
return hold(pool, signal)
|
|
625
|
+
}
|
|
626
|
+
return await new Promise((resolve, reject) => {
|
|
627
|
+
const grant = () => {
|
|
628
|
+
signal?.removeEventListener('abort', abandon)
|
|
629
|
+
resolve(hold(pool, signal))
|
|
630
|
+
}
|
|
631
|
+
const abandon = () => {
|
|
632
|
+
const at = pool.waiting.indexOf(grant)
|
|
633
|
+
if (at !== -1) pool.waiting.splice(at, 1)
|
|
634
|
+
reject(signal.reason)
|
|
635
|
+
}
|
|
636
|
+
pool.waiting.push(grant)
|
|
637
|
+
signal?.addEventListener('abort', abandon, { once: true })
|
|
638
|
+
})
|
|
639
|
+
},
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
//#endregion
|
|
644
|
+
|
|
424
645
|
/**
|
|
425
646
|
* The Cline Pass adapter. One instance serves the plugin's provider route(s);
|
|
426
647
|
* the harness model id IS the gateway's wire model id.
|
|
@@ -429,6 +650,7 @@ export class ClinePassAdapter extends LlmAdapter {
|
|
|
429
650
|
constructor(config) {
|
|
430
651
|
super()
|
|
431
652
|
this.config = config
|
|
653
|
+
this.upstreamSlots = createUpstreamSlots()
|
|
432
654
|
}
|
|
433
655
|
|
|
434
656
|
providerInfo(provider) {
|
|
@@ -444,11 +666,15 @@ export class ClinePassAdapter extends LlmAdapter {
|
|
|
444
666
|
const configured = connection.models.find((entry) => entry.id === model)
|
|
445
667
|
const discovered = this.config.discoveredContext?.(model)
|
|
446
668
|
const override = configured ?? { id: model }
|
|
669
|
+
// What the last official scan published about this model. Without it a model
|
|
670
|
+
// newer than this release resolves to the `text` fallback and the harness
|
|
671
|
+
// refuses every image for it, silently.
|
|
672
|
+
const published = this.config.publishedModel?.(model) ?? {}
|
|
447
673
|
const resolved = resolveModelMetadata(provider, model, override, {
|
|
448
674
|
contextWindow: discovered ?? connection.defaultContextWindow,
|
|
449
675
|
maxTokens: connection.maxTokens,
|
|
450
676
|
reasoning: connection.reasoningModels !== false,
|
|
451
|
-
})
|
|
677
|
+
}, published)
|
|
452
678
|
// A probe that saw this model's real endpoints overrides the published
|
|
453
679
|
// window: it is measured, not documented.
|
|
454
680
|
return Promise.resolve(discovered === undefined
|
|
@@ -494,12 +720,51 @@ export class ClinePassAdapter extends LlmAdapter {
|
|
|
494
720
|
const trace = []
|
|
495
721
|
let delivered = false
|
|
496
722
|
let lastError = null
|
|
723
|
+
/**
|
|
724
|
+
* When the caller saw its first chunk, in ms from the request's own start.
|
|
725
|
+
*
|
|
726
|
+
* Measured from `startedAt` rather than from the serving attempt so it can be
|
|
727
|
+
* read next to `ms`, which is also request-scoped: a failover attempt's dead
|
|
728
|
+
* time is part of what the user actually waited through. Zero means no chunk
|
|
729
|
+
* ever arrived, which is a fact worth keeping rather than a missing value.
|
|
730
|
+
*/
|
|
731
|
+
let firstChunkMs = 0
|
|
732
|
+
/**
|
|
733
|
+
* When the gateway's first byte arrived, in ms from the request's own start.
|
|
734
|
+
*
|
|
735
|
+
* `ttft` answers "when did the caller see something"; this answers "when did
|
|
736
|
+
* the gateway start talking at all". The pair separates two very different
|
|
737
|
+
* stalls: a gateway that accepted the request and then sent nothing (both
|
|
738
|
+
* late) from one that keeps the socket warm while the model thinks — a
|
|
739
|
+
* keep-alive comment or an early frame produces bytes without producing a
|
|
740
|
+
* chunk, and only having both tells those apart.
|
|
741
|
+
*/
|
|
742
|
+
let firstByteMs = 0
|
|
743
|
+
/**
|
|
744
|
+
* Accounts this request has already been refused by.
|
|
745
|
+
*
|
|
746
|
+
* The pool is offered as one account with a combined quota, so a refusal that
|
|
747
|
+
* is about the ACCOUNT — a spent window, a revoked key — moves to the next
|
|
748
|
+
* one instead of ending the request. `resolveAccount` answers `undefined` once
|
|
749
|
+
* nothing is left, which is when the real error is raised below.
|
|
750
|
+
*/
|
|
751
|
+
const refused = new Set()
|
|
497
752
|
for (const attempt of attempts) {
|
|
498
|
-
const account = await this.config.resolveAccount()
|
|
753
|
+
const account = await this.config.resolveAccount({ exclude: [...refused] })
|
|
754
|
+
if (account === undefined) break
|
|
499
755
|
const body = injectPrefs(buildRequestBody(options, connection, images), meta, attempt)
|
|
500
|
-
const attemptStarted = Date.now()
|
|
501
756
|
const consumer = new AbortController()
|
|
502
757
|
const upstream = options.signal === undefined ? consumer.signal : AbortSignal.any([options.signal, consumer.signal])
|
|
758
|
+
// The account's slot is taken before the attempt's clock starts: the wait
|
|
759
|
+
// is local, and counting it would teach the channel a latency it never had.
|
|
760
|
+
let release
|
|
761
|
+
try {
|
|
762
|
+
release = await this.upstreamSlots.acquire(account.name, connection.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS, upstream)
|
|
763
|
+
} catch (error) {
|
|
764
|
+
if (options.signal?.aborted) throw new LlmError('Cline Pass request aborted by caller', 'ABORTED', { cause: error })
|
|
765
|
+
throw error
|
|
766
|
+
}
|
|
767
|
+
const attemptStarted = Date.now()
|
|
503
768
|
const watchdog = idleWatchdog(upstream, connection.streamIdleTimeoutMs, STREAM_IDLE_TIMEOUT_CODE)
|
|
504
769
|
try {
|
|
505
770
|
let response
|
|
@@ -534,9 +799,22 @@ export class ClinePassAdapter extends LlmAdapter {
|
|
|
534
799
|
trace.push({ upstream: attempt.upstream ?? '(auto)', status: response.status, ms, note: detail.slice(0, 160) })
|
|
535
800
|
this.config.learnUpstream(options.model, attempt.upstream, classifyUpstreamError(detail), detail, ms)
|
|
536
801
|
lastError = new LlmError(`${detail || `HTTP ${response.status}`} [model=${options.model}]`, streamErrorCode(detail, response.status), { status: response.status })
|
|
537
|
-
// A rejected pin or a busy upstream is worth another candidate
|
|
538
|
-
//
|
|
539
|
-
|
|
802
|
+
// A rejected pin or a busy upstream is worth another candidate. An
|
|
803
|
+
// account-level refusal is worth another ACCOUNT: that is the whole
|
|
804
|
+
// point of a pool, and with one account the next attempt simply finds
|
|
805
|
+
// nothing left and this error is raised unchanged.
|
|
806
|
+
if (lastError.code === 'AUTH' || lastError.code === QUOTA_EXCEEDED_CODE) {
|
|
807
|
+
refused.add(account.name)
|
|
808
|
+
this.config.record(options.model, {
|
|
809
|
+
provider: null,
|
|
810
|
+
attempts: trace.map((row) => row.upstream),
|
|
811
|
+
ms: Date.now() - startedAt,
|
|
812
|
+
stream: true,
|
|
813
|
+
error: firstLine(lastError.message, 200),
|
|
814
|
+
code: lastError.code,
|
|
815
|
+
account: account.name,
|
|
816
|
+
})
|
|
817
|
+
}
|
|
540
818
|
continue
|
|
541
819
|
}
|
|
542
820
|
if (response.body === null || response.body === undefined) {
|
|
@@ -545,11 +823,28 @@ export class ClinePassAdapter extends LlmAdapter {
|
|
|
545
823
|
}
|
|
546
824
|
let yielded = false
|
|
547
825
|
const observed = { provider: null, canonical: null, pipeline: null }
|
|
548
|
-
|
|
826
|
+
// `body` in this scope is the request payload, so the response stream
|
|
827
|
+
// gets its own name.
|
|
828
|
+
const responseBody = tapFirstByte(response.body, () => {
|
|
829
|
+
if (firstByteMs === 0) firstByteMs = Date.now() - startedAt
|
|
830
|
+
})
|
|
831
|
+
const iterator = translate(tapRouting(parseServerSentEvents(responseBody), observed))[Symbol.asyncIterator]()
|
|
832
|
+
/**
|
|
833
|
+
* The usage frame, captured as it passes through.
|
|
834
|
+
*
|
|
835
|
+
* The gateway sends it in the last frames, after the content, so a record
|
|
836
|
+
* written when the stream ends has it — but only if the chunk was read on
|
|
837
|
+
* its way to the caller. `reasoningTokens` in particular is the number
|
|
838
|
+
* that explains a large time-to-first-chunk: it is thinking the caller
|
|
839
|
+
* cannot see, because this gateway does not stream it.
|
|
840
|
+
*/
|
|
841
|
+
let usage = null
|
|
549
842
|
try {
|
|
550
843
|
for (;;) {
|
|
551
844
|
const next = await watchdog.next(iterator)
|
|
552
845
|
if (next.done === true) break
|
|
846
|
+
if (next.value?.type === 'usage') usage = next.value.usage
|
|
847
|
+
if (firstChunkMs === 0) firstChunkMs = Date.now() - startedAt
|
|
553
848
|
yielded = true
|
|
554
849
|
yield next.value
|
|
555
850
|
}
|
|
@@ -562,21 +857,41 @@ export class ClinePassAdapter extends LlmAdapter {
|
|
|
562
857
|
const ms = Date.now() - attemptStarted
|
|
563
858
|
trace.push({ upstream: attempt.upstream ?? '(auto)', status: 200, ms, note: note.slice(0, 160) })
|
|
564
859
|
this.config.learnUpstream(options.model, attempt.upstream, classifyUpstreamError(note), note, ms)
|
|
565
|
-
// Content already reached the caller: this is the answer's failure,
|
|
566
|
-
//
|
|
567
|
-
//
|
|
568
|
-
|
|
860
|
+
// Content already reached the caller: this is the answer's failure, and
|
|
861
|
+
// no retry can un-deliver it. Before any content, an account-level
|
|
862
|
+
// refusal (a spent window, a revoked key) moves to the next account —
|
|
863
|
+
// the pool exists to absorb exactly that.
|
|
864
|
+
const code = error instanceof LlmError ? error.code : 'TRANSPORT'
|
|
865
|
+
if (yielded) {
|
|
569
866
|
this.config.record(options.model, {
|
|
570
867
|
provider: observed.provider,
|
|
571
868
|
canonical: observed.canonical,
|
|
572
869
|
attempts: trace.map((row) => row.upstream),
|
|
573
870
|
ms: Date.now() - startedAt,
|
|
871
|
+
ttfb: firstByteMs,
|
|
872
|
+
ttft: firstChunkMs,
|
|
574
873
|
stream: true,
|
|
575
874
|
error: note.slice(0, 200),
|
|
875
|
+
code,
|
|
576
876
|
account: account.name,
|
|
577
877
|
})
|
|
578
878
|
throw error instanceof LlmError ? error : new LlmError(note, 'TRANSPORT', { cause: error })
|
|
579
879
|
}
|
|
880
|
+
if (code === 'AUTH' || code === QUOTA_EXCEEDED_CODE) {
|
|
881
|
+
refused.add(account.name)
|
|
882
|
+
this.config.record(options.model, {
|
|
883
|
+
provider: observed.provider,
|
|
884
|
+
canonical: observed.canonical,
|
|
885
|
+
attempts: trace.map((row) => row.upstream),
|
|
886
|
+
ms: Date.now() - startedAt,
|
|
887
|
+
ttfb: firstByteMs,
|
|
888
|
+
ttft: firstChunkMs,
|
|
889
|
+
stream: true,
|
|
890
|
+
error: note.slice(0, 200),
|
|
891
|
+
code,
|
|
892
|
+
account: account.name,
|
|
893
|
+
})
|
|
894
|
+
}
|
|
580
895
|
lastError = error instanceof LlmError ? error : new LlmError(note, 'TRANSPORT', { cause: error })
|
|
581
896
|
continue
|
|
582
897
|
} finally {
|
|
@@ -588,19 +903,47 @@ export class ClinePassAdapter extends LlmAdapter {
|
|
|
588
903
|
}
|
|
589
904
|
}
|
|
590
905
|
delivered = true
|
|
591
|
-
|
|
906
|
+
// Which channel answered is read back from the stream, because a 200
|
|
907
|
+
// only proves someone answered. The router drops an unusable `only`
|
|
908
|
+
// without complaining, so "we asked for baseten" and "baseten served it"
|
|
909
|
+
// are different facts and only the second one is evidence.
|
|
910
|
+
const adherence = pinAdherence(attempt, { finalProvider: observed.provider })
|
|
911
|
+
const warnings = pinWarnings(attempt, meta)
|
|
912
|
+
trace.push({
|
|
913
|
+
upstream: attempt.upstream ?? '(auto)',
|
|
914
|
+
status: 200,
|
|
915
|
+
ms: Date.now() - attemptStarted,
|
|
916
|
+
note: adherence === 'not-adopted' || adherence === 'violated'
|
|
917
|
+
? `pin not adopted: served by ${observed.provider ?? '(unknown)'}`
|
|
918
|
+
: 'ok',
|
|
919
|
+
})
|
|
592
920
|
this.config.record(options.model, {
|
|
593
921
|
provider: observed.provider ?? (attempt.upstream ?? null),
|
|
594
922
|
canonical: observed.canonical,
|
|
595
923
|
attempts: trace.map((row) => row.upstream),
|
|
596
924
|
ms: Date.now() - startedAt,
|
|
925
|
+
ttfb: firstByteMs,
|
|
926
|
+
ttft: firstChunkMs,
|
|
597
927
|
stream: true,
|
|
598
928
|
error: null,
|
|
599
929
|
account: account.name,
|
|
930
|
+
adherence,
|
|
931
|
+
warnings,
|
|
932
|
+
// What the request cost in tokens, and how much of that the model spent
|
|
933
|
+
// thinking invisibly — the two numbers that explain a large `ttft`.
|
|
934
|
+
usage,
|
|
935
|
+
effort: options.reasoningEffort ?? '',
|
|
600
936
|
})
|
|
601
|
-
|
|
937
|
+
// Only a pin the router actually honored proves the channel works.
|
|
938
|
+
// Marking it usable no matter who answered is how a false "available"
|
|
939
|
+
// reached the verdict map the exclusion allow-list is built from — the
|
|
940
|
+
// map then vouched for the very channels it was meant to exclude.
|
|
941
|
+
if (attempt.upstream !== null && adherence === 'adopted') this.config.learnUpstream(options.model, attempt.upstream, 'ok', '', Date.now() - attemptStarted)
|
|
602
942
|
return
|
|
603
943
|
} finally {
|
|
944
|
+
// Held until the attempt ends, response body included, so the slot
|
|
945
|
+
// bounds open streams rather than requests waiting for headers.
|
|
946
|
+
release()
|
|
604
947
|
watchdog[Symbol.dispose]()
|
|
605
948
|
}
|
|
606
949
|
}
|
|
@@ -610,6 +953,8 @@ export class ClinePassAdapter extends LlmAdapter {
|
|
|
610
953
|
provider: null,
|
|
611
954
|
attempts: trace.map((row) => row.upstream),
|
|
612
955
|
ms: Date.now() - startedAt,
|
|
956
|
+
ttfb: firstByteMs,
|
|
957
|
+
ttft: firstChunkMs,
|
|
613
958
|
stream: true,
|
|
614
959
|
error: message.slice(0, 200),
|
|
615
960
|
account: null,
|
package/lib/catalog.js
CHANGED
|
@@ -168,17 +168,25 @@ const SEAM_MODALITIES = new Set(['text', 'image'])
|
|
|
168
168
|
/**
|
|
169
169
|
* Resolve the metadata seam fields for one model.
|
|
170
170
|
*
|
|
171
|
-
* Precedence per field: the configured override, then the
|
|
172
|
-
* then the route-wide fallback.
|
|
171
|
+
* Precedence per field: the configured override, then the shipped catalog, then
|
|
172
|
+
* what the last official scan published, then the route-wide fallback.
|
|
173
173
|
*
|
|
174
|
+
* The live scan sits below the shipped table rather than above it because the
|
|
175
|
+
* shipped entry is curated: models.dev reports `audio`/`video`/`pdf`, and the
|
|
176
|
+
* underscore-free spelling of a display name, neither of which the seam or the
|
|
177
|
+
* provider's own label wants. It exists to cover what the table cannot know —
|
|
178
|
+
* a model published after this release — so a multimodal model is never adopted
|
|
179
|
+
* as text-only just because it is new.
|
|
180
|
+
*
|
|
181
|
+
* @param discovered - metadata for this model from the last catalog scan.
|
|
174
182
|
* @returns a descriptor the seam's `normalizeModelInfo` accepts.
|
|
175
183
|
*/
|
|
176
|
-
export function resolveModelMetadata(provider, model, override, fallback) {
|
|
177
|
-
const published = catalogEntry(model) ?? {}
|
|
184
|
+
export function resolveModelMetadata(provider, model, override, fallback, discovered = {}) {
|
|
185
|
+
const published = { ...(discovered ?? {}), ...(catalogEntry(model) ?? {}) }
|
|
178
186
|
const name = pick(override?.name, published.name, model)
|
|
179
187
|
const contextWindow = pick(override?.contextWindow, published.contextWindow, fallback.contextWindow)
|
|
180
188
|
const maxTokens = pick(override?.maxTokens, published.maxTokens, fallback.maxTokens)
|
|
181
|
-
const modality = pick(override?.input, published.input, undefined)
|
|
189
|
+
const modality = pick(override?.input, published.input ?? published.inputModalities, undefined)
|
|
182
190
|
const reasoning = override?.reasoning ?? published.reasoning ?? fallback.reasoning
|
|
183
191
|
const offered = (modality ?? ['text']).filter((entry) => SEAM_MODALITIES.has(entry))
|
|
184
192
|
return {
|