dsh-router-core 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/docs/suppliers.md +19 -3
- package/lib/client-registry.js +13 -8
- package/lib/client.js +13 -8
- package/lib/index.js +611 -215
- package/lib/types/llm/adapter.d.ts +7 -1
- package/lib/types/router/account-pool.d.ts +54 -19
- package/lib/types/router/index.d.ts +5 -0
- package/lib/types/router/usage-store.d.ts +20 -3
- package/lib/types/shared.d.ts +2 -0
- package/package.json +1 -1
- package/src/client/RouterView.tsx +1 -1
- package/src/client/SupplierDetail.tsx +8 -8
- package/src/llm/adapter.test.ts +404 -1
- package/src/llm/adapter.ts +242 -34
- package/src/router/account-pool.test.ts +163 -61
- package/src/router/account-pool.ts +200 -68
- package/src/router/index.ts +137 -38
- package/src/router/models-cache.test.ts +87 -4
- package/src/router/precise-dispatch.test.ts +110 -7
- package/src/router/usage-store.ts +174 -74
- package/src/router/usage.test.ts +115 -0
- package/src/shared.ts +2 -0
- package/src/suppliers/loader.ts +2 -1
- package/src/suppliers/registry.test.ts +23 -3
- package/src/suppliers/registry.ts +7 -2
package/src/router/index.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { dirname, join } from 'node:path'
|
|
|
8
8
|
import type { ServerResponse } from 'node:http'
|
|
9
9
|
import type { Supplier, ChatRequest, ModelInfo, ModelWithEnabled, SupplierStatus, Combo } from './types.ts'
|
|
10
10
|
import type { ChatOnceResult } from '../suppliers/contract.ts'
|
|
11
|
+
import type { AccountState } from '../suppliers/contract.ts'
|
|
11
12
|
import { AccountPool } from './account-pool.ts'
|
|
12
13
|
import { SupplierConfigStore } from '../supplier-config.ts'
|
|
13
14
|
import { UsageStore } from './usage-store.ts'
|
|
@@ -21,6 +22,8 @@ interface ChatTrace {
|
|
|
21
22
|
attempts: number
|
|
22
23
|
/** 最后一次失败原因。 */
|
|
23
24
|
lastError?: string
|
|
25
|
+
/** 最后一次失败的状态(组合据此判断要不要「喘口气」再降级)。 */
|
|
26
|
+
lastState?: AccountState
|
|
24
27
|
}
|
|
25
28
|
|
|
26
29
|
/** 模型列表缓存有效期。 */
|
|
@@ -38,6 +41,36 @@ function writeJson(res: ServerResponse, status: number, body: unknown): void {
|
|
|
38
41
|
res.end(JSON.stringify(body))
|
|
39
42
|
}
|
|
40
43
|
|
|
44
|
+
/**
|
|
45
|
+
* 剥掉本供应商自己的 alias 前缀(只剥一层,且只认完整 `alias/` 开头)。
|
|
46
|
+
*
|
|
47
|
+
* 为什么不能用 lastIndexOf:模型 id 本身可以含斜杠(nvidia 的
|
|
48
|
+
* `deepseek-ai/deepseek-v4-flash-0731`),剥最后一段会把命名空间吃掉,
|
|
49
|
+
* 后面请求必然 404。只认「已知 alias + /」开头,其他原样返回。
|
|
50
|
+
*
|
|
51
|
+
* 注意:核心**只在记统计名时**用它。传给插件的 model 全名保持原样——
|
|
52
|
+
* 剥前缀是插件自己的事(各供应商插件都有一份同名实现)。
|
|
53
|
+
*/
|
|
54
|
+
function stripAlias(model: string, alias: string): string {
|
|
55
|
+
return alias !== '' && model.startsWith(`${alias}/`) ? model.slice(alias.length + 1) : model
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* 组合降级前的「喘口气」等待:瞬时故障给上游一个恢复窗口再换下一个模型。
|
|
60
|
+
*
|
|
61
|
+
* 学 9router combo.js 的同名处理(注释原话:fixes: combo falls through on
|
|
62
|
+
* transient 503)—— 过载的上游往往几百毫秒内就恢复,立刻换下一个模型会
|
|
63
|
+
* 让整个组合在几毫秒内被打穿,最后全灭返回 503。
|
|
64
|
+
*
|
|
65
|
+
* 只对**瞬时**故障等:连接层失败 / 上游不可用(对应 9router 的 502/503/504)。
|
|
66
|
+
* 模型不属于本供应商(no_such_model)绝不能等 —— 它不是故障,等纯属浪费。
|
|
67
|
+
*/
|
|
68
|
+
const TRANSIENT_SETTLE_MS = 2_000
|
|
69
|
+
|
|
70
|
+
function isTransient(state: AccountState): boolean {
|
|
71
|
+
return state === 'transport' || state === 'unavailable'
|
|
72
|
+
}
|
|
73
|
+
|
|
41
74
|
/** 丢弃响应的假 ServerResponse(测试模型用:记录状态+内容,用于判定成败)。 */
|
|
42
75
|
function sinkRes(): ServerResponse & { status(): number; body(): string } {
|
|
43
76
|
let status = 200
|
|
@@ -163,7 +196,9 @@ async function aggregateSSE(stream: ReadableStream<Uint8Array>): Promise<string>
|
|
|
163
196
|
}
|
|
164
197
|
|
|
165
198
|
/**
|
|
166
|
-
* 流式 SSE 逐帧归一化:剥掉上游多发一次的**空** tool_call
|
|
199
|
+
* 流式 SSE 逐帧归一化:剥掉上游多发一次的**空** tool_call 字段,
|
|
200
|
+
* 并记住上游有没有发终止帧(`data: [DONE]`)—— 断流时核心要据此补发,
|
|
201
|
+
* 见 `writeChatResult`。
|
|
167
202
|
*
|
|
168
203
|
* 为什么必须做:CodeBuddy 上游的工具调用第二个 delta 会**显式发空串**
|
|
169
204
|
* `{"function":{"name":"","arguments":"{...}"},"index":0}`(以及 `"id":""`)。
|
|
@@ -178,11 +213,16 @@ async function aggregateSSE(stream: ReadableStream<Uint8Array>): Promise<string>
|
|
|
178
213
|
* 只动 tool_calls 的空 name/id,其余原样透传;逐帧流式处理,不缓冲整条流
|
|
179
214
|
* (首字节延迟不受影响)。解析失败一律原样透传——宁可不改,也不能改坏。
|
|
180
215
|
*/
|
|
181
|
-
function normalizeSSEStream(stream: ReadableStream<Uint8Array>):
|
|
216
|
+
function normalizeSSEStream(stream: ReadableStream<Uint8Array>): {
|
|
217
|
+
stream: ReadableStream<Uint8Array>
|
|
218
|
+
/** 上游(或最后一帧残留)里出现过终止帧?调用方在流结束后读。 */
|
|
219
|
+
sawDone: () => boolean
|
|
220
|
+
} {
|
|
182
221
|
const enc = new TextEncoder()
|
|
183
222
|
const dec = new TextDecoder()
|
|
184
223
|
let buf = ''
|
|
185
|
-
|
|
224
|
+
let sawDone = false
|
|
225
|
+
const out = new ReadableStream<Uint8Array>({
|
|
186
226
|
start: async (ctrl) => {
|
|
187
227
|
const reader = stream.getReader()
|
|
188
228
|
try {
|
|
@@ -195,16 +235,26 @@ function normalizeSSEStream(stream: ReadableStream<Uint8Array>): ReadableStream<
|
|
|
195
235
|
while ((sep = buf.indexOf('\n\n')) >= 0) {
|
|
196
236
|
const frame = buf.slice(0, sep + 2)
|
|
197
237
|
buf = buf.slice(sep + 2)
|
|
238
|
+
if (isDoneFrame(frame)) sawDone = true
|
|
198
239
|
ctrl.enqueue(enc.encode(fixFrame(frame)))
|
|
199
240
|
}
|
|
200
241
|
}
|
|
201
|
-
if (buf !== '')
|
|
242
|
+
if (buf !== '') {
|
|
243
|
+
if (isDoneFrame(buf)) sawDone = true
|
|
244
|
+
ctrl.enqueue(enc.encode(fixFrame(buf)))
|
|
245
|
+
}
|
|
202
246
|
} finally {
|
|
203
247
|
reader.releaseLock()
|
|
204
248
|
}
|
|
205
249
|
ctrl.close()
|
|
206
250
|
},
|
|
207
251
|
})
|
|
252
|
+
return { stream: out, sawDone: () => sawDone }
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** 一帧 SSE 里有没有终止帧(宽松匹配:跨 chunk 的残留尾帧也认)。 */
|
|
256
|
+
function isDoneFrame(frame: string): boolean {
|
|
257
|
+
return /(^|\n)data:[ \t]*\[DONE\][ \t]*(\n|$)/.test(frame)
|
|
208
258
|
}
|
|
209
259
|
|
|
210
260
|
/** 修一帧 SSE:只剥空的 name/id,出任何问题都原样返回。 */
|
|
@@ -246,6 +296,9 @@ function fixFrame(frame: string): string {
|
|
|
246
296
|
* 先写头就等于把自己锁死在这一个账号上,组合就没法回退了——而客户端只拿到
|
|
247
297
|
* 一个空/截断的 SSE,表现为工具调用名是空串(`unknown tool ""`)。
|
|
248
298
|
*
|
|
299
|
+
* 截断时**补发终止帧**(见末尾 finally):客户端严格等 `[DONE]`,缺了就
|
|
300
|
+
* 整轮判失败,已流式吐出的内容全废。
|
|
301
|
+
*
|
|
249
302
|
* 天花板:一旦写出第一个字节就绑死(HTTP 语义),中途断流只能截断,
|
|
250
303
|
* 不能回退。要彻底解决得在写头前缓冲判定,代价是首字节延迟——未做。
|
|
251
304
|
*/
|
|
@@ -276,8 +329,11 @@ async function writeChatResult(res: ServerResponse, r: ChatOnceResult, wantsStre
|
|
|
276
329
|
// 流式:边透传边统计(统计在 tee 之外的旁路做,首字节延迟不受影响)
|
|
277
330
|
const tapped = probe === undefined ? undefined : tapStreamUsage(r.stream, startedAt)
|
|
278
331
|
let wroteAny = false
|
|
332
|
+
/** 响应已死(客户端断开/res 已销毁):等 drain 的写入要立刻放弃,别吊死。 */
|
|
333
|
+
let dead = false
|
|
279
334
|
const writeChunk = (chunk: Uint8Array): Promise<void> =>
|
|
280
335
|
new Promise<void>((resolve) => {
|
|
336
|
+
if (dead) { resolve(); return }
|
|
281
337
|
if (!wroteAny) {
|
|
282
338
|
// 第一个字节到手,此刻才提交响应头
|
|
283
339
|
res.writeHead(200, {
|
|
@@ -292,20 +348,37 @@ async function writeChatResult(res: ServerResponse, r: ChatOnceResult, wantsStre
|
|
|
292
348
|
wroteAny = true
|
|
293
349
|
}
|
|
294
350
|
if (res.write(chunk)) resolve()
|
|
295
|
-
else
|
|
351
|
+
else {
|
|
352
|
+
// 背压:等 drain。但连接可能在这期间断掉 —— 那时 'drain' 永不到来,
|
|
353
|
+
// 得靠 dead 标志(onError 里置位)兜住,否则这次请求会永远挂着。
|
|
354
|
+
const onDrain = (): void => { res.removeListener('error', onDrain); resolve() }
|
|
355
|
+
res.once('drain', onDrain)
|
|
356
|
+
}
|
|
296
357
|
})
|
|
297
358
|
|
|
298
359
|
// 流中途出错(客户端断开/上游断流)也要结束响应,否则连接悬挂
|
|
299
360
|
const onError = (): void => {
|
|
361
|
+
dead = true
|
|
300
362
|
res.destroy()
|
|
301
363
|
}
|
|
302
364
|
res.once('error', onError)
|
|
365
|
+
const norm = normalizeSSEStream(tapped?.stream ?? r.stream)
|
|
303
366
|
try {
|
|
304
|
-
await
|
|
367
|
+
await norm.stream.pipeTo(new WritableStream<Uint8Array>({ write: writeChunk }))
|
|
305
368
|
} catch {
|
|
306
369
|
// 流断了:一个字节都没写的话,调用方还能换号重试——交给上层决定
|
|
307
370
|
} finally {
|
|
308
371
|
res.removeListener('error', onError)
|
|
372
|
+
// 断流也要给客户端一个终止帧。客户端(dsh-llm adapter)严格等
|
|
373
|
+
// `[DONE]`,缺了就整轮判失败(`SSE payload stream ended without
|
|
374
|
+
// [DONE]`),已经流式吐出去的内容全部作废。上游自己发了就不重复发
|
|
375
|
+
// (多发一个 [DONE] 同样坑客户端)。
|
|
376
|
+
//
|
|
377
|
+
// 只在**已提交响应**时补:一个字节都没写 = 还能换号重试,补了就等于
|
|
378
|
+
// 把这次失败坐实成一个空响应,组合回退就没了。
|
|
379
|
+
if (wroteAny && !norm.sawDone() && !dead && !res.writableEnded && !res.destroyed) {
|
|
380
|
+
await writeChunk(DONE_FRAME)
|
|
381
|
+
}
|
|
309
382
|
if (wroteAny) res.end()
|
|
310
383
|
// 流走完才出得了 usage(上游多在最后一帧才发 usage)
|
|
311
384
|
if (probe !== undefined && tapped !== undefined) {
|
|
@@ -318,6 +391,9 @@ async function writeChatResult(res: ServerResponse, r: ChatOnceResult, wantsStre
|
|
|
318
391
|
return wroteAny
|
|
319
392
|
}
|
|
320
393
|
|
|
394
|
+
/** SSE 终止帧(断流兜底补发用)。 */
|
|
395
|
+
const DONE_FRAME = new TextEncoder().encode('data: [DONE]\n\n')
|
|
396
|
+
|
|
321
397
|
/** 从上游响应体提取错误信息(OpenAI {error:{message}} 或 {code,msg}/{message})。 */
|
|
322
398
|
function extractUpstreamError(body: string): string {
|
|
323
399
|
try {
|
|
@@ -696,18 +772,26 @@ export class Router {
|
|
|
696
772
|
for (let i = 0; i < combo.models.length; i++) {
|
|
697
773
|
const model = combo.models[(start + i) % combo.models.length]
|
|
698
774
|
if (model === undefined) continue
|
|
699
|
-
const
|
|
775
|
+
const trace: ChatTrace = { attempts: 0 }
|
|
776
|
+
const served = await this.chatWithModel(req, res, model, probe, trace)
|
|
700
777
|
if (served) {
|
|
701
778
|
settle(true)
|
|
702
779
|
return
|
|
703
780
|
}
|
|
781
|
+
// 瞬时故障:先给上游一个恢复窗口再换下一个模型。否则过载的上游
|
|
782
|
+
// 会被整个组合在几毫秒内依次打穿 —— 最后一次失败就是全灭 503。
|
|
783
|
+
if (trace.lastState !== undefined && isTransient(trace.lastState) && i < combo.models.length - 1) {
|
|
784
|
+
await new Promise((r) => setTimeout(r, TRANSIENT_SETTLE_MS))
|
|
785
|
+
}
|
|
704
786
|
}
|
|
705
787
|
settle(false, 'combo: all models unavailable')
|
|
706
788
|
writeJson(res, 503, openAIError('no_healthy_supplier', `combo ${JSON.stringify(req.model)}: all models unavailable`))
|
|
707
789
|
return
|
|
708
790
|
}
|
|
709
|
-
// 直接调用:模型全名 = alias/model
|
|
710
|
-
//
|
|
791
|
+
// 直接调用:模型全名 = alias/model。用别名反查供应商,精准调用。
|
|
792
|
+
// 查不到别名(无 `alias/` 前缀)就走 chatWithModel,那里会要求
|
|
793
|
+
// `supplierId,modelId` 形态;两者都不满足 = 缺供应商前缀 → 直接 503,
|
|
794
|
+
// 不再遍历所有供应商兜底(严格图6拓扑)。
|
|
711
795
|
const slash = req.model.indexOf('/')
|
|
712
796
|
const alias = slash > 0 ? req.model.slice(0, slash) : ''
|
|
713
797
|
const target = alias === '' ? undefined : this.supplierByAlias(alias)
|
|
@@ -744,7 +828,12 @@ export class Router {
|
|
|
744
828
|
* 不再挨个问供应商「这是不是你的模型」——精准调用,也避免同名模型串台。
|
|
745
829
|
* 兼容旧的裸 `modelId`(没有逗号):降级为遍历,行为同以前。
|
|
746
830
|
*/
|
|
747
|
-
|
|
831
|
+
/**
|
|
832
|
+
* @param outTrace 可选出参:把这次调用的追踪信息带回去(组合据此判断
|
|
833
|
+
* 要不要「喘口气」再降级)。裸 id 遍历路径不填(多供应商混在一起,
|
|
834
|
+
* 失败状态没有单一归属)。
|
|
835
|
+
*/
|
|
836
|
+
private async chatWithModel(req: ChatRequest, res: ServerResponse, model: string, probe?: UsageProbe, outTrace?: ChatTrace): Promise<boolean> {
|
|
748
837
|
const comma = model.indexOf(',')
|
|
749
838
|
const supplierId = comma > 0 ? model.slice(0, comma) : undefined
|
|
750
839
|
// 组合存的是供应商 id,但插件认的是自己的模型 id
|
|
@@ -759,29 +848,32 @@ export class Router {
|
|
|
759
848
|
clone.rawBody = req.rawBody
|
|
760
849
|
}
|
|
761
850
|
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
}
|
|
770
|
-
const t0 = Date.now()
|
|
771
|
-
const trace: ChatTrace = { attempts: 0 }
|
|
772
|
-
const served = await this.chatWithSupplier(s, clone, res, trace, probe)
|
|
773
|
-
this.logChat(req.model, served
|
|
774
|
-
? `${s.id}/${modelId} (${trace.uid ?? '?'}) ok${trace.attempts > 0 ? ` 重试${trace.attempts}次` : ''}`
|
|
775
|
-
: `${s.id}/${modelId} 失败 ${trace.lastError ?? 'no account'}`, Date.now() - t0)
|
|
776
|
-
return served
|
|
851
|
+
// 图6 严格拓扑:组合/路由里的每个模型都必须带 `supplierId,modelId` 前缀。
|
|
852
|
+
// 无逗号 = 配置缺供应商前缀(旧版裸 id 遗留),不再遍历所有供应商兜底——
|
|
853
|
+
// 遍历会让拼错的/缺前缀的模型静默落到某个供应商上,更难查。直接判失败。
|
|
854
|
+
if (supplierId === undefined) {
|
|
855
|
+
const msg = `model ${JSON.stringify(model)} 缺供应商前缀(应为 supplierId,modelId)`
|
|
856
|
+
this.logChat(req.model, msg, 0)
|
|
857
|
+
return false
|
|
777
858
|
}
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
859
|
+
// 精准:只调这一个供应商。查不到就是配置错了,直接失败(不遍历兜底)
|
|
860
|
+
const s = this.suppliers.find((x) => x.id === supplierId)
|
|
861
|
+
if (s === undefined) {
|
|
862
|
+
this.logChat(req.model, `supplier ${JSON.stringify(supplierId)} 不存在`, 0)
|
|
863
|
+
return false
|
|
864
|
+
}
|
|
865
|
+
const t0 = Date.now()
|
|
866
|
+
const trace: ChatTrace = { attempts: 0 }
|
|
867
|
+
const served = await this.chatWithSupplier(s, clone, res, trace, probe)
|
|
868
|
+
if (outTrace !== undefined) {
|
|
869
|
+
outTrace.attempts = trace.attempts
|
|
870
|
+
outTrace.lastError = trace.lastError
|
|
871
|
+
outTrace.lastState = trace.lastState
|
|
783
872
|
}
|
|
784
|
-
|
|
873
|
+
this.logChat(req.model, served
|
|
874
|
+
? `${s.id}/${modelId} (${trace.uid ?? '?'}) ok${trace.attempts > 0 ? ` 重试${trace.attempts}次` : ''}`
|
|
875
|
+
: `${s.id}/${modelId} 失败 ${trace.lastError ?? 'no account'}`, Date.now() - t0)
|
|
876
|
+
return served
|
|
785
877
|
}
|
|
786
878
|
|
|
787
879
|
/** 按别名找供应商(对外模型全名 = alias/model)。别名唯一,故最多命中一个。 */
|
|
@@ -806,7 +898,8 @@ export class Router {
|
|
|
806
898
|
const r = await s.chatOnce('', req)
|
|
807
899
|
if (!r.ok) {
|
|
808
900
|
if (r.state === 'no_such_model') return false // 不是我的模型:换供应商,不记账
|
|
809
|
-
|
|
901
|
+
if (trace !== undefined) trace.lastState = r.state
|
|
902
|
+
pool.noteFailure('', req.model, r.state, r.message)
|
|
810
903
|
return false
|
|
811
904
|
}
|
|
812
905
|
if (trace !== undefined) trace.uid = '(no-account)'
|
|
@@ -814,7 +907,10 @@ export class Router {
|
|
|
814
907
|
probe.supplier = s.id
|
|
815
908
|
// 记对外全名 alias/model:与直接调用路径一致。组合里存的是裸
|
|
816
909
|
// modelId,直接记会把同一个模型在 Top 榜上分裂成两行。
|
|
817
|
-
|
|
910
|
+
// 直接调 `alias/model` 时 req.model 已带前缀,先剥掉再拼——
|
|
911
|
+
// 否则会套成 `alias/alias/model`(插件会再剥一次,功能是好的,
|
|
912
|
+
// 只是统计名多一层,Top 榜照样分裂)。
|
|
913
|
+
probe.model = `${s.getAlias()}/${stripAlias(req.model, s.getAlias())}`
|
|
818
914
|
probe.uid = ''
|
|
819
915
|
}
|
|
820
916
|
return await writeChatResult(res, r, req.stream, probe, probe?.startedAt ?? 0)
|
|
@@ -823,7 +919,7 @@ export class Router {
|
|
|
823
919
|
// 不排除试过的就会原地打转——死循环等于整个服务挂住。
|
|
824
920
|
const tried = new Set<string>()
|
|
825
921
|
for (;;) {
|
|
826
|
-
const uid = pool.pick(s.accounts().filter((a) => !tried.has(a.uid)), cfg.poolOrder, cfg.poolStrategy)
|
|
922
|
+
const uid = pool.pick(s.accounts().filter((a) => !tried.has(a.uid)), cfg.poolOrder, cfg.poolStrategy, req.model)
|
|
827
923
|
if (uid === undefined) return false
|
|
828
924
|
tried.add(uid)
|
|
829
925
|
const r = await s.chatOnce(uid, req)
|
|
@@ -834,28 +930,31 @@ export class Router {
|
|
|
834
930
|
if (trace !== undefined) trace.lastError = 'no_such_model'
|
|
835
931
|
return false
|
|
836
932
|
}
|
|
837
|
-
pool.noteFailure(uid, r.state, r.message)
|
|
933
|
+
pool.noteFailure(uid, req.model, r.state, r.message)
|
|
838
934
|
if (trace !== undefined) {
|
|
839
935
|
trace.attempts += 1
|
|
936
|
+
trace.lastState = r.state
|
|
840
937
|
trace.lastError = `${r.state}: ${r.message}`
|
|
841
938
|
}
|
|
842
939
|
continue
|
|
843
940
|
}
|
|
844
|
-
pool.noteSuccess(uid)
|
|
941
|
+
pool.noteSuccess(uid, req.model)
|
|
845
942
|
if (trace !== undefined) trace.uid = uid
|
|
846
943
|
if (probe !== undefined) {
|
|
847
944
|
probe.supplier = s.id
|
|
848
945
|
// 记对外全名 alias/model:与直接调用路径一致。组合里存的是裸
|
|
849
946
|
// modelId,直接记会把同一个模型在 Top 榜上分裂成两行。
|
|
850
|
-
|
|
947
|
+
// 剥一次自己的前缀:直接调 `alias/model` 时 req.model 已带前缀。
|
|
948
|
+
probe.model = `${s.getAlias()}/${stripAlias(req.model, s.getAlias())}`
|
|
851
949
|
probe.uid = uid
|
|
852
950
|
}
|
|
853
951
|
const committed = await writeChatResult(res, r, req.stream, probe, probe?.startedAt ?? 0)
|
|
854
952
|
// 一个字节都没写(上游刚连上就断)→ 这个号不算数,换下一个重试
|
|
855
953
|
if (committed) return true
|
|
856
|
-
pool.noteFailure(uid, 'transport', 'stream failed before first byte')
|
|
954
|
+
pool.noteFailure(uid, req.model, 'transport', 'stream failed before first byte')
|
|
857
955
|
if (trace !== undefined) {
|
|
858
956
|
trace.attempts += 1
|
|
957
|
+
trace.lastState = 'transport'
|
|
859
958
|
trace.lastError = 'stream failed before first byte'
|
|
860
959
|
}
|
|
861
960
|
}
|
|
@@ -311,7 +311,15 @@ test('流在写出第一个字节前就断 → 组合回退到下一个模型',
|
|
|
311
311
|
assert.equal(heads, 1, '响应头只应提交一次')
|
|
312
312
|
})
|
|
313
313
|
|
|
314
|
-
|
|
314
|
+
/**
|
|
315
|
+
* 已写出字节后断流:诚实截断,不回退也不二次写头 —— 但要**补一个终止帧**。
|
|
316
|
+
*
|
|
317
|
+
* 客户端(dsh-llm adapter)严格等 `[DONE]` 才认为流正常结束;缺了就抛
|
|
318
|
+
* `SSE payload stream ended without [DONE]`,整轮判失败,已经吐出去的内容
|
|
319
|
+
* 全部作废。所以核心补终止帧:断流降级为「拿到截断内容并正常收尾」,
|
|
320
|
+
* 而不是「整轮白干」。
|
|
321
|
+
*/
|
|
322
|
+
test('已写出字节后断流 → 截断并补 [DONE],不回退也不二次写头', async () => {
|
|
315
323
|
const router = new Router('')
|
|
316
324
|
const half = supplier('half', [{ id: 'half-m' }])
|
|
317
325
|
half.s.chatOnce = async (): Promise<ChatOnceResult> => ({
|
|
@@ -351,10 +359,81 @@ test('已写出字节后断流 → 诚实截断,不回退也不二次写头',
|
|
|
351
359
|
await router.chatCompletions({ model: 'c2', stream: true, rawBody: JSON.stringify({ model: 'c2', messages: [] }) }, res)
|
|
352
360
|
|
|
353
361
|
assert.ok(out.includes('半截'), '已投递的字节要送达客户端')
|
|
354
|
-
assert.ok(
|
|
362
|
+
assert.ok(out.includes('data: [DONE]'), '断流要补终止帧,否则客户端整轮判失败')
|
|
363
|
+
assert.equal(out.match(/\[DONE\]/g)?.length ?? 0, 1, '终止帧只能有一个(重复会坑客户端)')
|
|
364
|
+
assert.ok(!out.includes('兜底'), '已提交就不该再接第二个模型的内容(会串流)')
|
|
355
365
|
assert.equal(heads, 1, '响应头只能提交一次')
|
|
356
366
|
})
|
|
357
367
|
|
|
368
|
+
/** 上游自己发了终止帧就别再补一个(重复 [DONE] 同样坑客户端)。 */
|
|
369
|
+
test('上游已发 [DONE] → 不重复补', async () => {
|
|
370
|
+
const router = new Router('')
|
|
371
|
+
const s = supplier('sup', [{ id: 'm' }])
|
|
372
|
+
s.s.chatOnce = async (): Promise<ChatOnceResult> => ({
|
|
373
|
+
ok: true,
|
|
374
|
+
stream: new ReadableStream<Uint8Array>({
|
|
375
|
+
start(c) {
|
|
376
|
+
c.enqueue(new TextEncoder().encode('data: {"choices":[{"delta":{"content":"完整"}}]}\n\ndata: [DONE]\n\n'))
|
|
377
|
+
c.close()
|
|
378
|
+
},
|
|
379
|
+
}),
|
|
380
|
+
})
|
|
381
|
+
addSupplier(router, s.s)
|
|
382
|
+
|
|
383
|
+
let out = ''
|
|
384
|
+
const res = {
|
|
385
|
+
writeHead: (): void => {},
|
|
386
|
+
write: (b: Uint8Array | string): boolean => { out += Buffer.from(b).toString(); return true },
|
|
387
|
+
end: (): void => {},
|
|
388
|
+
once: (): void => {},
|
|
389
|
+
removeListener: (): void => {},
|
|
390
|
+
destroy: (): void => {},
|
|
391
|
+
} as unknown as ServerResponse
|
|
392
|
+
await router.chatCompletions({ model: 'sup/m', stream: true, rawBody: JSON.stringify({ model: 'sup/m', messages: [] }) }, res)
|
|
393
|
+
|
|
394
|
+
assert.equal(out.match(/\[DONE\]/g)?.length ?? 0, 1, `终止帧不能重复,实际: ${JSON.stringify(out)}`)
|
|
395
|
+
})
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* 一个字节都没写出就断 → 绝不能补 [DONE]。
|
|
399
|
+
* 补了就等于把这次失败坐实成一个空响应,组合回退(换号/换模型)就废了。
|
|
400
|
+
*/
|
|
401
|
+
test('首字节前就断 → 不补 [DONE],留给组合回退', async () => {
|
|
402
|
+
const router = new Router('')
|
|
403
|
+
const dead = supplier('dead', [{ id: 'dead-m' }])
|
|
404
|
+
dead.s.chatOnce = async (): Promise<ChatOnceResult> => ({
|
|
405
|
+
ok: true,
|
|
406
|
+
stream: new ReadableStream<Uint8Array>({ start(c) { c.error(new Error('上游刚连上就断')) } }),
|
|
407
|
+
})
|
|
408
|
+
const live = supplier('live', [{ id: 'live-m' }])
|
|
409
|
+
live.s.chatOnce = async (): Promise<ChatOnceResult> => ({
|
|
410
|
+
ok: true,
|
|
411
|
+
stream: new ReadableStream<Uint8Array>({
|
|
412
|
+
start(c) {
|
|
413
|
+
c.enqueue(new TextEncoder().encode('data: {"choices":[{"delta":{"content":"兜底"}}]}\n\ndata: [DONE]\n\n'))
|
|
414
|
+
c.close()
|
|
415
|
+
},
|
|
416
|
+
}),
|
|
417
|
+
})
|
|
418
|
+
addSupplier(router, dead.s)
|
|
419
|
+
addSupplier(router, live.s)
|
|
420
|
+
assert.equal(router.createCombo('c3', 'fallback', ['dead,dead-m', 'live,live-m']).ok, true)
|
|
421
|
+
|
|
422
|
+
let out = ''
|
|
423
|
+
const res = {
|
|
424
|
+
writeHead: (): void => {},
|
|
425
|
+
write: (b: Uint8Array | string): boolean => { out += Buffer.from(b).toString(); return true },
|
|
426
|
+
end: (): void => {},
|
|
427
|
+
once: (): void => {},
|
|
428
|
+
removeListener: (): void => {},
|
|
429
|
+
destroy: (): void => {},
|
|
430
|
+
} as unknown as ServerResponse
|
|
431
|
+
await router.chatCompletions({ model: 'c3', stream: true, rawBody: JSON.stringify({ model: 'c3', messages: [] }) }, res)
|
|
432
|
+
|
|
433
|
+
assert.equal(out.match(/\[DONE\]/g)?.length ?? 0, 1, '只有兜底那个流该有终止帧')
|
|
434
|
+
assert.ok(out.includes('兜底'), '必须回退到下一个模型')
|
|
435
|
+
})
|
|
436
|
+
|
|
358
437
|
/**
|
|
359
438
|
* 对照组:插件若用 unavailable 表示「不是我的模型」就会污染账号——
|
|
360
439
|
* 这正是当初把 no_such_model 加进契约的原因(unavailable 在 RULES 里计数)。
|
|
@@ -368,7 +447,9 @@ test('对照:用 unavailable 表示「不是我的模型」会污染账号', a
|
|
|
368
447
|
({ ok: false, state: 'unavailable', message: 'not my model' })
|
|
369
448
|
addSupplier(router, other.s)
|
|
370
449
|
|
|
371
|
-
|
|
450
|
+
// 严格图6拓扑:路由必须带供应商前缀/别名,裸 model 不再遍历。
|
|
451
|
+
// 这里用 alias/model 精准命中 other(别名默认 = 供应商 id)。
|
|
452
|
+
const req = { model: 'other/other-m', stream: false, rawBody: JSON.stringify({ model: 'other/other-m', messages: [] }) }
|
|
372
453
|
for (let i = 0; i < 5; i++) await router.chatCompletions(req, fakeRes())
|
|
373
454
|
|
|
374
455
|
const acc = other.s.pool.decorate([{ uid: 'u1', credits: 0, state: 'ok' }])[0]
|
|
@@ -387,7 +468,9 @@ test('组合请求不污染无关供应商的账号(no_such_model 不记账)
|
|
|
387
468
|
addSupplier(router, other.s)
|
|
388
469
|
addSupplier(router, target.s)
|
|
389
470
|
|
|
390
|
-
|
|
471
|
+
// 严格图6拓扑:用 alias/model 精准命中 target(别名默认 = 供应商 id),
|
|
472
|
+
// other 完全不该被问到——这正是这条回归要守的。
|
|
473
|
+
const req = { model: 'target/want-m', stream: false, rawBody: JSON.stringify({ model: 'target/want-m', messages: [] }) }
|
|
391
474
|
const res = fakeRes()
|
|
392
475
|
// 连打 5 次(远超错误阈值 3)
|
|
393
476
|
for (let i = 0; i < 5; i++) await router.chatCompletions(req, res)
|
|
@@ -49,8 +49,11 @@ function spy(id: string, modelId: string, alias = id): Spy {
|
|
|
49
49
|
chatOnce: async (_uid, req) => {
|
|
50
50
|
calls.n += 1
|
|
51
51
|
const m = (JSON.parse(req.rawBody) as { model?: string }).model ?? ''
|
|
52
|
-
seen.push(m)
|
|
53
|
-
|
|
52
|
+
seen.push(m) // 记核心传来的**原样**全名,剥前缀是插件内部的事
|
|
53
|
+
// 真实插件会先剥掉自己的 alias 再认模型(各 suppliers/*/plugin.ts
|
|
54
|
+
// 都有一份 stripAlias);这里照做,否则直接调用路径永远匹配不上。
|
|
55
|
+
const base = m.startsWith(`${alias}/`) ? m.slice(alias.length + 1) : m
|
|
56
|
+
if (base !== modelId) return { ok: false, state: 'no_such_model', message: `unknown model ${m}` }
|
|
54
57
|
return { ok: true, status: 200, body: '{"ok":1}' }
|
|
55
58
|
},
|
|
56
59
|
dispose: () => {},
|
|
@@ -131,19 +134,22 @@ test('模型 id 含斜杠时不被吃掉(如 deepseek-ai/xxx)', async () =>
|
|
|
131
134
|
assert.deepEqual(a.seen, ['aaa/deepseek-ai/deepseek-v3'])
|
|
132
135
|
})
|
|
133
136
|
|
|
134
|
-
test('
|
|
137
|
+
test('组合存了缺供应商前缀的裸模型 id → 直接失败,不降级遍历(严格图6拓扑)', async () => {
|
|
135
138
|
const router = new Router('')
|
|
136
139
|
const a = spy('supA', 'a-model')
|
|
137
140
|
const b = spy('supB', 'b-model')
|
|
138
141
|
add(router, a.s); add(router, b.s)
|
|
139
|
-
// 模拟旧存储:裸 id,没有 supplierId 前缀
|
|
142
|
+
// 模拟旧存储:裸 id,没有 supplierId 前缀 —— 图6 下这是配置错误,不遍历兜底
|
|
140
143
|
;(router as unknown as { customCombos: Array<{ id: string; name: string; strategy: 'fallback'; models: string[] }> })
|
|
141
144
|
.customCombos.push({ id: 'legacy', name: 'legacy', strategy: 'fallback', models: ['b-model'] })
|
|
142
145
|
|
|
143
|
-
|
|
146
|
+
const errs: unknown[] = []
|
|
147
|
+
const res = { ...fakeRes(), end: (x?: string): unknown => { if (x) errs.push(x); return undefined } } as unknown as ServerResponse
|
|
148
|
+
await router.chatCompletions(reqWith('legacy'), res)
|
|
144
149
|
|
|
145
|
-
assert.equal(a.calls,
|
|
146
|
-
assert.equal(b.calls,
|
|
150
|
+
assert.equal(a.calls, 0, '不降级遍历,不该问 supA')
|
|
151
|
+
assert.equal(b.calls, 0, '裸 id 无前缀,不该被 supB 认领')
|
|
152
|
+
assert.equal(errs.length, 1, '缺前缀的模型应判失败(组合 503)')
|
|
147
153
|
})
|
|
148
154
|
|
|
149
155
|
test('组合存了不存在的供应商 id → 调用失败,不静默落到别的供应商', async () => {
|
|
@@ -196,3 +202,100 @@ test('空别名 → 生效别名回落到供应商 id', () => {
|
|
|
196
202
|
store.setAlias('supA', 'custom')
|
|
197
203
|
assert.equal(store.effectiveAliases().get('supA'), 'custom')
|
|
198
204
|
})
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* 为什么要有这一组:用量统计记的 model 名曾把 alias 前缀套了两层
|
|
208
|
+
* (`tx/tx/hy4-preview`、`trae/trae/DeepSeek-V4-Flash-Official`)。
|
|
209
|
+
*
|
|
210
|
+
* 成因:直接调 `alias/model` 时核心**原样**把全名传给插件(剥前缀是插件
|
|
211
|
+
* 的活,见上面那条断言),但记 `probe.model` 时又拼了一次 `${alias}/`
|
|
212
|
+
* ——于是统计里出现双前缀。功能不受影响(插件拿到的名字是对的),但
|
|
213
|
+
* Top 榜会把同一个模型**分裂成两行**(`tx/hy4-preview` 和
|
|
214
|
+
* `tx/tx/hy4-preview` 各占一行),看板数据失真。
|
|
215
|
+
*/
|
|
216
|
+
test('用量统计:直接调 alias/model 时 model 名不重复套前缀', async () => {
|
|
217
|
+
const router = new Router('')
|
|
218
|
+
const b = spy('supB', 'b-model', 'bbb')
|
|
219
|
+
add(router, b.s)
|
|
220
|
+
|
|
221
|
+
await router.chatCompletions(reqWith('bbb/b-model'), fakeRes())
|
|
222
|
+
|
|
223
|
+
const rec = router.usage.recentList(1)[0]
|
|
224
|
+
assert.equal(rec?.model, 'bbb/b-model', '统计名必须是 alias/model,不能是 alias/alias/model')
|
|
225
|
+
})
|
|
226
|
+
|
|
227
|
+
test('直接调裸模型 id(无前缀)→ 判失败,不再遍历兜底(严格图6拓扑)', async () => {
|
|
228
|
+
const router = new Router('')
|
|
229
|
+
const b = spy('supB', 'b-model', 'bbb')
|
|
230
|
+
add(router, b.s)
|
|
231
|
+
|
|
232
|
+
const errs: unknown[] = []
|
|
233
|
+
const res = { ...fakeRes(), end: (x?: string): unknown => { if (x) errs.push(x); return undefined } } as unknown as ServerResponse
|
|
234
|
+
await router.chatCompletions(reqWith('b-model'), res)
|
|
235
|
+
|
|
236
|
+
assert.equal(b.calls, 0, '裸 id 不该遍历到 supB')
|
|
237
|
+
assert.equal(errs.length, 1, '无前缀的裸模型应判失败(503)')
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* 为什么要有这一组:学 9router combo.js 的「瞬时故障降级前先喘口气」
|
|
242
|
+
* (其注释原话:fixes: combo falls through on transient 503)。
|
|
243
|
+
*
|
|
244
|
+
* 过载的上游往往几百毫秒内就恢复。立刻换下一个模型,会让整个组合在几
|
|
245
|
+
* 毫秒内被依次打穿 —— 最后一个模型也失败时,用户看到的就是全灭 503。
|
|
246
|
+
* 等待只针对**瞬时**故障:模型不属于本供应商(no_such_model)不是故障,
|
|
247
|
+
* 等它纯属浪费时间。
|
|
248
|
+
*/
|
|
249
|
+
|
|
250
|
+
/** 造一个总是以指定状态失败的供应商(可记调用时刻,用于验证等待)。 */
|
|
251
|
+
function failingSpy(id: string, modelId: string, state: 'transport' | 'unavailable' | 'no_such_model') {
|
|
252
|
+
const at: number[] = []
|
|
253
|
+
return {
|
|
254
|
+
calls: 0,
|
|
255
|
+
at,
|
|
256
|
+
s: {
|
|
257
|
+
id, name: id, priority: 0, pool: new AccountPool(),
|
|
258
|
+
status: () => ({ id, name: id, accounts: [{ uid: `${id}-u1`, credits: 0, state: 'ok' }] }),
|
|
259
|
+
listModels: async () => [{ id: modelId }],
|
|
260
|
+
modelsWithEnabled: async () => [{ id: modelId, enabled: true }],
|
|
261
|
+
getAlias: () => id,
|
|
262
|
+
accounts: () => [{ uid: `${id}-u1`, credits: 0, state: 'ok' }],
|
|
263
|
+
chatOnce: async (_uid: string, _req: { rawBody: string }) => {
|
|
264
|
+
at.push(Date.now())
|
|
265
|
+
return { ok: false, state, message: `${state} boom` } as ChatOnceResult
|
|
266
|
+
},
|
|
267
|
+
dispose: () => {},
|
|
268
|
+
},
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
test('组合:瞬时故障降级前会等一等,不是毫秒级打穿整个组合', async () => {
|
|
273
|
+
const router = new Router('')
|
|
274
|
+
const m1 = failingSpy('s1', 'm1', 'transport')
|
|
275
|
+
const m2 = failingSpy('s2', 'm2', 'unavailable')
|
|
276
|
+
add(router, m1.s); add(router, m2.s)
|
|
277
|
+
router.createCombo('combo', 'fallback', ['s1,m1', 's2,m2'])
|
|
278
|
+
|
|
279
|
+
const t0 = Date.now()
|
|
280
|
+
await router.chatCompletions(reqWith('combo'), fakeRes())
|
|
281
|
+
const elapsed = Date.now() - t0
|
|
282
|
+
|
|
283
|
+
// 两个都失败 → 全灭;但 s1 瞬时失败后应等一个窗口再试 s2
|
|
284
|
+
assert.equal(m1.at.length, 1)
|
|
285
|
+
assert.equal(m2.at.length, 1)
|
|
286
|
+
assert.ok(elapsed >= 1000, `瞬时故障降级前应等待(实际 ${elapsed}ms),否则组合会被瞬间打穿`)
|
|
287
|
+
})
|
|
288
|
+
|
|
289
|
+
test('组合:no_such_model 不是故障,降级不该等', async () => {
|
|
290
|
+
const router = new Router('')
|
|
291
|
+
const m1 = failingSpy('s1', 'm1', 'no_such_model')
|
|
292
|
+
const m2 = failingSpy('s2', 'm2', 'no_such_model')
|
|
293
|
+
add(router, m1.s); add(router, m2.s)
|
|
294
|
+
router.createCombo('combo', 'fallback', ['s1,m1', 's2,m2'])
|
|
295
|
+
|
|
296
|
+
const t0 = Date.now()
|
|
297
|
+
await router.chatCompletions(reqWith('combo'), fakeRes())
|
|
298
|
+
const elapsed = Date.now() - t0
|
|
299
|
+
|
|
300
|
+
assert.ok(elapsed < 1000, `模型不存在是配置问题不是故障,不该等(实际 ${elapsed}ms)`)
|
|
301
|
+
})
|