dsh-router-core 0.3.21 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +238 -63
- package/cordis.patch.yml +22 -0
- package/docs/ext.md +28 -5
- package/docs/pool-sticky-block.md +158 -0
- package/docs/suppliers.md +28 -5
- package/lib/client-registry.js +1300 -549
- package/lib/client.js +1300 -549
- package/lib/ext-test/index.js +47 -0
- package/lib/ext-test/locale/en.json +6 -0
- package/lib/ext-test/locale/zh.json +6 -0
- package/lib/ext-test/package.json +5 -0
- package/lib/http-state-C1vb3gW2.js +48 -0
- package/lib/index.js +1169 -254
- package/lib/suppliers/{nvidia.js → nvidia/index.js} +14 -6
- package/lib/suppliers/nvidia/locale/en.json +6 -0
- package/lib/suppliers/nvidia/locale/zh.json +6 -0
- package/lib/suppliers/nvidia/package.json +5 -0
- package/lib/suppliers/{opencode.js → opencode/index.js} +14 -6
- package/lib/suppliers/opencode/locale/en.json +6 -0
- package/lib/suppliers/opencode/locale/zh.json +6 -0
- package/lib/suppliers/opencode/package.json +5 -0
- package/lib/suppliers/{openrouter.js → openrouter/index.js} +14 -6
- package/lib/suppliers/openrouter/locale/en.json +6 -0
- package/lib/suppliers/openrouter/locale/zh.json +6 -0
- package/lib/suppliers/openrouter/package.json +5 -0
- package/lib/types/ext/contract.d.ts +9 -7
- package/lib/types/ext-test/index.d.ts +21 -0
- package/lib/types/ext-test/plugin.d.ts +23 -0
- package/lib/types/host-version.d.ts +15 -0
- package/lib/types/index.d.ts +6 -0
- package/lib/types/llm/adapter.d.ts +35 -4
- package/lib/types/router/account-pool.d.ts +53 -18
- package/lib/types/router/index.d.ts +65 -15
- package/lib/types/router/prefix-affinity.d.ts +18 -1
- package/lib/types/router/types.d.ts +7 -4
- package/lib/types/router/usage-store.d.ts +12 -0
- package/lib/types/shared.d.ts +25 -21
- package/lib/types/supplier-config.d.ts +15 -3
- package/lib/types/suppliers/contract.d.ts +37 -4
- package/lib/types/suppliers/http-state.d.ts +23 -0
- package/lib/types/suppliers/loader.d.ts +6 -4
- package/lib/types/suppliers/nvidia/index.d.ts +18 -0
- package/lib/types/suppliers/nvidia/plugin.d.ts +13 -0
- package/lib/types/suppliers/opencode/index.d.ts +18 -0
- package/lib/types/suppliers/opencode/plugin.d.ts +7 -0
- package/lib/types/suppliers/openrouter/index.d.ts +18 -0
- package/lib/types/suppliers/openrouter/plugin.d.ts +11 -0
- package/lib/types/suppliers/probe.d.ts +160 -0
- package/lib/types/suppliers/registry.d.ts +3 -2
- package/package.json +50 -16
- package/src/client/CombosTab.tsx +8 -53
- package/src/client/ExtDetail.tsx +32 -196
- package/src/client/ExtTab.tsx +25 -42
- package/src/client/ExtTestPanel.tsx +392 -0
- package/src/client/LastHitDock.tsx +141 -0
- package/src/client/RouterComponentsSection.tsx +241 -0
- package/src/client/RouterView.tsx +12 -5
- package/src/client/StatsTab.tsx +8 -6
- package/src/client/SupplierDetail.tsx +2 -34
- package/src/client/comp-button-scope.test.ts +58 -0
- package/src/client/comp-panel-layout.test.ts +58 -0
- package/src/client/env.d.ts +38 -1
- package/src/client/ext-panels.test.ts +44 -0
- package/src/client/ext-panels.ts +49 -0
- package/src/client/index.tsx +86 -0
- package/src/client/last-hit-store.ts +89 -0
- package/src/client/model-choice.test.ts +41 -0
- package/src/client/model-choice.ts +37 -0
- package/src/client/model-hint-copy.test.ts +7 -2
- package/src/client/model-hint-copy.ts +2 -2
- package/src/client/router-components.test.ts +114 -0
- package/src/client/router-components.ts +104 -0
- package/src/client/router.css +2592 -1635
- package/src/client/supplier-visibility.test.ts +57 -0
- package/src/client/supplier-visibility.ts +63 -0
- package/src/ext/contract.ts +9 -7
- package/src/ext-test/ext-test.test.ts +60 -0
- package/src/ext-test/index.ts +35 -0
- package/src/ext-test/locale/en.json +6 -0
- package/src/ext-test/locale/zh.json +6 -0
- package/src/ext-test/plugin.ts +34 -0
- package/src/ext-test/row.json +5 -0
- package/src/host-version.test.ts +61 -0
- package/src/host-version.ts +101 -0
- package/src/index.ts +165 -68
- package/src/llm/adapter.test.ts +175 -4
- package/src/llm/adapter.ts +109 -16
- package/src/router/account-pool.test.ts +189 -83
- package/src/router/account-pool.ts +120 -31
- package/src/router/disconnect-leak.test.ts +7 -12
- package/src/router/first-byte-budget.test.ts +160 -0
- package/src/router/index.ts +276 -62
- package/src/router/models-cache.test.ts +54 -9
- package/src/router/precise-dispatch.test.ts +6 -6
- package/src/router/prefix-affinity.ts +22 -1
- package/src/router/supplier-switch.test.ts +220 -0
- package/src/router/types.ts +7 -5
- package/src/router/usage-store.ts +16 -0
- package/src/router/usage.test.ts +1 -1
- package/src/shared.ts +25 -24
- package/src/supplier-config.ts +33 -13
- package/src/suppliers/contract.ts +56 -4
- package/src/suppliers/loader.test.ts +38 -4
- package/src/suppliers/loader.ts +33 -18
- package/src/suppliers/nvidia/index.ts +27 -0
- package/src/suppliers/nvidia/locale/en.json +6 -0
- package/src/suppliers/nvidia/locale/zh.json +6 -0
- package/src/suppliers/nvidia/row.json +5 -0
- package/src/suppliers/opencode/index.ts +27 -0
- package/src/suppliers/opencode/locale/en.json +6 -0
- package/src/suppliers/opencode/locale/zh.json +6 -0
- package/src/suppliers/opencode/row.json +5 -0
- package/src/suppliers/openrouter/index.ts +27 -0
- package/src/suppliers/openrouter/locale/en.json +6 -0
- package/src/suppliers/openrouter/locale/zh.json +6 -0
- package/src/suppliers/openrouter/row.json +5 -0
- package/src/suppliers/probe.test.ts +266 -0
- package/src/suppliers/probe.ts +475 -0
- package/src/suppliers/registry.ts +103 -23
- package/src/types.d.ts +2 -1
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
* 得落盘,届时加 stateFile 即可,接口不用变。
|
|
25
25
|
*/
|
|
26
26
|
import type { AccountState, SupplierAccountNow } from '../suppliers/contract.ts'
|
|
27
|
-
import {
|
|
27
|
+
import { sessionFingerprint } from './prefix-affinity.ts'
|
|
28
28
|
|
|
29
29
|
/** 各 AccountState 的处置:冷却策略 / 是否计入连续错误。 */
|
|
30
30
|
interface Rule {
|
|
@@ -182,9 +182,9 @@ export class AccountPool {
|
|
|
182
182
|
private rrCursor = 0
|
|
183
183
|
/** (supplier, model) → 当前驻留的号(只有**无亲和**的回退路径用它)。 */
|
|
184
184
|
private blocks = new Map<string, string>()
|
|
185
|
-
/** (supplier, model, uid) →
|
|
185
|
+
/** (supplier, model, uid[, session]) → 该号在某会话下的缓存质量样本。 */
|
|
186
186
|
private samples = new Map<string, Sample>()
|
|
187
|
-
/**
|
|
187
|
+
/** 同上键 → 缓存无效降权。 */
|
|
188
188
|
private demoted = new Map<string, DemoteEntry>()
|
|
189
189
|
/** (supplier, model) → Map<前缀指纹, uid>:前缀亲和的选号记忆(LRU 封顶)。 */
|
|
190
190
|
private affinity = new Map<string, Map<string, string>>()
|
|
@@ -197,6 +197,23 @@ export class AccountPool {
|
|
|
197
197
|
return `${this.supplierId}${SEP}${model}${SEP}${uid}`
|
|
198
198
|
}
|
|
199
199
|
|
|
200
|
+
/**
|
|
201
|
+
* 缓存质量样本/降权的键。
|
|
202
|
+
*
|
|
203
|
+
* **按会话隔离**(team 场景的关键):同一个号被多个会话共用时,各会话的
|
|
204
|
+
* 前缀会互相驱逐,把它们混在一起算命中率,会把「多会话互踩」误判成
|
|
205
|
+
* 「这个号缓存坏了」→ 错误降权 → 只剩一个号 → 那个号又被打低 → 降权串。
|
|
206
|
+
* 所以有会话身份时按 (model, uid, session) 记,降权只反映**该会话自己**
|
|
207
|
+
* 在这个号上的真实缓存表现。
|
|
208
|
+
*
|
|
209
|
+
* 无会话身份(外部 OpenAI 客户端,走无指纹回退路径)→ 保持 (model, uid),
|
|
210
|
+
* 与块轮询的原语义一致(那时本就是单会话视角)。
|
|
211
|
+
*/
|
|
212
|
+
private sampleKey(model: string, uid: string, session?: string): string {
|
|
213
|
+
const base = this.key(model, uid)
|
|
214
|
+
return session === undefined || session === '' ? base : `${base}${SEP}${session}`
|
|
215
|
+
}
|
|
216
|
+
|
|
200
217
|
private entry(model: string, uid: string): CooldownEntry {
|
|
201
218
|
const k = this.key(model, uid)
|
|
202
219
|
let e = this.cooldowns.get(k)
|
|
@@ -236,30 +253,32 @@ export class AccountPool {
|
|
|
236
253
|
}
|
|
237
254
|
|
|
238
255
|
/**
|
|
239
|
-
*
|
|
256
|
+
* 为「某个模型」选一个健康账号 —— **会话亲和选号,无策略旋钮**。
|
|
240
257
|
*
|
|
241
|
-
* round-robin
|
|
258
|
+
* 旧的两策略(fallback 取第一个 / round-robin 轮转)都删了:前者让所有
|
|
259
|
+
* 会话挤同一个号(team/多会话必互踩),后者的"轮转"与亲和"同会话永不换号"
|
|
260
|
+
* 直接冲突。正确模型只有一条:
|
|
242
261
|
*
|
|
243
262
|
* 1. 算请求的前缀指纹(messages 前几条),查绑定表 → 命中且该号健康 → 用它。
|
|
244
|
-
*
|
|
245
|
-
* 缓存里只写一份。块轮询治不了跨会话驱逐(多会话大前缀在同一号互踩,
|
|
246
|
-
* 实测 3 会话 3.8% vs 单会话 82.5%),亲和才是根治。
|
|
263
|
+
* 同一个会话的连续请求**永远落回同一个号**,前缀在该号缓存里只写一份。
|
|
247
264
|
* 2. 新指纹(或绑定的号不健康)→ 按游标挑一个号并**记下绑定**,之后就固定。
|
|
248
265
|
* 新会话用游标分发,所以不同会话天然散到不同号上(均衡)。
|
|
249
|
-
* 3.
|
|
250
|
-
*
|
|
266
|
+
* 3. 指纹算不出(拿不到 messages)→ **块轮询兜底**(§2 旧行为)。这不是
|
|
267
|
+
* "策略",是无会话身份时的替身。
|
|
268
|
+
* 4. 块统计不驱动轮转(亲和已保证驻留),只保留 `BLOCK_MAX` 次不达标 →
|
|
269
|
+
* 降权该号(缓存无效,可恢复)。
|
|
251
270
|
*
|
|
252
|
-
*
|
|
253
|
-
*
|
|
271
|
+
* 天花板:号数 < 并发会话数时必然有会话共用号 → 互踩,无法用选号手法消除
|
|
272
|
+
* (缓存空间是账号侧物理限制)。见 docs/pool-sticky-block.md §10。
|
|
254
273
|
*
|
|
255
274
|
* @param accounts 插件报告的「现在状态」(顺序即插件的自然顺序)
|
|
256
275
|
* @param poolOrder 用户在面板拖出来的顺序(核心管)
|
|
257
|
-
* @param strategy fallback 取第一个健康 / round-robin 前缀亲和
|
|
258
276
|
* @param modelId 当前要路由的模型(决定查哪个 (model, uid) 冷却单元)
|
|
259
|
-
* @param messages 请求体的 messages
|
|
277
|
+
* @param messages 请求体的 messages(算亲和指纹的内容兜底)
|
|
278
|
+
* @param session 宿主会话身份;给了就以它为主键(精确、零撞车)
|
|
260
279
|
* @returns 选中的 uid;无健康账号返回 undefined
|
|
261
280
|
*/
|
|
262
|
-
pick(accounts: SupplierAccountNow[], poolOrder: string[],
|
|
281
|
+
pick(accounts: SupplierAccountNow[], poolOrder: string[], modelId: string, messages?: unknown, session?: string): string | undefined {
|
|
263
282
|
const now = Date.now()
|
|
264
283
|
const byUid = new Map(accounts.map((a) => [a.uid, a]))
|
|
265
284
|
// 池顺序优先,未配置的按插件自然顺序追加
|
|
@@ -268,35 +287,98 @@ export class AccountPool {
|
|
|
268
287
|
...accounts.map((a) => a.uid).filter((uid) => !poolOrder.includes(uid)),
|
|
269
288
|
]
|
|
270
289
|
const healthy = ordered.filter((uid) => this.healthy(uid, modelId, now))
|
|
271
|
-
if (healthy.length === 0)
|
|
272
|
-
|
|
290
|
+
if (healthy.length === 0) {
|
|
291
|
+
this.lastWhy = '全池无健康号(都在冷却)'
|
|
292
|
+
return undefined
|
|
293
|
+
}
|
|
273
294
|
// 缓存无效降权:该号缓存质量样本长期不达标 → 跳过它(可到期恢复)
|
|
274
|
-
const usable = healthy.filter((uid) => !this.isDemoted(uid, modelId, now))
|
|
295
|
+
const usable = healthy.filter((uid) => !this.isDemoted(uid, modelId, now, session))
|
|
275
296
|
const pool = usable.length > 0 ? usable : healthy
|
|
276
297
|
// 前缀亲和:指纹有绑定且该号仍可用 → 直接复用(不进驻留逻辑)
|
|
277
|
-
const fp =
|
|
278
|
-
|
|
279
|
-
|
|
298
|
+
const fp = sessionFingerprint(session, messages)
|
|
299
|
+
const bound = fp === '' ? undefined : this.affinity.get(this.blockKey(modelId))?.get(fp)
|
|
300
|
+
if (bound !== undefined && pool.includes(bound)) {
|
|
301
|
+
this.lastWhy = `亲和命中 ${bound}(会话/前缀已绑定)`
|
|
302
|
+
return bound
|
|
303
|
+
}
|
|
304
|
+
// 走到这里说明没粘住:把「为什么没用原绑定」记清楚,否则每次换号都要靠猜
|
|
305
|
+
const why: string[] = []
|
|
306
|
+
if (fp === '') {
|
|
307
|
+
why.push('无指纹(拿不到 messages/会话)')
|
|
308
|
+
} else if (bound === undefined) {
|
|
309
|
+
why.push('该指纹无绑定(会话首次出现)')
|
|
310
|
+
} else {
|
|
311
|
+
if (!healthy.includes(bound)) {
|
|
312
|
+
const c = this.cooldowns.get(this.key(modelId, bound))
|
|
313
|
+
why.push(`原绑定 ${bound} 正在冷却${c?.reason ? `(${c.reason})` : ''}`)
|
|
314
|
+
} else if (!usable.includes(bound)) {
|
|
315
|
+
why.push(`原绑定 ${bound} 被降权(本会话样本不达标)`)
|
|
316
|
+
} else {
|
|
317
|
+
why.push(`原绑定 ${bound} 不在当前健康池`)
|
|
318
|
+
}
|
|
280
319
|
}
|
|
320
|
+
// 无指纹:沿用块轮询语义 —— 块内粘住(blockKeeps)则留守当前 block。
|
|
281
321
|
const block = this.blocks.get(this.blockKey(modelId))
|
|
282
322
|
if (fp === '' && block !== undefined && pool.includes(block) && this.blockKeeps(modelId, block)) {
|
|
323
|
+
this.lastWhy = `块驻留 ${block}(无指纹路径)`
|
|
283
324
|
return block
|
|
284
325
|
}
|
|
285
|
-
//
|
|
286
|
-
|
|
287
|
-
|
|
326
|
+
// 分配一个号并记住绑定。
|
|
327
|
+
//
|
|
328
|
+
// 有指纹(新会话):**优先铺开到未被其它会话占用的号**——同 model 下
|
|
329
|
+
// 每个 fresh 会话尽量独占一个号(缓存空间隔离、互不驱逐),号不够时
|
|
330
|
+
// 才回绕复用(共用号无害:样本按会话语义隔离,不会误降权)。
|
|
331
|
+
// 候选顺序:先「空闲号」,用完后才是「已被占用的号」。
|
|
332
|
+
// 无指纹:沿用块轮询(从 block 之后前进,原语义不变)。
|
|
333
|
+
const uid = fp !== ''
|
|
334
|
+
? this.allocSession(pool, modelId, fp)
|
|
335
|
+
: pool[(block === undefined ? this.rrCursor : Math.max(0, pool.indexOf(block) + 1)) % pool.length]!
|
|
288
336
|
this.rrCursor = (pool.indexOf(uid) + 1) % pool.length
|
|
289
337
|
this.blocks.set(this.blockKey(modelId), uid)
|
|
290
338
|
if (fp !== '') {
|
|
291
339
|
this.bind(modelId, fp, uid)
|
|
340
|
+
const free = this.freeUids(pool, modelId, fp)
|
|
341
|
+
why.push(free.length === 0 ? `号已铺满,按游标复用 ${uid}` : `铺开到空闲号 ${uid}`)
|
|
292
342
|
} else {
|
|
293
343
|
// 新驻留 = 新观察窗口(旧行为:块计数从 0 开始)。样本是跨驻留累计的,
|
|
294
344
|
// 不清的话上一个大样本会立刻把新驻留判走。
|
|
295
345
|
this.samples.delete(this.key(modelId, uid))
|
|
346
|
+
why.push(`块推进到 ${uid}`)
|
|
296
347
|
}
|
|
348
|
+
this.lastWhy = why.join(';')
|
|
349
|
+
return uid
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* 新会话分配:优先返回**未被其它会话占用**的号(按游标推进),
|
|
354
|
+
* 空闲号用尽才回绕到已占用的号(此时按游标复用,语义等同旧行为)。
|
|
355
|
+
*
|
|
356
|
+
* 「占用」= 该 model 的绑定表里,某个 **其它** 指纹当前指向该 uid。
|
|
357
|
+
* 同一会话自己的旧绑定不算占用(它已经在上面的亲和分支处理过,走到这里
|
|
358
|
+
* 说明原绑定不可用,允许它改绑到自己的其它号或空闲号)。
|
|
359
|
+
*/
|
|
360
|
+
private allocSession(pool: string[], modelId: string, fp: string): string {
|
|
361
|
+
const free = this.freeUids(pool, modelId, fp)
|
|
362
|
+
const candidates = free.length > 0 ? free : pool
|
|
363
|
+
const uid = candidates[this.rrCursor % candidates.length]!
|
|
297
364
|
return uid
|
|
298
365
|
}
|
|
299
366
|
|
|
367
|
+
/** 当前 model 下、本指纹之外没有任何其它会话绑定的号(按 pool 原顺序)。 */
|
|
368
|
+
private freeUids(pool: string[], modelId: string, fp: string): string[] {
|
|
369
|
+
const taken = new Set<string>()
|
|
370
|
+
const m = this.affinity.get(this.blockKey(modelId))
|
|
371
|
+
if (m !== undefined) {
|
|
372
|
+
for (const [k, v] of m) {
|
|
373
|
+
if (k !== fp) taken.add(v)
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
return pool.filter((uid) => !taken.has(uid))
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/** 上一次 pick 的选号理由(诊断用:解释"为什么没粘住原号")。 */
|
|
380
|
+
lastWhy = ''
|
|
381
|
+
|
|
300
382
|
/** 记下「前缀指纹 → 号」的绑定(LRU:超上限丢最久未用的)。 */
|
|
301
383
|
private bind(modelId: string, fp: string, uid: string): void {
|
|
302
384
|
const k = this.blockKey(modelId)
|
|
@@ -314,9 +396,14 @@ export class AccountPool {
|
|
|
314
396
|
}
|
|
315
397
|
}
|
|
316
398
|
|
|
317
|
-
/**
|
|
318
|
-
|
|
319
|
-
|
|
399
|
+
/**
|
|
400
|
+
* 该号在**本会话**(有身份时)本模型上是否被判定「缓存无效」(降权中)。
|
|
401
|
+
*
|
|
402
|
+
* 判据按会话隔离(见 sampleKey):降权只惩罚「这个会话在这个号上真的
|
|
403
|
+
* 反复不命中」,不会因别的会话共用同号互踩而被牵连。
|
|
404
|
+
*/
|
|
405
|
+
private isDemoted(uid: string, modelId: string, now: number, session?: string): boolean {
|
|
406
|
+
const e = this.demoted.get(this.sampleKey(modelId, uid, session))
|
|
320
407
|
return e !== undefined && e.until > now
|
|
321
408
|
}
|
|
322
409
|
|
|
@@ -344,14 +431,16 @@ export class AccountPool {
|
|
|
344
431
|
* 记录一次成功请求的缓存命中情况(降权判据的反馈信号)。
|
|
345
432
|
*
|
|
346
433
|
* 只有**成功**请求才计入:失败没产生缓存,计入会污染命中率并导致错降权。
|
|
347
|
-
* 统计按 (model, uid)
|
|
348
|
-
*
|
|
434
|
+
* 统计按 (model, uid[, session]) 记,**不管这个号是不是「当前驻留」**——
|
|
435
|
+
* 亲和开启后多个会话各绑各的号交错服务,按驻留记会漏掉大部分真实反馈。
|
|
436
|
+
* 按会话隔离(见 sampleKey)则保证「多会话共用同号互踩」不会误伤降权。
|
|
349
437
|
* @param uid 实际服务的号
|
|
350
438
|
* @param modelId 模型(块粒度)
|
|
351
439
|
* @param cachedTokens 上游报的命中 token 数;0 = 全量重算
|
|
440
|
+
* @param session 宿主会话身份;有则按会话隔离统计
|
|
352
441
|
*/
|
|
353
|
-
noteCache(uid: string, modelId: string, cachedTokens: number): void {
|
|
354
|
-
const k = this.
|
|
442
|
+
noteCache(uid: string, modelId: string, cachedTokens: number, session?: string): void {
|
|
443
|
+
const k = this.sampleKey(modelId, uid, session)
|
|
355
444
|
const s = this.samples.get(k) ?? { served: 0, hits: 0 }
|
|
356
445
|
s.served += 1
|
|
357
446
|
if (cachedTokens > 0) s.hits += 1
|
|
@@ -158,28 +158,24 @@ test('客户端断开:上游流 cancel 链要穿过 tapStreamUsage / normalize
|
|
|
158
158
|
assert.equal(leak.cancelled(), true, '取消必须穿过两层 wrapper 到达上游')
|
|
159
159
|
})
|
|
160
160
|
|
|
161
|
-
test('
|
|
161
|
+
test('聚合失败:上游流必须被回收(解锁,不能永久锁死)', async () => {
|
|
162
162
|
// 这一条测的**不是**「cancel 回调有没有触发」——那测不出东西:
|
|
163
163
|
// 实测结论(两个都验过):
|
|
164
164
|
// - 流 error 后调 cancel(),只会以同一错误 reject,**不触发** cancel 回调
|
|
165
165
|
// (`pull()` 抛错会把流置为 errored;上游拆连接则流直接 error)
|
|
166
166
|
// - 这两种情况下 undici **已经自己关了连接**,不需要我们 cancel
|
|
167
|
-
// 所以聚合失败路径真正要保证的是
|
|
167
|
+
// 所以聚合失败路径真正要保证的是 **上游流不再被锁着**:锁着的话上层
|
|
168
168
|
// (writeChatResult 的失败路径 / 未来的取消逻辑)想 cancel 都会抛
|
|
169
169
|
// `Invalid state: ReadableStream is locked`,连接就永久漏了。
|
|
170
|
-
|
|
170
|
+
//
|
|
171
|
+
// 注意拓扑:核心现在隔着「首字节截止」wrapper(withFirstByteDeadline)读上游,
|
|
172
|
+
// 所以判据只能是**最终状态**(locked=false),不能再断言核心直接 releaseLock
|
|
173
|
+
// 上游的 reader —— 中间隔了一层,核心只会解锁 wrapper 那个 reader。
|
|
171
174
|
const enc = new TextEncoder()
|
|
172
175
|
const failing = new ReadableStream<Uint8Array>({
|
|
173
176
|
start(c) { c.enqueue(enc.encode('data: {"choices":[{"delta":{"content":"a"}}]}\n\n')) },
|
|
174
177
|
pull() { throw new Error('aggregate read failed') },
|
|
175
178
|
})
|
|
176
|
-
const origGetReader = failing.getReader.bind(failing)
|
|
177
|
-
;(failing as unknown as { getReader: () => unknown }).getReader = () => {
|
|
178
|
-
const r = origGetReader()
|
|
179
|
-
const origRelease = r.releaseLock.bind(r)
|
|
180
|
-
r.releaseLock = () => { releaseLocked = true; origRelease() }
|
|
181
|
-
return r
|
|
182
|
-
}
|
|
183
179
|
|
|
184
180
|
const router = new Router('')
|
|
185
181
|
router.add(supplierWith(failing) as never)
|
|
@@ -195,8 +191,7 @@ test('聚合失败:reader 必须被解绑(否则连接还活着时上层无
|
|
|
195
191
|
)
|
|
196
192
|
await new Promise((r) => setTimeout(r, 150))
|
|
197
193
|
|
|
198
|
-
assert.equal(
|
|
199
|
-
assert.equal(failing.locked, false, '流必须处于未锁定状态(可被 cancel / 已被回收)')
|
|
194
|
+
assert.equal(failing.locked, false, '上游流必须处于未锁定状态(可被 cancel / 已被回收)')
|
|
200
195
|
})
|
|
201
196
|
|
|
202
197
|
test('fixFrame:把后续 delta 里重发成 null 的 id/name 剥掉(否则外部 OpenAI 客户端会丢工具名)', async () => {
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 首字节预算测试 —— 组合的一条腿「连上了、200 了、就是不吐字节」时必须降级。
|
|
3
|
+
*
|
|
4
|
+
* 2026-09-24 夜里的现场(usage.json / 三个 session 的 step 时间线互相对齐):
|
|
5
|
+
* money 组合第一腿 codebuddy-en 卡住,ttfb 分别 84s / 122s / 151s / 244s / 246s /
|
|
6
|
+
* 250s / 254s / 255s,**8 次全部 ok: true**——全都最终成功了,只是晚了四分钟。
|
|
7
|
+
* 用户干等 200 多秒手动暂停。
|
|
8
|
+
*
|
|
9
|
+
* 为什么以前不会降级:停滞既不是成功也不是失败,所以
|
|
10
|
+
* - 插件的 120s 只守「连接 + 响应头」(fetch 一返回就撤表),守不到 body
|
|
11
|
+
* - 核心写响应的路径没有计时器,组合链卡在这一条腿上一动不动
|
|
12
|
+
* - 该号也不会被冷却(没失败 → noteFailure 不触发)→ 下次还挑它
|
|
13
|
+
*
|
|
14
|
+
* 这里守的是修复后的语义:**预算到就判失败,链往前走**。没装这个闸门时
|
|
15
|
+
* 下面两条测试都会在 5s 的看门狗里超时变红(而不是把测试进程挂死)。
|
|
16
|
+
*/
|
|
17
|
+
import { test } from 'node:test'
|
|
18
|
+
import assert from 'node:assert/strict'
|
|
19
|
+
import type { ServerResponse } from 'node:http'
|
|
20
|
+
import { Router, withFirstByteDeadline } from './index.ts'
|
|
21
|
+
import { AccountPool } from './account-pool.ts'
|
|
22
|
+
import type { ModelInfo } from './types.ts'
|
|
23
|
+
import type { AccountPool as Pool } from './account-pool.ts'
|
|
24
|
+
import type { ChatOnceResult, SupplierAccountNow, SupplierStatusNow } from '../suppliers/contract.ts'
|
|
25
|
+
|
|
26
|
+
const enc = new TextEncoder()
|
|
27
|
+
|
|
28
|
+
/** 永不吐字节的流(模拟「200 响应头到手,body 停摆」),记录有没有被 cancel。 */
|
|
29
|
+
function stallingStream(): { stream: ReadableStream<Uint8Array>; cancelled: () => boolean } {
|
|
30
|
+
let cancelled = false
|
|
31
|
+
const stream = new ReadableStream<Uint8Array>({
|
|
32
|
+
cancel() { cancelled = true },
|
|
33
|
+
})
|
|
34
|
+
return { stream, cancelled: () => cancelled }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** 立刻吐一帧 + [DONE] 的正常流。 */
|
|
38
|
+
function healthyStream(text: string): ReadableStream<Uint8Array> {
|
|
39
|
+
return new ReadableStream<Uint8Array>({
|
|
40
|
+
start(ctrl) {
|
|
41
|
+
ctrl.enqueue(enc.encode(`data: {"choices":[{"delta":{"content":"${text}"}}]}\n\n`))
|
|
42
|
+
ctrl.enqueue(enc.encode('data: [DONE]\n\n'))
|
|
43
|
+
ctrl.close()
|
|
44
|
+
},
|
|
45
|
+
})
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function supplier(id: string, uids: string[], chatOnce: (uid: string) => Promise<ChatOnceResult>) {
|
|
49
|
+
const accounts: SupplierAccountNow[] = uids.map((uid) => ({ uid, credits: 0, state: 'ok' as const }))
|
|
50
|
+
return {
|
|
51
|
+
id, name: id, priority: 0, pool: new AccountPool(id) as Pool,
|
|
52
|
+
status: (): SupplierStatusNow => ({ id, name: id, accounts }),
|
|
53
|
+
listModels: async (): Promise<ModelInfo[]> => [{ id: 'm1' }],
|
|
54
|
+
getAlias: () => id,
|
|
55
|
+
accounts: () => accounts,
|
|
56
|
+
chatOnce,
|
|
57
|
+
dispose: () => {},
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function captureRes(): { res: ServerResponse; body: () => string } {
|
|
62
|
+
const chunks: string[] = []
|
|
63
|
+
const text = (chunk: unknown): string =>
|
|
64
|
+
chunk instanceof Uint8Array ? new TextDecoder().decode(chunk) : String(chunk)
|
|
65
|
+
const res = {
|
|
66
|
+
writableEnded: false,
|
|
67
|
+
destroyed: false,
|
|
68
|
+
writeHead: (): unknown => undefined,
|
|
69
|
+
write: (chunk?: unknown): boolean => { chunks.push(text(chunk)); return true },
|
|
70
|
+
end: (chunk?: unknown): unknown => { if (chunk !== undefined) chunks.push(text(chunk)); return undefined },
|
|
71
|
+
once: (): unknown => undefined,
|
|
72
|
+
removeListener: (): unknown => undefined,
|
|
73
|
+
destroy: (): unknown => undefined,
|
|
74
|
+
} as unknown as ServerResponse
|
|
75
|
+
return { res, body: () => chunks.join('') }
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const req = {
|
|
79
|
+
model: 'c',
|
|
80
|
+
stream: true,
|
|
81
|
+
rawBody: JSON.stringify({ model: 'c', messages: [{ role: 'user', content: 'hi' }], stream: true }),
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** 给等待加看门狗:闸门没装时这里是 5s 超时(红),而不是把测试挂死。 */
|
|
85
|
+
async function within<T>(ms: number, run: () => Promise<T>): Promise<T> {
|
|
86
|
+
return await Promise.race([
|
|
87
|
+
run(),
|
|
88
|
+
new Promise<T>((_, reject) => setTimeout(() => reject(new Error(`组合 ${ms}ms 内没有降级`)), ms)),
|
|
89
|
+
])
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
test('第一腿停滞:预算到就判失败,落到下一条腿(而不是把请求挂到客户端自己放弃)', async () => {
|
|
93
|
+
const stall = stallingStream()
|
|
94
|
+
const router = new Router('', undefined, undefined, { firstByteBudgetMs: 150 })
|
|
95
|
+
router.add(supplier('supA', ['a1'], async () => ({ ok: true, stream: stall.stream })) as never)
|
|
96
|
+
router.add(supplier('supB', ['b1'], async () => ({ ok: true, stream: healthyStream('ok-from-B') })) as never)
|
|
97
|
+
assert.equal(router.createCombo('c', ['supA,m1', 'supB,m1']).ok, true)
|
|
98
|
+
|
|
99
|
+
const { res, body } = captureRes()
|
|
100
|
+
const t0 = Date.now()
|
|
101
|
+
await within(5_000, () => router.chatCompletions(req, res))
|
|
102
|
+
const elapsed = Date.now() - t0
|
|
103
|
+
|
|
104
|
+
assert.match(body(), /ok-from-B/, '必须由第二条腿服务(第一腿 200 了但不吐字节)')
|
|
105
|
+
assert.equal(stall.cancelled(), true, '停滞的上游流必须被 cancel,否则连接原样挂着')
|
|
106
|
+
// 2s 是组合「喘口气」等待(TRANSIENT_SETTLE_MS)——停滞按 transport 归类,
|
|
107
|
+
// 该等就得等;这里断言它确实走了「失败→降级」这条路,而不是瞬间成功。
|
|
108
|
+
assert.ok(elapsed >= 2_000, `应先等喘口气窗口再降级,实际 ${elapsed}ms`)
|
|
109
|
+
assert.ok(elapsed < 5_000, `降级必须发生在预算+喘口气附近,实际 ${elapsed}ms`)
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
test('预算是「每条腿」一份:腿内第二个号不再各等一次(否则 60s × 号数)', async () => {
|
|
113
|
+
const stallA1 = stallingStream()
|
|
114
|
+
const stallA2 = stallingStream()
|
|
115
|
+
let a1 = 0
|
|
116
|
+
let a2 = 0
|
|
117
|
+
const router = new Router('', undefined, undefined, { firstByteBudgetMs: 150 })
|
|
118
|
+
router.add(supplier('supA', ['a1', 'a2'], async (uid) => {
|
|
119
|
+
if (uid === 'a1') { a1 += 1; return { ok: true, stream: stallA1.stream } }
|
|
120
|
+
a2 += 1
|
|
121
|
+
return { ok: true, stream: stallA2.stream }
|
|
122
|
+
}) as never)
|
|
123
|
+
router.add(supplier('supB', ['b1'], async () => ({ ok: true, stream: healthyStream('ok-from-B') })) as never)
|
|
124
|
+
assert.equal(router.createCombo('c', ['supA,m1', 'supB,m1']).ok, true)
|
|
125
|
+
|
|
126
|
+
const { res, body } = captureRes()
|
|
127
|
+
const t0 = Date.now()
|
|
128
|
+
await within(5_000, () => router.chatCompletions(req, res))
|
|
129
|
+
const elapsed = Date.now() - t0
|
|
130
|
+
|
|
131
|
+
assert.match(body(), /ok-from-B/)
|
|
132
|
+
assert.equal(a1, 1, '第一个号被调了一次')
|
|
133
|
+
assert.equal(a2, 0, '预算已耗尽:同一个号池里的第二个号不该再各等 150ms(停滞是上游级的)')
|
|
134
|
+
assert.ok(elapsed < 5_000, `不能按号数翻倍地等,实际 ${elapsed}ms`)
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
test('wrapper:第一个字节到手后不再计时(长生成不能被砍)', async () => {
|
|
138
|
+
let ctrlRef: ReadableStreamDefaultController<Uint8Array> | undefined
|
|
139
|
+
const src = new ReadableStream<Uint8Array>({ start: (c) => { ctrlRef = c } })
|
|
140
|
+
const reader = withFirstByteDeadline(src, 60).getReader()
|
|
141
|
+
|
|
142
|
+
ctrlRef?.enqueue(enc.encode('first'))
|
|
143
|
+
assert.equal(dec((await reader.read()).value!), 'first')
|
|
144
|
+
// 停 3 倍预算才吐第二块:过了截止点也必须原样透传
|
|
145
|
+
await new Promise((r) => setTimeout(r, 200))
|
|
146
|
+
ctrlRef?.enqueue(enc.encode('second'))
|
|
147
|
+
assert.equal(dec((await reader.read()).value!), 'second')
|
|
148
|
+
await reader.cancel()
|
|
149
|
+
})
|
|
150
|
+
|
|
151
|
+
test('wrapper:源流一个字节都不吐就到期 → 流判失败(上层据此换号/换模型)', async () => {
|
|
152
|
+
const { stream, cancelled } = stallingStream()
|
|
153
|
+
const reader = withFirstByteDeadline(stream, 60).getReader()
|
|
154
|
+
await assert.rejects(() => reader.read(), /no first byte within 60ms/)
|
|
155
|
+
assert.equal(cancelled(), true, '到期时必须把取消传给源流(否则上游连接原样挂着)')
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
function dec(v: Uint8Array): string {
|
|
159
|
+
return new TextDecoder().decode(v)
|
|
160
|
+
}
|