dsh-agent-board 1.9.0 → 1.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -8
- package/index.mjs +1 -0
- package/lib/client/board-list.js +1 -1
- package/lib/client/dashboard.js +52 -7
- package/lib/client/task-detail.js +2 -2
- package/lib/client.js +55 -10
- package/lib/core.mjs +48 -12
- package/lib/dispatch.mjs +135 -1142
- package/lib/hung.mjs +202 -0
- package/lib/model-health.mjs +553 -0
- package/lib/notify.mjs +37 -4
- package/lib/policy.mjs +19 -9
- package/lib/rpc.mjs +7 -3
- package/lib/settle.mjs +954 -0
- package/package.json +1 -1
- package/test/core.test.mjs +339 -77
- package/test/decompose.test.mjs +70 -16
- package/test/fault-e2e-circuit.test.mjs +187 -0
- package/test/fault-e2e-fallback-chain.test.mjs +249 -0
- package/test/fault-e2e-hung.test.mjs +504 -0
- package/test/fault-e2e-recovery.test.mjs +255 -0
- package/test/fault-e2e-softgate.test.mjs +206 -0
- package/test/fault-e2e-stale-dir.test.mjs +226 -0
- package/test/fault-injection-docs.test.mjs +132 -0
- package/test/fault-provider.test.mjs +458 -0
- package/test/helpers/fault-provider.mjs +475 -0
- package/test/helpers/mock-ctx.mjs +9 -0
- package/test/hung-frame-signal.test.mjs +630 -0
- package/test/integration.test.mjs +41 -0
- package/test/lane-gui-bysize.test.mjs +410 -0
- package/test/lane-models.test.mjs +134 -0
- package/test/popover-clamp.test.mjs +92 -0
- package/test/runtime-health.test.mjs +2 -1
- package/test/scorecard-data.test.mjs +2 -1
- package/test/settle-stop-interrupt.test.mjs +294 -3
- package/test/ui-zoom-client.test.mjs +9 -4
|
@@ -397,6 +397,47 @@ test('集成f·回执聚合过滤:45s 窗口聚合为一条,已归档任务
|
|
|
397
397
|
} finally { await env.cleanup() }
|
|
398
398
|
})
|
|
399
399
|
|
|
400
|
+
// ===== 场景 f2:拆解官结算(task-mvrqiu6x)——失败分支 🚨 系统异常区照旧,split 成功才出「🧩 已拆解」分区 =====
|
|
401
|
+
// draft 源卡 → task_decompose 工具 → decomposer 一次性 run 落定(无 structured)→ settleDecompose 失败臂
|
|
402
|
+
// pushSysNote(sysNotesBuf 易失随回执冲刷);开窗任务的完成回执到点冲刷时一并投递,且不产生「🧩 已拆解」回执。
|
|
403
|
+
test('集成f2·拆解官结算:draft 源卡 task_decompose → structured 缺失 → 🚨 系统异常区,无 🧩 回执', async () => {
|
|
404
|
+
var env = createMockCtx()
|
|
405
|
+
try {
|
|
406
|
+
plugin.apply(env.ctx)
|
|
407
|
+
// draft 源卡(Team 模式语义:先拆后发,源卡停 draft 待用户确认发布)
|
|
408
|
+
var r = await env.rpc('create-task', { sessionId: env.sid, id: 'task-dc1', title: '集成需求卡', description: '集成套件拆解源卡', draft: true })
|
|
409
|
+
assert.equal(r.ok, true, 'draft 源卡创建')
|
|
410
|
+
assert.equal(r.task.status, 'draft')
|
|
411
|
+
// 拆解官 spawn(一次性 run,label decomposer:task-dc1)
|
|
412
|
+
var dc = await env.tools.get('task_decompose').execute({ taskId: 'task-dc1' })
|
|
413
|
+
assert.equal(dc.ok, true, 'task_decompose 成功: ' + JSON.stringify(dc))
|
|
414
|
+
var vrun = await env.waitFor(function () { return env.lastRun('decomposer:task-dc1') }, 'decomposer spawn 发生')
|
|
415
|
+
assert.equal(vrun.req.label, 'decomposer:task-dc1')
|
|
416
|
+
// run 落定但无 structured(宿主/provider 不支持 outputSchema)→ settleDecompose 失败臂 pushSysNote(不带 taskId)
|
|
417
|
+
vrun.resolveWith('拆解官自由文本,无结构化提案')
|
|
418
|
+
await env.waitFor(function () { var t = env.task('task-dc1'); return t && (t.messages || []).some(function (m) { return /拆解官未返回结构化提案/.test(m.text) }) }, '失败原因落源卡消息')
|
|
419
|
+
assert.equal(env.task('task-dc1').status, 'draft', '源卡状态不变')
|
|
420
|
+
// 开窗任务:完成回执入队(pushReceipt 挂 45s 冲刷窗口),失败 sysNote 随窗冲刷
|
|
421
|
+
await createTask(env, 'task-dc2', { pipeline: 'work' })
|
|
422
|
+
await env.clock.advance(50)
|
|
423
|
+
await env.waitFor(function () { return env.continuables.length >= 1 }, '开窗任务 spawn 发生')
|
|
424
|
+
// 等派发收尾落卡(claimedBy=真实子会话):完成回执的 isDispatched 判定依赖它——
|
|
425
|
+
// 抢在落卡前 board_report 会落成「未派发完成」,回执被正确地静默(本用例要验的是投递面)。
|
|
426
|
+
var child = env.continuables[0]
|
|
427
|
+
await env.waitFor(function () { var t = env.task('task-dc2'); return t && t.status === 'in-progress' && t.claimedBy === child.childId ? t : null }, '开窗任务落卡 in-progress')
|
|
428
|
+
var rep = await env.tools.get('board_report').execute({ taskId: 'task-dc2', kind: 'complete', summary: 'f2 交付', changes: '', selfTest: '', diffStat: '' })
|
|
429
|
+
assert.equal(rep.ok, true)
|
|
430
|
+
// 45s 聚合窗口到点 → 冲刷:完成回执 + 🚨 系统异常(失败分支不产生「🧩 已拆解」回执)
|
|
431
|
+
await env.clock.advance(45000)
|
|
432
|
+
await env.waitFor(function () { return env.sent.length >= 1 }, '失败信号随回执冲刷投递')
|
|
433
|
+
var text = env.sent[0].content[0].text
|
|
434
|
+
assert.ok(text.indexOf('⚠️ 系统异常 1 条:') >= 0, '失败分支仍落 ⚠️ 系统异常区')
|
|
435
|
+
assert.ok(text.indexOf('🚨 拆解官 run 未产出可用提案(源卡 task-dc1)') >= 0, '失败文案逐字')
|
|
436
|
+
assert.ok(text.indexOf('已拆解') < 0, '失败不产生「🧩 已拆解」回执')
|
|
437
|
+
assert.ok(text.indexOf('f2 交付') >= 0, '完成回执(开窗任务)照常投递')
|
|
438
|
+
} finally { await env.cleanup() }
|
|
439
|
+
})
|
|
440
|
+
|
|
400
441
|
// ===== 场景 g:歧义通知 25s 去抖——连报两次只投最新一条 =====
|
|
401
442
|
// 同一任务 25s 窗口内两次上报歧义:第一次调度被第二次顶替(escNotifyTimers 身份不匹配静默丢弃),
|
|
402
443
|
// 到点只投一条且内容是最新疑问(deliverEscalation 投递时才读 escalation.question)。
|
|
@@ -448,6 +448,41 @@ function mkProbeBoard() {
|
|
|
448
448
|
d.laneModels.heavy.worker = ['p/b', 'p/a']; d.fallbackModels = ['p/c']
|
|
449
449
|
return d
|
|
450
450
|
}
|
|
451
|
+
// EMPTY_RESPONSE 宿主逐字文案(dsh-llm-pi-ai mapStopReason:model "<m>" returned a completed
|
|
452
|
+
// response with no content;dsh-llm-deepseek 另作 '<…> returned no content')——探测侧按文本特征识别。
|
|
453
|
+
var EMPTY_RESPONSE_TEXT = 'model "p/a" returned a completed response with no content'
|
|
454
|
+
// 探测响应脚本桩(task-mvrfp1rj):script 按「第 N 次 llm.stream 调用」取一步
|
|
455
|
+
// 'stop' 正常完成(finish/stop)
|
|
456
|
+
// 'empty' 宿主空响应(finish/error + failure.code=EMPTY_RESPONSE,逐字文案见上)
|
|
457
|
+
// { empty: t } 自定义文本的空响应(验 quota 类文本由既有嗅探接管)
|
|
458
|
+
// { error: t } 真错误(finish/error,无 code——超时/限流/连接等形态)
|
|
459
|
+
// 'nofinish' 流结束但零 finish 块
|
|
460
|
+
// 'hang' 永不完成(挂起直到 signal abort——task-mvsv8zym 验超时/重试超时形态)
|
|
461
|
+
function mkProbeScriptCtx(calls, script) {
|
|
462
|
+
var llm = {
|
|
463
|
+
stream: function (options) {
|
|
464
|
+
var step = script[Math.min(calls.length, script.length - 1)]
|
|
465
|
+
calls.push(options)
|
|
466
|
+
return (async function * () {
|
|
467
|
+
if (step === 'nofinish') return
|
|
468
|
+
if (step === 'hang') {
|
|
469
|
+
await new Promise(function (_, reject) {
|
|
470
|
+
if (options.signal.aborted) return reject(new Error('aborted'))
|
|
471
|
+
options.signal.addEventListener('abort', function () { reject(new Error('aborted')) })
|
|
472
|
+
})
|
|
473
|
+
return
|
|
474
|
+
}
|
|
475
|
+
if (step === 'empty' || (step && step.empty)) {
|
|
476
|
+
yield { type: 'finish', reason: { kind: 'error', failure: { message: (step && step.empty) || EMPTY_RESPONSE_TEXT, code: 'EMPTY_RESPONSE' } } }
|
|
477
|
+
return
|
|
478
|
+
}
|
|
479
|
+
if (step && step.error) { yield { type: 'finish', reason: { kind: 'error', failure: { message: step.error } } }; return }
|
|
480
|
+
yield { type: 'finish', reason: { kind: 'stop' } }
|
|
481
|
+
})()
|
|
482
|
+
},
|
|
483
|
+
}
|
|
484
|
+
return { get: function (name) { return name === 'llm' ? llm : null } }
|
|
485
|
+
}
|
|
451
486
|
|
|
452
487
|
// ===== ⑬ 主动嗅探:极短 llm.stream 直调 + 仅配置集合 + 关闭零请求 + 429 复用冷却 =====
|
|
453
488
|
test('⑬主动探测:maxTokens=1 无系统/工具/子代理,仅已配置去重集合;probeEnabled=false 零请求', async () => {
|
|
@@ -470,6 +505,7 @@ test('⑬主动探测:maxTokens=1 无系统/工具/子代理,仅已配置去
|
|
|
470
505
|
assert.equal(h.state, 'healthy')
|
|
471
506
|
assert.ok(h.lastProbe && h.lastSeen)
|
|
472
507
|
assert.equal(typeof h.latencyMs, 'number')
|
|
508
|
+
assert.equal(calls.filter(function (x) { return x.maxTokens === 8 }).length, 0, '一次成功不触发重试(8 token 重试只在空响应后出现)')
|
|
473
509
|
|
|
474
510
|
var offCalls = [], offBoard = mkProbeBoard(); offBoard.probeEnabled = false
|
|
475
511
|
var offState = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
@@ -488,6 +524,8 @@ test('⑬b主动探测失败:429 finish error 复用冷却登记,不污染 b
|
|
|
488
524
|
var result = await mkDispatch(board, mkProbeCtx(calls, '429 RATE_LIMIT'), state).probeConfiguredModels(SID, board)
|
|
489
525
|
assert.equal(result.probed, 1)
|
|
490
526
|
assert.equal(result.healthy, 0)
|
|
527
|
+
assert.equal(calls.length, 1, '真错误一次定性:不触发空响应 8 token 重试')
|
|
528
|
+
assert.equal(calls[0].maxTokens, 1)
|
|
491
529
|
assert.ok(state.modelCooldown['p/a'] > Date.now())
|
|
492
530
|
assert.equal(state.modelHealth[SID]['p/a'].state, 'cooldown')
|
|
493
531
|
assert.ok(state.modelHealth[SID]['p/a'].lastProbe)
|
|
@@ -495,6 +533,375 @@ test('⑬b主动探测失败:429 finish error 复用冷却登记,不污染 b
|
|
|
495
533
|
} finally { console.error = origErr }
|
|
496
534
|
})
|
|
497
535
|
|
|
536
|
+
// ===== ⑬c 空响应重试(task-mvrfp1rj):首探 maxTokens=1 空 → 8 重试有内容 = 🟢 探测通过 =====
|
|
537
|
+
// 语义矩阵(用户 2026-10-28 拍板):①1 token 空→8 重试一次;②重试有内容→🟢;④真错误/超时/无 finish 不重试。
|
|
538
|
+
test('⑬c空响应重试:首探 maxTokens=1 空 → maxTokens=8 重试有内容 → 🟢探测通过,不喂冷却', async () => {
|
|
539
|
+
var calls = [], board = seed(SID); board.workerModel = 'p/a'
|
|
540
|
+
var state = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
541
|
+
var result = await mkDispatch(board, mkProbeScriptCtx(calls, ['empty', 'stop']), state).probeConfiguredModels(SID, board)
|
|
542
|
+
assert.equal(result.probed, 1)
|
|
543
|
+
assert.equal(result.healthy, 1, '重试有内容 = 探测通过(存活)')
|
|
544
|
+
assert.equal(calls.length, 2, '空响应触发且只触发一次重试')
|
|
545
|
+
assert.deepEqual(calls.map(function (x) { return x.maxTokens }), [1, 8], '首探 1 token → 重试 8 token')
|
|
546
|
+
for (var i = 0; i < calls.length; i++) {
|
|
547
|
+
assert.equal(calls[i].messages[0].content[0].text, 'ping')
|
|
548
|
+
assert.equal('system' in calls[i], false)
|
|
549
|
+
assert.equal('tools' in calls[i], false)
|
|
550
|
+
assert.equal('sessionId' in calls[i], false, '重试同样是最小直调(不创建/绑定子会话)')
|
|
551
|
+
}
|
|
552
|
+
var h = state.modelHealth[SID]['p/a']
|
|
553
|
+
assert.equal(h.state, 'healthy')
|
|
554
|
+
assert.equal(h.observed, 'healthy')
|
|
555
|
+
assert.match(h.observedNote, /主动探测通过/)
|
|
556
|
+
assert.equal('p/a' in state.modelCooldown, false, '存活不喂冷却')
|
|
557
|
+
assert.equal(Object.keys(state.badModels).length, 0, '探测通过不碰熔断表')
|
|
558
|
+
})
|
|
559
|
+
|
|
560
|
+
// ===== ⑬d 空响应分治(task-mvrqdw4k·用户拍板,修正 task-mvrfp1rj 的「重试仍空=不可调用」旧口径)=====
|
|
561
|
+
// 可用性实证(三重,决定性):v4.1-flash 正产出当轮对话、两次「失败」皆我方固定 60s 挂死闸误杀(已修自适应)
|
|
562
|
+
// ——它是慢不是死,纯空响应=供应商怪癖判 🟡degraded 存活(不喂冷却/路由仍可选用);额度/凭据/连接信号=真错维持失败臂。
|
|
563
|
+
test('⑬d空响应分治:重试仍空(纯 EMPTY_RESPONSE)判 🟡degraded 存活不喂冷却;quota 文本仍由既有嗅探接管走失败臂', async () => {
|
|
564
|
+
var calls = [], board = seed(SID); board.workerModel = 'p/a'
|
|
565
|
+
var state = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
566
|
+
var origErr = console.error; console.error = function () {}
|
|
567
|
+
try {
|
|
568
|
+
var result = await mkDispatch(board, mkProbeScriptCtx(calls, ['empty', 'empty']), state).probeConfiguredModels(SID, board)
|
|
569
|
+
assert.equal(result.probed, 1)
|
|
570
|
+
assert.equal(result.healthy, 0, '纯空响应不记 🟢存活(healthy 计数不含它)')
|
|
571
|
+
assert.equal(result.degraded, 1, '纯空响应=供应商怪癖,记 🟡degraded(degraded 计数含它)')
|
|
572
|
+
assert.deepEqual(calls.map(function (x) { return x.maxTokens }), [1, 8], '重试节奏不变(首探 1 token → 重试 8 token)')
|
|
573
|
+
var h = state.modelHealth[SID]['p/a']
|
|
574
|
+
assert.equal(h.state, 'degraded', '🟡降级存活态(syncModelHealth degraded 派生)')
|
|
575
|
+
assert.equal(h.observed, 'degraded', 'observeModel 记降级不记失败')
|
|
576
|
+
assert.match(h.observedNote, /链路存活但极简请求零内容/, '降级 note 标明供应商怪癖(含「零内容」)')
|
|
577
|
+
assert.match(h.observedNote, /maxTokens=8 重试仍空/, '文案保留重试口径便于排查')
|
|
578
|
+
assert.ok(h.lastProbe, '降级也留 lastProbe(探测确实跑过)')
|
|
579
|
+
assert.equal('p/a' in state.modelCooldown, false, '纯空响应不喂冷却嗅探(冷却表零登记)')
|
|
580
|
+
assert.equal(Object.keys(state.badModels).length, 0, '探测不碰 badModels 熔断表(与既有口径一致)')
|
|
581
|
+
|
|
582
|
+
// 同一「重试仍空」路径,重试文本带额度字样 → 失败臂,既有嗅探正则接管(quota 优先级高于空响应判定,契约保留)
|
|
583
|
+
var calls2 = [], state2 = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
584
|
+
var quotaText = '429 额度不足:' + EMPTY_RESPONSE_TEXT
|
|
585
|
+
var r2 = await mkDispatch(board, mkProbeScriptCtx(calls2, ['empty', { empty: quotaText }]), state2).probeConfiguredModels(SID, board)
|
|
586
|
+
assert.equal(r2.healthy, 0)
|
|
587
|
+
assert.equal(r2.degraded, 0, '额度信号=真错,不判 degraded(quota 文本优先级高于空响应判定)')
|
|
588
|
+
assert.deepEqual(calls2.map(function (x) { return x.maxTokens }), [1, 8])
|
|
589
|
+
assert.ok(state2.modelCooldown['p/a'] > Date.now(), 'quota 类文本由 parseModelCooldown 接管(失败臂照旧喂嗅探)')
|
|
590
|
+
assert.equal(state2.modelHealth[SID]['p/a'].state, 'cooldown')
|
|
591
|
+
assert.match(state2.modelHealth[SID]['p/a'].cooldownNote, /额度限流/)
|
|
592
|
+
|
|
593
|
+
// 凭据类(task-mvrqdw4k 矩阵第 3 行):空响应文本带凭据字样 → 失败臂(该供应商全模型不可用),不判 degraded
|
|
594
|
+
var calls3 = [], state3 = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
595
|
+
var credText = '401 invalid api key:' + EMPTY_RESPONSE_TEXT
|
|
596
|
+
var r3 = await mkDispatch(board, mkProbeScriptCtx(calls3, ['empty', { empty: credText }]), state3).probeConfiguredModels(SID, board)
|
|
597
|
+
assert.equal(r3.healthy, 0)
|
|
598
|
+
assert.equal(r3.degraded, 0, '凭据信号=真错,不判 degraded')
|
|
599
|
+
assert.equal(state3.modelHealth[SID]['p/a'].state, 'unknown', '凭据类走失败臂(observed=unknown)')
|
|
600
|
+
assert.match(state3.modelHealth[SID]['p/a'].observedNote, /主动探测失败/)
|
|
601
|
+
assert.equal('p/a' in state3.modelCooldown, false, '凭据文本不命中额度/连接嗅探(不误登记冷却)')
|
|
602
|
+
|
|
603
|
+
// 连接信号(矩阵第 4 行):空响应文本带连接字样 → 失败臂,连接嗅探照旧登记短冷却(语义逐字不变)
|
|
604
|
+
var calls4 = [], state4 = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
605
|
+
var connText = 'ECONNRESET:' + EMPTY_RESPONSE_TEXT
|
|
606
|
+
var r4 = await mkDispatch(board, mkProbeScriptCtx(calls4, ['empty', { empty: connText }]), state4).probeConfiguredModels(SID, board)
|
|
607
|
+
assert.equal(r4.healthy, 0)
|
|
608
|
+
assert.equal(r4.degraded, 0, '连接信号=真错,不判 degraded(连接/超时语义逐字不变)')
|
|
609
|
+
assert.ok(state4.modelCooldown['p/a'] > Date.now(), '连接类文本由 parseModelCooldown 接管(连接短冷却)')
|
|
610
|
+
assert.equal(state4.modelHealth[SID]['p/a'].state, 'cooldown')
|
|
611
|
+
} finally { console.error = origErr }
|
|
612
|
+
})
|
|
613
|
+
|
|
614
|
+
// ===== ⑯ degraded 放行(task-mvrqdw4k):healthyModel 只拦 circuit/cooldown,degraded 必须放行 = 路由仍可选用 =====
|
|
615
|
+
// 闭环论证:额度真耗尽有两条独立通道兜底——①宿主直接抛 QUOTA_EXCEEDED(探测即降级冷却);②若网关伪装成空响应,
|
|
616
|
+
// 真派发 run 会失败 → recentRuns 软闸(≥2 全败)跳过它——探测放宽不放跑真坏模型(探测管可调性、行为闸管质量)。
|
|
617
|
+
test('⑯degraded放行:降级态模型仍被路由选用(healthyModel 不拦 degraded),公开视图同口径透出 🟡', async () => {
|
|
618
|
+
var spawnCalls = []
|
|
619
|
+
var board = seed(SID)
|
|
620
|
+
board.tasks = [mkTask({ size: 'heavy' })] // in-progress + spawn-pending(mkTask 缺省,spawn 落闸重验要求)
|
|
621
|
+
board.laneModels.heavy.worker = ['p/a']
|
|
622
|
+
var state = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
623
|
+
// 直种 degraded 观察态(probeConfiguredModels 空响应分治的同款产物,⑬d 已断言落账路径)
|
|
624
|
+
state.modelHealth[SID] = { 'p/a': { state: 'degraded', observed: 'degraded', observedNote: '主动探测:链路存活但极简请求零内容(EMPTY_RESPONSE 供应商怪癖)', lastProbe: '2026-10-28T00:00:00.000Z', lastSeen: '2026-10-28T00:00:00.000Z', note: '主动探测:链路存活但极简请求零内容', recentRuns: [] } }
|
|
625
|
+
var origErr = console.error; console.error = function () {}
|
|
626
|
+
try {
|
|
627
|
+
var sr = await mkDispatch(board, { subagents: mkActivationSubagents(spawnCalls) }, state).spawnOneShot(SID, board.tasks[0], 'worker')
|
|
628
|
+
assert.ok(sr)
|
|
629
|
+
assert.deepEqual(spawnCalls[0].request.agentOptions, { provider: 'p', model: 'a' }, 'degraded 模型仍被路由选用(healthyModel 只拦 circuit/cooldown)')
|
|
630
|
+
assert.equal(sr.modelSource, 'lane', '档位槽直取(degraded 放行:未降下一跳/未走备用链/未回父模型)')
|
|
631
|
+
// 公开健康视图(get-tasks 数据源 modelHealthView):degraded 派生同口径透出,UI 才能显示 🟡降级
|
|
632
|
+
var view = modelHealthView(board, state.modelHealth[SID], { sid: SID, modelCooldown: state.modelCooldown, badModels: state.badModels })
|
|
633
|
+
assert.equal(view['p/a'].state, 'degraded', '公开视图透出 degraded(仲裁放行的单行派生)')
|
|
634
|
+
assert.match(view['p/a'].note, /链路存活但极简请求零内容/, '视图 note 随降级态透出')
|
|
635
|
+
} finally { console.error = origErr }
|
|
636
|
+
})
|
|
637
|
+
|
|
638
|
+
// ===== ⑬e 真失败不重试(task-mvrfp1rj):无 finish / 超时 语义逐字不变,且不触发 8 token 重试 =====
|
|
639
|
+
test('⑬e真失败不重试:无 finish 与超时仍判探测失败(语义逐字不变),不触发空响应重试', async () => {
|
|
640
|
+
var calls = [], board = seed(SID); board.workerModel = 'p/a'
|
|
641
|
+
var state = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
642
|
+
var origErr = console.error; console.error = function () {}
|
|
643
|
+
try {
|
|
644
|
+
var r1 = await mkDispatch(board, mkProbeScriptCtx(calls, ['nofinish']), state).probeConfiguredModels(SID, board)
|
|
645
|
+
assert.equal(r1.healthy, 0)
|
|
646
|
+
assert.equal(calls.length, 1, '无 finish 一次定性,不重试')
|
|
647
|
+
assert.match(state.modelHealth[SID]['p/a'].observedNote, /未返回 finish/)
|
|
648
|
+
|
|
649
|
+
// 超时:计时器到点 abort(宿主真实行为),probe timeout 文案照旧,同样不重试
|
|
650
|
+
var calls2 = [], state2 = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
651
|
+
var hangCtx = {
|
|
652
|
+
timer: {
|
|
653
|
+
timeout: function (fn) { var t = setTimeout(fn, 0); return function () { clearTimeout(t) } },
|
|
654
|
+
interval: function () { return function () {} }, // 心跳注册(本用例不需要真跑)
|
|
655
|
+
},
|
|
656
|
+
get: function (name) {
|
|
657
|
+
if (name !== 'llm') return null
|
|
658
|
+
return {
|
|
659
|
+
stream: function (options) {
|
|
660
|
+
calls2.push(options)
|
|
661
|
+
return (async function * () {
|
|
662
|
+
await new Promise(function (_, reject) {
|
|
663
|
+
if (options.signal.aborted) return reject(new Error('aborted'))
|
|
664
|
+
options.signal.addEventListener('abort', function () { reject(new Error('aborted')) })
|
|
665
|
+
})
|
|
666
|
+
})()
|
|
667
|
+
},
|
|
668
|
+
}
|
|
669
|
+
},
|
|
670
|
+
}
|
|
671
|
+
var r2 = await mkDispatch(board, hangCtx, state2).probeConfiguredModels(SID, board)
|
|
672
|
+
assert.equal(r2.healthy, 0)
|
|
673
|
+
assert.equal(calls2.length, 1, '超时一次定性,不重试')
|
|
674
|
+
assert.match(state2.modelHealth[SID]['p/a'].observedNote, /probe timeout/, '超时文案沿用既有口径')
|
|
675
|
+
assert.equal('p/a' in state2.modelCooldown, false, 'probe timeout 文本不命中冷却嗅探(既有口径)')
|
|
676
|
+
} finally { console.error = origErr }
|
|
677
|
+
})
|
|
678
|
+
|
|
679
|
+
// ===== ⑰ 探测预算独立(task-mvsv8zym):每尝试各 20s 非共享递减 + 重试超时沿用首探判定 + 超时不覆盖既有观测 =====
|
|
680
|
+
// 事故实证(2026-10-29):旧 15s 共享 deadline 下慢模型首探耗 ~10s 空响应,重试只剩 4.6s 必超时,
|
|
681
|
+
// 被我方预算饿死误记「probe timeout」=模型失败(误导主窗口两次)。记录型计时器捕获预算窗口 ms,不真等 20s。
|
|
682
|
+
function mkRecordTimerCtx(base, delays) {
|
|
683
|
+
base.timer = {
|
|
684
|
+
timeout: function (fn, ms) { delays.push(ms); var t = setTimeout(fn, 0); return function () { clearTimeout(t) } },
|
|
685
|
+
interval: function () { return function () {} }, // 心跳注册(本用例不需要真跑)
|
|
686
|
+
}
|
|
687
|
+
return base
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
test('⑰a探测预算独立:每尝试各 20s 非共享递减;慢模型首探空+重试超时→沿用首探判定 🟡degraded 不喂冷却', async () => {
|
|
691
|
+
var calls = [], delays = [], board = seed(SID); board.workerModel = 'p/a'
|
|
692
|
+
var state = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
693
|
+
var origErr = console.error; console.error = function () {}
|
|
694
|
+
try {
|
|
695
|
+
// 慢模型复刻(事故实证形态):首探空响应 + 重试挂起到超时;记录型立即触发计时器
|
|
696
|
+
var ctx = mkRecordTimerCtx(mkProbeScriptCtx(calls, ['empty', 'hang']), delays)
|
|
697
|
+
var result = await mkDispatch(board, ctx, state).probeConfiguredModels(SID, board)
|
|
698
|
+
assert.equal(result.probed, 1)
|
|
699
|
+
assert.equal(result.healthy, 0)
|
|
700
|
+
assert.equal(result.degraded, 1, '重试超时沿用首探判定:首探纯空响应=🟡degraded,不再误记 fail')
|
|
701
|
+
assert.deepEqual(calls.map(function (x) { return x.maxTokens }), [1, 8], '重试节奏不变(首探 1 token → 重试 8 token)')
|
|
702
|
+
assert.deepEqual(delays, [20000, 20000], '每次尝试独立预算:两次计时器窗口各 20s(旧共享制第二次只拿残余递减值)')
|
|
703
|
+
var h = state.modelHealth[SID]['p/a']
|
|
704
|
+
assert.equal(h.state, 'degraded', '🟡降级存活态(非 fail/cooldown/unknown)')
|
|
705
|
+
assert.equal(h.observed, 'degraded')
|
|
706
|
+
assert.match(h.observedNote, /重试预算耗尽(慢模型),沿用首探判定/, '文案标明沿用首探判定')
|
|
707
|
+
assert.match(h.observedNote, /链路存活但极简请求零内容/, '降级 note 保留供应商怪癖口径')
|
|
708
|
+
assert.ok(h.lastProbe, 'degraded 也留 lastProbe(探测确实跑过)')
|
|
709
|
+
assert.equal('p/a' in state.modelCooldown, false, '重试超时+纯空响应不喂冷却')
|
|
710
|
+
assert.equal(Object.keys(state.badModels).length, 0, '不碰熔断表')
|
|
711
|
+
} finally { console.error = origErr }
|
|
712
|
+
})
|
|
713
|
+
|
|
714
|
+
test('⑰b重试超时+首探真错信号:沿用首探判定走失败臂,额度喂冷却/凭据不误登记', async () => {
|
|
715
|
+
var board = seed(SID); board.workerModel = 'p/a'
|
|
716
|
+
var origErr = console.error; console.error = function () {}
|
|
717
|
+
try {
|
|
718
|
+
// 额度类:首探空响应文本带额度字样 + 重试超时 → 沿用首探判定=fail → 嗅探登记冷却
|
|
719
|
+
var calls = [], delays = [], state = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
720
|
+
var quotaText = '429 额度不足:' + EMPTY_RESPONSE_TEXT
|
|
721
|
+
var ctx = mkRecordTimerCtx(mkProbeScriptCtx(calls, [{ empty: quotaText }, 'hang']), delays)
|
|
722
|
+
var r = await mkDispatch(board, ctx, state).probeConfiguredModels(SID, board)
|
|
723
|
+
assert.equal(r.healthy, 0)
|
|
724
|
+
assert.equal(r.degraded, 0, '额度信号=真错,沿用首探判定不判 degraded')
|
|
725
|
+
assert.deepEqual(delays, [20000, 20000], '重试同样拿独立 20s 预算')
|
|
726
|
+
assert.ok(state.modelCooldown['p/a'] > Date.now(), '沿用首探判定后仍喂嗅探(quota 真错)')
|
|
727
|
+
assert.equal(state.modelHealth[SID]['p/a'].state, 'cooldown')
|
|
728
|
+
assert.match(state.modelHealth[SID]['p/a'].observedNote, /重试预算耗尽(慢模型),沿用首探判定/)
|
|
729
|
+
|
|
730
|
+
// 凭据类:沿用首探判定走失败臂但不误登记冷却(CREDENTIAL 不命中额度/连接嗅探)
|
|
731
|
+
var calls2 = [], delays2 = [], state2 = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
732
|
+
var credText = '401 invalid api key:' + EMPTY_RESPONSE_TEXT
|
|
733
|
+
var ctx2 = mkRecordTimerCtx(mkProbeScriptCtx(calls2, [{ empty: credText }, 'hang']), delays2)
|
|
734
|
+
var r2 = await mkDispatch(board, ctx2, state2).probeConfiguredModels(SID, board)
|
|
735
|
+
assert.equal(r2.healthy, 0)
|
|
736
|
+
assert.equal(r2.degraded, 0, '凭据信号=真错,沿用首探判定不判 degraded')
|
|
737
|
+
assert.equal(state2.modelHealth[SID]['p/a'].state, 'unknown', '凭据类走失败臂(observed=unknown)')
|
|
738
|
+
assert.match(state2.modelHealth[SID]['p/a'].observedNote, /重试预算耗尽(慢模型),沿用首探判定/)
|
|
739
|
+
assert.equal('p/a' in state2.modelCooldown, false, '凭据文本不命中额度/连接嗅探(不误登记冷却)')
|
|
740
|
+
} finally { console.error = origErr }
|
|
741
|
+
})
|
|
742
|
+
|
|
743
|
+
test('⑰c超时不覆盖既有观测:先 healthy 再超时→仍 healthy+note 更新+延迟保留;首探超时新条目仍 unknown', async () => {
|
|
744
|
+
var board = seed(SID); board.workerModel = 'p/a'
|
|
745
|
+
var origErr = console.error; console.error = function () {}
|
|
746
|
+
try {
|
|
747
|
+
// 先 healthy:正常探测通过
|
|
748
|
+
var state = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
749
|
+
var calls = []
|
|
750
|
+
await mkDispatch(board, mkProbeScriptCtx(calls, ['stop']), state).probeConfiguredModels(SID, board)
|
|
751
|
+
var h = state.modelHealth[SID]['p/a']
|
|
752
|
+
assert.equal(h.state, 'healthy')
|
|
753
|
+
assert.match(h.observedNote, /主动探测通过/)
|
|
754
|
+
var latBefore = h.latencyMs
|
|
755
|
+
assert.equal(typeof latBefore, 'number')
|
|
756
|
+
// 再超时:hang + 立即触发计时器 → 判定保留,observed/state/latency 不倒退
|
|
757
|
+
var calls2 = [], delays2 = []
|
|
758
|
+
var ctx2 = mkRecordTimerCtx(mkProbeScriptCtx(calls2, ['hang']), delays2)
|
|
759
|
+
await mkDispatch(board, ctx2, state).probeConfiguredModels(SID, board)
|
|
760
|
+
assert.deepEqual(calls2.map(function (x) { return x.maxTokens }), [1], '首探超时无重试(⑬e 契约保留)')
|
|
761
|
+
assert.equal(h.state, 'healthy', '超时不覆盖既有 healthy 观测(不倒退成 unknown)')
|
|
762
|
+
assert.equal(h.observed, 'healthy')
|
|
763
|
+
assert.equal(h.latencyMs, latBefore, '超时不覆写既有延迟(保留原 healthy 观测延迟)')
|
|
764
|
+
assert.match(h.observedNote, /主动探测超时(慢模型,判定保留)/, 'note 更新为超时+判定保留')
|
|
765
|
+
assert.match(h.observedNote, /probe timeout/, '保留超时细节便于排查')
|
|
766
|
+
assert.ok(h.lastProbe, '超时也留 lastProbe(探测确实跑过)')
|
|
767
|
+
|
|
768
|
+
// 首探超时+无既有观测:新条目仍 unknown + note 更新(不误记 healthy/fail/cooldown)
|
|
769
|
+
var state2 = { modelHealth: {}, modelCooldown: {}, badModels: {} }
|
|
770
|
+
var calls3 = []
|
|
771
|
+
var ctx3 = mkRecordTimerCtx(mkProbeScriptCtx(calls3, ['hang']), [])
|
|
772
|
+
var r3 = await mkDispatch(board, ctx3, state2).probeConfiguredModels(SID, board)
|
|
773
|
+
assert.equal(r3.healthy, 0)
|
|
774
|
+
var h2 = state2.modelHealth[SID]['p/a']
|
|
775
|
+
assert.equal(h2.state, 'unknown', '首探超时新条目=unknown(不误记 fail)')
|
|
776
|
+
assert.equal(h2.observed, 'unknown')
|
|
777
|
+
assert.match(h2.observedNote, /主动探测超时(慢模型,判定保留)/)
|
|
778
|
+
assert.equal('p/a' in state2.modelCooldown, false, 'probe timeout 不命中冷却嗅探(既有口径)')
|
|
779
|
+
} finally { console.error = origErr }
|
|
780
|
+
})
|
|
781
|
+
|
|
782
|
+
// ===== ⑬f 探测恢复熔断解除(task-mvvw71nc):连续 2 次探测通过翻绿,红态可证活 =====
|
|
783
|
+
// 用户 2026-10-31 实测:「配置模型熔断后状态变红,再下一个探测周期是否会变回来,实测没有」——旧 ok 臂只写
|
|
784
|
+
// observed='healthy',从不碰 badModels 熔断表,syncModelHealth 覆盖闸每次重查熔断表压回 circuit(探测能证
|
|
785
|
+
// 「可调用」却解除不了「不可调」判定)。修法(用户拍板「乙+连击」):探测 ok 臂累计 probeOkStreak,连续 2 次
|
|
786
|
+
// (15min 双周期确认)→ delete badModels + 连击清零 + 重派生 + 通知;中断(失败/超时/degraded)计数清零;
|
|
787
|
+
// 单次成功不复位(不放过「能回 1 token 但跑长任务就崩」的偶发病模型)。
|
|
788
|
+
function mkRecoverState(k) {
|
|
789
|
+
var st = { modelHealth: {}, modelCooldown: {}, badModels: {}, probeOkStreak: {}, modelFailStreak: {} }
|
|
790
|
+
st.badModels[k] = true
|
|
791
|
+
st.modelFailStreak[k] = 3
|
|
792
|
+
st.modelHealth[SID] = {}
|
|
793
|
+
st.modelHealth[SID]['p/a'] = { state: 'circuit', observed: 'healthy', observedNote: '主动探测通过', lastProbe: '2026-10-30T00:00:00.000Z', lastSeen: '2026-10-30T00:00:00.000Z', circuitNote: '连续 3 次 spawn/续跑即被宿主拆除(模型失败),已熔断', recentRuns: [] }
|
|
794
|
+
return st
|
|
795
|
+
}
|
|
796
|
+
// 捕获 pushSysNote 的派发桩(mkDispatch 的 pushSysNote 是 no-op,恢复通知投递断言需要显式捕获);
|
|
797
|
+
// 接线与 mkDispatch 同形,state 各表按引用共享(probeOkStreak 由测试预置,工厂就地取用同一引用)。
|
|
798
|
+
function mkRecoverDispatch(board, ctxOver, state, notes) {
|
|
799
|
+
var shared = Object.assign({ knownSessions: {}, dispatchedEver: {}, teamModeCache: {}, activeRuns: {} }, state)
|
|
800
|
+
return createDispatch(
|
|
801
|
+
Object.assign({ fs: {}, effect: function () {}, get: function () { return null } }, ctxOver || {}),
|
|
802
|
+
shared,
|
|
803
|
+
{
|
|
804
|
+
rt: async function () { return board }, wt: async function () {},
|
|
805
|
+
mutateLocked: async function (sid, fn) { return fn(board) },
|
|
806
|
+
kickCycle: function () {}, rootForSession: function () { return { id: SID } },
|
|
807
|
+
sessionCwd: function () { return '' }, withTimeout: function (p) { return p }, runsFor: function (sid) { return shared.activeRuns[sid] || (shared.activeRuns[sid] = {}) },
|
|
808
|
+
feedbackOn: function () { return true }, pushSysNote: function (sid, text) { notes.push({ sid: sid, text: String(text) }) }, maybeNotify: function () {}, notifyTaskDone: function () {},
|
|
809
|
+
})
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
test('⑬f探测恢复熔断解除:单次通过不复位;连续 2 次→badModels 清除+连击清零+翻 healthy+通知投递', async () => {
|
|
813
|
+
var board = seed(SID); board.workerModel = 'p/a'
|
|
814
|
+
var k = SID + '|p/a'
|
|
815
|
+
var origErr = console.error; console.error = function () {}
|
|
816
|
+
try {
|
|
817
|
+
// ① 已熔断 + 探测成功 1 次 → 仍 circuit(计数 1,badModels/连击未动,零通知)
|
|
818
|
+
var calls = [], notes = [], st = mkRecoverState(k)
|
|
819
|
+
var d = mkRecoverDispatch(board, mkProbeScriptCtx(calls, ['stop']), st, notes)
|
|
820
|
+
var r1 = await d.probeConfiguredModels(SID, board)
|
|
821
|
+
assert.equal(r1.healthy, 1)
|
|
822
|
+
assert.equal(st.badModels[k], true, '单次探测成功不解除熔断(过滤偶发响应)')
|
|
823
|
+
assert.equal(st.probeOkStreak[k], 1, '恢复连击累计 1')
|
|
824
|
+
assert.equal(st.modelFailStreak[k], 3, '熔断连击未被单次探测清零')
|
|
825
|
+
assert.equal(st.modelHealth[SID]['p/a'].state, 'circuit', '覆盖闸仍压回 circuit(半程红态)')
|
|
826
|
+
assert.equal(st.modelHealth[SID]['p/a'].observed, 'healthy', 'observed 已翻 healthy(覆盖态仍 circuit)')
|
|
827
|
+
assert.equal(notes.length, 0, '未复位不发通知')
|
|
828
|
+
|
|
829
|
+
// ② 连续第 2 次 → 解除:badModels 清除 + 连击清零 + 恢复计数清零 + state 翻 healthy + 通知投递
|
|
830
|
+
var r2 = await d.probeConfiguredModels(SID, board)
|
|
831
|
+
assert.equal(r2.healthy, 1)
|
|
832
|
+
assert.equal(k in st.badModels, false, '连续 2 次探测通过 → 解除熔断')
|
|
833
|
+
assert.equal(k in st.modelFailStreak, false, '熔断连击清零(复位对称操作)')
|
|
834
|
+
assert.equal(k in st.probeOkStreak, false, '恢复连击复位后清零')
|
|
835
|
+
assert.equal(st.modelHealth[SID]['p/a'].state, 'healthy', '熔断解除后重派生翻绿(🔴→🟢)')
|
|
836
|
+
assert.equal(st.modelHealth[SID]['p/a'].observed, 'healthy')
|
|
837
|
+
assert.equal(notes.length, 1, '解除熔断投递系统通知')
|
|
838
|
+
assert.equal(notes[0].sid, SID)
|
|
839
|
+
assert.equal(notes[0].text, '模型 p/a 连续 2 次探测通过,熔断已解除,恢复调度')
|
|
840
|
+
|
|
841
|
+
// ⑤ 解除后再探测成功:未熔断 → 零副作用(不重建计数/不重复通知)
|
|
842
|
+
await d.probeConfiguredModels(SID, board)
|
|
843
|
+
assert.equal(k in st.probeOkStreak, false, '未熔断模型探测成功零副作用(不建多余计数)')
|
|
844
|
+
assert.equal(notes.length, 1, '未熔断不再发通知')
|
|
845
|
+
} finally { console.error = origErr }
|
|
846
|
+
})
|
|
847
|
+
|
|
848
|
+
test('⑬f2探测恢复中断清零:中途一次失败→计数清零重新计;degraded 🟡 不计入不复位', async () => {
|
|
849
|
+
var board = seed(SID); board.workerModel = 'p/a'
|
|
850
|
+
var k = SID + '|p/a'
|
|
851
|
+
var origErr = console.error; console.error = function () {}
|
|
852
|
+
try {
|
|
853
|
+
// ③ 成功 1 次 → 失败 1 次(计数清零)→ 成功 1 次:仍 circuit 且计数 1(必须真的连续,偶发成功不叠加)
|
|
854
|
+
var calls = [], notes = [], st = mkRecoverState(k)
|
|
855
|
+
var d = mkRecoverDispatch(board, mkProbeScriptCtx(calls, ['stop', { error: 'boom' }, 'stop', 'stop']), st, notes)
|
|
856
|
+
await d.probeConfiguredModels(SID, board)
|
|
857
|
+
assert.equal(st.probeOkStreak[k], 1)
|
|
858
|
+
var rf = await d.probeConfiguredModels(SID, board)
|
|
859
|
+
assert.equal(rf.healthy, 0)
|
|
860
|
+
assert.equal(k in st.probeOkStreak, false, '中途一次探测失败 → 恢复计数清零')
|
|
861
|
+
assert.equal(st.badModels[k], true, '失败不复位熔断')
|
|
862
|
+
await d.probeConfiguredModels(SID, board)
|
|
863
|
+
assert.equal(st.probeOkStreak[k], 1, '清零后重新计(不是累计 2)')
|
|
864
|
+
assert.equal(st.badModels[k], true, '仍需再连续一次才解除')
|
|
865
|
+
assert.equal(st.modelHealth[SID]['p/a'].state, 'circuit')
|
|
866
|
+
assert.equal(notes.length, 0)
|
|
867
|
+
await d.probeConfiguredModels(SID, board)
|
|
868
|
+
assert.equal(k in st.badModels, false, '清零重计后连续 2 次才解除')
|
|
869
|
+
assert.equal(notes.length, 1)
|
|
870
|
+
|
|
871
|
+
// ④ degraded 🟡(纯空响应供应商怪癖)非探测成功 → 不复位且计数清零(只有 🟢 通过才算证活)
|
|
872
|
+
var calls2 = [], notes2 = [], st2 = mkRecoverState(k)
|
|
873
|
+
var d2 = mkRecoverDispatch(board, mkProbeScriptCtx(calls2, ['stop', 'empty', 'empty']), st2, notes2)
|
|
874
|
+
await d2.probeConfiguredModels(SID, board)
|
|
875
|
+
assert.equal(st2.probeOkStreak[k], 1)
|
|
876
|
+
var rd = await d2.probeConfiguredModels(SID, board)
|
|
877
|
+
assert.equal(rd.degraded, 1, '纯空响应判 🟡degraded 存活')
|
|
878
|
+
assert.equal(k in st2.probeOkStreak, false, 'degraded 不计入恢复连击(清零)')
|
|
879
|
+
assert.equal(st2.badModels[k], true, 'degraded 不复位熔断')
|
|
880
|
+
assert.equal(st2.modelHealth[SID]['p/a'].state, 'circuit', '熔断覆盖态仍在(degraded 未解除)')
|
|
881
|
+
assert.equal(notes2.length, 0, 'degraded 不发解除通知')
|
|
882
|
+
|
|
883
|
+
// ④b 探测超时(判定保留臂)同样非成功 → 计数清零不复位
|
|
884
|
+
var calls4 = [], notes4 = [], st4 = mkRecoverState(k)
|
|
885
|
+
await mkRecoverDispatch(board, mkProbeScriptCtx(calls4, ['stop']), st4, notes4).probeConfiguredModels(SID, board)
|
|
886
|
+
assert.equal(st4.probeOkStreak[k], 1)
|
|
887
|
+
var ctx4 = mkRecordTimerCtx(mkProbeScriptCtx(calls4, ['hang']), [])
|
|
888
|
+
await mkRecoverDispatch(board, ctx4, st4, notes4).probeConfiguredModels(SID, board)
|
|
889
|
+
assert.equal(k in st4.probeOkStreak, false, '探测超时(判定保留)同样不计入恢复连击')
|
|
890
|
+
assert.equal(st4.badModels[k], true, '超时不复位熔断')
|
|
891
|
+
assert.equal(st4.modelHealth[SID]['p/a'].state, 'circuit')
|
|
892
|
+
assert.equal(notes4.length, 0)
|
|
893
|
+
|
|
894
|
+
// ⑤ 未熔断模型:探测成功零副作用(不建计数/不通知)
|
|
895
|
+
var calls3 = [], notes3 = [], st3 = { modelHealth: {}, modelCooldown: {}, badModels: {}, probeOkStreak: {}, modelFailStreak: {} }
|
|
896
|
+
var d3 = mkRecoverDispatch(board, mkProbeScriptCtx(calls3, ['stop', 'stop']), st3, notes3)
|
|
897
|
+
await d3.probeConfiguredModels(SID, board)
|
|
898
|
+
await d3.probeConfiguredModels(SID, board)
|
|
899
|
+
assert.deepEqual(Object.keys(st3.probeOkStreak), [], '未熔断模型探测成功零副作用(不建多余计数)')
|
|
900
|
+
assert.deepEqual(Object.keys(st3.badModels), [])
|
|
901
|
+
assert.equal(notes3.length, 0, '未熔断无通知')
|
|
902
|
+
} finally { console.error = origErr }
|
|
903
|
+
})
|
|
904
|
+
|
|
498
905
|
async function runVerifierHealth(result) {
|
|
499
906
|
var settle = null, pending = new Promise(function (resolve) { settle = resolve })
|
|
500
907
|
var board = seed(SID); board.verifierModel = 'p/v'; board.tasks = [mkTask({ status: 'verifying' })]
|
|
@@ -535,6 +942,9 @@ test('⑭被动健康面:Verifier 成功 run 记健康/延迟/近5次,失败
|
|
|
535
942
|
test('⑮模型路由健康点:🟢🟡🔴⚪ + 悬浮详情 + 主动探测配置 + build 产物标记', () => {
|
|
536
943
|
assert.ok(src.indexOf('function ModelHealthDot(props)') >= 0)
|
|
537
944
|
for (const icon of ['🟢', '🟡', '🔴', '⚪']) assert.ok(src.indexOf(icon) >= 0, '缺健康点图标 ' + icon)
|
|
945
|
+
// task-mvrqdw4k 错误码分治:degraded(🟡降级,与 cooldown 同图标不同文案)挂 MODEL_HEALTH_META,源与产物双锁定
|
|
946
|
+
assert.ok(src.indexOf("degraded: { icon: '🟡', label: '降级'") >= 0, '健康点缺 🟡降级档(task-mvrqdw4k 错误码分治)')
|
|
947
|
+
assert.ok(built.indexOf("degraded: { icon: '🟡', label: '降级'") >= 0, 'client.js 缺 🟡降级档(需 npm run build-client)')
|
|
538
948
|
assert.ok(src.indexOf("role: 'img'") >= 0 && src.indexOf("'aria-label': props.model + ' ' + label") >= 0, '健康点缺可访问名称')
|
|
539
949
|
for (const text of ['最近探测:', '最近信号:', '最近延迟:', '次 run:']) assert.ok(src.indexOf(text) >= 0, '悬浮详情缺 ' + text)
|
|
540
950
|
assert.match(src, /React\.createElement\(ModelHealthDot, \{ model: m, health: props\.health \}\)/, '链芯片未接健康点')
|
|
@@ -592,3 +592,137 @@ test('备用链F5:无备用链配置时冷却降档/熔断降跳与现状逐
|
|
|
592
592
|
} finally { console.error = origErr }
|
|
593
593
|
})
|
|
594
594
|
|
|
595
|
+
// ===== 连败软闸 + 目录失效闸(task-mvd3li6):healthyModel 增读 modelHealth recentRuns + 模型目录 =====
|
|
596
|
+
// 断言清单(对应任务验收口径):
|
|
597
|
+
// S1 连败软闸:recentRuns ≥2 且全败 → 取模跳过该模型(槽内次选赢);1 次失败不闸(行为不变);软熔断不落 badModels
|
|
598
|
+
// S2 一次成功复位:成功条目进 recentRuns 滚动窗 → 「全败」不成立 → 该模型重新被取(聚合表归零语义复用)
|
|
599
|
+
// D1 目录失效闸·卡级特例:模型已从模型目录移除 → 跳过该级降下一跳 + 卡 messages 留痕「配置已失效:X(已从模型目录移除)」
|
|
600
|
+
// D2 目录失效闸·档位槽:槽内目录外模型跳过取次选;槽全目录外 → 代表入链被存在闸拦下降下一跳
|
|
601
|
+
// D3 目录失效闸·备用链:链上目录外项跳过取下一个健康者(留痕)
|
|
602
|
+
// D4 目录不可得(llm 缺失/空目录/provider 枚举失败)→ 目录闸整体跳过,行为逐字不变(保守语义不误伤)
|
|
603
|
+
// 病模型种法:state.modelHealth 直种 health entry(recentRuns 为聚合表近 5 次 run 成败,settleRunRecord 喂给)
|
|
604
|
+
function seedRunsHealth(state, model, runs) {
|
|
605
|
+
if (!state.modelHealth) state.modelHealth = {}
|
|
606
|
+
if (!state.modelHealth[SID]) state.modelHealth[SID] = {}
|
|
607
|
+
state.modelHealth[SID][model] = { state: 'unknown', observed: 'unknown', lastSeen: null, lastProbe: null, note: '测试种入', observedNote: '测试种入', recentRuns: runs }
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
test('连败软闸S1:recentRuns ≥2 全败 → 槽内取模跳过该模型次选赢(软熔断不落 badModels);单次失败不闸', async function () {
|
|
611
|
+
var origErr = console.error; console.error = function () {}
|
|
612
|
+
try {
|
|
613
|
+
// 槽 [first, second],first 近 2 次全败 → 软闸 → second 赢(modelSource 仍 lane)
|
|
614
|
+
var calls = []
|
|
615
|
+
var b = laneArrBoard({ size: 'heavy' }, { heavy: { worker: ['lw/first', 'lw/second'], verifier: [] } })
|
|
616
|
+
var st = { badModels: {} }
|
|
617
|
+
seedRunsHealth(st, 'lw/first', [{ ok: false, at: '2026-01-01T00:00:00.000Z', latencyMs: 78000 }, { ok: false, at: '2026-01-01T00:01:00.000Z', latencyMs: null }])
|
|
618
|
+
var sr = await mkDispatch(b, { subagents: mkActivationSubagents(calls) }, st).spawnOneShot(SID, b.tasks[0], 'worker')
|
|
619
|
+
assert.ok(sr)
|
|
620
|
+
assert.deepEqual(calls[0].request.agentOptions, { provider: 'lw', model: 'second' }, '连败两次 → 槽内跳过该模型次选赢')
|
|
621
|
+
assert.equal(sr.modelSource, 'lane', '仍记 lane 跳(软闸=槽内健康判定,不走降档)')
|
|
622
|
+
assert.equal(Object.keys(st.badModels).length, 0, '软熔断不落 badModels 黑名单(与硬熔断语义分治)')
|
|
623
|
+
// 单次失败(< 2)不闸:first 照取(取模行为不变)
|
|
624
|
+
var calls2 = []
|
|
625
|
+
var b2 = laneArrBoard({ size: 'heavy' }, { heavy: { worker: ['lw/first', 'lw/second'], verifier: [] } })
|
|
626
|
+
var st2 = { badModels: {} }
|
|
627
|
+
seedRunsHealth(st2, 'lw/first', [{ ok: false, at: '2026-01-01T00:00:00.000Z', latencyMs: 78000 }])
|
|
628
|
+
await mkDispatch(b2, { subagents: mkActivationSubagents(calls2) }, st2).spawnOneShot(SID, b2.tasks[0], 'worker')
|
|
629
|
+
assert.deepEqual(calls2[0].request.agentOptions, { provider: 'lw', model: 'first' }, '单次失败未达软闸线,取模行为不变')
|
|
630
|
+
} finally { console.error = origErr }
|
|
631
|
+
})
|
|
632
|
+
|
|
633
|
+
test('连败软闸S2:一次成功即复位——成功条目进 recentRuns 后该模型重新被取(聚合表归零语义复用)', async function () {
|
|
634
|
+
var origErr = console.error; console.error = function () {}
|
|
635
|
+
try {
|
|
636
|
+
var calls = []
|
|
637
|
+
var b = laneArrBoard({ size: 'heavy' }, { heavy: { worker: ['lw/first', 'lw/second'], verifier: [] } })
|
|
638
|
+
var st = { badModels: {} }
|
|
639
|
+
seedRunsHealth(st, 'lw/first', [{ ok: false, at: '2026-01-01T00:00:00.000Z', latencyMs: 78000 }, { ok: true, at: '2026-01-01T00:01:00.000Z', latencyMs: 80 }])
|
|
640
|
+
var sr = await mkDispatch(b, { subagents: mkActivationSubagents(calls) }, st).spawnOneShot(SID, b.tasks[0], 'worker')
|
|
641
|
+
assert.ok(sr)
|
|
642
|
+
assert.deepEqual(calls[0].request.agentOptions, { provider: 'lw', model: 'first' }, '成功条目进窗 → 「全败」不成立 → 软闸复位 first 重新被取')
|
|
643
|
+
assert.equal(sr.modelSource, 'lane')
|
|
644
|
+
} finally { console.error = origErr }
|
|
645
|
+
})
|
|
646
|
+
|
|
647
|
+
test('目录闸D1:卡级特例模型已从模型目录移除 → 跳过该级降下一跳 + 卡 messages 留痕(特例只豁免健康闸不豁免存在闸)', async function () {
|
|
648
|
+
var origErr = console.error; console.error = function () {}
|
|
649
|
+
try {
|
|
650
|
+
var calls = []
|
|
651
|
+
var b = laneBoard({ model: 'gone/gone-m', size: 'heavy' })
|
|
652
|
+
await mkDispatch(b, { subagents: mkActivationSubagents(calls), get: mkLlmCtx({ lw: [{ id: 'lane-w' }], bw: [{ id: 'board-w' }] }).get }).spawnOneShot(SID, b.tasks[0], 'worker')
|
|
653
|
+
assert.deepEqual(calls[0].request.agentOptions, { provider: 'lw', model: 'lane-w' }, '卡级特例目录外 → 跳过该级降 lane')
|
|
654
|
+
assert.equal(b.tasks[0].runs[0].modelSource, 'lane')
|
|
655
|
+
var sm = (b.tasks[0].messages || []).filter(function (m) { return m.kind === 'stale-model' })
|
|
656
|
+
assert.equal(sm.length, 1, '卡 messages 记一条配置失效留痕')
|
|
657
|
+
assert.match(sm[0].text, /配置已失效:gone\/gone-m(已从模型目录移除)/, '留痕含模型与失效说明')
|
|
658
|
+
} finally { console.error = origErr }
|
|
659
|
+
})
|
|
660
|
+
|
|
661
|
+
test('目录闸D2:档位槽目录外模型跳过取次选;槽全目录外 → 代表入链被存在闸拦下降下一跳(留痕)', async function () {
|
|
662
|
+
var origErr = console.error; console.error = function () {}
|
|
663
|
+
try {
|
|
664
|
+
// 形态1:槽 [gone/gone-a, lw/second],目录含 lw/second → 槽内跳过 gone-a 取 second
|
|
665
|
+
var calls = []
|
|
666
|
+
var b1 = laneArrBoard({ size: 'heavy' }, { heavy: { worker: ['gone/gone-a', 'lw/second'], verifier: [] } })
|
|
667
|
+
await mkDispatch(b1, { subagents: mkActivationSubagents(calls), get: mkLlmCtx({ lw: [{ id: 'second' }], bw: [{ id: 'board-w' }] }).get }).spawnOneShot(SID, b1.tasks[0], 'worker')
|
|
668
|
+
assert.deepEqual(calls[0].request.agentOptions, { provider: 'lw', model: 'second' }, '槽内目录外模型跳过,次选赢')
|
|
669
|
+
// 形态2:槽全目录外 → 首项代表入链 → cands 循环存在闸拦下 → 降板级
|
|
670
|
+
var calls2 = []
|
|
671
|
+
var b2 = laneArrBoard({ size: 'heavy' }, { heavy: { worker: ['gone/gone-a', 'gone/gone-b'], verifier: [] } })
|
|
672
|
+
await mkDispatch(b2, { subagents: mkActivationSubagents(calls2), get: mkLlmCtx({ lw: [{ id: 'lane-w' }], bw: [{ id: 'board-w' }] }).get }).spawnOneShot(SID, b2.tasks[0], 'worker')
|
|
673
|
+
assert.deepEqual(calls2[0].request.agentOptions, { provider: 'bw', model: 'board-w' }, '槽全目录外 → 代表被存在闸拦下降板级')
|
|
674
|
+
var sm2 = (b2.tasks[0].messages || []).filter(function (m) { return m.kind === 'stale-model' })
|
|
675
|
+
assert.equal(sm2.length, 1, '代表级留痕一条(per 模型去重)')
|
|
676
|
+
assert.match(sm2[0].text, /配置已失效:gone\/gone-a(已从模型目录移除)/)
|
|
677
|
+
} finally { console.error = origErr }
|
|
678
|
+
})
|
|
679
|
+
|
|
680
|
+
test('目录闸D3:备用链目录外项跳过取下一个健康者 + 留痕;候选健康时备用链零触发(行为不变)', async function () {
|
|
681
|
+
var origErr = console.error; console.error = function () {}
|
|
682
|
+
try {
|
|
683
|
+
// 候选(lane)冷却 → 备用链 [gone/fb-a, fb/fb-b],目录含 fb/fb-b → 跳过 gone/fb-a 取 fb-b(留痕一条)
|
|
684
|
+
var calls = []
|
|
685
|
+
var b = laneBoard({ size: 'heavy' }, { fallbackModels: ['gone/fb-a', 'fb/fb-b'] })
|
|
686
|
+
var st = { modelCooldown: { 'lw/lane-w': Date.now() + 3600000 } }
|
|
687
|
+
await mkDispatch(b, { subagents: mkActivationSubagents(calls), get: mkLlmCtx({ lw: [{ id: 'lane-w' }], fb: [{ id: 'fb-b' }] }).get }, st).spawnOneShot(SID, b.tasks[0], 'worker')
|
|
688
|
+
assert.deepEqual(calls[0].request.agentOptions, { provider: 'fb', model: 'fb-b' }, '备用链目录外项跳过取下一个健康者')
|
|
689
|
+
assert.equal(b.tasks[0].runs[0].modelSource, 'fallback')
|
|
690
|
+
var sm = (b.tasks[0].messages || []).filter(function (m) { return m.kind === 'stale-model' })
|
|
691
|
+
assert.equal(sm.length, 1, '备用链目录外留痕一条')
|
|
692
|
+
assert.match(sm[0].text, /配置已失效:gone\/fb-a(已从模型目录移除)/)
|
|
693
|
+
// 候选健康(无冷却/熔断)→ 备用链不触发(零留痕零跳过,行为不变)
|
|
694
|
+
var calls2 = []
|
|
695
|
+
var b2 = laneBoard({ size: 'heavy' }, { fallbackModels: ['gone/fb-a'] })
|
|
696
|
+
await mkDispatch(b2, { subagents: mkActivationSubagents(calls2), get: mkLlmCtx({ lw: [{ id: 'lane-w' }] }).get }).spawnOneShot(SID, b2.tasks[0], 'worker')
|
|
697
|
+
assert.deepEqual(calls2[0].request.agentOptions, { provider: 'lw', model: 'lane-w' }, '候选健康 → 备用链不触发')
|
|
698
|
+
assert.equal((b2.tasks[0].messages || []).filter(function (m) { return m.kind === 'stale-model' }).length, 0, '零留痕')
|
|
699
|
+
} finally { console.error = origErr }
|
|
700
|
+
})
|
|
701
|
+
|
|
702
|
+
test('目录闸D4:目录不可得(llm 缺失/空目录/provider 枚举失败)→ 目录闸整体跳过行为逐字不变(保守语义)', async function () {
|
|
703
|
+
var origErr = console.error; console.error = function () {}
|
|
704
|
+
try {
|
|
705
|
+
// 形态1:llm 缺失(get → null,老宿主)→ 卡级特例照取(目录闸跳过,行为不变)
|
|
706
|
+
var calls = []
|
|
707
|
+
var b1 = laneBoard({ model: 'cm/card-m', size: 'heavy' })
|
|
708
|
+
await mkDispatch(b1, { subagents: mkActivationSubagents(calls) }).spawnOneShot(SID, b1.tasks[0], 'worker')
|
|
709
|
+
assert.deepEqual(calls[0].request.agentOptions, { provider: 'cm', model: 'card-m' }, 'llm 缺失 → 目录闸跳过,卡级特例照取')
|
|
710
|
+
assert.equal((b1.tasks[0].messages || []).filter(function (m) { return m.kind === 'stale-model' }).length, 0, '零留痕')
|
|
711
|
+
// 形态2:枚举结果为空目录 → 保守跳过(不误判全候选目录外)
|
|
712
|
+
var calls2 = []
|
|
713
|
+
var b2 = laneBoard({ model: 'cm/card-m', size: 'heavy' })
|
|
714
|
+
await mkDispatch(b2, { subagents: mkActivationSubagents(calls2), get: mkLlmCtx({}).get }).spawnOneShot(SID, b2.tasks[0], 'worker')
|
|
715
|
+
assert.deepEqual(calls2[0].request.agentOptions, { provider: 'cm', model: 'card-m' }, '空目录 → 保守跳过,行为不变')
|
|
716
|
+
// 形态3:provider 枚举抛错(目录不完整)→ 保守跳过
|
|
717
|
+
var calls3 = []
|
|
718
|
+
var b3 = laneBoard({ model: 'cm/card-m', size: 'heavy' })
|
|
719
|
+
var boomLlm = { get: function (name) {
|
|
720
|
+
if (name !== 'llm') return null
|
|
721
|
+
return { listProviders: function () { return [{ id: 'broken' }] }, listModels: async function () { throw new Error('gateway boom') } }
|
|
722
|
+
} }
|
|
723
|
+
await mkDispatch(b3, { subagents: mkActivationSubagents(calls3), get: boomLlm.get }).spawnOneShot(SID, b3.tasks[0], 'worker')
|
|
724
|
+
assert.deepEqual(calls3[0].request.agentOptions, { provider: 'cm', model: 'card-m' }, 'provider 枚举失败 → 保守跳过,行为不变')
|
|
725
|
+
assert.equal((b3.tasks[0].messages || []).filter(function (m) { return m.kind === 'stale-model' }).length, 0, '零留痕')
|
|
726
|
+
} finally { console.error = origErr }
|
|
727
|
+
})
|
|
728
|
+
|