@arcaneorion/dsh-model-channel-manager 0.3.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.js CHANGED
@@ -1,6 +1,9 @@
1
1
  import z from "@deepseek-ai/schemastery";
2
+ import { HealthStore } from './health-store.js';
2
3
  export const name = 'model-channel-manager';
3
4
  export const inject = ['llm', 'timer'];
5
+ // storageDomain 由 dsh-base 组合(json 后端,root=storages/);不进 inject 硬依赖——
6
+ // 缺栈的 profile 降级为「健康流水不持久化」而不是拒绝启动。
4
7
  const ROUTE_PREFIX = 'roundrobin/';
5
8
  const GROUP_ID_RE = /^[a-z0-9][a-z0-9-]*$/;
6
9
  const NS_CONFIG = 'model-channels';
@@ -9,6 +12,8 @@ const NS_HEALTH = 'model-channel-health';
9
12
  // 因此原先的 model-channels / model-channel-health 两个命名空间合并为本插件的 Config
10
13
  // (字段 groups / providerOrder / effortMemory / health),命名空间 id 即本行 id。
11
14
  const HEALTH_SCHEMA = z.object({
15
+ // records/speedResults 仅作旧版存量迁移的读取源(迁入 domain 后不再写入);
16
+ // 权威数据在 storageDomain 的 model_channel_health 单元。
12
17
  records: z.dict(z.array(z.any())).default({}),
13
18
  speedResults: z.dict(z.array(z.any())).default({}),
14
19
  runtime: z.dict(z.any()).default({}),
@@ -18,6 +23,10 @@ const HEALTH_SCHEMA = z.object({
18
23
  testResults: z.dict(z.any()).default({}),
19
24
  lastTestHandledNonce: z.number().default(0),
20
25
  legacyMigrated: z.any(),
26
+ healthMigrated: z.any(),
27
+ // 小投影:host 聚合好的健康摘要(client 渲染用),digestAt 为生成时刻
28
+ digest: z.array(z.any()).default([]),
29
+ digestAt: z.number().default(0),
21
30
  }).loose(true);
22
31
  export const Config = z.object({
23
32
  groups: z.array(z.any()).default([]).volatile(),
@@ -116,6 +125,10 @@ export function apply(ctx, config) {
116
125
  timeoutMs: Number.isFinite(rg.timeoutMs) && rg.timeoutMs > 0 ? rg.timeoutMs : 30000,
117
126
  cooldownMs: Number.isFinite(rg.cooldownMs) && rg.cooldownMs >= 0 ? rg.cooldownMs : 60000,
118
127
  maxRetriesPerCandidate: Number.isSafeInteger(rg.maxRetriesPerCandidate) && rg.maxRetriesPerCandidate >= 0 ? rg.maxRetriesPerCandidate : 2,
128
+ // F01 配套(审计 §5.3):整次请求的总预算。旧实现最坏 2×N×(R+1) 次尝试
129
+ // (默认 R=2 → 6N 次),4 候选组可拖数分钟。默认 10 分钟;超预算
130
+ // 后不再开新尝试,直接 CHANNEL_FULL_FAIL 终结。
131
+ totalBudgetMs: Number.isFinite(rg.totalBudgetMs) && rg.totalBudgetMs > 0 ? rg.totalBudgetMs : 600000,
119
132
  speedTest: {
120
133
  enabled: st.enabled !== false,
121
134
  sortKey: ['ttft', 'latency', 'hybrid', 'smart'].includes(st.sortKey) ? st.sortKey : 'ttft',
@@ -286,7 +299,6 @@ export function apply(ctx, config) {
286
299
  const recordHealth = (gid, cand, entry) => {
287
300
  const now = Date.now();
288
301
  const key = gid || 'global';
289
- const list = state.records[key] || (state.records[key] = []);
290
302
  const rec = { ts: now, provider: cand.provider, model: cand.model, ok: entry.ok, ttftMs: entry.ttftMs, latencyMs: entry.latencyMs, code: entry.code || null };
291
303
  // 上游真实 token 用量(来自 usage StreamChunk,计费口径与 dsh-token-meter 一致):
292
304
  // 展平写入记录条目;缺失/非法字段不写,旧记录按 0 处理(向后兼容)。
@@ -298,74 +310,81 @@ export function apply(ctx, config) {
298
310
  if (Number.isFinite(u.cacheWriteTokens)) rec.cacheWriteTokens = u.cacheWriteTokens;
299
311
  if (Number.isFinite(u.reasoningTokens)) rec.reasoningTokens = u.reasoningTokens;
300
312
  }
313
+ // 内存镜像(聚合与 smart 键读内存即可)+ domain 追加(事实持久化)。
314
+ // domain 写入走 per-record put(写一条只动一个渠道桶文档),
315
+ // 不再有 2s 防抖与 volatile 快照覆盖的竞态窗口。
316
+ const list = state.records[key] || (state.records[key] = []);
301
317
  list.push(rec);
302
318
  const cutoff = now - 7 * 24 * 3600 * 1000;
303
- // 每键保留 300 条:健康页签 30m/24h 视图与选择器置顶绰绰有余,同时把
304
- // settings.yaml 的体量压到可读(2000 条/键时该文件一度膨胀到 2 万行)
305
319
  state.records[key] = list.filter((e) => e.ts >= cutoff).slice(-300);
306
- pendingRecords.push({ key, rec });
307
- persistHealth();
320
+ if (healthStore !== null) {
321
+ healthStore.appendEvent(key, rec).catch((e) => console.warn('[model-channel-manager] health event append failed:', e && e.message));
322
+ }
323
+ scheduleDigest();
308
324
  };
309
325
  const persistSpeedResults = async (r) => {
326
+ if (healthStore !== null) {
327
+ await healthStoreSpeed(r).catch(() => { });
328
+ }
310
329
  if (bus === null) return;
311
- await bus.writeHealth({ speedResults: clone(r), runtime: await persistRuntime() });
312
- };
313
- // ---------- 健康持久化节流:合并 2s 窗口内的写入,避免每个请求全量重写 settings ----------
314
- let healthFlushHandle = null;
315
- let healthDirty = false;
316
- // 尚未落盘的流水:reloadFromConfig 每次 volatile-update 都用配置快照整体覆盖 state.records,
317
- // 而 flush 有 2s 防抖——不缓冲的话,刚记下的一笔会在落盘前被内存覆盖(实测「测试成功不入账」就是这个)。
318
- let pendingRecords = [];
319
- const flushHealthNow = async () => {
320
- if (bus === null) return;
321
- const flushed = pendingRecords.slice();
322
- await bus.writeHealth({ records: clone(pullRecords()), speedResults: clone(pullSpeedResults()), runtime: await persistRuntime() });
323
- // 写入成功后才清账;失败就留着,等下一次 flush 重试
324
- pendingRecords = pendingRecords.filter((p) => !flushed.includes(p));
330
+ await bus.writeHealth({ runtime: await persistRuntime() });
325
331
  };
326
- const persistHealth = () => {
327
- healthDirty = true;
328
- if (healthFlushHandle === null) {
329
- healthFlushHandle = ctx.timeout(() => {
330
- healthFlushHandle = null;
331
- if (!healthDirty) return;
332
- healthDirty = false;
333
- flushHealthNow().catch(() => { });
334
- }, 2000);
332
+ const healthStoreSpeed = async (r) => {
333
+ for (const [groupId, rows] of Object.entries(r || {})) {
334
+ await healthStore.putSpeedRows(groupId, rows);
335
335
  }
336
336
  };
337
- ctx.effect(() => () => {
338
- // 插件卸载时尽力冲刷最后一批健康数据(settings 服务仍在,失败静默)
339
- if (healthFlushHandle !== null) {
340
- healthFlushHandle();
341
- healthFlushHandle = null;
342
- }
343
- if (healthDirty) {
344
- healthDirty = false;
345
- flushHealthNow().catch(() => { });
346
- }
347
- }, 'model-channel health flush');
348
337
  // ---------- 引擎:单候选尝试 ----------
338
+ // F01/F02(审计):超时必须能真正结束一次尝试。
339
+ // - 旧行为:Promise.race 超时后 await inner.return(),但 return() 排在挂起的
340
+ // next() 之后——上游挂着时 failover 被拖住(H02:15ms 超时 80ms 后仍未切换)。
341
+ // - 新行为:每次尝试独立 AbortController。pi-ai 适配器把 options.signal 经
342
+ // AbortSignal.any 融进 watchdog 并传给上游 HTTP(signal: watchdog.signal),
343
+ // abort 即真正取消网络请求。超时路径:先 abort → 有界等待(3s)排干 → 放弃
344
+ // 等待继续 failover(放弃的迭代器由适配器的 finally 自行清理)。
345
+ // - 生命周期(F02):成功 return / 失败 / 消费者提前退出统一走 finally 的
346
+ // 有界 closeInner,不再只在 catch 里清理。
347
+ const ATTEMPT_CLOSE_BUDGET_MS = 3000;
349
348
  async function* streamAttempt(cfg, cand, options, attemptStart) {
350
349
  if (cand.provider.startsWith(ROUTE_PREFIX))
351
350
  throw { code: 'INVALID_CANDIDATE', message: 'candidate provider must not be a virtual route' };
352
351
  const llm = ctx.llm;
353
- const inner = llm.stream(Object.assign({}, options, { provider: cand.provider, model: cand.model }));
352
+ // 用户取消信号透传 + 尝试自身的超时中止,二者融合为本次尝试的 signal
353
+ const attemptController = new AbortController();
354
+ const outerSignal = options.signal;
355
+ const relayAbort = () => { try { attemptController.abort(outerSignal.reason); } catch (_e) { } };
356
+ if (outerSignal) {
357
+ if (outerSignal.aborted)
358
+ relayAbort();
359
+ else
360
+ outerSignal.addEventListener('abort', relayAbort, { once: true });
361
+ }
362
+ const inner = llm.stream(Object.assign({}, options, { provider: cand.provider, model: cand.model, signal: attemptController.signal }));
354
363
  const timeoutMs = dynamicTimeoutMs(cfg, candKey(cand));
355
364
  let buffer = [];
356
365
  let started = false;
357
366
  let ttftMs = null;
358
367
  let lastAt = Date.now();
359
368
  let lastUsage = null;
360
- const closeInner = async () => { try {
361
- const c = inner.return ? inner.return() : null;
362
- if (c && c.then)
363
- await c;
364
- }
365
- catch (_e) { } };
369
+ let closed = false;
370
+ const closeInner = async () => {
371
+ if (closed)
372
+ return;
373
+ closed = true;
374
+ try {
375
+ const c = inner.return ? inner.return() : null;
376
+ if (c && c.then) {
377
+ await Promise.race([
378
+ c,
379
+ ctx.timeout(ATTEMPT_CLOSE_BUDGET_MS).then(() => { attemptController.abort({ code: 'TIMEOUT', message: 'attempt close budget exceeded' }); }, () => { }),
380
+ ]);
381
+ }
382
+ }
383
+ catch (_e) { }
384
+ };
366
385
  try {
367
386
  while (true) {
368
- if (options.signal && options.signal.aborted)
387
+ if (outerSignal && outerSignal.aborted)
369
388
  throw { code: 'ABORTED', message: 'request aborted' };
370
389
  const remaining = Math.max(0, timeoutMs - (Date.now() - (started ? lastAt : attemptStart)));
371
390
  if (remaining <= 0)
@@ -377,6 +396,13 @@ export function apply(ctx, config) {
377
396
  try {
378
397
  next = await Promise.race([inner.next(), guard.promise]);
379
398
  }
399
+ catch (raceErr) {
400
+ // 超时(或任何 next 失败):先中止本次尝试的上游请求再上抛——
401
+ // 这是 F01 的核心:abort 信号会穿透 pi-ai 取消真实 HTTP 请求
402
+ if (raceErr && raceErr.code === 'TIMEOUT')
403
+ attemptController.abort(raceErr);
404
+ throw raceErr;
405
+ }
380
406
  finally {
381
407
  // 超时路径同样要 dispose:ctx.effect 的注册表条目只有显式调用 disposer 才会移除
382
408
  guard.dispose();
@@ -431,33 +457,144 @@ export function apply(ctx, config) {
431
457
  }
432
458
  }
433
459
  catch (err) {
434
- await closeInner();
435
460
  if (started && err && typeof err === 'object')
436
461
  err.emitted = true;
437
462
  if (err && err.code === 'TIMEOUT')
438
463
  recordHealth(cand.provider, cand, { ok: false, ttftMs, latencyMs: null, code: 'TIMEOUT', usage: lastUsage });
439
464
  throw err;
440
465
  }
466
+ finally {
467
+ // F02:所有出口(成功 return / 失败 / 消费者提前 return)统一有界关闭;
468
+ // relayAbort 监听器随本次尝试终结移除(防泄漏:outerSignal 可能长命)
469
+ if (outerSignal)
470
+ outerSignal.removeEventListener('abort', relayAbort);
471
+ await closeInner();
472
+ }
441
473
  }
474
+ // ---------- F08 最小切片:候选能力过滤(防「成功路径改变用户输入」) ----------
475
+ // 审计 H14:虚拟模型声明 image,候选只支持 text——宿主把图片静默替换为
476
+ // "[image omitted...]" 后仍返回 stop 成功(用户输入被改变却报成功)。
477
+ // 审计 H06:非推理候选被强塞 effort → dispatch 前被拒 → CHANNEL_FULL_FAIL。
478
+ // 策略:请求含图/带 effort 时,过滤「明确声明不支持」的候选;能力未知
479
+ // (inputModalities 缺失 = 未知,只有显式排除才是负能力——宿主契约)不过滤,
480
+ // 保持原 failover 行为。解析失败不阻塞(同样视作未知)。
481
+ // 缓存 60s TTL:pi-ai 的 resolveModel 是内存快照解析(已核安装版源码),
482
+ // 但 async 接口 + 每次请求都调不划算;能力基本静态,TTL 足够。
483
+ const capabilityCache = new Map(); // key: provider::model → { at, info }
484
+ const CAPABILITY_TTL_MS = 60000;
485
+ const resolveCapability = async (cand) => {
486
+ const key = candKey(cand);
487
+ const hit = capabilityCache.get(key);
488
+ if (hit && Date.now() - hit.at < CAPABILITY_TTL_MS)
489
+ return hit.info;
490
+ try {
491
+ const info = await ctx.llm.resolveModelInfo(cand.provider, cand.model);
492
+ const entry = { at: Date.now(), info };
493
+ capabilityCache.set(key, entry);
494
+ return info;
495
+ }
496
+ catch (_e) {
497
+ return null; // 解析失败 = 未知,不过滤
498
+ }
499
+ };
500
+ /** 请求是否包含图片输入(粗判:消息 content 里有 image 块)。 */
501
+ const requestHasImage = (options) => {
502
+ try {
503
+ return (options.messages || []).some((m) => Array.isArray(m === null || m === void 0 ? void 0 : m.content) && m.content.some((p) => p && p.type === 'image'));
504
+ }
505
+ catch (_e) {
506
+ return false;
507
+ }
508
+ };
509
+ /** 过滤掉明确不支持本次请求能力的候选;全部被滤或未知时退回原列表(不因过滤而失去候选)。 */
510
+ const filterByCapability = async (cands, options) => {
511
+ const needImage = requestHasImage(options);
512
+ const needReasoning = typeof options.reasoningEffort === 'string' && options.reasoningEffort.length > 0 && options.reasoningEffort !== 'off';
513
+ if (!needImage && !needReasoning)
514
+ return cands;
515
+ const skipped = [];
516
+ const kept = [];
517
+ for (const cand of cands) {
518
+ const info = await resolveCapability(cand);
519
+ if (info === null) {
520
+ kept.push(cand); // 未知:不过滤
521
+ continue;
522
+ }
523
+ if (needImage) {
524
+ const mods = info.inputModalities;
525
+ if (Array.isArray(mods) && mods.length > 0 && !mods.includes('image')) {
526
+ skipped.push({ candidate: candKey(cand), reason: 'image input not supported' });
527
+ continue; // 明确排除 image
528
+ }
529
+ }
530
+ if (needReasoning && info.reasoning === undefined) {
531
+ skipped.push({ candidate: candKey(cand), reason: 'reasoning not supported' });
532
+ continue; // 明确无 reasoning 档位
533
+ }
534
+ kept.push(cand);
535
+ }
536
+ // 全被滤掉 = 声明与所有候选冲突:退回原列表(让请求去撞,报真实错误)
537
+ // 而不是直接 NO_CANDIDATES 假装没有渠道
538
+ if (skipped.length > 0 && kept.length > 0) {
539
+ // 记录到组事件(cfg 此时未知,用 provider 维度的 console 代替;组事件在
540
+ // streamGroup 拿到 cfg 后会补记 capability-filter)
541
+ console.log('[model-channel-manager] capability filter:', skipped.map((s) => s.candidate + ' (' + s.reason + ')').join(', '));
542
+ }
543
+ return kept.length > 0 ? kept : cands;
544
+ };
545
+
442
546
  // ---------- 引擎:组级故障转移循环 ----------
443
547
  async function* streamGroup(cfg, options) {
444
548
  const rt = groupRuntime(cfg.id);
445
- const order = orderedCandidates(cfg);
549
+ let order = orderedCandidates(cfg);
446
550
  if (order.length === 0) {
447
551
  pushEvent(cfg, 'no-candidates', {});
448
552
  yield failChunk('channel group "' + cfg.id + '" has no candidates', 'NO_CANDIDATES');
449
553
  return;
450
554
  }
555
+ // F14(审计 H07):enabled=true 但从未测过速的组,首次真实使用时后台触发一次
556
+ // 自动测速(不阻塞、不 await 当前请求)。旧实现只有 boot 时的 onFirstUse 分支,
557
+ // 常规请求路径完全没有自动测速入口——开启开关后从未生效。
558
+ if (cfg.speedTest.enabled && (state.speedResults[cfg.id] || []).length === 0 && !rt.speedTestRunning) {
559
+ rt.speedTestRunning = true; // 先占位防重入(runSpeedTest 内部还会再查)
560
+ rt.speedTestRunning = false;
561
+ runSpeedTest(cfg).catch(() => { });
562
+ }
563
+ // F08:按本次请求能力过滤(含图/带 effort 时)。过滤发生在测速排序之后,
564
+ // 保序不改变相对优先级。
565
+ const before = order.length;
566
+ order = await filterByCapability(order, options);
567
+ if (order.length < before)
568
+ pushEvent(cfg, 'capability-filter', { skipped: before - order.length, kept: order.length });
451
569
  const strategy = cfg.strategy || 'sticky';
452
- let cursor = strategy === 'primary' ? 0 : rt.currentIndex % order.length;
570
+ // F13(审计 H13):round-robin 的指针改为「选择时原子预留」——旧实现成功后才
571
+ // 推进 rt.currentIndex,两个并发请求都从 first 开始(并发不均分)。
572
+ // 现在:选定时立即占位推进(同步读改写,JS 单线程内原子),失败不回滚
573
+ // (失败反馈走冷却,指针保持前进语义——轮询分布比严格顺序更重要)。
574
+ if (strategy === 'round-robin') {
575
+ cursor = rt.currentIndex % order.length;
576
+ rt.currentIndex = (cursor + 1) % order.length; // 本次请求已占用 cursor 槽位
577
+ }
578
+ else {
579
+ cursor = strategy === 'primary' ? 0 : rt.currentIndex % order.length;
580
+ }
453
581
  let fullRounds = 0;
454
582
  let lastFail = null;
455
583
  const failedKeys = [];
584
+ // 总请求预算(F01 配套):组级起点。超预算不再开新尝试。
585
+ const groupStart = Date.now();
586
+ const budgetMs = cfg.totalBudgetMs || 600000;
587
+ const budgetLeft = () => budgetMs - (Date.now() - groupStart);
456
588
  while (true) {
457
589
  if (options.signal && options.signal.aborted) {
458
590
  yield abortedChunk('request aborted');
459
591
  return;
460
592
  }
593
+ if (budgetLeft() <= 0) {
594
+ pushEvent(cfg, 'budget-exhausted', { budgetMs, failures: failedKeys.slice() });
595
+ yield failChunk('channel group "' + cfg.id + '" exhausted its total budget of ' + budgetMs + 'ms: ' + (lastFail || 'multiple failures'), 'CHANNEL_BUDGET_EXCEEDED');
596
+ return;
597
+ }
461
598
  const round = [];
462
599
  for (let i = 0; i < order.length; i++)
463
600
  round.push(order[(cursor + i) % order.length]);
@@ -480,6 +617,8 @@ export function apply(ctx, config) {
480
617
  yield abortedChunk('request aborted');
481
618
  return;
482
619
  }
620
+ if (budgetLeft() <= 0)
621
+ break; // 预算耗尽:不开新尝试,交给组层终结
483
622
  const attemptStart = Date.now();
484
623
  try {
485
624
  const attempt = streamAttempt(cfg, cand, options, attemptStart);
@@ -488,12 +627,12 @@ export function apply(ctx, config) {
488
627
  succeeded = true;
489
628
  candidateOk = true;
490
629
  rt.cooldowns.delete(key);
630
+ // F13:round-robin 的推进已移到选定时(见上方原子预留);
631
+ // 成功路径只维护 sticky/primary 的语义。
491
632
  const pos = order.findIndex((c) => candKey(c) === key);
492
- if (strategy === 'round-robin')
493
- rt.currentIndex = (pos + 1) % order.length;
494
- else if (strategy === 'sticky')
633
+ if (strategy === 'sticky')
495
634
  rt.currentIndex = pos;
496
- else
635
+ else if (strategy === 'primary')
497
636
  rt.currentIndex = 0;
498
637
  if (failedKeys.length > 0)
499
638
  pushEvent(cfg, 'failover', { from: failedKeys[failedKeys.length - 1], to: key, reason: 'candidate change after ' + failedKeys.length + ' failure(s)' });
@@ -547,24 +686,47 @@ export function apply(ctx, config) {
547
686
  }
548
687
  }
549
688
  // ---------- 测速 ----------
689
+ // F15 修复:旧行为每 chunk 重置 guard(单次等待上限而非总时限,H12:timeoutMs=40
690
+ // 流每 20ms 输出,84ms 后仍成功)+ 超时不中止上游。现在:总时限 deadline +
691
+ // 超时 abort 上游(与 streamAttempt 同一模式)+ 统一 finally 关闭。
550
692
  async function measureCandidate(cfg, cand, st) {
551
693
  const llm = ctx.llm;
552
- const inner = llm.stream({ provider: cand.provider, model: cand.model, messages: [{ role: 'user', content: [{ type: 'text', text: st.prompt }] }], maxTokens: st.maxTokens, sessionId: 'mcm-speedtest-' + (st.nonce ?? Date.now()) });
694
+ const controller = new AbortController();
695
+ const deadline = Date.now() + (st.timeoutMs || 60000);
696
+ const inner = llm.stream({ provider: cand.provider, model: cand.model, messages: [{ role: 'user', content: [{ type: 'text', text: st.prompt }] }], maxTokens: st.maxTokens, sessionId: 'mcm-speedtest-' + (st.nonce ?? Date.now()), signal: controller.signal });
553
697
  const start = Date.now();
554
698
  let ttft = null;
555
- const closeInner = async () => { try {
556
- const c = inner.return ? inner.return() : null;
557
- if (c && c.then)
558
- await c;
559
- }
560
- catch (_e) { } };
699
+ let closed = false;
700
+ const closeInner = async () => {
701
+ if (closed)
702
+ return;
703
+ closed = true;
704
+ try {
705
+ const c = inner.return ? inner.return() : null;
706
+ if (c && c.then) {
707
+ await Promise.race([
708
+ c,
709
+ ctx.timeout(ATTEMPT_CLOSE_BUDGET_MS).then(() => { try { controller.abort({ code: 'TIMEOUT', message: 'speedtest close budget exceeded' }); } catch (_e) { } }, () => { }),
710
+ ]);
711
+ }
712
+ }
713
+ catch (_e) { }
714
+ };
561
715
  try {
562
716
  while (true) {
563
- const guard = timed(Math.max(1, st.timeoutMs), () => ({ code: 'TIMEOUT', message: 'speedtest timed out after ' + st.timeoutMs + 'ms' }));
717
+ const remaining = deadline - Date.now();
718
+ if (remaining <= 0)
719
+ throw { code: 'TIMEOUT', message: 'speedtest timed out after ' + st.timeoutMs + 'ms (total)' };
720
+ const guard = timed(Math.max(1, remaining), () => ({ code: 'TIMEOUT', message: 'speedtest timed out after ' + st.timeoutMs + 'ms (total)' }));
564
721
  let next;
565
722
  try {
566
723
  next = await Promise.race([inner.next(), guard.promise]);
567
724
  }
725
+ catch (raceErr) {
726
+ if (raceErr && raceErr.code === 'TIMEOUT')
727
+ try { controller.abort(raceErr); } catch (_e) { }
728
+ throw raceErr;
729
+ }
568
730
  finally {
569
731
  guard.dispose();
570
732
  }
@@ -582,9 +744,11 @@ export function apply(ctx, config) {
582
744
  }
583
745
  }
584
746
  catch (err) {
585
- await closeInner();
586
747
  throw { code: (err && err.code) || 'STREAM_ERROR', message: (err && err.message) || 'speedtest failed' };
587
748
  }
749
+ finally {
750
+ await closeInner();
751
+ }
588
752
  }
589
753
  async function runSpeedTest(cfg) {
590
754
  const rt = groupRuntime(cfg.id);
@@ -702,25 +866,48 @@ export function apply(ctx, config) {
702
866
  // ---------- 单模型真实请求测试(client 经 settings 总线下发,走 DSH 真实 llm.stream 链路) ----------
703
867
  async function runModelTest(provider, model, prompt, maxTokens) {
704
868
  const llm = ctx.llm;
705
- const inner = llm.stream({ provider, model, messages: [{ role: 'user', content: [{ type: 'text', text: prompt || '你好' }] }], maxTokens: maxTokens || 512, sessionId: 'mcm-modeltest-' + Date.now() });
869
+ // F15(与 measureCandidate 同一模式):总时限 60s(旧行为每 chunk 重置 guard,
870
+ // 持续输出的请求永不超时而 client 已放弃)+ 超时 abort 上游 + 统一 finally 关闭。
871
+ const MODEL_TEST_TOTAL_MS = 60000;
872
+ const controller = new AbortController();
873
+ const deadline = Date.now() + MODEL_TEST_TOTAL_MS;
874
+ const inner = llm.stream({ provider, model, messages: [{ role: 'user', content: [{ type: 'text', text: prompt || '你好' }] }], maxTokens: maxTokens || 512, sessionId: 'mcm-modeltest-' + Date.now(), signal: controller.signal });
706
875
  const start = Date.now();
707
876
  let ttft = null;
708
877
  let text = '';
709
878
  let reasoning = '';
710
879
  let lastUsage = null;
711
- const closeInner = async () => { try {
712
- const c = inner.return ? inner.return() : null;
713
- if (c && c.then)
714
- await c;
715
- }
716
- catch (_e) { } };
880
+ let closed = false;
881
+ const closeInner = async () => {
882
+ if (closed)
883
+ return;
884
+ closed = true;
885
+ try {
886
+ const c = inner.return ? inner.return() : null;
887
+ if (c && c.then) {
888
+ await Promise.race([
889
+ c,
890
+ ctx.timeout(ATTEMPT_CLOSE_BUDGET_MS).then(() => { try { controller.abort({ code: 'TIMEOUT', message: 'model test close budget exceeded' }); } catch (_e) { } }, () => { }),
891
+ ]);
892
+ }
893
+ }
894
+ catch (_e) { }
895
+ };
717
896
  try {
718
897
  while (true) {
719
- const guard = timed(60000, () => ({ code: 'TIMEOUT', message: 'model test timed out after 60000ms' }));
898
+ const remaining = deadline - Date.now();
899
+ if (remaining <= 0)
900
+ throw { code: 'TIMEOUT', message: 'model test timed out after ' + MODEL_TEST_TOTAL_MS + 'ms (total)' };
901
+ const guard = timed(Math.max(1, remaining), () => ({ code: 'TIMEOUT', message: 'model test timed out after ' + MODEL_TEST_TOTAL_MS + 'ms (total)' }));
720
902
  let next;
721
903
  try {
722
904
  next = await Promise.race([inner.next(), guard.promise]);
723
905
  }
906
+ catch (raceErr) {
907
+ if (raceErr && raceErr.code === 'TIMEOUT')
908
+ try { controller.abort(raceErr); } catch (_e) { }
909
+ throw raceErr;
910
+ }
724
911
  finally {
725
912
  guard.dispose();
726
913
  }
@@ -751,28 +938,35 @@ export function apply(ctx, config) {
751
938
  }
752
939
  }
753
940
  catch (err) {
754
- await closeInner();
755
941
  throw err;
756
942
  }
943
+ finally {
944
+ await closeInner();
945
+ }
757
946
  }
758
947
  function handleTestRequest(next) {
759
948
  const req = next.testRequest;
760
- const last = Math.max(next.lastTestHandledNonce || 0, claimedTestNonce);
761
- if (!req || typeof req.provider !== 'string' || typeof req.model !== 'string' || typeof req.nonce !== 'number' || req.nonce === last)
949
+ // nonce 统一为字符串 UUID(0.3.2 起 client 换 randomUUID,防同毫秒碰撞);
950
+ // 旧数字 nonce(Date.now()%1e9)仍被接受:typeof 兼容两种,比较用 !==。
951
+ const nonceOf = (v) => (typeof v === 'number' || typeof v === 'string') ? v : null;
952
+ const reqNonce = nonceOf(req && req.nonce);
953
+ const lastNum = typeof next.lastTestHandledNonce === 'number' ? next.lastTestHandledNonce : null;
954
+ const last = lastNum !== null ? Math.max(lastNum, typeof claimedTestNonce === 'number' ? claimedTestNonce : 0) : (claimedTestNonce || null);
955
+ if (!req || typeof req.provider !== 'string' || typeof req.model !== 'string' || reqNonce === null || reqNonce === last)
762
956
  return;
763
- claimedTestNonce = req.nonce;
957
+ claimedTestNonce = reqNonce;
764
958
  // nonce 的落盘推迟到本次测试得出结论之后:只有「真跑过」才算 handled。
765
959
  // 启动早于凭据服务就绪时会以 MISSING_CREDENTIAL 失败,那要释放认领重试;
766
960
  // 若提前写了 nonce,重试会被自己的持久值挡掉。
767
961
  const settle = (entry) => {
768
962
  if (bus !== null)
769
- bus.writeHealth({ lastTestHandledNonce: req.nonce }).catch((e) => console.error('[model-channel-manager] lastTestHandledNonce 写入失败:', e));
963
+ bus.writeHealth({ lastTestHandledNonce: reqNonce }).catch((e) => console.error('[model-channel-manager] lastTestHandledNonce 写入失败:', e));
770
964
  return setResult(entry);
771
965
  };
772
966
  // 0.2 的配置写入是「整字段落盘」,不再有 path-ops;结果集按当前值合并后整段写回。
773
967
  const setResult = async (entry) => {
774
- const key = String(req.nonce);
775
- const value = Object.assign({}, entry, { nonce: req.nonce, provider: req.provider, model: req.model });
968
+ const key = String(reqNonce);
969
+ const value = Object.assign({}, entry, { nonce: reqNonce, provider: req.provider, model: req.model });
776
970
  const cur = healthOf().testResults || {};
777
971
  if (bus !== null)
778
972
  await bus.writeHealth({ testResults: Object.assign({}, cur, { [key]: value }) }).catch((e) => console.error('[model-channel-manager] testResults 写入失败:', e));
@@ -801,7 +995,7 @@ export function apply(ctx, config) {
801
995
  }).catch(async (e) => {
802
996
  if (notReady(e)) {
803
997
  // 凭据服务尚未就绪(典型是启动瞬间的重放):不写成假 error,释放认领并延后重试
804
- claimedTestNonce = 0;
998
+ claimedTestNonce = null;
805
999
  scheduleReplay('test', () => reloadFromConfig(false));
806
1000
  return;
807
1001
  }
@@ -810,7 +1004,20 @@ export function apply(ctx, config) {
810
1004
  }
811
1005
  function rewireRoutes() {
812
1006
  const llm = ctx.llm;
813
- const next = pullConfig().groups.map((g) => routeOfGroup(g.id));
1007
+ // F16(审计 H08):normalizeConfig 对非法/重复 ID 是静默 continue——保存 3 组
1008
+ // 实际只注册 1 条路由,界面仍显示「已保存」。这里把被丢弃的组大声报出来:
1009
+ // 配置里有 N 组但只有 M 组成为路由时,用户能从日志看到哪组、为什么被丢。
1010
+ const rawGroups = Array.isArray(state.config && state.config.groups) ? state.config.groups : [];
1011
+ const normalized = pullConfig().groups;
1012
+ if (rawGroups.length > normalized.length) {
1013
+ const keptIds = new Set(normalized.map((g) => g.id));
1014
+ const dropped = rawGroups
1015
+ .map((g) => g && g.id)
1016
+ .filter((id) => typeof id === 'string' && !keptIds.has(id));
1017
+ if (dropped.length > 0)
1018
+ console.warn('[model-channel-manager] 以下轮询组未通过校验,已被丢弃(不会成为路由):', dropped.join(', '), '——组 ID 规则:小写字母/数字/连字符,且不重复');
1019
+ }
1020
+ const next = normalized.map((g) => routeOfGroup(g.id));
814
1021
  const current = llm.listProviders().map((p) => p.id).filter((id) => id.startsWith(ROUTE_PREFIX));
815
1022
  if (next.length === 0) {
816
1023
  // 空配置:LLM 服务拒绝注册零 provider 的适配器——推迟首次注册;已注册则清空路由
@@ -823,13 +1030,96 @@ export function apply(ctx, config) {
823
1030
  else if (JSON.stringify(current) !== JSON.stringify(next))
824
1031
  adapterHandle.replace(next);
825
1032
  }
1033
+ // ---------- 健康数据 domain 存储 + 存量迁移 + 小投影 ----------
1034
+ // healthStore:storageDomain 存在时启用(dsh-base 组合了 json 后端)。
1035
+ // 启动序列:先开 domain → 迁移 settings 存量 → reloadFromConfig 从 domain 读权威数据。
1036
+ let healthStore = null;
1037
+ // digest:写进 settings health 子树的小投影(client 健康页渲染用)。
1038
+ // 聚合上移 host,client 不再拉原始流水全量 describe——settings 里不再出现 records。
1039
+ let digestTimer = null;
1040
+ const DIGEST_INTERVAL = 5000;
1041
+ const buildDigest = () => {
1042
+ // 与旧 client HealthPanel 聚合口径一致:total/success/ttft/latency/token(计费口径
1043
+ // = input + cacheRead + cacheWrite + output),外加 lastTs/lastOk/lastCode
1044
+ const by = new Map();
1045
+ for (const list of Object.values(state.records)) {
1046
+ for (const e of list) {
1047
+ const key = e.provider + '::' + e.model;
1048
+ let a = by.get(key);
1049
+ if (a === undefined) {
1050
+ a = { provider: e.provider, model: e.model, total: 0, success: 0, ttftSum: 0, latSum: 0, tokIn: 0, tokOut: 0, tokCache: 0, lastTs: 0, lastOk: null, lastCode: null };
1051
+ by.set(key, a);
1052
+ }
1053
+ a.total++;
1054
+ const inTok = e.inputTokens || 0;
1055
+ const outTok = e.outputTokens || 0;
1056
+ const cacheTok = (e.cacheReadTokens || 0) + (e.cacheWriteTokens || 0);
1057
+ a.tokIn += inTok;
1058
+ a.tokOut += outTok;
1059
+ a.tokCache += cacheTok;
1060
+ if (e.ok) {
1061
+ a.success++;
1062
+ if (e.ttftMs != null && e.ttftMs >= 0) a.ttftSum += e.ttftMs;
1063
+ if (e.latencyMs != null && e.latencyMs >= 0) a.latSum += e.latencyMs;
1064
+ }
1065
+ if ((e.ts || 0) > a.lastTs) {
1066
+ a.lastTs = e.ts || 0;
1067
+ a.lastOk = e.ok;
1068
+ a.lastCode = e.ok ? null : (e.code || null); // lastCode 随 lastTs 对齐(不再按遍历序覆盖)
1069
+ }
1070
+ }
1071
+ }
1072
+ return [...by.values()].map((a) => ({
1073
+ provider: a.provider, model: a.model, total: a.total, success: a.success,
1074
+ ttftAvg: a.success > 0 ? Math.round(a.ttftSum / a.success) : null,
1075
+ latAvg: a.success > 0 ? Math.round(a.latSum / a.success) : null,
1076
+ tokIn: a.tokIn, tokOut: a.tokOut, tokCache: a.tokCache,
1077
+ lastTs: a.lastTs, lastOk: a.lastOk, lastCode: a.lastCode,
1078
+ }));
1079
+ };
1080
+ const flushDigest = () => {
1081
+ digestTimer = null;
1082
+ if (bus !== null) {
1083
+ bus.writeHealth({ digest: buildDigest(), digestAt: Date.now() }).catch(() => { });
1084
+ }
1085
+ };
1086
+ const scheduleDigest = () => {
1087
+ if (digestTimer === null) {
1088
+ digestTimer = ctx.timeout(flushDigest, DIGEST_INTERVAL);
1089
+ }
1090
+ };
1091
+ ctx.effect(() => () => {
1092
+ if (digestTimer !== null) {
1093
+ digestTimer();
1094
+ digestTimer = null;
1095
+ }
1096
+ flushDigest();
1097
+ }, 'model-channel health digest');
1098
+ // settings 存量 records/speedResults → domain 一次性迁移(桶已存在即跳过,domain 权威)。
1099
+ const migrateHealthToDomain = async () => {
1100
+ if (healthStore === null || bus === null) return;
1101
+ const health = healthOf();
1102
+ if (health.healthMigrated === true) return;
1103
+ let ok = false;
1104
+ try {
1105
+ ok = await healthStore.migrateFrom(health.records || {}, health.speedResults || {});
1106
+ }
1107
+ catch (_e) {
1108
+ ok = false; // 下次启动再试
1109
+ }
1110
+ if (!ok) return; // store 已关闭(卸载竞态):不写标记,避免标记与数据永久不一致
1111
+ // 同一次合并写:迁移标记 + 清空 settings 里的旧存量(否则 writeHealth 的
1112
+ // Object.assign 会把 records/speedResults 永远带下去,profile 膨胀问题未真正解决)
1113
+ bus.writeHealth({ healthMigrated: true, records: {}, speedResults: {} }).catch(() => { });
1114
+ };
1115
+
826
1116
  // ---------- settings 总线接入(响应式:settings 服务异步初始化,apply 时查询太早) ----------
827
1117
  let bus = null;
828
1118
  let runtimeRestored = false;
829
1119
  // 非持久去重:回写要等当前 HMR 事务结束才落盘,这中间 volatile-update 可能带着同一个 nonce 再来,
830
1120
  // 内存里先认领,避免同一请求被重复执行(并顺带消掉重复的回写风暴)。
831
- let claimedTestNonce = 0;
832
- let claimedSpeedNonce = 0;
1121
+ let claimedTestNonce = null; // string UUID 或旧数字;null = 未认领
1122
+ let claimedSpeedNonce = null; // string UUID 或旧数字;null = 未认领
833
1123
  // 启动早于凭据服务就绪:此时重放测试/测速会以 MISSING_CREDENTIAL 失败。
834
1124
  // 这类错误是「环境还没准备好」而不是「渠道故障」,必须释放认领、延后重试,不能写成假 error。
835
1125
  const notReady = (e) => {
@@ -846,18 +1136,19 @@ export function apply(ctx, config) {
846
1136
  ctx.timeout(() => { fn(); }, ms);
847
1137
  };
848
1138
  // 从实例配置读取全部状态;volatile 提交后由 loader/volatile-update 触发重载。
1139
+ // 0.3.1 起 records/speedResults 的权威在 storageDomain(healthStore);
1140
+ // settings health 子树只承载小投影(digest 给 client 渲染)+ 运行态 + 任务哨,
1141
+ // volatile 快照覆盖不再能抹掉任何流水(F11/F23 的根因消除)。
849
1142
  function reloadFromConfig(initial) {
850
1143
  const cfg = cfgOf();
851
1144
  state.config = clone(cfg) || { groups: [] };
852
1145
  const health = healthOf();
853
- state.records = clone(health.records || {});
854
- // 配置快照覆盖后,把尚未落盘的流水补回来(ts+provider+model 相同视为同一笔,不重复)
855
- for (const p of pendingRecords) {
856
- const list = state.records[p.key] || (state.records[p.key] = []);
857
- if (!list.some((e) => e.ts === p.rec.ts && e.provider === p.rec.provider && e.model === p.rec.model))
858
- list.push(p.rec);
859
- }
860
- state.speedResults = clone(health.speedResults || {});
1146
+ state.records = healthStore !== null
1147
+ ? clone(healthStore.allEventBuckets())
1148
+ : clone(health.records || {});
1149
+ state.speedResults = healthStore !== null
1150
+ ? clone(healthStore.allSpeedBuckets())
1151
+ : clone(health.speedResults || {});
861
1152
  state.testResults = clone(health.testResults || {});
862
1153
  if (initial && !runtimeRestored) {
863
1154
  restoreRuntime(health.runtime || {});
@@ -872,20 +1163,23 @@ export function apply(ctx, config) {
872
1163
  runtime.delete(id);
873
1164
  rewireRoutes();
874
1165
  const req = health.speedRequest;
875
- const last = Math.max(health.lastHandledNonce || 0, claimedSpeedNonce);
876
- if (req && typeof req.group === 'string' && typeof req.nonce === 'number' && req.nonce !== last) {
877
- claimedSpeedNonce = req.nonce;
1166
+ // nonce 兼容字符串 UUID / 旧数字(与 handleTestRequest 同一策略)
1167
+ const speedNonce = (typeof (req && req.nonce) === 'number' || typeof (req && req.nonce) === 'string') ? req.nonce : null;
1168
+ const lastNum = typeof health.lastHandledNonce === 'number' ? health.lastHandledNonce : null;
1169
+ const last = lastNum !== null ? Math.max(lastNum, typeof claimedSpeedNonce === 'number' ? claimedSpeedNonce : 0) : (claimedSpeedNonce || null);
1170
+ if (req && typeof req.group === 'string' && speedNonce !== null && speedNonce !== last) {
1171
+ claimedSpeedNonce = speedNonce;
878
1172
  const cfgRow = pullConfig().groups.find((g) => g.id === req.group);
879
1173
  // nonce 落盘推迟到本次测速得出结论之后(同 handleTestRequest 的理由):
880
1174
  // 提前写会把「没就绪」的重试用自己的持久值挡掉。
881
1175
  const settleSpeed = () => {
882
1176
  if (bus !== null)
883
- bus.writeHealth({ lastHandledNonce: req.nonce }).catch(() => { });
1177
+ bus.writeHealth({ lastHandledNonce: speedNonce }).catch(() => { });
884
1178
  };
885
1179
  if (cfgRow)
886
1180
  runSpeedTest(cfgRow).then((r) => {
887
1181
  if (r && r.deferred) {
888
- claimedSpeedNonce = 0;
1182
+ claimedSpeedNonce = null;
889
1183
  scheduleReplay('speed', () => reloadFromConfig(false));
890
1184
  return;
891
1185
  }
@@ -893,7 +1187,7 @@ export function apply(ctx, config) {
893
1187
  settleSpeed();
894
1188
  }).catch((e) => {
895
1189
  if (notReady(e)) {
896
- claimedSpeedNonce = 0;
1190
+ claimedSpeedNonce = null;
897
1191
  scheduleReplay('speed', () => reloadFromConfig(false));
898
1192
  return;
899
1193
  }
@@ -940,12 +1234,54 @@ export function apply(ctx, config) {
940
1234
  cfgScope: { get: () => cfgOf() },
941
1235
  healthScope: { get: () => healthOf() },
942
1236
  };
943
- reloadFromConfig(true);
1237
+ // healthStore 可用(dsh-base 的 storage 栈在场)时:开 domain → 迁移存量 →
1238
+ // 用 domain 权威数据重建内存镜像 → boot;否则同步走旧路径。
1239
+ // ctx.inject 回调不是 async 函数,domain 初始化用 promise 链表达。
1240
+ // 卸载竞态防护:then 回调先验 fiber 活性,inactive 时回滚 healthStore 并跳过
1241
+ // effect 注册(对 inactive fiber 注册 effect 会抛 INACTIVE_EFFECT,若被吞掉则
1242
+ // domain 永不 close,facility 名字被占 → HMR 重载后 already-open 静默降级)。
1243
+ // boot 也挪进链尾:domain 路径下 reloadFromConfig(true) 异步排队,若 boot 先跑,
1244
+ // state.speedResults 尚为空 → onFirstUse 组每次启动都重测;且 state.config 为空
1245
+ // 时 rewireRoutes 推迟,启动早期虚拟路由短暂不存在。
1246
+ const domainFacility = ctx.get('storageDomain');
1247
+ const fiberUid = ctx.fiber && ctx.fiber.uid;
1248
+ const fiberAlive = () => ctx.fiber !== undefined && ctx.fiber.uid === fiberUid && ctx.fiber.state !== 4 /* disposed */;
1249
+ if (domainFacility !== undefined) {
1250
+ const store = new HealthStore(ctx);
1251
+ Promise.resolve()
1252
+ .then(() => store.open())
1253
+ .then(() => {
1254
+ if (!fiberAlive()) {
1255
+ // 已卸载:回滚赋值,由 closeAll 兜底回收 domain
1256
+ void store.close().catch(() => { });
1257
+ return;
1258
+ }
1259
+ healthStore = store;
1260
+ ctx.effect(() => () => { void store.close(); }, 'model-channel health domain close');
1261
+ })
1262
+ .catch((e) => {
1263
+ console.warn('[model-channel-manager] storageDomain open failed, health records stay in-memory only:', e && e.message);
1264
+ })
1265
+ .then(() => (healthStore !== null ? migrateHealthToDomain().catch(() => { }) : null))
1266
+ .then(() => {
1267
+ if (fiberAlive())
1268
+ reloadFromConfig(true);
1269
+ })
1270
+ .then(() => {
1271
+ if (fiberAlive())
1272
+ boot();
1273
+ })
1274
+ .catch(() => { }); // 卸载竞态下 reload/boot 内部可能 throw,链尾兜底防 unhandled rejection
1275
+ }
1276
+ else {
1277
+ reloadFromConfig(true);
1278
+ boot();
1279
+ }
944
1280
  ctx.on('loader/volatile-update', () => {
945
1281
  reloadFromConfig(false);
946
1282
  console.log('[model-channel-manager] config hot-reloaded, routes:', pullConfig().groups.map((g) => g.id).join(', ') || '(none)');
947
1283
  });
948
- boot();
1284
+ // boot 已在 domain ready 链尾(或降级分支)调用,此处不再调
949
1285
  });
950
1286
  // ---------- 全局 LLM 请求健康拦截 (涵盖所有非虚拟路由的真实渠道模型调用) ----------
951
1287
  ctx.on('llm/stream', async function* (options, next) {
@@ -1047,6 +1383,13 @@ export function apply(ctx, config) {
1047
1383
  const text = await fsSvc.readText(target);
1048
1384
  const legacy = JSON.parse(text);
1049
1385
  const migrated = normalizeConfig(legacy);
1386
+ // F10(审计 H15):读取是 await 的——这个窗口里用户可能已经保存了
1387
+ // 新组。写前重检:现在有组了就不迁(用户的新配置优先于旧文件复活)。
1388
+ if (pullConfig().groups.length !== 0) {
1389
+ console.log('[model-channel-manager] legacy migration skipped: user config arrived during read');
1390
+ bus.writeHealth({ legacyMigrated: true }).catch(() => { });
1391
+ return;
1392
+ }
1050
1393
  if (migrated.groups.length > 0) {
1051
1394
  await bus.settings.update(SELF_NS, { groups: migrated.groups });
1052
1395
  state.config = { groups: migrated.groups };