@arcaneorion/dsh-model-channel-manager 0.3.14 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -317
- package/package.json +18 -4
- package/src/channel-state.js +196 -0
- package/src/client.js +126 -307
- package/src/health-store.js +11 -20
- package/src/index.js +174 -622
package/src/index.js
CHANGED
|
@@ -1,85 +1,20 @@
|
|
|
1
1
|
import z from "@deepseek-ai/schemastery";
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
2
3
|
import { HealthStore } from './health-store.js';
|
|
4
|
+
import { ChannelController, RuntimeStore, TaskRegistry, aggregateHealth, createRpcHandler } from './channel-state.js';
|
|
3
5
|
export const name = 'model-channel-manager';
|
|
4
6
|
export const inject = ['llm', 'timer'];
|
|
5
7
|
// storageDomain 由 dsh-base 组合(json 后端,root=storages/);不进 inject 硬依赖——
|
|
6
8
|
// 缺栈的 profile 降级为「健康流水不持久化」而不是拒绝启动。
|
|
7
9
|
const ROUTE_PREFIX = 'roundrobin/';
|
|
8
10
|
const GROUP_ID_RE = /^[a-z0-9][a-z0-9-]*$/;
|
|
9
|
-
|
|
10
|
-
const NS_HEALTH = 'model-channel-health';
|
|
11
|
-
// dsh 0.2:settings 不再是可动态注册的命名空间总线,而是「插件行实例配置」的表单投影。
|
|
12
|
-
// 因此原先的 model-channels / model-channel-health 两个命名空间合并为本插件的 Config
|
|
13
|
-
// (字段 groups / providerOrder / effortMemory / health),命名空间 id 即本行 id。
|
|
14
|
-
const HEALTH_SCHEMA = z.object({
|
|
15
|
-
// records/speedResults 仅作旧版存量迁移的读取源(迁入 domain 后不再写入);
|
|
16
|
-
// 权威数据在 storageDomain 的 model_channel_health 单元。
|
|
17
|
-
records: z.dict(z.array(z.any())).default({}),
|
|
18
|
-
speedResults: z.dict(z.array(z.any())).default({}),
|
|
19
|
-
runtime: z.dict(z.any()).default({}),
|
|
20
|
-
speedRequest: z.any(),
|
|
21
|
-
// nonce 必须容忍 number 与 string 两种历史形态:0.3.2 的 client 写过 UUID 字符串,
|
|
22
|
-
// 而 schema 当时声明 z.number() → 字符串一旦落盘,**整个 health 子树校验失败**,
|
|
23
|
-
// 被 .loose(true) 兜底替换成默认值:records/digest 在运行时全部消失、迁移空转。
|
|
24
|
-
// (0.3.11 线上事故根因;回归见 tests/health-schema-tolerance.test.cjs)
|
|
25
|
-
lastHandledNonce: z.union([z.number(), z.string()]).default(0),
|
|
26
|
-
testRequest: z.any(),
|
|
27
|
-
testResults: z.dict(z.any()).default({}),
|
|
28
|
-
lastTestHandledNonce: z.union([z.number(), z.string()]).default(0),
|
|
29
|
-
legacyMigrated: z.any(),
|
|
30
|
-
healthMigrated: z.any(),
|
|
31
|
-
// 小投影:host 聚合好的健康摘要(client 渲染用),digestAt 为生成时刻
|
|
32
|
-
digest: z.array(z.any()).default([]),
|
|
33
|
-
digestAt: z.number().default(0),
|
|
34
|
-
}).loose(true);
|
|
11
|
+
// health is a read-only migration input; it is never exposed by Settings forms.
|
|
35
12
|
export const Config = z.object({
|
|
36
13
|
groups: z.array(z.any()).default([]).volatile(),
|
|
37
14
|
providerOrder: z.array(z.string()).default([]).volatile(),
|
|
38
15
|
effortMemory: z.dict(z.string()).default({}).volatile(),
|
|
39
|
-
health:
|
|
16
|
+
health: z.any(),
|
|
40
17
|
}).loose(true);
|
|
41
|
-
// ---------- 任务请求的 consume-once 判定(纯函数,便于回归测试) ----------
|
|
42
|
-
// nonce 规范化为字符串键:number(现行 client 的随机 53-bit 整数)与 string(0.3.2–0.3.8 的 UUID)
|
|
43
|
-
// 都能比较;非法形态返回 null。
|
|
44
|
-
export const nonceKey = (v) => (typeof v === 'number' || typeof v === 'string') ? String(v) : null;
|
|
45
|
-
// schema 默认值 0 表示「从未处理过」,不能当一个真实 nonce 用
|
|
46
|
-
export const settledNonceKey = (v) => {
|
|
47
|
-
const k = nonceKey(v);
|
|
48
|
-
return (k === null || k === '0') ? null : k;
|
|
49
|
-
};
|
|
50
|
-
/**
|
|
51
|
-
* 是否应当消费这个 nonce(唯一入口,测试与测速共用)。
|
|
52
|
-
* 关键点:只用「严格等值 + 已消费集合」,不做任何数值大小推断——
|
|
53
|
-
* 旧实现用 Math.max(…) 比较数值大小来判等,而 nonce 是随机 53-bit,
|
|
54
|
-
* 「上一轮 > 本轮」时(≈50%)guard 失效,同一请求会在每次 volatile-update
|
|
55
|
-
* 被重新消费(重复上游请求 + 结果条目互相污染)。
|
|
56
|
-
* hasTerminal 为真(finishedAt / status ok|error)时也视为已消费:结果已落地,
|
|
57
|
-
* 重放只会得到重复计费与覆盖。
|
|
58
|
-
*/
|
|
59
|
-
export function shouldConsumeNonce({ consumed, key, claimed, settled, hasTerminal }) {
|
|
60
|
-
if (key === null || key === undefined) return false;
|
|
61
|
-
if (hasTerminal) return false;
|
|
62
|
-
if (key === claimed || key === settled) return false;
|
|
63
|
-
return !consumed.has(key);
|
|
64
|
-
}
|
|
65
|
-
/**
|
|
66
|
-
* 启动清扫的分类(纯函数):新进程里已存在的测试条目分为两类——
|
|
67
|
-
* aborted : status=running 且没有 finishedAt = 上次进程遗留的在飞任务(重载打断),
|
|
68
|
-
* 必须补写 ABORTED 终态,否则客户端只会等到 66s 假超时(审计 R3);
|
|
69
|
-
* repaired : status=running 但有 finishedAt(终态字段已在)= status 被并发整树回写
|
|
70
|
-
* 改坏的僵尸条目,按 ok 归一状态位(审计 #2 的线上物证形态)。
|
|
71
|
-
*/
|
|
72
|
-
export function selectStaleTestEntries(testResults) {
|
|
73
|
-
const aborted = [];
|
|
74
|
-
const repaired = [];
|
|
75
|
-
for (const [key, e] of Object.entries(testResults || {})) {
|
|
76
|
-
if (!e || typeof e !== 'object') continue;
|
|
77
|
-
if (e.status !== 'running') continue;
|
|
78
|
-
if (!e.finishedAt) aborted.push(key);
|
|
79
|
-
else if (e.ok === true || e.code || e.error) repaired.push(key);
|
|
80
|
-
}
|
|
81
|
-
return { aborted, repaired };
|
|
82
|
-
}
|
|
83
18
|
export function apply(ctx, config) {
|
|
84
19
|
// 本行在 profile 中的 entry id 就是 settings 命名空间;缺失时回落到包名。
|
|
85
20
|
const SELF_NS = (ctx.fiber && ctx.fiber.entry && ctx.fiber.entry.options && ctx.fiber.entry.options.id) || 'model-channel-manager';
|
|
@@ -95,7 +30,7 @@ export function apply(ctx, config) {
|
|
|
95
30
|
}
|
|
96
31
|
return fallback;
|
|
97
32
|
};
|
|
98
|
-
const
|
|
33
|
+
const legacyHealth = config.health || {};
|
|
99
34
|
const cfgOf = () => ({
|
|
100
35
|
groups: refOf('groups', []) || [],
|
|
101
36
|
providerOrder: refOf('providerOrder', []) || [],
|
|
@@ -117,7 +52,27 @@ export function apply(ctx, config) {
|
|
|
117
52
|
const groupOfRoute = (p) => (p != null && p.startsWith(ROUTE_PREFIX) ? p.slice(ROUTE_PREFIX.length) : null);
|
|
118
53
|
const routeOfGroup = (id) => ROUTE_PREFIX + id;
|
|
119
54
|
const runtime = new Map();
|
|
120
|
-
const state = { config:
|
|
55
|
+
const state = { config: cfgOf(), records: clone(legacyHealth.records || {}), speedResults: clone(legacyHealth.speedResults || {}) };
|
|
56
|
+
let healthStore = null;
|
|
57
|
+
let runtimeStore = null;
|
|
58
|
+
let storageReady = Promise.resolve();
|
|
59
|
+
let storageStatus = 'memory';
|
|
60
|
+
let storageError = null;
|
|
61
|
+
let closed = false;
|
|
62
|
+
let runtimeTimer = null;
|
|
63
|
+
let cleanupLegacy = () => {};
|
|
64
|
+
const pendingEvents = [];
|
|
65
|
+
const saveRuntime = async (rows = tasks.list()) => {
|
|
66
|
+
if (runtimeStore) await runtimeStore.save({ runtime: persistRuntime(), tasks: rows });
|
|
67
|
+
};
|
|
68
|
+
const tasks = new TaskRegistry({ persist: saveRuntime });
|
|
69
|
+
const scheduleRuntime = () => {
|
|
70
|
+
if (runtimeTimer || closed) return;
|
|
71
|
+
runtimeTimer = ctx.timeout(() => {
|
|
72
|
+
runtimeTimer = null;
|
|
73
|
+
saveRuntime().catch(error => { storageError = error.message; });
|
|
74
|
+
}, 5000);
|
|
75
|
+
};
|
|
121
76
|
const groupRuntime = (id) => {
|
|
122
77
|
let g = runtime.get(id);
|
|
123
78
|
if (g === undefined) {
|
|
@@ -138,14 +93,14 @@ export function apply(ctx, config) {
|
|
|
138
93
|
let timer = null;
|
|
139
94
|
const promise = new Promise((_, reject) => { rejectFn = reject; });
|
|
140
95
|
const fire = (err) => { if (settled) return; settled = true; rejectFn(err); };
|
|
141
|
-
ctx.effect(() => {
|
|
96
|
+
const release = ctx.effect(() => {
|
|
142
97
|
timer = setTimeout(() => fire(makeError()), ms);
|
|
143
98
|
return () => {
|
|
144
99
|
if (timer !== null) clearTimeout(timer);
|
|
145
100
|
fire({ code: 'ABORTED', message: 'plugin reloaded while this attempt was in flight' });
|
|
146
101
|
};
|
|
147
102
|
}, 'model-channel timeout guard');
|
|
148
|
-
return { promise, dispose: () => {
|
|
103
|
+
return { promise, dispose: () => { settled = true; release(); } };
|
|
149
104
|
};
|
|
150
105
|
// ---------- 配置归一化(与动态原型逐行一致) ----------
|
|
151
106
|
function normalizeConfig(raw) {
|
|
@@ -318,11 +273,12 @@ export function apply(ctx, config) {
|
|
|
318
273
|
function pushEvent(cfg, kind, payload) {
|
|
319
274
|
const rt = groupRuntime(cfg.id);
|
|
320
275
|
rt.events.push(Object.assign({ ts: Date.now(), group: cfg.id, kind }, payload || {}));
|
|
276
|
+
scheduleRuntime();
|
|
321
277
|
if (rt.events.length > 200)
|
|
322
278
|
rt.events.splice(0, rt.events.length - 200);
|
|
323
279
|
}
|
|
324
|
-
// ----------
|
|
325
|
-
const persistRuntime =
|
|
280
|
+
// ---------- 路由运行态快照 ----------
|
|
281
|
+
const persistRuntime = () => {
|
|
326
282
|
const out = {};
|
|
327
283
|
for (const [id, rt] of runtime) {
|
|
328
284
|
const now = Date.now();
|
|
@@ -331,6 +287,7 @@ export function apply(ctx, config) {
|
|
|
331
287
|
cooldowns: Array.from(rt.cooldowns.entries()).map(([key, until]) => ({ key, until, active: until > now })),
|
|
332
288
|
events: rt.events.slice(-40),
|
|
333
289
|
lastSpeedTestAt: rt.lastSpeedTestAt,
|
|
290
|
+
speedTestRunning: rt.speedTestRunning,
|
|
334
291
|
};
|
|
335
292
|
}
|
|
336
293
|
return out;
|
|
@@ -356,7 +313,7 @@ export function apply(ctx, config) {
|
|
|
356
313
|
const recordHealth = (gid, cand, entry) => {
|
|
357
314
|
const now = Date.now();
|
|
358
315
|
const key = gid || 'global';
|
|
359
|
-
const rec = { ts: now, provider: cand.provider, model: cand.model, ok: entry.ok, ttftMs: entry.ttftMs, latencyMs: entry.latencyMs, code: entry.code || null };
|
|
316
|
+
const rec = { id: randomUUID(), ts: now, provider: cand.provider, model: cand.model, ok: entry.ok, ttftMs: entry.ttftMs, latencyMs: entry.latencyMs, code: entry.code || null };
|
|
360
317
|
// 上游真实 token 用量(来自 usage StreamChunk,计费口径与 dsh-token-meter 一致):
|
|
361
318
|
// 展平写入记录条目;缺失/非法字段不写,旧记录按 0 处理(向后兼容)。
|
|
362
319
|
const u = entry.usage;
|
|
@@ -375,16 +332,15 @@ export function apply(ctx, config) {
|
|
|
375
332
|
const cutoff = now - 7 * 24 * 3600 * 1000;
|
|
376
333
|
state.records[key] = list.filter((e) => e.ts >= cutoff).slice(-300);
|
|
377
334
|
if (healthStore !== null) {
|
|
378
|
-
healthStore.appendEvent(key, rec).catch((e) => console.warn('[model-channel-manager] health event append failed:', e
|
|
379
|
-
}
|
|
380
|
-
|
|
335
|
+
healthStore.appendEvent(key, rec).catch((e) => { storageError = e.message; console.warn('[model-channel-manager] health event append failed:', e.message); });
|
|
336
|
+
} else if (storageStatus === 'loading') pendingEvents.push({ key, rec });
|
|
337
|
+
scheduleRuntime();
|
|
381
338
|
};
|
|
382
339
|
const persistSpeedResults = async (r) => {
|
|
383
340
|
if (healthStore !== null) {
|
|
384
341
|
await healthStoreSpeed(r).catch(() => { });
|
|
385
342
|
}
|
|
386
|
-
|
|
387
|
-
await bus.writeHealth({ runtime: await persistRuntime() });
|
|
343
|
+
await saveRuntime();
|
|
388
344
|
};
|
|
389
345
|
const healthStoreSpeed = async (r) => {
|
|
390
346
|
for (const [groupId, rows] of Object.entries(r || {})) {
|
|
@@ -628,6 +584,10 @@ export function apply(ctx, config) {
|
|
|
628
584
|
// 推进 rt.currentIndex,两个并发请求都从 first 开始(并发不均分)。
|
|
629
585
|
// 现在:选定时立即占位推进(同步读改写,JS 单线程内原子),失败不回滚
|
|
630
586
|
// (失败反馈走冷却,指针保持前进语义——轮询分布比严格顺序更重要)。
|
|
587
|
+
// 0.3.15:`cursor` 必须先声明——0.3.5 重构时把原来的 `let cursor = …`
|
|
588
|
+
// 改成了分支内赋值却没保留声明,ESM 严格模式下每次走虚拟路由都抛
|
|
589
|
+
// `ReferenceError: cursor is not defined`(本轮运行失败)。
|
|
590
|
+
let cursor = 0;
|
|
631
591
|
if (strategy === 'round-robin') {
|
|
632
592
|
cursor = rt.currentIndex % order.length;
|
|
633
593
|
rt.currentIndex = (cursor + 1) % order.length; // 本次请求已占用 cursor 槽位
|
|
@@ -746,9 +706,16 @@ export function apply(ctx, config) {
|
|
|
746
706
|
// F15 修复:旧行为每 chunk 重置 guard(单次等待上限而非总时限,H12:timeoutMs=40
|
|
747
707
|
// 流每 20ms 输出,84ms 后仍成功)+ 超时不中止上游。现在:总时限 deadline +
|
|
748
708
|
// 超时 abort 上游(与 streamAttempt 同一模式)+ 统一 finally 关闭。
|
|
749
|
-
async function measureCandidate(cfg, cand, st) {
|
|
709
|
+
async function measureCandidate(cfg, cand, st, signal) {
|
|
750
710
|
const llm = ctx.llm;
|
|
751
711
|
const controller = new AbortController();
|
|
712
|
+
const cancelled = new Promise((_, reject) => {
|
|
713
|
+
const abort = () => { controller.abort(signal.reason); reject(signal.reason); };
|
|
714
|
+
if (signal?.aborted) abort();
|
|
715
|
+
else signal?.addEventListener('abort', abort, { once: true });
|
|
716
|
+
controller.detach = () => signal?.removeEventListener('abort', abort);
|
|
717
|
+
});
|
|
718
|
+
cancelled.catch(() => {});
|
|
752
719
|
const deadline = Date.now() + (st.timeoutMs || 60000);
|
|
753
720
|
const inner = llm.stream({ provider: cand.provider, model: cand.model, messages: [{ role: 'user', content: [{ type: 'text', text: st.prompt }] }], maxTokens: st.maxTokens, sessionId: 'mcm-speedtest-' + (st.nonce ?? Date.now()), signal: controller.signal });
|
|
754
721
|
const start = Date.now();
|
|
@@ -777,7 +744,7 @@ export function apply(ctx, config) {
|
|
|
777
744
|
const guard = timed(Math.max(1, remaining), () => ({ code: 'TIMEOUT', message: 'speedtest timed out after ' + st.timeoutMs + 'ms (total)' }));
|
|
778
745
|
let next;
|
|
779
746
|
try {
|
|
780
|
-
next = await Promise.race([inner.next(), guard.promise]);
|
|
747
|
+
next = await Promise.race([inner.next(), guard.promise, cancelled]);
|
|
781
748
|
}
|
|
782
749
|
catch (raceErr) {
|
|
783
750
|
if (raceErr && raceErr.code === 'TIMEOUT')
|
|
@@ -804,10 +771,11 @@ export function apply(ctx, config) {
|
|
|
804
771
|
throw { code: (err && err.code) || 'STREAM_ERROR', message: (err && err.message) || 'speedtest failed' };
|
|
805
772
|
}
|
|
806
773
|
finally {
|
|
774
|
+
controller.detach();
|
|
807
775
|
await closeInner();
|
|
808
776
|
}
|
|
809
777
|
}
|
|
810
|
-
async function runSpeedTest(cfg) {
|
|
778
|
+
async function runSpeedTest(cfg, signal) {
|
|
811
779
|
const rt = groupRuntime(cfg.id);
|
|
812
780
|
if (rt.speedTestRunning)
|
|
813
781
|
return { ok: false, reason: 'already-running' };
|
|
@@ -819,6 +787,7 @@ export function apply(ctx, config) {
|
|
|
819
787
|
const results = [];
|
|
820
788
|
const concurrency = Math.max(1, st.concurrency || 3);
|
|
821
789
|
for (let i = 0; i < order.length; i += concurrency) {
|
|
790
|
+
signal?.throwIfAborted();
|
|
822
791
|
const wave = order.slice(i, i + concurrency);
|
|
823
792
|
if (wave.length === 0)
|
|
824
793
|
break;
|
|
@@ -826,9 +795,10 @@ export function apply(ctx, config) {
|
|
|
826
795
|
let lastErr = null;
|
|
827
796
|
for (let t = 0; t <= (st.retries || 0); t++) {
|
|
828
797
|
try {
|
|
829
|
-
return await measureCandidate(cfg, cand, st);
|
|
798
|
+
return await measureCandidate(cfg, cand, st, signal);
|
|
830
799
|
}
|
|
831
800
|
catch (err) {
|
|
801
|
+
signal?.throwIfAborted();
|
|
832
802
|
lastErr = err;
|
|
833
803
|
if (t < (st.retries || 0))
|
|
834
804
|
await ctx.timeout(1500);
|
|
@@ -920,8 +890,8 @@ export function apply(ctx, config) {
|
|
|
920
890
|
},
|
|
921
891
|
};
|
|
922
892
|
let adapterHandle = null;
|
|
923
|
-
// ----------
|
|
924
|
-
async function runModelTest(provider, model, prompt, maxTokens) {
|
|
893
|
+
// ---------- 单模型任务:通过 DSH llm.stream 执行 ----------
|
|
894
|
+
async function runModelTest(provider, model, prompt, maxTokens, signal) {
|
|
925
895
|
const llm = ctx.llm;
|
|
926
896
|
// F15(与 measureCandidate 同一模式):总时限 45s(旧行为每 chunk 重置 guard,
|
|
927
897
|
// 持续输出的请求永不超时而 client 已放弃)+ 超时 abort 上游 + 统一 finally 关闭。
|
|
@@ -929,6 +899,13 @@ export function apply(ctx, config) {
|
|
|
929
899
|
// 「终态写入 + 下一次 describe」,跑满就会撞上假超时(审计 #7)。
|
|
930
900
|
const MODEL_TEST_TOTAL_MS = 45000;
|
|
931
901
|
const controller = new AbortController();
|
|
902
|
+
const cancelled = new Promise((_, reject) => {
|
|
903
|
+
const abort = () => { controller.abort(signal.reason); reject(signal.reason); };
|
|
904
|
+
if (signal?.aborted) abort();
|
|
905
|
+
else signal?.addEventListener('abort', abort, { once: true });
|
|
906
|
+
controller.detach = () => signal?.removeEventListener('abort', abort);
|
|
907
|
+
});
|
|
908
|
+
cancelled.catch(() => {});
|
|
932
909
|
const deadline = Date.now() + MODEL_TEST_TOTAL_MS;
|
|
933
910
|
const inner = llm.stream({ provider, model, messages: [{ role: 'user', content: [{ type: 'text', text: prompt || '你好' }] }], maxTokens: maxTokens || 512, sessionId: 'mcm-modeltest-' + Date.now(), signal: controller.signal });
|
|
934
911
|
const start = Date.now();
|
|
@@ -960,7 +937,7 @@ export function apply(ctx, config) {
|
|
|
960
937
|
const guard = timed(Math.max(1, remaining), () => ({ code: 'TIMEOUT', message: 'model test timed out after ' + MODEL_TEST_TOTAL_MS + 'ms (total)' }));
|
|
961
938
|
let next;
|
|
962
939
|
try {
|
|
963
|
-
next = await Promise.race([inner.next(), guard.promise]);
|
|
940
|
+
next = await Promise.race([inner.next(), guard.promise, cancelled]);
|
|
964
941
|
}
|
|
965
942
|
catch (raceErr) {
|
|
966
943
|
if (raceErr && raceErr.code === 'TIMEOUT')
|
|
@@ -979,16 +956,15 @@ export function apply(ctx, config) {
|
|
|
979
956
|
if (chunk && chunk.type === 'usage' && chunk.usage && typeof chunk.usage === 'object')
|
|
980
957
|
lastUsage = chunk.usage;
|
|
981
958
|
if (chunk && chunk.type === 'text-delta' && typeof chunk.text === 'string') {
|
|
982
|
-
text
|
|
959
|
+
text = (text + chunk.text).slice(0, 2000);
|
|
983
960
|
}
|
|
984
961
|
else if (chunk && chunk.type === 'reasoning-delta' && typeof chunk.text === 'string') {
|
|
985
|
-
reasoning
|
|
962
|
+
reasoning = (reasoning + chunk.text).slice(0, 2000);
|
|
986
963
|
}
|
|
987
964
|
if (isTerminalChunk(chunk)) {
|
|
988
965
|
const reason = chunk.reason || {};
|
|
989
966
|
if (isSuccessReason(reason)) {
|
|
990
|
-
//
|
|
991
|
-
// (失败那笔由拦截器的 catch 分支记,所以此前只有失败会进健康流水)
|
|
967
|
+
// Internal model tests bypass the global interceptor and record exactly once here.
|
|
992
968
|
recordHealth(provider, { provider, model }, { ok: true, ttftMs: ttft, latencyMs: Date.now() - start, code: null, usage: lastUsage });
|
|
993
969
|
return { ok: true, ttftMs: ttft, latencyMs: Date.now() - start, text: text.slice(0, 2000), reasoning: reasoning.slice(0, 2000) };
|
|
994
970
|
}
|
|
@@ -997,151 +973,15 @@ export function apply(ctx, config) {
|
|
|
997
973
|
}
|
|
998
974
|
}
|
|
999
975
|
catch (err) {
|
|
976
|
+
if (!controller.signal.aborted && err?.code !== 'ABORTED')
|
|
977
|
+
recordHealth(provider, { provider, model }, { ok: false, ttftMs: ttft, latencyMs: Date.now() - start, code: err.code || 'STREAM_ERROR', usage: lastUsage });
|
|
1000
978
|
throw err;
|
|
1001
979
|
}
|
|
1002
980
|
finally {
|
|
981
|
+
controller.detach();
|
|
1003
982
|
await closeInner();
|
|
1004
983
|
}
|
|
1005
984
|
}
|
|
1006
|
-
// 启动清扫(审计 R3):新进程里任何「status running 但没有 finishedAt」的测试条目
|
|
1007
|
-
// 必然是上一次进程遗留的(进程内在飞任务都有 fiber 生命周期,重载会打断它们,
|
|
1008
|
-
// 旧闭包的收尾写入往往以 "Configuration entry is no longer available" 失败)。
|
|
1009
|
-
// 补写 ABORTED 终态,客户端就不必等到 66s 假超时。
|
|
1010
|
-
// 同时修复「running + finishedAt」的僵尸条目(status 被并发整树回写改坏),
|
|
1011
|
-
// 按 ok 归一状态位。
|
|
1012
|
-
const sweepStaleTestResults = () => {
|
|
1013
|
-
if (bus === null) return;
|
|
1014
|
-
const all = healthOf().testResults || {};
|
|
1015
|
-
const { aborted, repaired } = selectStaleTestEntries(all);
|
|
1016
|
-
for (const key of aborted) {
|
|
1017
|
-
bus.writeHealthLeaf(['testResults', key], Object.assign({}, all[key], {
|
|
1018
|
-
status: 'error', code: 'ABORTED',
|
|
1019
|
-
error: 'host 在测试执行期间重启/重载,结果未写入;请重试',
|
|
1020
|
-
finishedAt: Date.now(),
|
|
1021
|
-
})).catch((err) => console.warn('[model-channel-manager] stale test result sweep failed:', err && err.message));
|
|
1022
|
-
}
|
|
1023
|
-
for (const key of repaired) {
|
|
1024
|
-
bus.writeHealthLeaf(['testResults', key], Object.assign({}, all[key], { status: all[key].ok === true ? 'ok' : 'error' }))
|
|
1025
|
-
.catch((err) => console.warn('[model-channel-manager] zombie test result repair failed:', err && err.message));
|
|
1026
|
-
}
|
|
1027
|
-
if (aborted.length > 0 || repaired.length > 0)
|
|
1028
|
-
console.log('[model-channel-manager] test results swept after restart: aborted =', aborted.length, ', repaired =', repaired.length);
|
|
1029
|
-
};
|
|
1030
|
-
// 兜底重试(审计 R2):testRequest 存在、却既没被认领也没结算 —— 典型是 loader 的
|
|
1031
|
-
// volatile 提交被静默拒绝(document 落盘、fiber 引用不更新、不发 volatile-update),
|
|
1032
|
-
// 请求会被彻底忽略。这里每个 nonce 主动重试一次并留下日志,不再无声失败。
|
|
1033
|
-
const sweepPendingTestRequest = () => {
|
|
1034
|
-
if (bus === null) return;
|
|
1035
|
-
const health = healthOf();
|
|
1036
|
-
const req = health.testRequest;
|
|
1037
|
-
const key = nonceKey(req && req.nonce);
|
|
1038
|
-
if (key === null) return;
|
|
1039
|
-
const entry = (health.testResults || {})[key];
|
|
1040
|
-
const hasTerminal = !!(entry && (entry.finishedAt || entry.status === 'ok' || entry.status === 'error'));
|
|
1041
|
-
// 复用与消费路径同一个(已被测试覆盖的)判定:返回 true = 这个请求还没被消费
|
|
1042
|
-
const needsSweep = shouldConsumeNonce({
|
|
1043
|
-
consumed: consumedTestNonces,
|
|
1044
|
-
key,
|
|
1045
|
-
claimed: nonceKey(claimedTestNonce),
|
|
1046
|
-
settled: settledNonceKey(health.lastTestHandledNonce),
|
|
1047
|
-
hasTerminal,
|
|
1048
|
-
});
|
|
1049
|
-
if (!needsSweep || pendingSweepAttempted.has(key)) return;
|
|
1050
|
-
markConsumed(pendingSweepAttempted, key);
|
|
1051
|
-
console.warn('[model-channel-manager] testRequest nonce', key,
|
|
1052
|
-
'既未结算也未消费——疑似 volatile 提交被静默拒绝,主动重试一次(若仍无结果,请看宿主 stdout 的 loader 警告)');
|
|
1053
|
-
reloadFromConfig(false);
|
|
1054
|
-
};
|
|
1055
|
-
function handleTestRequest(next) {
|
|
1056
|
-
const req = next.testRequest;
|
|
1057
|
-
// nonce 接受 number(现行 client 的随机 53-bit 整数)与 string(0.3.2–0.3.8 的 UUID)
|
|
1058
|
-
// 两种历史形态;去重一律按字符串键严格比较,不再做数值大小推断。
|
|
1059
|
-
const nonceOf = (v) => (typeof v === 'number' || typeof v === 'string') ? v : null;
|
|
1060
|
-
const reqNonce = nonceOf(req && req.nonce);
|
|
1061
|
-
const reqKey = nonceKey(reqNonce);
|
|
1062
|
-
const claimedKey = nonceKey(claimedTestNonce);
|
|
1063
|
-
const settled = settledNonceKey(next.lastTestHandledNonce);
|
|
1064
|
-
const entry = reqKey !== null ? (next.testResults || {})[reqKey] : null;
|
|
1065
|
-
// 终态判定用 finishedAt(写入后不会被覆盖),不只信 status:
|
|
1066
|
-
// 整树快照回写曾把 status 改回 running 却保留 finishedAt/code/error(0.3.11 事故)。
|
|
1067
|
-
const hasTerminal = !!(entry && (entry.finishedAt || entry.status === 'ok' || entry.status === 'error'));
|
|
1068
|
-
if (!req || typeof req.provider !== 'string' || typeof req.model !== 'string' || reqKey === null) {
|
|
1069
|
-
// 字段非法:显式记一次,别再静默吞掉(审计 R2)
|
|
1070
|
-
if (req && reqKey !== null && !warnedTestNonces.has(reqKey)) {
|
|
1071
|
-
markConsumed(warnedTestNonces, reqKey);
|
|
1072
|
-
console.warn('[model-channel-manager] testRequest 字段非法,已忽略(需要 provider/model/nonce):', JSON.stringify(req).slice(0, 200));
|
|
1073
|
-
}
|
|
1074
|
-
return;
|
|
1075
|
-
}
|
|
1076
|
-
if (Date.now() < testBackoffUntil) return; // 退避窗口内不消费(等重放)
|
|
1077
|
-
if (!shouldConsumeNonce({ consumed: consumedTestNonces, key: reqKey, claimed: claimedKey, settled, hasTerminal })) {
|
|
1078
|
-
// 未消费的显式解释:请求存在但既没被认领、也没终态 → 多半是 volatile 提交被
|
|
1079
|
-
// loader 静默拒绝(document 已落盘、fiber 引用不更新、不发 volatile-update,
|
|
1080
|
-
// 审计 R2),此时请求会被彻底忽略且没有任何可见报错。
|
|
1081
|
-
if (!hasTerminal && reqKey !== claimedKey && !warnedTestNonces.has(reqKey)) {
|
|
1082
|
-
markConsumed(warnedTestNonces, reqKey);
|
|
1083
|
-
console.warn('[model-channel-manager] testRequest 未被消费: nonce', reqKey,
|
|
1084
|
-
'(已结算或已消费命中)——若本次测试始终没有结果,检查宿主 stdout 是否有 loader 的 "volatile config update failed"');
|
|
1085
|
-
}
|
|
1086
|
-
return;
|
|
1087
|
-
}
|
|
1088
|
-
markConsumed(consumedTestNonces, reqKey);
|
|
1089
|
-
claimedTestNonce = reqNonce;
|
|
1090
|
-
// nonce 的落盘推迟到本次测试得出结论之后:只有「真跑过」才算 handled。
|
|
1091
|
-
// 启动早于凭据服务就绪时会以 MISSING_CREDENTIAL 失败,那要释放认领重试;
|
|
1092
|
-
// 若提前写了 nonce,重试会被自己的持久值挡掉。
|
|
1093
|
-
const settle = (entry) => {
|
|
1094
|
-
if (bus !== null)
|
|
1095
|
-
bus.writeHealth({ lastTestHandledNonce: reqNonce }).catch((e) => console.error('[model-channel-manager] lastTestHandledNonce 写入失败:', e));
|
|
1096
|
-
return setResult(entry);
|
|
1097
|
-
};
|
|
1098
|
-
// 0.3.12:结果按「单键叶写」落盘(path-ops),不再把整个 testResults 读出来整段写回。
|
|
1099
|
-
// 整段写回的 payload 是调用时刻的快照,落盘晚于并发的终态写时会把该 nonce 的
|
|
1100
|
-
// status 改回 running(深合并保留 finishedAt/code/error)——线上事故的直接成因。
|
|
1101
|
-
// 叶写只动这一个 nonce 的条目,物理上不可能覆盖别人的、也不可能自我回退。
|
|
1102
|
-
const setResult = async (entry) => {
|
|
1103
|
-
const key = String(reqNonce);
|
|
1104
|
-
const value = Object.assign({}, entry, { nonce: reqNonce, provider: req.provider, model: req.model });
|
|
1105
|
-
if (bus !== null)
|
|
1106
|
-
await bus.writeHealthLeaf(['testResults', key], value).catch((e) => console.error('[model-channel-manager] testResults 写入失败:', e));
|
|
1107
|
-
};
|
|
1108
|
-
// 修剪低频执行:只在条目数超限时砍到 50。用 unset 叶删(update 的深合并删不掉键)
|
|
1109
|
-
const maybePrune = () => {
|
|
1110
|
-
const all = healthOf().testResults || {};
|
|
1111
|
-
const entries = Object.entries(all);
|
|
1112
|
-
if (entries.length <= 50) return;
|
|
1113
|
-
const removed = entries
|
|
1114
|
-
.sort((a, b) => ((b[1] && b[1].finishedAt) || 0) - ((a[1] && a[1].finishedAt) || 0))
|
|
1115
|
-
.slice(50)
|
|
1116
|
-
.map(([k]) => k);
|
|
1117
|
-
if (removed.length === 0 || bus === null) return;
|
|
1118
|
-
bus.writeHealthOps(removed.map((k) => ({ op: 'unset', path: ['health', 'testResults', k] })))
|
|
1119
|
-
.catch((e) => console.error('[model-channel-manager] testResults 修剪失败:', e));
|
|
1120
|
-
};
|
|
1121
|
-
const done = async (r) => {
|
|
1122
|
-
try {
|
|
1123
|
-
await settle(Object.assign({}, r, { finishedAt: Date.now() }));
|
|
1124
|
-
maybePrune();
|
|
1125
|
-
}
|
|
1126
|
-
catch (_e) { }
|
|
1127
|
-
};
|
|
1128
|
-
void setResult({ status: 'running', startedAt: Date.now() });
|
|
1129
|
-
runModelTest(req.provider, req.model, req.prompt, req.maxTokens).then(async (r) => {
|
|
1130
|
-
replayAttempts.delete('test');
|
|
1131
|
-
await done(Object.assign({ status: 'ok' }, r));
|
|
1132
|
-
}).catch(async (e) => {
|
|
1133
|
-
if (notReady(e)) {
|
|
1134
|
-
// 凭据服务尚未就绪(典型是启动瞬间的重放):不写成假 error,释放认领并延后重试
|
|
1135
|
-
claimedTestNonce = null;
|
|
1136
|
-
releaseConsumed(consumedTestNonces, reqKey); // 释放后才能靠重放再消费
|
|
1137
|
-
releaseConsumed(warnedTestNonces, reqKey);
|
|
1138
|
-
testBackoffUntil = Date.now() + 5000; // 退避窗口:期间任何 volatile-update 都不再消费
|
|
1139
|
-
scheduleReplay('test', () => reloadFromConfig(false));
|
|
1140
|
-
return;
|
|
1141
|
-
}
|
|
1142
|
-
await done({ status: 'error', code: (e && e.code) || 'STREAM_ERROR', error: (e && e.message) || String(e) });
|
|
1143
|
-
});
|
|
1144
|
-
}
|
|
1145
985
|
function rewireRoutes() {
|
|
1146
986
|
const llm = ctx.llm;
|
|
1147
987
|
// F16(审计 H08):normalizeConfig 对非法/重复 ID 是静默 continue——保存 3 组
|
|
@@ -1170,348 +1010,116 @@ export function apply(ctx, config) {
|
|
|
1170
1010
|
else if (JSON.stringify(current) !== JSON.stringify(next))
|
|
1171
1011
|
adapterHandle.replace(next);
|
|
1172
1012
|
}
|
|
1173
|
-
//
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
let healthStore = null;
|
|
1177
|
-
// digest:写进 settings health 子树的小投影(client 健康页渲染用)。
|
|
1178
|
-
// 聚合上移 host,client 不再拉原始流水全量 describe——settings 里不再出现 records。
|
|
1179
|
-
let digestTimer = null;
|
|
1180
|
-
let lastDigestJson = null; // 上次落盘的 digest 内容(相同则跳过,减少 revision 噪声)
|
|
1181
|
-
let lastDigestWriteAt = 0;
|
|
1182
|
-
const DIGEST_INTERVAL = 5000;
|
|
1183
|
-
const buildDigest = () => {
|
|
1184
|
-
// 与旧 client HealthPanel 聚合口径一致:total/success/ttft/latency/token(计费口径
|
|
1185
|
-
// = input + cacheRead + cacheWrite + output),外加 lastTs/lastOk/lastCode
|
|
1186
|
-
const by = new Map();
|
|
1187
|
-
for (const list of Object.values(state.records)) {
|
|
1188
|
-
for (const e of list) {
|
|
1189
|
-
const key = e.provider + '::' + e.model;
|
|
1190
|
-
let a = by.get(key);
|
|
1191
|
-
if (a === undefined) {
|
|
1192
|
-
a = { provider: e.provider, model: e.model, total: 0, success: 0, ttftSum: 0, latSum: 0, tokIn: 0, tokOut: 0, tokCache: 0, lastTs: 0, lastOk: null, lastCode: null };
|
|
1193
|
-
by.set(key, a);
|
|
1194
|
-
}
|
|
1195
|
-
a.total++;
|
|
1196
|
-
const inTok = e.inputTokens || 0;
|
|
1197
|
-
const outTok = e.outputTokens || 0;
|
|
1198
|
-
const cacheTok = (e.cacheReadTokens || 0) + (e.cacheWriteTokens || 0);
|
|
1199
|
-
a.tokIn += inTok;
|
|
1200
|
-
a.tokOut += outTok;
|
|
1201
|
-
a.tokCache += cacheTok;
|
|
1202
|
-
if (e.ok) {
|
|
1203
|
-
a.success++;
|
|
1204
|
-
if (e.ttftMs != null && e.ttftMs >= 0) a.ttftSum += e.ttftMs;
|
|
1205
|
-
if (e.latencyMs != null && e.latencyMs >= 0) a.latSum += e.latencyMs;
|
|
1206
|
-
}
|
|
1207
|
-
if ((e.ts || 0) > a.lastTs) {
|
|
1208
|
-
a.lastTs = e.ts || 0;
|
|
1209
|
-
a.lastOk = e.ok;
|
|
1210
|
-
a.lastCode = e.ok ? null : (e.code || null); // lastCode 随 lastTs 对齐(不再按遍历序覆盖)
|
|
1211
|
-
}
|
|
1212
|
-
}
|
|
1213
|
-
}
|
|
1214
|
-
return [...by.values()].map((a) => ({
|
|
1215
|
-
provider: a.provider, model: a.model, total: a.total, success: a.success,
|
|
1216
|
-
ttftAvg: a.success > 0 ? Math.round(a.ttftSum / a.success) : null,
|
|
1217
|
-
latAvg: a.success > 0 ? Math.round(a.latSum / a.success) : null,
|
|
1218
|
-
tokIn: a.tokIn, tokOut: a.tokOut, tokCache: a.tokCache,
|
|
1219
|
-
lastTs: a.lastTs, lastOk: a.lastOk, lastCode: a.lastCode,
|
|
1220
|
-
}));
|
|
1221
|
-
};
|
|
1222
|
-
const flushDigest = () => {
|
|
1223
|
-
digestTimer = null;
|
|
1224
|
-
if (bus !== null) {
|
|
1225
|
-
const digest = buildDigest();
|
|
1226
|
-
// 0.3.14:内容没变就不写。settings 行的 revision 是 raw config 的 JSON 指纹
|
|
1227
|
-
// (dsh-settings describe:revision += raw !== previous.raw),每一次无意义的写
|
|
1228
|
-
// 都会让 client 手里的 revision 过期,表现为保存时的 settings/conflict。
|
|
1229
|
-
// 60s 心跳保证 digestAt 不会永远停住。
|
|
1230
|
-
const json = JSON.stringify(digest);
|
|
1231
|
-
if (json === lastDigestJson && Date.now() - lastDigestWriteAt < 60000) return;
|
|
1232
|
-
lastDigestJson = json;
|
|
1233
|
-
lastDigestWriteAt = Date.now();
|
|
1234
|
-
bus.writeHealth({ digest, digestAt: Date.now() }).catch(() => { });
|
|
1235
|
-
}
|
|
1236
|
-
};
|
|
1237
|
-
const scheduleDigest = () => {
|
|
1238
|
-
if (digestTimer === null) {
|
|
1239
|
-
digestTimer = ctx.timeout(flushDigest, DIGEST_INTERVAL);
|
|
1240
|
-
}
|
|
1241
|
-
};
|
|
1242
|
-
ctx.effect(() => () => {
|
|
1243
|
-
if (digestTimer !== null) {
|
|
1244
|
-
digestTimer();
|
|
1245
|
-
digestTimer = null;
|
|
1246
|
-
}
|
|
1247
|
-
flushDigest();
|
|
1248
|
-
}, 'model-channel health digest');
|
|
1249
|
-
// settings 存量 records/speedResults → domain 一次性迁移(桶已存在即跳过,domain 权威)。
|
|
1250
|
-
// 清空旧键必须走 path-ops:update 是深合并,`records: {}` 清不掉已有键——
|
|
1251
|
-
// 0.3.8 用合并写「清空」实际没删,旧流水一直留在 profile 里(每 5s describe 都要带着它)。
|
|
1252
|
-
const migrateHealthToDomain = async () => {
|
|
1253
|
-
if (healthStore === null || bus === null) return;
|
|
1254
|
-
const health = healthOf();
|
|
1255
|
-
const alreadyMigrated = health.healthMigrated === true;
|
|
1256
|
-
const hasRecords = health.records !== undefined && Object.keys(health.records || {}).length > 0;
|
|
1257
|
-
const hasSpeed = health.speedResults !== undefined && Object.keys(health.speedResults || {}).length > 0;
|
|
1258
|
-
if (!hasRecords && !hasSpeed) {
|
|
1259
|
-
// 无存量:只补哨兵(幂等)
|
|
1260
|
-
if (!alreadyMigrated) {
|
|
1261
|
-
await bus.writeHealthOps([{ op: 'set', path: ['health', 'healthMigrated'], value: true }])
|
|
1262
|
-
.catch((e) => console.warn('[model-channel-manager] health marker write failed:', e && e.message));
|
|
1263
|
-
}
|
|
1264
|
-
return;
|
|
1265
|
-
}
|
|
1266
|
-
// 有存量就必须导入(合并幂等)——不能只看 marker:schema 缺陷期间 marker 可能已被
|
|
1267
|
-
// 误写而数据从未导入(0.3.11 线上事故),照 marker 跳过再清理 = 直接删掉旧账。
|
|
1268
|
-
let ok = false;
|
|
1269
|
-
try {
|
|
1270
|
-
ok = await healthStore.migrateFrom(health.records || {}, health.speedResults || {});
|
|
1271
|
-
}
|
|
1272
|
-
catch (e) {
|
|
1273
|
-
console.warn('[model-channel-manager] legacy health import failed (will retry next boot):', e && e.message);
|
|
1274
|
-
return;
|
|
1275
|
-
}
|
|
1276
|
-
if (!ok) return; // store 已关闭(卸载竞态):不清理,避免标记与数据永久不一致
|
|
1277
|
-
// 导入成功才真删除旧键(path-ops;update 的深合并清不掉)+ 落哨兵
|
|
1278
|
-
const ops = [];
|
|
1279
|
-
if (hasRecords) ops.push({ op: 'unset', path: ['health', 'records'] });
|
|
1280
|
-
if (hasSpeed) ops.push({ op: 'unset', path: ['health', 'speedResults'] });
|
|
1281
|
-
if (!alreadyMigrated) ops.push({ op: 'set', path: ['health', 'healthMigrated'], value: true });
|
|
1282
|
-
if (ops.length === 0) return;
|
|
1283
|
-
await bus.writeHealthOps(ops).catch((e) => console.warn('[model-channel-manager] legacy health cleanup failed:', e && e.message));
|
|
1284
|
-
console.log('[model-channel-manager] legacy health imported into domain and cleared from settings');
|
|
1285
|
-
};
|
|
1286
|
-
|
|
1287
|
-
// ---------- settings 总线接入(响应式:settings 服务异步初始化,apply 时查询太早) ----------
|
|
1288
|
-
let bus = null;
|
|
1289
|
-
let runtimeRestored = false;
|
|
1290
|
-
// 非持久去重:回写要等当前 HMR 事务结束才落盘,这中间 volatile-update 可能带着同一个 nonce 再来,
|
|
1291
|
-
// 内存里先认领,避免同一请求被重复执行(并顺带消掉重复的回写风暴)。
|
|
1292
|
-
//
|
|
1293
|
-
// 0.3.12:改为严格 consume-once(字符串键集合),不再依赖 nonce 的数值单调性。
|
|
1294
|
-
// 旧实现 last = Math.max(上一轮结算值, 本次认领值),而两个 nonce 都是随机 53-bit:
|
|
1295
|
-
// 「上一轮 > 本轮」时判等直接失效(≈50%),于是测试在每次 loader/volatile-update
|
|
1296
|
-
// (包括宿主自己每 5s 的 digest 回写)都被重新消费——重复上游请求(真实计费)
|
|
1297
|
-
// 且结果条目互相污染。线上物证:同 nonce 一条记录同时带
|
|
1298
|
-
// `code TIMEOUT / 60000ms` 与 `ok:true / text`(单次执行不可能)。
|
|
1299
|
-
// 回归见 tests/task-dedup.test.cjs。
|
|
1300
|
-
let claimedTestNonce = null; // string UUID 或旧数字;null = 未认领
|
|
1301
|
-
let claimedSpeedNonce = null; // string UUID 或旧数字;null = 未认领
|
|
1302
|
-
let lastHealthKeyCount = -1; // health 子树键数(用于探测「被兜底清空」,审计 R2/#4)
|
|
1303
|
-
const consumedTestNonces = new Set(); // 已消费过的 nonce 字符串键(含本次)
|
|
1304
|
-
const consumedSpeedNonces = new Set();
|
|
1305
|
-
const warnedTestNonces = new Set(); // 每个 nonce 只提醒一次(防日志刷屏)
|
|
1306
|
-
const pendingSweepAttempted = new Set(); // 兜底重试每个 nonce 只做一次
|
|
1307
|
-
const CONSUMED_NONCE_CAP = 256; // 上限:只用于防重复消费,不需要无限历史
|
|
1308
|
-
// notReady(凭据服务未就绪)释放认领后的退避窗口:期间不再消费,等 scheduleReplay 到点。
|
|
1309
|
-
// 否则释放后任何一次 volatile-update 都会立刻再消费,并发起多个 60s 上游请求(审计 #6)。
|
|
1310
|
-
let testBackoffUntil = 0;
|
|
1311
|
-
let speedBackoffUntil = 0;
|
|
1312
|
-
const markConsumed = (set, key) => {
|
|
1313
|
-
set.add(key);
|
|
1314
|
-
if (set.size > CONSUMED_NONCE_CAP) {
|
|
1315
|
-
const oldest = set.values().next().value;
|
|
1316
|
-
set.delete(oldest);
|
|
1317
|
-
}
|
|
1318
|
-
};
|
|
1319
|
-
const releaseConsumed = (set, key) => { if (key !== null) set.delete(key); };
|
|
1320
|
-
// 启动早于凭据服务就绪:此时重放测试/测速会以 MISSING_CREDENTIAL 失败。
|
|
1321
|
-
// 这类错误是「环境还没准备好」而不是「渠道故障」,必须释放认领、延后重试,不能写成假 error。
|
|
1322
|
-
const notReady = (e) => {
|
|
1323
|
-
const code = (e && e.code) || '';
|
|
1324
|
-
const msg = String((e && (e.message || e.error)) || '');
|
|
1325
|
-
return code === 'MISSING_CREDENTIAL' || code === 'INVALID_CREDENTIAL' || /no credential|credential .*not set|is not set — store/i.test(msg);
|
|
1326
|
-
};
|
|
1327
|
-
const replayAttempts = new Map();
|
|
1328
|
-
const scheduleReplay = (key, fn, ms = 5000) => {
|
|
1329
|
-
const n = (replayAttempts.get(key) || 0) + 1;
|
|
1330
|
-
if (n > 24)
|
|
1331
|
-
return;
|
|
1332
|
-
replayAttempts.set(key, n);
|
|
1333
|
-
ctx.timeout(() => { fn(); }, ms);
|
|
1334
|
-
};
|
|
1335
|
-
// 从实例配置读取全部状态;volatile 提交后由 loader/volatile-update 触发重载。
|
|
1336
|
-
// 0.3.1 起 records/speedResults 的权威在 storageDomain(healthStore);
|
|
1337
|
-
// settings health 子树只承载小投影(digest 给 client 渲染)+ 运行态 + 任务哨,
|
|
1338
|
-
// volatile 快照覆盖不再能抹掉任何流水(F11/F23 的根因消除)。
|
|
1339
|
-
function reloadFromConfig(initial) {
|
|
1340
|
-
const cfg = cfgOf();
|
|
1341
|
-
state.config = clone(cfg) || { groups: [] };
|
|
1342
|
-
const health = healthOf();
|
|
1343
|
-
// 意外清空告警(审计 R2/#4):health 子树本来有内容,运行时却变成空对象 =
|
|
1344
|
-
// schema 校验失败被 .loose(true) 兜底(0.3.11 事故形态),必须大声报出来。
|
|
1345
|
-
const healthKeys = Object.keys(health || {}).length;
|
|
1346
|
-
if (healthKeys === 0 && lastHealthKeyCount > 0) {
|
|
1347
|
-
console.warn('[model-channel-manager] health 子树在运行时变空(上一轮有 ' + lastHealthKeyCount + ' 个键)——极可能是 schema 校验失败被 .loose(true) 静默兜底,请检查最近一次写入的字段类型');
|
|
1348
|
-
}
|
|
1349
|
-
lastHealthKeyCount = healthKeys;
|
|
1350
|
-
state.records = healthStore !== null
|
|
1351
|
-
? clone(healthStore.allEventBuckets())
|
|
1352
|
-
: clone(health.records || {});
|
|
1353
|
-
state.speedResults = healthStore !== null
|
|
1354
|
-
? clone(healthStore.allSpeedBuckets())
|
|
1355
|
-
: clone(health.speedResults || {});
|
|
1356
|
-
state.testResults = clone(health.testResults || {});
|
|
1357
|
-
if (initial && !runtimeRestored) {
|
|
1358
|
-
restoreRuntime(health.runtime || {});
|
|
1359
|
-
runtimeRestored = true;
|
|
1360
|
-
}
|
|
1013
|
+
// Configuration changes only rebuild routes; they never reload runtime snapshots.
|
|
1014
|
+
const reloadFromConfig = () => {
|
|
1015
|
+
state.config = clone(cfgOf());
|
|
1361
1016
|
const live = pullConfig().groups;
|
|
1362
|
-
for (const
|
|
1363
|
-
|
|
1364
|
-
// 清理已删除组的运行时残留,避免 runtime 持久化无限累积
|
|
1365
|
-
for (const id of Array.from(runtime.keys()))
|
|
1366
|
-
if (!live.some((g) => g.id === id))
|
|
1367
|
-
runtime.delete(id);
|
|
1017
|
+
for (const group of live) groupRuntime(group.id);
|
|
1018
|
+
for (const id of runtime.keys()) if (!live.some(group => group.id === id)) runtime.delete(id);
|
|
1368
1019
|
rewireRoutes();
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
}).catch((e) => {
|
|
1398
|
-
if (notReady(e)) {
|
|
1399
|
-
claimedSpeedNonce = null;
|
|
1400
|
-
releaseConsumed(consumedSpeedNonces, speedKey);
|
|
1401
|
-
speedBackoffUntil = Date.now() + 5000; // 退避:同上
|
|
1402
|
-
scheduleReplay('speed', () => reloadFromConfig(false));
|
|
1403
|
-
return;
|
|
1404
|
-
}
|
|
1405
|
-
settleSpeed();
|
|
1406
|
-
console.error('[model-channel-manager] speedtest error:', e);
|
|
1407
|
-
});
|
|
1408
|
-
else
|
|
1409
|
-
settleSpeed();
|
|
1410
|
-
}
|
|
1411
|
-
handleTestRequest(health);
|
|
1020
|
+
};
|
|
1021
|
+
const notReady = error => /credential/i.test(String(error?.code || '') + ' ' + String(error?.message || ''));
|
|
1022
|
+
restoreRuntime(legacyHealth.runtime || {});
|
|
1023
|
+
tasks.restore(Object.entries(legacyHealth.testResults || {}).map(([id, row]) => ({ ...row, id, kind: 'test', fingerprint: 'legacy' })));
|
|
1024
|
+
ctx.on('loader/volatile-update', reloadFromConfig);
|
|
1025
|
+
reloadFromConfig();
|
|
1026
|
+
|
|
1027
|
+
// No settings service is needed for runtime reads or model tasks.
|
|
1028
|
+
{
|
|
1029
|
+
const handler = createRpcHandler({
|
|
1030
|
+
tasks,
|
|
1031
|
+
ready: () => storageReady,
|
|
1032
|
+
snapshot: () => ({
|
|
1033
|
+
windows: aggregateHealth(state.records), digestAt: Date.now(),
|
|
1034
|
+
speedResults: clone(state.speedResults), runtime: persistRuntime(),
|
|
1035
|
+
storage: { mode: storageStatus, error: storageError },
|
|
1036
|
+
retention: { days: 7, perBucketLimit: 300 },
|
|
1037
|
+
}),
|
|
1038
|
+
test: (request, signal) => runModelTest(request.provider, request.model, request.prompt, request.maxTokens, signal),
|
|
1039
|
+
speed: async (request, signal) => {
|
|
1040
|
+
const group = pullConfig().groups.find(group => group.id === request.group);
|
|
1041
|
+
if (!group) throw Object.assign(new Error('轮询组不存在,请先保存配置'), { code: 'GROUP_NOT_FOUND' });
|
|
1042
|
+
const result = await runSpeedTest(group, signal);
|
|
1043
|
+
if (!result.ok) throw Object.assign(new Error(result.deferred ? '凭据尚未就绪,请配置后重试' : '该组正在测速'), { code: result.deferred ? 'MISSING_CREDENTIAL' : 'BUSY' });
|
|
1044
|
+
return result;
|
|
1045
|
+
},
|
|
1046
|
+
});
|
|
1047
|
+
ctx.plugin(ChannelController, { handler });
|
|
1412
1048
|
}
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
if (ops.length === 0) return;
|
|
1445
|
-
return outsideTransaction(() => settings.mutate(SELF_NS, ops, undefined));
|
|
1049
|
+
|
|
1050
|
+
ctx.inject(['storageDomain'], dctx => {
|
|
1051
|
+
const store = new HealthStore(dctx);
|
|
1052
|
+
const profile = ctx.get('profileContext');
|
|
1053
|
+
const persistent = new RuntimeStore(dctx, (profile?.dir || 'default') + ':' + SELF_NS);
|
|
1054
|
+
storageStatus = 'loading';
|
|
1055
|
+
storageReady = (async () => {
|
|
1056
|
+
await store.open();
|
|
1057
|
+
const saved = await persistent.open();
|
|
1058
|
+
// Import before deleting Config, including requests retained as an archive only.
|
|
1059
|
+
await persistent.archive(clone(legacyHealth));
|
|
1060
|
+
await store.migrateFrom(legacyHealth.records || {}, legacyHealth.speedResults || {});
|
|
1061
|
+
await store.migrateFrom(state.records, state.speedResults);
|
|
1062
|
+
while (pendingEvents.length) {
|
|
1063
|
+
const batch = {};
|
|
1064
|
+
for (const { key, rec } of pendingEvents.splice(0)) (batch[key] ||= []).push(rec);
|
|
1065
|
+
await store.migrateFrom(batch, {});
|
|
1066
|
+
}
|
|
1067
|
+
if (closed) { await store.close(); await persistent.close(); return; }
|
|
1068
|
+
healthStore = store;
|
|
1069
|
+
runtimeStore = persistent;
|
|
1070
|
+
state.records = clone(store.allEventBuckets());
|
|
1071
|
+
state.speedResults = clone(store.allSpeedBuckets());
|
|
1072
|
+
if (saved) { restoreRuntime(saved.runtime); tasks.restore(saved.tasks); }
|
|
1073
|
+
await saveRuntime();
|
|
1074
|
+
storageStatus = 'persistent';
|
|
1075
|
+
queueMicrotask(() => cleanupLegacy());
|
|
1076
|
+
})().catch(async error => {
|
|
1077
|
+
storageStatus = 'memory';
|
|
1078
|
+
storageError = '健康数据暂存内存,旧配置保留:' + error.message;
|
|
1079
|
+
console.warn('[model-channel-manager]', storageError);
|
|
1446
1080
|
});
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1081
|
+
dctx.effect(() => async () => {
|
|
1082
|
+
await storageReady;
|
|
1083
|
+
await tasks.close();
|
|
1084
|
+
await saveRuntime().catch(error => { storageError = error.message; });
|
|
1085
|
+
if (healthStore === store) healthStore = null;
|
|
1086
|
+
if (runtimeStore === persistent) runtimeStore = null;
|
|
1087
|
+
await store.close();
|
|
1088
|
+
await persistent.close();
|
|
1089
|
+
if (!closed) { tasks.closed = false; storageStatus = 'memory'; }
|
|
1090
|
+
}, 'model-channel persistent stores');
|
|
1091
|
+
});
|
|
1092
|
+
// This one-time edit uses ConfigEditor because health is intentionally not live-editable.
|
|
1093
|
+
ctx.inject(['configEditor'], ectx => {
|
|
1094
|
+
const hmr = ctx.get('hmr');
|
|
1095
|
+
const cleanup = async () => {
|
|
1096
|
+
await storageReady;
|
|
1097
|
+
if (closed || storageStatus !== 'persistent' || !Object.keys(legacyHealth).length) return;
|
|
1098
|
+
const entry = ectx.configEditor.entries().find(row => row.options.id === SELF_NS);
|
|
1099
|
+
if (!entry || !Object.hasOwn(entry.options.config || {}, 'health')) return;
|
|
1100
|
+
await ectx.configEditor.edit(entry, raw => {
|
|
1101
|
+
if (JSON.stringify(raw.health) !== JSON.stringify(legacyHealth)) throw new Error('旧健康数据在迁移期间发生变化,已保留配置,请重新加载完成迁移');
|
|
1102
|
+
const next = { ...raw };
|
|
1103
|
+
delete next.health;
|
|
1104
|
+
return next;
|
|
1105
|
+
});
|
|
1463
1106
|
};
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
sweepStaleTestResults();
|
|
1471
|
-
reloadFromConfig(true);
|
|
1472
|
-
ctx.on('loader/volatile-update', () => {
|
|
1473
|
-
reloadFromConfig(false);
|
|
1474
|
-
console.log('[model-channel-manager] config hot-reloaded, routes:', pullConfig().groups.map((g) => g.id).join(', ') || '(none)');
|
|
1475
|
-
});
|
|
1476
|
-
boot();
|
|
1477
|
-
scheduleDigest(); // 首屏投影:让健康页不用等下一次真实请求
|
|
1478
|
-
// 未消费请求的兜底轮询(审计 R2):每 5s 看一眼 testRequest 是否被静默丢弃
|
|
1479
|
-
ctx.effect(() => {
|
|
1480
|
-
const timer = setInterval(() => { try { sweepPendingTestRequest(); } catch (_e) { } }, 5000);
|
|
1481
|
-
return () => clearInterval(timer);
|
|
1482
|
-
}, 'model-channel pending-request sweep');
|
|
1483
|
-
// storageDomain 是可选服务,必须经响应式 inject 取得「已声明依赖」的上下文:
|
|
1484
|
-
// 直接 ctx.get('storageDomain') 只能拿到未注入的裸服务,随后任何属性访问都会被
|
|
1485
|
-
// Cordis 代理拒绝(实测 "cannot get property storageDomain without inject",
|
|
1486
|
-
// domain 永远打不开、健康流水静默降级为内存)。inject 回调在服务可用时触发;
|
|
1487
|
-
// profile 没有 storage 栈时它不触发,插件保持内存模式,路由与面板不受影响。
|
|
1488
|
-
ctx.inject(['storageDomain'], (dctx) => {
|
|
1489
|
-
const store = new HealthStore(dctx);
|
|
1490
|
-
Promise.resolve()
|
|
1491
|
-
.then(() => store.open())
|
|
1492
|
-
.then(() => {
|
|
1493
|
-
if (!fiberAlive()) {
|
|
1494
|
-
void store.close().catch(() => { }); // 已卸载:由 facility closeAll 兜底回收
|
|
1495
|
-
return;
|
|
1496
|
-
}
|
|
1497
|
-
healthStore = store;
|
|
1498
|
-
ctx.effect(() => () => { void store.close(); }, 'model-channel health domain close');
|
|
1499
|
-
console.log('[model-channel-manager] health domain ready: model_channel_health');
|
|
1500
|
-
})
|
|
1501
|
-
.catch((e) => {
|
|
1502
|
-
console.warn('[model-channel-manager] storageDomain open failed, health records stay in-memory only:', e && e.message);
|
|
1503
|
-
})
|
|
1504
|
-
.then(() => (healthStore !== null ? migrateHealthToDomain().catch(() => { }) : null))
|
|
1505
|
-
.then(() => {
|
|
1506
|
-
if (!fiberAlive()) return;
|
|
1507
|
-
reloadFromConfig(true); // domain 权威数据覆盖内存镜像
|
|
1508
|
-
scheduleDigest(); // 立刻把投影刷新到 settings(健康页首屏可读)
|
|
1509
|
-
})
|
|
1510
|
-
.catch(() => { }); // 卸载竞态下 reload 内部可能 throw,链尾兜底防 unhandled rejection
|
|
1511
|
-
});
|
|
1107
|
+
const launch = () => { void cleanup().catch(error => { storageError = error.message; console.warn('[model-channel-manager] migration cleanup:', error.message); }); };
|
|
1108
|
+
cleanupLegacy = () => {
|
|
1109
|
+
if (hmr?.executing?.getStore()) hmr.executing.exit(launch);
|
|
1110
|
+
else launch();
|
|
1111
|
+
};
|
|
1112
|
+
cleanupLegacy();
|
|
1512
1113
|
});
|
|
1114
|
+
ctx.effect(() => async () => {
|
|
1115
|
+
closed = true;
|
|
1116
|
+
if (runtimeTimer) runtimeTimer();
|
|
1117
|
+
await tasks.close();
|
|
1118
|
+
}, 'model-channel task shutdown');
|
|
1119
|
+
|
|
1513
1120
|
// ---------- 全局 LLM 请求健康拦截 (涵盖所有非虚拟路由的真实渠道模型调用) ----------
|
|
1514
1121
|
ctx.on('llm/stream', async function* (options, next) {
|
|
1122
|
+
if (/^mcm-(modeltest|speedtest)-/.test(options?.sessionId || '')) { yield* next(); return; }
|
|
1515
1123
|
// 如果 options.provider 是虚拟轮询路由(以 roundrobin/ 开头),则跳过被动采集(避免双计)
|
|
1516
1124
|
if (options && typeof options.provider === 'string' && options.provider.startsWith(ROUTE_PREFIX)) {
|
|
1517
1125
|
for await (const chunk of next()) {
|
|
@@ -1578,61 +1186,5 @@ export function apply(ctx, config) {
|
|
|
1578
1186
|
}
|
|
1579
1187
|
}, { global: true, prepend: true });
|
|
1580
1188
|
|
|
1581
|
-
|
|
1582
|
-
async function boot() {
|
|
1583
|
-
// 动态原型迁移异步进行,不阻塞路由注册(迁移落盘后 cfg watcher 会热重建路由)
|
|
1584
|
-
void migrateLegacyConfig();
|
|
1585
|
-
rewireRoutes();
|
|
1586
|
-
for (const g of pullConfig().groups) {
|
|
1587
|
-
if (g.speedTest.enabled && g.speedTest.onFirstUse) {
|
|
1588
|
-
const rt = groupRuntime(g.id);
|
|
1589
|
-
if ((state.speedResults[g.id] || []).length === 0)
|
|
1590
|
-
runSpeedTest(g).catch(() => { });
|
|
1591
|
-
}
|
|
1592
|
-
}
|
|
1593
|
-
console.log('[model-channel-manager] booted, groups:', pullConfig().groups.map((g) => g.id).join(', ') || '(none)');
|
|
1594
|
-
}
|
|
1595
|
-
// 工作区 .channel-manager/config.json -> 本行实例配置(一次性)。
|
|
1596
|
-
// 完成后写 legacyMigrated 哨兵,防止「用户清空全部组 → 重启 → 旧配置复活」;
|
|
1597
|
-
// fs/sandboxPolicy 未就绪时短暂重试,而不是静默放弃直到下次重启。
|
|
1598
|
-
async function migrateLegacyConfig() {
|
|
1599
|
-
if (bus === null || pullConfig().groups.length !== 0)
|
|
1600
|
-
return;
|
|
1601
|
-
if (healthOf().legacyMigrated === true)
|
|
1602
|
-
return;
|
|
1603
|
-
for (let attempt = 0; attempt < 6; attempt++) {
|
|
1604
|
-
const fsSvc = ctx.get('fs');
|
|
1605
|
-
const sp = ctx.get('sandboxPolicy');
|
|
1606
|
-
const root = sp ? sp.workspaceRoot : null;
|
|
1607
|
-
if (fsSvc !== undefined && root) {
|
|
1608
|
-
try {
|
|
1609
|
-
const target = await fsSvc.resolve(root + '/.channel-manager/config.json');
|
|
1610
|
-
const text = await fsSvc.readText(target);
|
|
1611
|
-
const legacy = JSON.parse(text);
|
|
1612
|
-
const migrated = normalizeConfig(legacy);
|
|
1613
|
-
// F10(审计 H15):读取是 await 的——这个窗口里用户可能已经保存了
|
|
1614
|
-
// 新组。写前重检:现在有组了就不迁(用户的新配置优先于旧文件复活)。
|
|
1615
|
-
if (pullConfig().groups.length !== 0) {
|
|
1616
|
-
console.log('[model-channel-manager] legacy migration skipped: user config arrived during read');
|
|
1617
|
-
bus.writeHealth({ legacyMigrated: true }).catch(() => { });
|
|
1618
|
-
return;
|
|
1619
|
-
}
|
|
1620
|
-
if (migrated.groups.length > 0) {
|
|
1621
|
-
await bus.writeConfig({ groups: migrated.groups });
|
|
1622
|
-
state.config = { groups: migrated.groups };
|
|
1623
|
-
console.log('[model-channel-manager] migrated legacy config from workspace .channel-manager/config.json');
|
|
1624
|
-
}
|
|
1625
|
-
}
|
|
1626
|
-
catch (_e) { /* 无遗留配置,忽略 */ }
|
|
1627
|
-
bus.writeHealth({ legacyMigrated: true }).catch(() => { });
|
|
1628
|
-
return;
|
|
1629
|
-
}
|
|
1630
|
-
try {
|
|
1631
|
-
await ctx.timeout(5000);
|
|
1632
|
-
}
|
|
1633
|
-
catch (_e) {
|
|
1634
|
-
return; // 插件已卸载,停止重试
|
|
1635
|
-
}
|
|
1636
|
-
}
|
|
1637
|
-
}
|
|
1189
|
+
console.log('[model-channel-manager] ready, groups:', pullConfig().groups.map(group => group.id).join(', ') || '(none)');
|
|
1638
1190
|
}
|