mslxdff 0.1.159 → 0.1.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/cli_help_mini.md +3 -1
- package/package.json +1 -1
- package/src/autostart.js +3 -0
- package/src/chat-pipeline/auto-race.js +1 -1
- package/src/chat-pipeline/empty-turn.js +119 -0
- package/src/chat-pipeline/index.js +4 -1
- package/src/chat-pipeline/serial-trial.js +244 -210
- package/src/cli/commands/daemon.js +4 -4
- package/src/cli/commands/model/list-providers.js +1 -1
- package/src/cli/commands/provider/globalqwenwork-login.js +126 -0
- package/src/cli/commands/provider/index.js +2 -0
- package/src/cli/commands/provider/models.js +4 -1
- package/src/cli/commands/stats.js +20 -5
- package/src/cli/commands/system.js +2 -2
- package/src/cli/format.js +6 -0
- package/src/cli/policy.js +2 -2
- package/src/cli/provider-row.js +2 -2
- package/src/daemon.js +7 -2
- package/src/logs.js +14 -1
- package/src/model-trace.js +5 -2
- package/src/providers/AGENTS.md +46 -0
- package/src/providers/classify.js +1 -1
- package/src/providers/globalqwenwork/account-store.js +141 -0
- package/src/providers/globalqwenwork/constants.js +73 -0
- package/src/providers/globalqwenwork/cosy.js +123 -0
- package/src/providers/globalqwenwork/crypto.js +220 -0
- package/src/providers/globalqwenwork/http.js +22 -0
- package/src/providers/globalqwenwork/index.js +331 -0
- package/src/providers/globalqwenwork/payload.js +145 -0
- package/src/providers/globalqwenwork/rsa.js +56 -0
- package/src/providers/globalqwenwork/sse.js +270 -0
- package/src/providers/globalqwenwork/stream.js +132 -0
- package/src/providers/globalqwenwork/upstream.js +122 -0
- package/src/providers/globalqwenwork.js +1 -0
- package/src/providers/registry.js +8 -0
- package/src/providers/zcode/chat.js +11 -10
- package/src/providers/zcode/const.js +1 -1
- package/src/providers/zcode/context-shape.js +162 -0
- package/src/providers/zcode/headers.js +38 -0
- package/src/providers/zcode/sse.js +19 -2
- package/src/routes/AGENTS.md +37 -0
- package/src/routes/chat/broadband-handler.js +3 -2
- package/src/routes/chat/exhausted-handler.js +23 -8
- package/src/routes/chat/hedge-handler.js +3 -0
- package/src/routes/chat/local-handler.js +5 -1
- package/src/routes/chat/peer-handler.js +2 -0
- package/src/routes/chat/relay-pipeline.js +40 -14
- package/src/routes/chat/via-route-handler.js +2 -0
- package/src/routes/helpers.js +20 -4
- package/src/routes/stream-hold.js +138 -0
- package/src/routes/stream-scan.js +278 -0
- package/src/routes/stream.js +106 -158
- package/src/runtime/lifecycle-forensics.js +116 -0
- package/src/runtime/lifecycle-log.js +23 -0
- package/src/runtime/provider-gate.js +4 -3
- package/src/talk-log.js +226 -0
- package/src/timeline.js +5 -2
- package/src/usage/record.js +8 -1
- package/src/usage/report.js +39 -2
package/src/routes/stream.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { performance } from "node:perf_hooks";
|
|
2
2
|
import { applyFallbackHeaders, enrichNonStreamJson, enrichSseChunkText } from "./fallback.js";
|
|
3
3
|
import { json } from "./helpers.js";
|
|
4
|
-
import {
|
|
5
|
-
|
|
4
|
+
import { extractUsageFromSseText } from "../metrics.js";
|
|
5
|
+
import { scanSseChunk, scanNonStreamBody, preflightMs, createTalkBucket, captureTalkFallback, isEmptyTurnDetail, holdableChunk, extractRetryAfterSec, isTrivialFrame } from "./stream-scan.js"; // 逐帧/逐体观测累加器 + 锚点偏移 + 对话正文 + 空轮判据 + 错误包络暂扣 + 可撤销前缀判据(纯函数全在 stream-scan)
|
|
6
|
+
import { talkLogEnabled } from "../talk-log.js"; // 环形对话日志开关(默认开)
|
|
7
|
+
import { holdEndEnabled, createRevocablePrefix, endEmptyTurnStream } from "./stream-hold.js"; // 空轮可撤销前缀 / 延后封口 / 终局收场(拆出去是为了不破 stream.js 20KB 硬门)
|
|
6
8
|
// SDK 通道(TextEncoder)产出 Uint8Array,legacy 通道为 Buffer;
|
|
7
9
|
// 统一转文本,避免 [DONE]/finish_reason/usage/chars 统计在 SDK 路径下静默失效。
|
|
8
10
|
function chunkText(chunk) {
|
|
@@ -30,79 +32,6 @@ function hasPayload(text) {
|
|
|
30
32
|
return false;
|
|
31
33
|
}
|
|
32
34
|
|
|
33
|
-
// 错误包络 chunk 判定(纯函数):data 行 JSON 含顶层 .error 对象、或只有 finish_reason=error
|
|
34
|
-
// 的空帧,且整 chunk 无任何 content/tool_calls/reasoning 输出 → 返回错误摘要(建议暂扣),
|
|
35
|
-
// 否则返回 null(透传)。解析失败一律透传(默认保安全)。
|
|
36
|
-
// 背景:网关把上游失败包成 HTTP 200 SSE;暂扣后下游流式 UI 不再展示瞬时错误
|
|
37
|
-
//(如 Vertex 503 + google fallback 400),本轮走空转重试,客户端只看到最终结果。
|
|
38
|
-
// Note: 流内错误包络捕获与 retryAfterSeconds 错误直出 — 见 .agents/notes/implemented/bug-fix/2026-09-28-retry-after-surfacing-error.md
|
|
39
|
-
function extractRetryAfterSec(obj) {
|
|
40
|
-
if (!obj) return null;
|
|
41
|
-
if (typeof obj === "number") return obj;
|
|
42
|
-
if (typeof obj === "string") {
|
|
43
|
-
const m = obj.match(/"retryAfterSeconds"\s*:\s*(\d+)/i) || obj.match(/\bretryAfterSeconds\s*[:=]\s*(\d+)/i);
|
|
44
|
-
if (m) return Number(m[1]);
|
|
45
|
-
try {
|
|
46
|
-
const parsed = JSON.parse(obj);
|
|
47
|
-
return extractRetryAfterSec(parsed);
|
|
48
|
-
} catch {}
|
|
49
|
-
}
|
|
50
|
-
if (typeof obj === "object") {
|
|
51
|
-
if (Number.isFinite(obj.retryAfterSeconds)) return Number(obj.retryAfterSeconds);
|
|
52
|
-
if (obj.message) {
|
|
53
|
-
const r = extractRetryAfterSec(obj.message);
|
|
54
|
-
if (r != null) return r;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
return null;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
function holdableChunk(txt) {
|
|
61
|
-
if (typeof txt !== "string" || !txt.includes("data:")) return null;
|
|
62
|
-
let sawData = false;
|
|
63
|
-
let sawEventError = false;
|
|
64
|
-
let retrySec = null;
|
|
65
|
-
const errs = [];
|
|
66
|
-
for (const rawLine of txt.split("\n")) {
|
|
67
|
-
const t = rawLine.trim();
|
|
68
|
-
if (t.startsWith("event:") && t.slice(6).trim() === "error") {
|
|
69
|
-
sawEventError = true;
|
|
70
|
-
continue;
|
|
71
|
-
}
|
|
72
|
-
if (!t.startsWith("data:")) continue;
|
|
73
|
-
const d = t.slice(5).trim();
|
|
74
|
-
if (!d || d === "[DONE]") continue;
|
|
75
|
-
let j = null;
|
|
76
|
-
try { j = JSON.parse(d); } catch { return null; }
|
|
77
|
-
if (!j || typeof j !== "object") return null;
|
|
78
|
-
sawData = true;
|
|
79
|
-
const c0 = (Array.isArray(j.choices) && j.choices[0]) || {};
|
|
80
|
-
const delta = c0.delta || {};
|
|
81
|
-
const msg = c0.message || {};
|
|
82
|
-
const out = delta.content || msg.content || delta.tool_calls || msg.tool_calls || delta.reasoning || msg.reasoning;
|
|
83
|
-
if (out && !(Array.isArray(out) && out.length === 0)) return null;
|
|
84
|
-
|
|
85
|
-
const r = extractRetryAfterSec(j);
|
|
86
|
-
if (r != null) {
|
|
87
|
-
retrySec = r;
|
|
88
|
-
continue;
|
|
89
|
-
}
|
|
90
|
-
if (j.error && typeof j.error === "object") {
|
|
91
|
-
const m = j.error.message || j.error.code || "";
|
|
92
|
-
if (m) errs.push(String(m).slice(0, 300));
|
|
93
|
-
} else if (sawEventError && j.message) {
|
|
94
|
-
errs.push(String(j.message).slice(0, 300));
|
|
95
|
-
} else if (c0.finish_reason === "error") {
|
|
96
|
-
errs.push("finish_reason=error 空帧");
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
if (!sawData) return null;
|
|
100
|
-
if (retrySec != null) return `上游供应商触发 retryAfterSeconds: ${retrySec} ,请等候重试`;
|
|
101
|
-
if (errs.length) return errs.join(" | ");
|
|
102
|
-
if (sawEventError) return "event: error 空帧";
|
|
103
|
-
return null;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
35
|
// 自持 reader 优先:for-await 会锁定 ReadableStream,使 body.cancel() 必 reject(真流上等于空操作),
|
|
107
36
|
// 超时/断下游要真能掐上游必须走 reader.cancel()
|
|
108
37
|
async function* bodyChunks(body, reader) {
|
|
@@ -149,28 +78,36 @@ export const MAX_STREAM_MS = (() => {
|
|
|
149
78
|
return Number.isInteger(n) && n > 0 ? n : 0;
|
|
150
79
|
})();
|
|
151
80
|
|
|
152
|
-
export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort, streamTimeoutMs = STREAM_TIMEOUT_MS, keepaliveMs = KEEPALIVE_MS, fallback } = {}) {
|
|
81
|
+
export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort, streamTimeoutMs = STREAM_TIMEOUT_MS, keepaliveMs = KEEPALIVE_MS, fallback, attemptStartMs } = {}) {
|
|
153
82
|
const t0 = performance.now();
|
|
83
|
+
// 上报锚点偏移:只加给「上报用时长」(usage 行),闸门计时仍从上面的 t0 起算,两者不得混用
|
|
84
|
+
const preflight = preflightMs(t0, attemptStartMs);
|
|
154
85
|
const contentType = upRes.headers.get("content-type") || "";
|
|
155
86
|
// 需同时满足:客户端要流 + 上游真的是 SSE;避免 muse-spark 聚合 JSON 被误判为流式,或 workbuddy SSE 被聚合
|
|
156
87
|
const isStream = Boolean(body?.stream) && contentType.includes("text/event-stream");
|
|
157
|
-
|
|
158
|
-
//
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
88
|
+
// 重入同一条连接(空轮留口后同模型重拉 / 换候选)时 headers 可能已 flush(前一发发过 keepalive 注释帧):
|
|
89
|
+
// 此时 statusCode 与自定义头都不可再设,否则 setHeader 抛 ERR_HTTP_HEADERS_SENT,重拉直接炸在半路
|
|
90
|
+
if (!res.headersSent) {
|
|
91
|
+
res.statusCode = upRes.status;
|
|
92
|
+
// propagate workbuddy uid / allowlist headers
|
|
93
|
+
try {
|
|
94
|
+
const uid = upRes.headers.get("x-mslxdff-workbuddy-uid");
|
|
95
|
+
if (uid) res.setHeader("x-mslxdff-workbuddy-uid", uid);
|
|
96
|
+
const reason = upRes.headers.get("x-mslxdff-workbuddy-reason");
|
|
97
|
+
if (reason) res.setHeader("x-mslxdff-workbuddy-reason", reason);
|
|
98
|
+
const allow = upRes.headers.get("x-mslxdff-allowlist");
|
|
99
|
+
if (allow) res.setHeader("x-mslxdff-allowlist", allow);
|
|
100
|
+
const engine = upRes.headers.get("x-mslxdff-upstream-engine");
|
|
101
|
+
if (engine) res.setHeader("x-mslxdff-upstream-engine", engine);
|
|
102
|
+
} catch {}
|
|
103
|
+
if (fallback) applyFallbackHeaders(res, fallback);
|
|
104
|
+
}
|
|
170
105
|
|
|
171
106
|
let ttf = null;
|
|
172
107
|
let interrupted = false;
|
|
173
108
|
let finishedNormally = false;
|
|
109
|
+
let commitPendingPrefix = () => {};
|
|
110
|
+
let flushHeldTail = () => {}; // 把暂扣的 [DONE] 补写出去(错误帧必须能先站在它前面)
|
|
174
111
|
const detail = {
|
|
175
112
|
receivedChunks: 0,
|
|
176
113
|
receivedBytes: 0,
|
|
@@ -187,11 +124,14 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
187
124
|
downstreamClosed: false,
|
|
188
125
|
usage: null,
|
|
189
126
|
chars: 0,
|
|
127
|
+
reasoningChars: 0, // 思考内容字符数(与 chars 分列,相加会重复计数)
|
|
190
128
|
toolCalls: 0,
|
|
191
129
|
chatShaped: false,
|
|
130
|
+
talk: talkLogEnabled() ? createTalkBucket() : null, // 环形对话日志正文桶(关闭时为 null,零开销)
|
|
192
131
|
heldErrorChunks: 0,
|
|
193
132
|
upstreamErrorText: null,
|
|
194
133
|
recoveries: 0,
|
|
134
|
+
wrotePayload: false, // 是否已向下游写出真实数据帧(注释帧不算):决定空转还能不能靠重试救
|
|
195
135
|
};
|
|
196
136
|
let prevChunkAt = t0;
|
|
197
137
|
// 断下游即掐上游:流式分支装配真实取消,非流式保持 no-op
|
|
@@ -204,6 +144,8 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
204
144
|
}
|
|
205
145
|
};
|
|
206
146
|
res.on("close", onClose);
|
|
147
|
+
// 入口即已断开/已收场:先落账,否则这一发会把留口当成「下游还活着」,重拉写给消失的读者
|
|
148
|
+
if (res.writableEnded || res.destroyed) detail.downstreamClosed = true;
|
|
207
149
|
|
|
208
150
|
if (isStream) {
|
|
209
151
|
// failover 重入时 headers 可能已发(前一个候选只发过 keepalive 注释帧就被掐)——已发则不可再设
|
|
@@ -224,6 +166,12 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
224
166
|
let first = true;
|
|
225
167
|
let wroteAny = false;
|
|
226
168
|
let wrotePayload = false; // 真实数据帧(注释帧不算):决定能否安全 failover
|
|
169
|
+
const revocableHold = holdEndEnabled(); // HOLD_END=0 时连缓冲都不做(逐字节复现旧行为,含时机)
|
|
170
|
+
// 可撤销前缀缓冲(策略见 stream-hold.js):模型产出出现前,空 delta 帧与 [DONE] 先不入下游,
|
|
171
|
+
// 这样同模型重拉 / 换候选的正文还能顺着同一条连接送出去;上限兜底防异常形状囤内存。
|
|
172
|
+
const prefix = createRevocablePrefix(res, detail);
|
|
173
|
+
commitPendingPrefix = () => prefix.commit();
|
|
174
|
+
flushHeldTail = () => prefix.flushTail();
|
|
227
175
|
let timedOut = false;
|
|
228
176
|
let stalled = false;
|
|
229
177
|
let tooLong = false;
|
|
@@ -289,48 +237,11 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
289
237
|
if (!detail.upstreamErrorText) detail.upstreamErrorText = String(holdErr).slice(0, 500);
|
|
290
238
|
}
|
|
291
239
|
const isPayload = hasPayload(txt);
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
if (!detail.chatShaped && (txt.includes('"choices"') || txt.includes('"delta"') || txt.includes('"finish_reason"') || txt.includes('"usage"') || txt.includes('"prompt_tokens"') || txt.includes("[DONE]"))) detail.chatShaped = true;
|
|
298
|
-
// 工具调用计数(空数组不算):tool_calls 无正文是合法 agent 轮,
|
|
299
|
-
// 空转闸门必须豁免它,否则所有工具轮都会被误判为空轮——见 relay-pipeline 4b。
|
|
300
|
-
const tc = txt.match(/"tool_calls"\s*:\s*\[\s*\{/g);
|
|
301
|
-
if (tc) detail.toolCalls = (detail.toolCalls || 0) + tc.length;
|
|
302
|
-
// 尝试提取 usage(流式末帧):口径收口到 metrics.js,与未流式分支共用
|
|
303
|
-
if (txt.includes("\"usage\"") || txt.includes("\"prompt_tokens\"")) {
|
|
304
|
-
try {
|
|
305
|
-
const lines = txt.split("\n");
|
|
306
|
-
for (const line of lines) {
|
|
307
|
-
const t = line.trim();
|
|
308
|
-
if (!t.startsWith("data:")) continue;
|
|
309
|
-
const d = t.slice(5).trim();
|
|
310
|
-
if (d === "[DONE]" || !d) continue;
|
|
311
|
-
// 行级隔离:单行坏 JSON 不拖累同 chunk 其余行
|
|
312
|
-
try {
|
|
313
|
-
const j = JSON.parse(d);
|
|
314
|
-
if (!j || typeof j !== "object") continue;
|
|
315
|
-
const u = extractUsageFromJson(j);
|
|
316
|
-
if (u) detail.usage = u;
|
|
317
|
-
// 兜底 chars:从 choices 文本长度累加
|
|
318
|
-
const delta = j.choices?.[0]?.delta?.content || j.choices?.[0]?.message?.content || "";
|
|
319
|
-
if (delta) detail.chars += String(delta).length;
|
|
320
|
-
} catch { /* 单行坏帧忽略 */ }
|
|
321
|
-
}
|
|
322
|
-
} catch {}
|
|
323
|
-
} else {
|
|
324
|
-
// 非 usage 的普通 delta 也累 chars
|
|
325
|
-
try {
|
|
326
|
-
const ms = txt.match(/"content"\s*:\s*"([^"]*)"/g);
|
|
327
|
-
if (ms) for (const mm of ms) {
|
|
328
|
-
const c = JSON.parse(`{${mm}}`);
|
|
329
|
-
if (c.content) detail.chars += String(c.content).length;
|
|
330
|
-
}
|
|
331
|
-
} catch {}
|
|
332
|
-
}
|
|
333
|
-
} catch { /* ignore */ }
|
|
240
|
+
const prevChars = detail.chars, prevTools = detail.toolCalls, prevReason = detail.reasoningChars;
|
|
241
|
+
scanSseChunk(detail, txt);
|
|
242
|
+
// 「模型产出」=正文/工具调用/思考三者本轮有新增。思考也算:已发出的思考撤不回,
|
|
243
|
+
// 从它出现的那刻起这次尝试就提交了(现网空轮多数正是「思考刷满、正文为零」)。
|
|
244
|
+
const gained = Number(detail.chars) > prevChars || Number(detail.toolCalls) > prevTools || Number(detail.reasoningChars) > prevReason;
|
|
334
245
|
// 首块/空闲超时后上游仍吐出了真实数据 → 只是慢,不是死:撤销超时判定,照常转发
|
|
335
246
|
//(cancel 是异步的,竞态窗口内已到达的数据是纯收益;丢掉是纯损失)
|
|
336
247
|
// 注释帧(keepalive)不算:否则对端只要在发心跳,闸门就永远解除
|
|
@@ -362,11 +273,24 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
362
273
|
} catch {}
|
|
363
274
|
}
|
|
364
275
|
if (!holdErr) {
|
|
365
|
-
|
|
366
|
-
if (isPayload)
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
276
|
+
const size = Buffer.isBuffer(outChunk) ? outChunk.length : (outChunk?.length ?? len);
|
|
277
|
+
if (!isPayload) {
|
|
278
|
+
// 注释帧等非数据帧:与改前一致直写(客户端视作噪声,不影响可撤销性)
|
|
279
|
+
wroteAny = true;
|
|
280
|
+
prefix.writeNow(outChunk, size);
|
|
281
|
+
} else if (gained || !revocableHold || !isTrivialFrame(txt)) {
|
|
282
|
+
// 有模型产出(正文/工具/思考)、留口已关、或看不懂的异形帧:按序补写暂扣前缀 + 本帧,
|
|
283
|
+
// 这条流就此提交不可撤销(透传契约要求解析失败的帧必须原样走,绝不为救空轮扣下)
|
|
284
|
+
wroteAny = true;
|
|
285
|
+
wrotePayload = true; detail.wrotePayload = true;
|
|
286
|
+
prefix.submit(outChunk, size);
|
|
287
|
+
} else if (!prefix.hold(outChunk, size, txt.includes("[DONE]"))) {
|
|
288
|
+
// 可撤销前缀:空 delta 暂扣(判为空轮时整段撤销,留给重拉的正文);
|
|
289
|
+
// [DONE] 无论是否已提交都进 tail 槽 —— 错误帧必须能排在它前面,否则客户端一见 [DONE] 就收尾
|
|
290
|
+
wroteAny = true;
|
|
291
|
+
wrotePayload = true; detail.wrotePayload = true;
|
|
292
|
+
prefix.submit(outChunk, size); // 暂扣超上限(异常形状)→ 被迫提交,退回旧行为
|
|
293
|
+
}
|
|
370
294
|
}
|
|
371
295
|
armStall();
|
|
372
296
|
}
|
|
@@ -386,14 +310,14 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
386
310
|
if ((timedOut || stalled || tooLong) && !wrotePayload) {
|
|
387
311
|
res.removeListener("close", onClose);
|
|
388
312
|
// Note: 超时是显式字段(timedOut),别再用 status 数值当信号 — 见 .agents/notes/implemented/architecture/2026-09-17-relay-timedout-explicit-and-metrics-seam.md
|
|
389
|
-
return { status: 504, timedOut: true, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: true, interrupted: false, detail };
|
|
313
|
+
return { status: 504, timedOut: true, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: true, interrupted: false, preflightMs: preflight, detail };
|
|
390
314
|
}
|
|
391
315
|
if ((stalled || tooLong) && wrotePayload) {
|
|
392
316
|
interrupted = true;
|
|
393
317
|
detail.exitReason = detail.exitReason || (stalled ? "stall" : "max");
|
|
394
|
-
|
|
318
|
+
flushHeldTail(); // 断流收场也要把 [DONE] 按序补上,别让客户端等一个不存在的终止符
|
|
395
319
|
try { res.end(); } catch { /* ignore */ }
|
|
396
|
-
return { status: 200, timedOut: false, ttfMs: ttf, totalMs: Math.round(performance.now() - t0), aborted: false, interrupted, detail };
|
|
320
|
+
return { status: 200, timedOut: false, ttfMs: ttf, totalMs: Math.round(performance.now() - t0), aborted: false, interrupted, preflightMs: preflight, detail };
|
|
397
321
|
}
|
|
398
322
|
} else {
|
|
399
323
|
detail.exitReason = "empty-body";
|
|
@@ -401,9 +325,30 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
401
325
|
const totalMs = Math.round(performance.now() - t0);
|
|
402
326
|
if (!detail.exitReason) detail.exitReason = "normal";
|
|
403
327
|
finishedNormally = true;
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
328
|
+
// 留口期间不摘 close 监听(close 只发一次,摘了客户端窗口内断开就永久盲窗、照烧额度)
|
|
329
|
+
const heldOpen = holdEndEnabled() && detail.wrotePayload !== true && !detail.downstreamClosed && isEmptyTurnDetail(detail);
|
|
330
|
+
if (!heldOpen) res.removeListener("close", onClose);
|
|
331
|
+
let terminalForm = null;
|
|
332
|
+
if (heldOpen) {
|
|
333
|
+
detail.exitReason = "empty-turn-hold";
|
|
334
|
+
} else {
|
|
335
|
+
if (!detail.downstreamClosed) {
|
|
336
|
+
commitPendingPrefix(); // 不留口 = 补写暂扣前缀(HOLD_END=0 时逐字节复现改前行为)
|
|
337
|
+
// 思考刷满 max_tokens、正文为零(现网主流):思考撤不回,错误帧必须排在 [DONE] 之前才看得见
|
|
338
|
+
const reasoningOnly = Number(detail.chars) === 0 && Number(detail.toolCalls) === 0 && Number(detail.reasoningChars) > 0;
|
|
339
|
+
if (reasoningOnly && holdEndEnabled()) {
|
|
340
|
+
detail.exitReason = "reasoning-only";
|
|
341
|
+
terminalForm = "sse-error-tail";
|
|
342
|
+
endEmptyTurnStream(res, `EMPTY_MODEL_RESPONSE: 只产出思考 ${detail.reasoningChars} 字、正文为零(额度被思考吃满),已重试仍无正文 — raise max_tokens or rephrase`, () => flushHeldTail());
|
|
343
|
+
} else {
|
|
344
|
+
flushHeldTail(); // 没有错误帧要插队,[DONE] 按原序发出
|
|
345
|
+
try { res.end(); } catch { /* ignore */ }
|
|
346
|
+
}
|
|
347
|
+
} else {
|
|
348
|
+
try { res.end(); } catch { /* ignore */ }
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
return { status: 200, timedOut: false, ttfMs: ttf, totalMs, aborted: false, interrupted: false, heldOpen, terminalForm, preflightMs: preflight, detail };
|
|
407
352
|
}
|
|
408
353
|
|
|
409
354
|
finishedNormally = true;
|
|
@@ -411,28 +356,31 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
411
356
|
const text = await upRes.text();
|
|
412
357
|
detail.receivedBytes = Buffer.byteLength(text);
|
|
413
358
|
detail.exitReason = "normal-non-stream";
|
|
414
|
-
//
|
|
359
|
+
// 非流式同样「先判后写」:此刻 headers 还没 flush,body 一写下去就再无退路。
|
|
360
|
+
let enriched = null;
|
|
415
361
|
try {
|
|
416
362
|
const parsed = JSON.parse(text);
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
//
|
|
422
|
-
|
|
423
|
-
const
|
|
424
|
-
if (
|
|
425
|
-
|
|
363
|
+
scanNonStreamBody(detail, parsed);
|
|
364
|
+
enriched = enrichNonStreamJson(parsed, fallback);
|
|
365
|
+
} catch { /* 上游给的不是可解析 JSON:走下面的原文透传 */ }
|
|
366
|
+
if (enriched === null) {
|
|
367
|
+
// 纯文本时按长度估 chars;若文本其实是 SSE(client 未要流但上游给流)仍尝试提 usage
|
|
368
|
+
try { detail.chars = text.length; } catch {}
|
|
369
|
+
try { const u = extractUsageFromSseText(text); if (u) detail.usage = u; } catch {}
|
|
370
|
+
try { if (detail.talk) captureTalkFallback(detail.talk, text); } catch {}
|
|
371
|
+
} else if (holdEndEnabled() && !detail.downstreamClosed && isEmptyTurnDetail(detail)) {
|
|
372
|
+
detail.exitReason = "empty-turn-hold";
|
|
373
|
+
return { status: upRes.status, timedOut: false, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: false, interrupted: false, heldOpen: true, preflightMs: preflight, detail };
|
|
374
|
+
}
|
|
375
|
+
detail.wrotePayload = true; // 真写下去了:从此撤不回,也不再判空轮
|
|
376
|
+
const nsTotalMs = Math.round(performance.now() - t0);
|
|
377
|
+
if (enriched !== null) {
|
|
426
378
|
json(res, upRes.status, enriched);
|
|
427
|
-
}
|
|
379
|
+
} else {
|
|
428
380
|
res.statusCode = upRes.status;
|
|
429
381
|
res.setHeader("Content-Type", contentType || "text/plain");
|
|
430
382
|
res.end(text);
|
|
431
|
-
// 纯文本时按长度估 chars;若文本其实是 SSE(client 未要流但上游给流)仍尝试提 usage
|
|
432
|
-
try { detail.chars = text.length; } catch {}
|
|
433
|
-
try { const u = extractUsageFromSseText(text); if (u) detail.usage = u; } catch {}
|
|
434
383
|
}
|
|
435
384
|
// 非流式 ttf 视为 total(一次性返回)
|
|
436
|
-
|
|
437
|
-
return { status: upRes.status, timedOut: false, ttfMs: totalMs, totalMs, aborted: false, interrupted: false, detail };
|
|
385
|
+
return { status: upRes.status, timedOut: false, ttfMs: nsTotalMs, totalMs: nsTotalMs, aborted: false, interrupted: false, heldOpen: false, preflightMs: preflight, detail };
|
|
438
386
|
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// daemon 上一次死亡的验尸官 —— 死亡证明从来不能由死者开:Windows 的关机/TerminateProcess
|
|
2
|
+
// 不给被杀进程跑任何 JS 的机会,硬死在遗言层面是零痕迹的(见 crosscut 叶"四种死法")。
|
|
3
|
+
// 所以抓捕靠三方各出一块拼图:
|
|
4
|
+
// ① 杀者留牌:stopDaemon / autostart 清场等"我方所杀"在动手前写 daemon.exit-marker
|
|
5
|
+
// (带死者 pid/版本/意图)——反正 Windows 下被杀者说不出话,牌只能杀者挂;
|
|
6
|
+
// ② 活体信标:daemon 心跳同步写 daemon.alive,钉死"最后一次确证活着"的时刻(死亡窗口下界);
|
|
7
|
+
// ③ 开机验尸:新进程在 installLifecycleLog(早于 writePid 覆盖 pid 文件)读
|
|
8
|
+
// pid/alive/marker + os.uptime() 回推本机开机时刻,纯函数分桶,结论落 errors.log。
|
|
9
|
+
// 分类恒给 diedBetween(死亡窗口)与 detail(中文归因),unknown/reboot/kill 各归各桶,
|
|
10
|
+
// 宁可报"不可解释的强杀"也不假装干净。全部 IO 吞异常——验尸失败绝不挡启动。
|
|
11
|
+
import { mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
|
12
|
+
import { dirname, join } from "node:path";
|
|
13
|
+
import os from "node:os";
|
|
14
|
+
|
|
15
|
+
const SLACK_MS = 15_000; // 时钟/顺序抖动容差:牌的 at 不早于最后心跳−slack 才算"死于本牌"
|
|
16
|
+
|
|
17
|
+
export function aliveFile(dir) { return join(dir, "daemon.alive"); }
|
|
18
|
+
export function markerFile(dir) { return join(dir, "daemon.exit-marker"); }
|
|
19
|
+
export function pidPath(dir) { return join(dir, "daemon.pid"); }
|
|
20
|
+
|
|
21
|
+
function readJsonSafe(file) {
|
|
22
|
+
try { return JSON.parse(readFileSync(file, "utf8")); } catch { return null; }
|
|
23
|
+
}
|
|
24
|
+
function writeJsonSafe(file, obj) {
|
|
25
|
+
try {
|
|
26
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
27
|
+
writeFileSync(file, JSON.stringify(obj), { mode: 0o600 });
|
|
28
|
+
return true;
|
|
29
|
+
} catch { return false; }
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** 活体信标:{pid, version, at, uptimeMs, rssMb}。daemon 模式专属写(debug 前台不碰,防污染死亡窗口)。 */
|
|
33
|
+
export function writeAlive(dir, info) { return writeJsonSafe(aliveFile(dir), { at: Date.now(), ...info }); }
|
|
34
|
+
export function readAlive(dir) { return readJsonSafe(aliveFile(dir)); }
|
|
35
|
+
|
|
36
|
+
/** 杀者牌:动手前写,必须自带死者身份(stopDaemon 随后会 unlink pid 文件)。 */
|
|
37
|
+
export function writeExitMarker(dir, { reason, prevPid, prevVersion = null, byPid = process.pid, code = null, at } = {}) {
|
|
38
|
+
return writeJsonSafe(markerFile(dir), { reason, prevPid, prevVersion, byPid, code, at: at ?? Date.now() });
|
|
39
|
+
}
|
|
40
|
+
/** 读后即删:一张牌只许生效一代,防止下代异常死亡被旧牌误导归因。 */
|
|
41
|
+
export function consumeExitMarker(dir) {
|
|
42
|
+
const m = readJsonSafe(markerFile(dir));
|
|
43
|
+
try { if (m) unlinkSync(markerFile(dir)); } catch {}
|
|
44
|
+
return m;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function readPidFrom(dir) {
|
|
48
|
+
try {
|
|
49
|
+
const n = Number(String(readFileSync(pidPath(dir), "utf8")).trim().split("\n")[0]);
|
|
50
|
+
return Number.isInteger(n) && n > 0 ? n : null;
|
|
51
|
+
} catch { return null; }
|
|
52
|
+
}
|
|
53
|
+
export function readPidVersionFrom(dir) {
|
|
54
|
+
try {
|
|
55
|
+
const l = String(readFileSync(pidPath(dir), "utf8")).split("\n");
|
|
56
|
+
return l.length > 1 && l[1].trim() ? l[1].trim() : null;
|
|
57
|
+
} catch { return null; }
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** 开机时刻 = now − 整机 uptime。重启后它是硬事实,任何进程都伪造不了。 */
|
|
61
|
+
export function bootTimeMs(now = Date.now()) {
|
|
62
|
+
try { return now - Math.round(os.uptime() * 1000); } catch { return null; }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* 纯函数分桶(可测,不碰 fs):
|
|
67
|
+
* none 无前任痕迹(首装 / uninstall 清干净后)
|
|
68
|
+
* clean-exit 死者自己跑完了 exit 钩子(Linux 信号优雅退出 / process.exit)
|
|
69
|
+
* killed-by-cli 新鲜杀者牌:via=stop|restart|upgrade|debug-takeover|uninstall|autostart-cleanup…
|
|
70
|
+
* reboot 无牌,但开机时刻晚于最后信标 → 整机重启带走
|
|
71
|
+
* holder-alive 旧 pid 仍活(抢端口/接管现场)
|
|
72
|
+
* unknown 有旧 pid 无信标(早于验尸功能的旧版本残留),死亡窗口给不出
|
|
73
|
+
* unexplained-kill 机器没重启、无牌、信标戛然而止 → taskkill/任务管理器/硬杀
|
|
74
|
+
*/
|
|
75
|
+
export function classifyPrevExit({ prevPid = null, prevVersion = null, alive = null, marker = null, bootAtMs = null, now = Date.now(), selfPid = 0, prevStillRunning = false } = {}) {
|
|
76
|
+
const lastAlive = Number.isFinite(alive?.at) ? alive.at : null;
|
|
77
|
+
if (!prevPid && !alive && !marker) return { verdict: "none", detail: "无前任 daemon 痕迹" };
|
|
78
|
+
const freshMarker = marker && Number.isFinite(marker.at) && (lastAlive == null || marker.at >= lastAlive - SLACK_MS);
|
|
79
|
+
if (freshMarker && marker.reason === "process-exit") {
|
|
80
|
+
return { verdict: "clean-exit", prevPid: marker.prevPid ?? prevPid, prevVersion: marker.prevVersion ?? prevVersion, at: marker.at, diedBetween: [lastAlive, marker.at], detail: `前任进程自写遗书正常退出 (pid=${marker.prevPid ?? prevPid} exit code=${marker.code ?? "?"})` };
|
|
81
|
+
}
|
|
82
|
+
if (freshMarker) {
|
|
83
|
+
return { verdict: "killed-by-cli", via: marker.reason, prevPid: marker.prevPid ?? prevPid, prevVersion: marker.prevVersion ?? prevVersion, byPid: marker.byPid ?? null, at: marker.at, diedBetween: [lastAlive, marker.at], detail: `被 CLI/清理方 (pid=${marker.byPid ?? "?"}) 以 ${marker.reason} 名义主动杀死` };
|
|
84
|
+
}
|
|
85
|
+
if (lastAlive != null && bootAtMs != null && bootAtMs > lastAlive + SLACK_MS) {
|
|
86
|
+
return { verdict: "reboot", prevPid, prevVersion, lastAliveAt: lastAlive, bootAt: bootAtMs, diedBetween: [lastAlive, bootAtMs], detail: "死者零遗言,且本机开机时刻晚于最后心跳 → 整机重启带走(Windows 关机不跑 exit 钩子,属预期盲区,靠本桶收口)" };
|
|
87
|
+
}
|
|
88
|
+
if (prevPid && prevStillRunning && prevPid !== selfPid) {
|
|
89
|
+
return { verdict: "holder-alive", prevPid, prevVersion, detail: `旧 pid=${prevPid} 仍在世(抢端口/接管现场,本进程=${selfPid})` };
|
|
90
|
+
}
|
|
91
|
+
if (lastAlive == null) {
|
|
92
|
+
return { verdict: "unknown", prevPid, prevVersion, detail: "有旧 pid 无活体信标(前任早于验尸功能上线),只能确认已死" };
|
|
93
|
+
}
|
|
94
|
+
return { verdict: "unexplained-kill", prevPid, prevVersion, lastAliveAt: lastAlive, diedBetween: [lastAlive, now], detail: "机器未重启、无杀者牌、信标戛然而止:taskkill/任务管理器/硬杀或断电级崩溃(exit 钩子未跑)" };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function defaultIsPidAlive(pid) {
|
|
98
|
+
try { process.kill(pid, 0); return true; } catch (err) { return err.code !== "ESRCH"; }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** 开机验尸入口:读齐三方证据 → 收牌(读后即删)→ 分桶。collect 与 report 一起回,供调用方落日志。 */
|
|
102
|
+
export function collectAutopsy({ dir, selfPid = process.pid, now = Date.now(), isPidAliveFn } = {}) {
|
|
103
|
+
const prevPid = readPidFrom(dir);
|
|
104
|
+
const alive = readAlive(dir);
|
|
105
|
+
const input = {
|
|
106
|
+
prevPid,
|
|
107
|
+
prevVersion: readPidVersionFrom(dir),
|
|
108
|
+
alive,
|
|
109
|
+
marker: consumeExitMarker(dir),
|
|
110
|
+
bootAtMs: bootTimeMs(now),
|
|
111
|
+
now,
|
|
112
|
+
selfPid,
|
|
113
|
+
prevStillRunning: !!prevPid && prevPid !== selfPid ? (isPidAliveFn ?? defaultIsPidAlive)(prevPid) : false,
|
|
114
|
+
};
|
|
115
|
+
return { input, report: classifyPrevExit(input) };
|
|
116
|
+
}
|
|
@@ -3,8 +3,11 @@
|
|
|
3
3
|
// 被信号杀(有 `SIGTERM received` 行)、崩溃(有 uncaughtException 行)、
|
|
4
4
|
// 被 SIGKILL/任务管理器杀(只有未送达的 exit 行 → 看最后一条 heartbeat 定位死亡时刻)、
|
|
5
5
|
// OOM 积累(heartbeat 的 rss 曲线)。
|
|
6
|
+
// 验尸三件套(详见 src/runtime/lifecycle-forensics.js):daemon.alive 活体信标 + daemon.exit-marker 杀者牌 + 开机分桶 → errors.log(type=daemon-exit)。
|
|
6
7
|
// 顶层异常只记录不退出(代理服务无持久状态,活着比干净退出更有价值;异常多为请求级漏网)。
|
|
7
8
|
import { appendEvent } from "../logs.js";
|
|
9
|
+
import { daemonDir } from "../daemon.js";
|
|
10
|
+
import { collectAutopsy, writeAlive, writeExitMarker } from "./lifecycle-forensics.js";
|
|
8
11
|
|
|
9
12
|
const HEARTBEAT_MS = Math.max(60_000, Number(process.env.MSLXDFF_HEARTBEAT_MS) || 30 * 60 * 1000);
|
|
10
13
|
|
|
@@ -39,25 +42,45 @@ export function installLifecycleLog({ version, bus, logs } = {}) {
|
|
|
39
42
|
}));
|
|
40
43
|
emit("lifecycle-start", { pid: process.pid, ppid: process.ppid, version, node: process.version });
|
|
41
44
|
|
|
45
|
+
// 开机验尸:上一次死亡归因(七桶),结论落 errors.log 供事后排查——必须赶在 server-lifecycle writePid 覆盖旧 pid 文件之前
|
|
46
|
+
const dir = daemonDir();
|
|
47
|
+
const isDaemon = process.env.MSLXDFF_DAEMON === "1";
|
|
48
|
+
try {
|
|
49
|
+
const { report } = collectAutopsy({ dir, selfPid: process.pid });
|
|
50
|
+
if (report.verdict !== "none") {
|
|
51
|
+
const w = report.diedBetween && report.diedBetween[0] ? ` window=${new Date(report.diedBetween[0]).toISOString()}..${report.diedBetween[1] ? new Date(report.diedBetween[1]).toISOString() : "?"}` : "";
|
|
52
|
+
log(`[lifecycle] autopsy prev-run verdict=${report.verdict}${w} ${report.detail}`);
|
|
53
|
+
try { logs?.appendError?.({ type: "daemon-exit", model: "daemon", status: 0, verdict: report.verdict, via: report.via ?? null, prevPid: report.prevPid ?? null, prevVersion: report.prevVersion ?? null, byPid: report.byPid ?? null, lastAliveAt: report.lastAliveAt ?? null, diedBetween: report.diedBetween ?? null, bootAt: report.bootAt ?? null, detail: report.detail }); } catch {}
|
|
54
|
+
emit("lifecycle-autopsy", { verdict: report.verdict, via: report.via ?? null, prevPid: report.prevPid ?? null });
|
|
55
|
+
}
|
|
56
|
+
} catch {}
|
|
57
|
+
// 活体信标:启动即写一条,死亡窗口下界不必等 30min 心跳
|
|
58
|
+
if (isDaemon) { try { writeAlive(dir, { pid: process.pid, version, uptimeMs: 0, rssMb: rssMb() }); } catch {} }
|
|
59
|
+
|
|
42
60
|
process.on("uncaughtException", (e) => {
|
|
43
61
|
const detail = String(e?.stack || e?.message || e).slice(0, 2000);
|
|
44
62
|
log(`[lifecycle] uncaughtException (kept running): ${detail}`);
|
|
45
63
|
emit("lifecycle-uncaught", { detail: detail.slice(0, 500) });
|
|
64
|
+
try { logs?.appendError?.({ type: "daemon-uncaught", model: "daemon", status: 0, message: `uncaughtException (kept running): ${detail.slice(0, 400)}` }); } catch {}
|
|
46
65
|
});
|
|
47
66
|
process.on("unhandledRejection", (r) => {
|
|
48
67
|
const detail = String(r?.stack || r?.message || r).slice(0, 2000);
|
|
49
68
|
log(`[lifecycle] unhandledRejection (kept running): ${detail}`);
|
|
50
69
|
emit("lifecycle-unhandled-rejection", { detail: detail.slice(0, 500) });
|
|
70
|
+
try { logs?.appendError?.({ type: "daemon-unhandled", model: "daemon", status: 0, message: `unhandledRejection (kept running): ${detail.slice(0, 400)}` }); } catch {}
|
|
51
71
|
});
|
|
52
72
|
process.on("exit", (code) => {
|
|
53
73
|
const uptimeSec = Math.round((Date.now() - t0) / 1000);
|
|
54
74
|
log(`[lifecycle] exit code=${code} uptime=${uptimeSec}s rss=${rssMb()}mb`);
|
|
75
|
+
// 遗书:exit 钩子跑得动 = 非硬杀(Linux 信号善终 / process.exit);Windows 强杀场景永远看不到这行,缺它本身就是证据
|
|
76
|
+
try { writeExitMarker(dir, { reason: "process-exit", prevPid: process.pid, prevVersion: version, byPid: process.pid, code }); } catch {}
|
|
55
77
|
});
|
|
56
78
|
process.on("SIGHUP", () => log(`[lifecycle] SIGHUP received (ignored, pid ${process.pid})`));
|
|
57
79
|
|
|
58
80
|
const hb = setInterval(() => {
|
|
59
81
|
// 心跳只落 daemon.log(不写 events.log:48 条/天,避免污染结构化事件流)
|
|
60
82
|
log(lifecycleHeartbeatLine({ pid: process.pid, uptimeMin: Math.round((Date.now() - t0) / 60000), rssMb: rssMb() }));
|
|
83
|
+
if (isDaemon) { try { writeAlive(dir, { pid: process.pid, version, uptimeMs: Date.now() - t0, rssMb: rssMb() }); } catch {} }
|
|
61
84
|
}, HEARTBEAT_MS);
|
|
62
85
|
hb.unref?.();
|
|
63
86
|
return { heartbeatMs: HEARTBEAT_MS };
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// Note: auth 号型不要求 baseUrl 是硬约束(旧门禁 baseUrl&&keys 曾静默跳过整家 qoder)— 见 .agents/notes/implemented/bug-fix/2026-09-21-auth-doc-provider-gate.md
|
|
5
5
|
|
|
6
6
|
/** 凭证在 auth 目录、无需 baseUrl 的「auth 号型」定制供应商(端点由各自 constants 决定)。 */
|
|
7
|
-
export const AUTH_DOC_PROVIDER_IDS = ["workbuddy", "traework", "qoder", "qwenwork"];
|
|
7
|
+
export const AUTH_DOC_PROVIDER_IDS = ["workbuddy", "traework", "qoder", "qwenwork", "globalqwenwork"];
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* - auth 号型:keys 或 auths 或 auth 目录有号 → 启用(baseUrl 可空)
|
|
@@ -24,8 +24,9 @@ export async function loadAuthDocs(gid) {
|
|
|
24
24
|
? await import("../providers/traework/account-store.js")
|
|
25
25
|
: gid === "qwenwork"
|
|
26
26
|
? await import("../providers/qwenwork/account-store.js")
|
|
27
|
-
:
|
|
28
|
-
|
|
27
|
+
: gid === "globalqwenwork"
|
|
28
|
+
? await import("../providers/globalqwenwork/account-store.js")
|
|
29
|
+
: await import("../providers/qoder/account-store.js");
|
|
29
30
|
return store.listAccountDocs();
|
|
30
31
|
} catch {
|
|
31
32
|
return [];
|