@wu529778790/open-im 1.10.9-beta.3 → 1.10.9-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-sdk-adapter.js +59 -22
- package/dist/adapters/registry.js +3 -0
- package/dist/codebuddy/cli-runner.js +31 -2
- package/dist/codex/cli-runner.js +28 -2
- package/dist/dingtalk/client.js +2 -1
- package/dist/dingtalk/event-handler.js +1 -1
- package/dist/index.js +33 -0
- package/dist/qq/client.js +7 -1
- package/dist/queue/request-queue.js +11 -10
- package/dist/shared/ai-task.d.ts +14 -0
- package/dist/shared/ai-task.js +57 -9
- package/dist/shared/process-kill.d.ts +24 -0
- package/dist/shared/process-kill.js +79 -0
- package/dist/shared/reconnect.d.ts +28 -0
- package/dist/shared/reconnect.js +56 -0
- package/dist/shared/task-cleanup.d.ts +16 -0
- package/dist/shared/task-cleanup.js +34 -1
- package/dist/telegram/client.js +7 -1
- package/dist/telemetry/telemetry-upload.js +1 -1
- package/dist/wework/client.js +17 -5
- package/dist/wework/event-handler.js +3 -0
- package/dist/workbuddy/centrifuge-client.d.ts +4 -0
- package/dist/workbuddy/centrifuge-client.js +76 -28
- package/dist/workbuddy/client.js +39 -2
- package/package.json +1 -1
|
@@ -49,6 +49,8 @@ loadUserPluginSettings();
|
|
|
49
49
|
// 存储所有活跃的 SDKSession 对象,key 为 sessionId
|
|
50
50
|
// 使用 Map 而不是 Set,因为我们需要通过 sessionId 获取 session
|
|
51
51
|
const activeSessions = new Map();
|
|
52
|
+
// 记录每个 session 创建/恢复时的 workDir,防止跨 workDir 复用已固定 cwd 的子进程
|
|
53
|
+
const sessionWorkDirs = new Map();
|
|
52
54
|
// 存储正在进行的流式迭代器,用于中断
|
|
53
55
|
const activeStreams = new Set();
|
|
54
56
|
// 空闲会话清理:跟踪最后使用时间,定期清除超时会话
|
|
@@ -92,34 +94,53 @@ const cleanupInterval = setInterval(() => {
|
|
|
92
94
|
activeSessions.delete(id);
|
|
93
95
|
}
|
|
94
96
|
sessionLastUsed.delete(id);
|
|
97
|
+
sessionWorkDirs.delete(id);
|
|
95
98
|
log.info(`Cleaned up idle session (unused ${Math.round((now - lastUsed) / 60000)}min): ${id}`);
|
|
96
99
|
}
|
|
97
100
|
}
|
|
98
101
|
}, CLEANUP_INTERVAL_MS);
|
|
99
102
|
cleanupInterval.unref(); // 不阻止进程退出
|
|
100
103
|
/**
|
|
101
|
-
*
|
|
104
|
+
* 串行化进程级 process.chdir() —— 同一时刻仅一个 chdir 生效。
|
|
102
105
|
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
* concurrent requests don't race on the working directory.
|
|
106
|
+
* SDK V2 的 createSession/resumeSession 不接受 cwd 参数;且 send()/stream()
|
|
107
|
+
* 会以「调用时的 process.cwd()」派生 Claude Code 子进程。必须用互斥锁串行化
|
|
108
|
+
* 「整个 turn 的 cwd 切换」,否则并发多用户会让工具跑错目录。
|
|
107
109
|
*
|
|
108
|
-
* **TODO:**
|
|
109
|
-
* supports a `cwd` option in createSession/resumeSession. Track upstream:
|
|
110
|
+
* **TODO:** SDK 支持 cwd 选项后移除此锁。upstream:
|
|
110
111
|
* https://github.com/anthropics/claude-agent-sdk/issues
|
|
111
112
|
*/
|
|
112
113
|
let chdirMutex = Promise.resolve();
|
|
113
114
|
function withChdirMutex(fn) {
|
|
114
115
|
const previous = chdirMutex;
|
|
115
|
-
let
|
|
116
|
-
chdirMutex = new Promise((r) => {
|
|
117
|
-
return previous.then(() => {
|
|
116
|
+
let release;
|
|
117
|
+
chdirMutex = new Promise((r) => { release = r; });
|
|
118
|
+
return previous.then(async () => {
|
|
118
119
|
try {
|
|
119
|
-
return fn();
|
|
120
|
+
return await fn();
|
|
120
121
|
}
|
|
121
122
|
finally {
|
|
122
|
-
|
|
123
|
+
release();
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* 在持有全局 chdir 互斥锁期间,把进程 cwd 切到 workDir 执行 fn,结束后恢复。
|
|
129
|
+
* 用于包裹 session.send()+stream(),确保子进程在正确 workDir 派生。
|
|
130
|
+
*/
|
|
131
|
+
function runWithWorkDir(workDir, fn) {
|
|
132
|
+
return withChdirMutex(async () => {
|
|
133
|
+
const originalCwd = process.cwd();
|
|
134
|
+
if (workDir && workDir !== originalCwd) {
|
|
135
|
+
process.chdir(workDir);
|
|
136
|
+
}
|
|
137
|
+
try {
|
|
138
|
+
return await fn();
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
if (workDir && workDir !== originalCwd) {
|
|
142
|
+
process.chdir(originalCwd);
|
|
143
|
+
}
|
|
123
144
|
}
|
|
124
145
|
});
|
|
125
146
|
}
|
|
@@ -201,18 +222,21 @@ async function getOrCreateSession(sessionId, workDir, model, permissionMode, onS
|
|
|
201
222
|
process.chdir(workDir);
|
|
202
223
|
}
|
|
203
224
|
if (sessionId) {
|
|
204
|
-
// 优先复用内存中已有的 SDKSession
|
|
225
|
+
// 优先复用内存中已有的 SDKSession,避免每次都启动新进程。
|
|
226
|
+
// 仅当 workDir 与创建时一致才复用:否则子进程 cwd 已固定在旧目录,
|
|
227
|
+
// 需走 resume 重新派生(在当前 workDir 启动新子进程)。
|
|
205
228
|
const existing = activeSessions.get(sessionId);
|
|
206
|
-
if (existing) {
|
|
229
|
+
if (existing && sessionWorkDirs.get(sessionId) === workDir) {
|
|
207
230
|
log.info(`Reusing existing in-memory session: ${sessionId}`);
|
|
208
231
|
sessionLastUsed.set(sessionId, Date.now());
|
|
209
232
|
return { session: existing, sessionId };
|
|
210
233
|
}
|
|
211
|
-
//
|
|
234
|
+
// 内存中没有(或 workDir 变了),尝试通过 resume 恢复(会启动新 CLI 进程)
|
|
212
235
|
try {
|
|
213
236
|
log.info(`Attempting to resume session: ${sessionId}`);
|
|
214
237
|
session = unstable_v2_resumeSession(sessionId, sessionOptions);
|
|
215
238
|
activeSessions.set(sessionId, session);
|
|
239
|
+
sessionWorkDirs.set(sessionId, workDir);
|
|
216
240
|
sessionLastUsed.set(sessionId, Date.now());
|
|
217
241
|
log.info(`Successfully resumed session: ${sessionId}`);
|
|
218
242
|
return { session, sessionId };
|
|
@@ -228,6 +252,7 @@ async function getOrCreateSession(sessionId, workDir, model, permissionMode, onS
|
|
|
228
252
|
// 暂时返回 undefined,稍后在 init 消息中获取
|
|
229
253
|
const tempId = `pending-${++sessionSeq}`;
|
|
230
254
|
activeSessions.set(tempId, session);
|
|
255
|
+
sessionWorkDirs.set(tempId, workDir);
|
|
231
256
|
sessionLastUsed.set(tempId, Date.now());
|
|
232
257
|
log.info(`Created new session (tempId: ${tempId})`);
|
|
233
258
|
return { session, sessionId: tempId, wasReused: false };
|
|
@@ -267,6 +292,7 @@ export class ClaudeSDKAdapter {
|
|
|
267
292
|
}
|
|
268
293
|
activeSessions.clear();
|
|
269
294
|
sessionLastUsed.clear();
|
|
295
|
+
sessionWorkDirs.clear();
|
|
270
296
|
}
|
|
271
297
|
/**
|
|
272
298
|
* Remove a specific session from the in-memory cache and close it.
|
|
@@ -281,6 +307,7 @@ export class ClaudeSDKAdapter {
|
|
|
281
307
|
catch { /* ignore */ }
|
|
282
308
|
activeSessions.delete(sessionId);
|
|
283
309
|
sessionLastUsed.delete(sessionId);
|
|
310
|
+
sessionWorkDirs.delete(sessionId);
|
|
284
311
|
log.info(`Explicitly removed session: ${sessionId}`);
|
|
285
312
|
}
|
|
286
313
|
}
|
|
@@ -320,12 +347,16 @@ export class ClaudeSDKAdapter {
|
|
|
320
347
|
}
|
|
321
348
|
runningSessions.add(returnedId);
|
|
322
349
|
trackedRunningId = returnedId;
|
|
323
|
-
//
|
|
324
|
-
|
|
325
|
-
//
|
|
326
|
-
const stream =
|
|
327
|
-
|
|
328
|
-
|
|
350
|
+
// 在持有 chdir 锁期间完成 send() + stream() 获取:SDK V2 会以当前
|
|
351
|
+
// process.cwd() 派生 Claude Code 子进程,必须保证此刻 cwd 为 workDir。
|
|
352
|
+
// 锁串行化后,并发多用户的子进程不会在错误的目录派生。
|
|
353
|
+
const stream = await runWithWorkDir(workDir, async () => {
|
|
354
|
+
await session.send(prompt);
|
|
355
|
+
const s = session.stream();
|
|
356
|
+
currentStream = s;
|
|
357
|
+
activeStreams.add(s);
|
|
358
|
+
return s;
|
|
359
|
+
});
|
|
329
360
|
let accumulated = '';
|
|
330
361
|
let accumulatedThinking = '';
|
|
331
362
|
const toolStats = {};
|
|
@@ -348,13 +379,19 @@ export class ClaudeSDKAdapter {
|
|
|
348
379
|
// 更新 sessionId 映射
|
|
349
380
|
// 清理 pending 临时 ID(actualSessionId 尚未赋值时用 pendingTempId)
|
|
350
381
|
const idToClean = actualSessionId ?? pendingTempId;
|
|
382
|
+
const inheritedWorkDir = idToClean ? sessionWorkDirs.get(idToClean) : undefined;
|
|
351
383
|
if (idToClean?.startsWith('pending-')) {
|
|
352
384
|
activeSessions.delete(idToClean);
|
|
353
385
|
}
|
|
354
386
|
activeSessions.set(newSessionId, session);
|
|
387
|
+
if (inheritedWorkDir !== undefined) {
|
|
388
|
+
sessionWorkDirs.set(newSessionId, inheritedWorkDir);
|
|
389
|
+
}
|
|
355
390
|
sessionLastUsed.set(newSessionId, Date.now());
|
|
356
|
-
if (idToClean)
|
|
391
|
+
if (idToClean) {
|
|
357
392
|
sessionLastUsed.delete(idToClean);
|
|
393
|
+
sessionWorkDirs.delete(idToClean);
|
|
394
|
+
}
|
|
358
395
|
// 更新 runningSessions:移除旧 ID,添加新 ID
|
|
359
396
|
if (idToClean)
|
|
360
397
|
runningSessions.delete(idToClean);
|
|
@@ -3,6 +3,7 @@ import { ClaudeSDKAdapter, configureClaudeSdkSessionIdle } from './claude-sdk-ad
|
|
|
3
3
|
import { CodexAdapter } from './codex-adapter.js';
|
|
4
4
|
import { CodeBuddyAdapter } from './codebuddy-adapter.js';
|
|
5
5
|
import { createLogger } from '../logger.js';
|
|
6
|
+
import { destroyAllLiveChildren } from '../shared/process-kill.js';
|
|
6
7
|
const log = createLogger('Registry');
|
|
7
8
|
const adapters = new Map();
|
|
8
9
|
export function initAdapters(config) {
|
|
@@ -30,5 +31,7 @@ export function getAdapter(aiCommand) {
|
|
|
30
31
|
}
|
|
31
32
|
export function cleanupAdapters() {
|
|
32
33
|
ClaudeSDKAdapter.destroy();
|
|
34
|
+
// 强制终止仍在运行的 CLI 子进程(Codex/CodeBuddy),避免僵尸 / 孤儿
|
|
35
|
+
destroyAllLiveChildren();
|
|
33
36
|
adapters.clear();
|
|
34
37
|
}
|
|
@@ -3,6 +3,7 @@ import { accessSync, constants } from 'node:fs';
|
|
|
3
3
|
import { isAbsolute, join } from 'node:path';
|
|
4
4
|
import { createLogger } from '../logger.js';
|
|
5
5
|
import { processEnvForNonClaudeCliChild } from '../config/file-io.js';
|
|
6
|
+
import { killProcessTree, trackChild } from '../shared/process-kill.js';
|
|
6
7
|
const log = createLogger('CodeBuddyCli');
|
|
7
8
|
export function buildCodeBuddyArgs(prompt, sessionId, options) {
|
|
8
9
|
const args = ['--print', '--output-format', 'stream-json'];
|
|
@@ -200,10 +201,14 @@ export function runCodeBuddy(cliPath, prompt, sessionId, workDir, callbacks, opt
|
|
|
200
201
|
log.info(`Spawning CodeBuddy CLI: path=${normalizedCliPath}, cwd=${workDir}, session=${sessionId ?? 'new'}, args=${args.join(' ')}`);
|
|
201
202
|
const child = spawn(spawnCmd, spawnArgs, {
|
|
202
203
|
cwd: workDir,
|
|
204
|
+
// Unix 上放入独立进程组,使 abort/关停能用 process.kill(-pid) 杀掉整棵子树(含 MCP/shell 孙进程)
|
|
205
|
+
detached: process.platform !== 'win32',
|
|
203
206
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
204
207
|
env,
|
|
205
208
|
windowsHide: process.platform === 'win32',
|
|
206
209
|
});
|
|
210
|
+
trackChild(child);
|
|
211
|
+
let cliTimeoutHandle = null;
|
|
207
212
|
let accumulated = '';
|
|
208
213
|
let accumulatedThinking = '';
|
|
209
214
|
let completed = false;
|
|
@@ -311,6 +316,10 @@ export function runCodeBuddy(cliPath, prompt, sessionId, workDir, callbacks, opt
|
|
|
311
316
|
}
|
|
312
317
|
});
|
|
313
318
|
child.on('close', (code) => {
|
|
319
|
+
if (cliTimeoutHandle) {
|
|
320
|
+
clearTimeout(cliTimeoutHandle);
|
|
321
|
+
cliTimeoutHandle = null;
|
|
322
|
+
}
|
|
314
323
|
if (completed)
|
|
315
324
|
return;
|
|
316
325
|
if (stdoutState.buffer.trim()) {
|
|
@@ -343,16 +352,36 @@ export function runCodeBuddy(cliPath, prompt, sessionId, workDir, callbacks, opt
|
|
|
343
352
|
});
|
|
344
353
|
});
|
|
345
354
|
child.on('error', (err) => {
|
|
355
|
+
if (cliTimeoutHandle) {
|
|
356
|
+
clearTimeout(cliTimeoutHandle);
|
|
357
|
+
cliTimeoutHandle = null;
|
|
358
|
+
}
|
|
346
359
|
if (completed)
|
|
347
360
|
return;
|
|
348
361
|
completed = true;
|
|
349
362
|
callbacks.onError(`Failed to start CodeBuddy CLI: ${err.message}`);
|
|
350
363
|
});
|
|
364
|
+
// 墙钟超时:防止 CLI 挂死永久占用用户队列槽
|
|
365
|
+
const cliTimeoutMs = Number(process.env.OPEN_IM_CLI_TIMEOUT_MS) || 30 * 60 * 1000;
|
|
366
|
+
cliTimeoutHandle = setTimeout(() => {
|
|
367
|
+
if (completed)
|
|
368
|
+
return;
|
|
369
|
+
log.warn(`CodeBuddy CLI 超过 ${cliTimeoutMs}ms,强制终止 (pid=${child.pid})`);
|
|
370
|
+
completed = true;
|
|
371
|
+
killProcessTree(child);
|
|
372
|
+
callbacks.onError(`CodeBuddy CLI 运行超时(${Math.round(cliTimeoutMs / 1000)}s),已终止。请重试。`);
|
|
373
|
+
}, cliTimeoutMs);
|
|
374
|
+
cliTimeoutHandle.unref();
|
|
351
375
|
return {
|
|
352
376
|
abort: () => {
|
|
377
|
+
if (completed)
|
|
378
|
+
return;
|
|
353
379
|
completed = true;
|
|
354
|
-
if (
|
|
355
|
-
|
|
380
|
+
if (cliTimeoutHandle) {
|
|
381
|
+
clearTimeout(cliTimeoutHandle);
|
|
382
|
+
cliTimeoutHandle = null;
|
|
383
|
+
}
|
|
384
|
+
killProcessTree(child);
|
|
356
385
|
},
|
|
357
386
|
};
|
|
358
387
|
}
|
package/dist/codex/cli-runner.js
CHANGED
|
@@ -7,6 +7,7 @@ import { dirname, join } from 'node:path';
|
|
|
7
7
|
import { createInterface } from 'node:readline';
|
|
8
8
|
import { createLogger } from '../logger.js';
|
|
9
9
|
import { processEnvForNonClaudeCliChild } from '../config/file-io.js';
|
|
10
|
+
import { killProcessTree, trackChild } from '../shared/process-kill.js';
|
|
10
11
|
const log = createLogger('CodexCli');
|
|
11
12
|
const windowsCodexLaunchCache = new Map();
|
|
12
13
|
const SUPPORTED_IMAGE_EXTENSIONS = new Set([
|
|
@@ -201,10 +202,13 @@ export function runCodex(cliPath, prompt, sessionId, workDir, callbacks, options
|
|
|
201
202
|
: args;
|
|
202
203
|
const child = spawn(spawnCmd, spawnArgs, {
|
|
203
204
|
cwd: workDir,
|
|
205
|
+
// Unix 上放入独立进程组,使 abort/关停能用 process.kill(-pid) 杀掉整棵子树(含 MCP/shell 孙进程)
|
|
206
|
+
detached: process.platform !== 'win32',
|
|
204
207
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
205
208
|
env,
|
|
206
209
|
windowsHide: process.platform === 'win32',
|
|
207
210
|
});
|
|
211
|
+
trackChild(child);
|
|
208
212
|
child.stdin?.write(prompt);
|
|
209
213
|
child.stdin?.end();
|
|
210
214
|
let accumulated = '';
|
|
@@ -327,7 +331,12 @@ export function runCodex(cliPath, prompt, sessionId, workDir, callbacks, options
|
|
|
327
331
|
let exitCode = null;
|
|
328
332
|
let rlClosed = false;
|
|
329
333
|
let childClosed = false;
|
|
334
|
+
let cliTimeoutHandle = null;
|
|
330
335
|
const finalize = () => {
|
|
336
|
+
if (cliTimeoutHandle) {
|
|
337
|
+
clearTimeout(cliTimeoutHandle);
|
|
338
|
+
cliTimeoutHandle = null;
|
|
339
|
+
}
|
|
331
340
|
if (!rlClosed || !childClosed)
|
|
332
341
|
return;
|
|
333
342
|
if (completed)
|
|
@@ -387,12 +396,29 @@ export function runCodex(cliPath, prompt, sessionId, workDir, callbacks, options
|
|
|
387
396
|
childClosed = true;
|
|
388
397
|
finalize();
|
|
389
398
|
});
|
|
399
|
+
// 墙钟超时:防止 CLI 挂死(网络卡住、工具死循环等)永久占用用户队列槽
|
|
400
|
+
const cliTimeoutMs = Number(process.env.OPEN_IM_CLI_TIMEOUT_MS) || 30 * 60 * 1000;
|
|
401
|
+
cliTimeoutHandle = setTimeout(() => {
|
|
402
|
+
if (completed)
|
|
403
|
+
return;
|
|
404
|
+
log.warn(`Codex CLI 超过 ${cliTimeoutMs}ms,强制终止 (pid=${child.pid})`);
|
|
405
|
+
completed = true;
|
|
406
|
+
rl.close();
|
|
407
|
+
killProcessTree(child);
|
|
408
|
+
callbacks.onError(`Codex CLI 运行超时(${Math.round(cliTimeoutMs / 1000)}s),已终止。请重试。`);
|
|
409
|
+
}, cliTimeoutMs);
|
|
410
|
+
cliTimeoutHandle.unref();
|
|
390
411
|
return {
|
|
391
412
|
abort: () => {
|
|
413
|
+
if (completed)
|
|
414
|
+
return;
|
|
392
415
|
completed = true;
|
|
416
|
+
if (cliTimeoutHandle) {
|
|
417
|
+
clearTimeout(cliTimeoutHandle);
|
|
418
|
+
cliTimeoutHandle = null;
|
|
419
|
+
}
|
|
393
420
|
rl.close();
|
|
394
|
-
|
|
395
|
-
child.kill('SIGTERM');
|
|
421
|
+
killProcessTree(child);
|
|
396
422
|
},
|
|
397
423
|
};
|
|
398
424
|
}
|
package/dist/dingtalk/client.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { DWClient, TOPIC_ROBOT } from 'dingtalk-stream';
|
|
2
2
|
import { createLogger } from '../logger.js';
|
|
3
|
+
import { jitteredDelay } from '../shared/reconnect.js';
|
|
3
4
|
import { registerSessionWebhook as registerWebhook, clearWebhooks, sendText as webhookSendText, sendMarkdown as webhookSendMarkdown, downloadRobotMessageFile as webhookDownloadFile, sendProactiveText as webhookSendProactive, } from './webhook.js';
|
|
4
5
|
import { prepareStreamingCard as cardPrepare, updateStreamingCard as cardUpdate, finishStreamingCard as cardFinish, createAndDeliverCard as cardCreateAndDeliver, updateCardInstance as cardUpdateInstance, sendRobotInteractiveCard as cardSendInteractive, updateRobotInteractiveCard as cardUpdateInteractive, clearUnionIdCache, } from './streaming-card.js';
|
|
5
6
|
const log = createLogger('DingTalk');
|
|
@@ -111,7 +112,7 @@ export async function initDingTalk(cfg, eventHandler) {
|
|
|
111
112
|
lastErr = err;
|
|
112
113
|
if (is429(err) && attempt < maxTries) {
|
|
113
114
|
log.warn(`DingTalk gateway 429 (attempt ${attempt}/${maxTries}), retrying in ${retryDelayMs / 1000}s...`);
|
|
114
|
-
await new Promise((r) => setTimeout(r, retryDelayMs));
|
|
115
|
+
await new Promise((r) => setTimeout(r, jitteredDelay(retryDelayMs)));
|
|
115
116
|
continue;
|
|
116
117
|
}
|
|
117
118
|
throw new Error(formatDingTalkInitError(err));
|
|
@@ -163,7 +163,7 @@ export function setupDingTalkHandlers(config, sessionManager) {
|
|
|
163
163
|
sessionManager,
|
|
164
164
|
sender: { sendTextReply, sendDirectorySelection },
|
|
165
165
|
});
|
|
166
|
-
const { accessControl, requestQueue, runningTasks
|
|
166
|
+
const { accessControl, requestQueue, runningTasks } = ctx;
|
|
167
167
|
// DingTalk-specific sender callbacks for the factory
|
|
168
168
|
const dingtalkSender = {
|
|
169
169
|
sendThinkingMessage: async (chatId, replyToMessageId, toolId) => {
|
package/dist/index.js
CHANGED
|
@@ -27,11 +27,13 @@ import { sendTextReply as sendWorkBuddyTextReply } from "./workbuddy/message-sen
|
|
|
27
27
|
import { initAdapters, cleanupAdapters } from "./adapters/registry.js";
|
|
28
28
|
import { SessionManager } from "./session/session-manager.js";
|
|
29
29
|
import { loadActiveChats, getActiveChatId, flushActiveChats, } from "./shared/active-chats.js";
|
|
30
|
+
import { destroyAllLiveChildren } from "./shared/process-kill.js";
|
|
30
31
|
import { initLogger, createLogger, closeLogger, emitStructuredEvent, shutdownLoggerTelemetry, } from "./logger.js";
|
|
31
32
|
import { APP_HOME, SHUTDOWN_PORT } from "./constants.js";
|
|
32
33
|
import { createRequire } from "node:module";
|
|
33
34
|
import { escapePathForMarkdown } from "./shared/utils.js";
|
|
34
35
|
import { applyOpenImGitCoauthorToProcessEnv } from "./shared/git-coauthor.js";
|
|
36
|
+
import { emitInterruptedTerminals } from "./shared/task-cleanup.js";
|
|
35
37
|
const require = createRequire(import.meta.url);
|
|
36
38
|
const { version: APP_VERSION } = require("../package.json");
|
|
37
39
|
const log = createLogger("Main");
|
|
@@ -262,6 +264,15 @@ export async function main() {
|
|
|
262
264
|
const uptimeSec = Math.floor((Date.now() - startedAt) / 1000);
|
|
263
265
|
const m = Math.floor(uptimeSec / 60);
|
|
264
266
|
const shutdownMsg = buildShutdownMessage(m);
|
|
267
|
+
// 在任何异步等待之前,先为仍在运行的任务补发终态遥测事件。
|
|
268
|
+
// 这样即使后续 await(通知发送、platform.stop)期间进程被第二次信号杀死,
|
|
269
|
+
// ai.task.start 也有对应的 interrupted 终态,避免遥测里的 miss。
|
|
270
|
+
for (const platform of successfulPlatforms) {
|
|
271
|
+
const handle = activeHandles.get(platform);
|
|
272
|
+
if (handle?.runningTasks && handle.runningTasks.size > 0) {
|
|
273
|
+
emitInterruptedTerminals(handle.runningTasks);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
265
276
|
// Send notification only to successfully initialized platforms
|
|
266
277
|
for (const platform of successfulPlatforms) {
|
|
267
278
|
await sendLifecycleNotification(platform, shutdownMsg).catch((err) => {
|
|
@@ -299,6 +310,16 @@ export async function main() {
|
|
|
299
310
|
};
|
|
300
311
|
process.on("SIGINT", () => shutdown().catch(() => process.exit(1)));
|
|
301
312
|
process.on("SIGTERM", () => shutdown().catch(() => process.exit(1)));
|
|
313
|
+
// 兜底:进程退出(含异常路径,如未捕获异常 / SIGKILL)时强制清理 CLI 子进程,
|
|
314
|
+
// 避免僵尸 / 孤儿。正常 shutdown 已在 cleanupAdapters() 里清理过。
|
|
315
|
+
process.on("exit", () => {
|
|
316
|
+
try {
|
|
317
|
+
destroyAllLiveChildren();
|
|
318
|
+
}
|
|
319
|
+
catch {
|
|
320
|
+
/* 退出期间 best effort */
|
|
321
|
+
}
|
|
322
|
+
});
|
|
302
323
|
// Global error handlers to prevent unhandled crashes
|
|
303
324
|
process.on("unhandledRejection", (reason) => {
|
|
304
325
|
log.error("Unhandled Promise rejection (this indicates a bug — the affected request may hang without a response):", reason);
|
|
@@ -312,6 +333,18 @@ export async function main() {
|
|
|
312
333
|
return;
|
|
313
334
|
}
|
|
314
335
|
log.error("Uncaught exception (process will exit):", err);
|
|
336
|
+
// 进程即将因未捕获异常退出:补发在途任务的终态,避免 miss
|
|
337
|
+
for (const platform of successfulPlatforms) {
|
|
338
|
+
const handle = activeHandles.get(platform);
|
|
339
|
+
if (handle?.runningTasks && handle.runningTasks.size > 0) {
|
|
340
|
+
try {
|
|
341
|
+
emitInterruptedTerminals(handle.runningTasks);
|
|
342
|
+
}
|
|
343
|
+
catch {
|
|
344
|
+
/* best effort:不能因遥测失败再次抛出 */
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
315
348
|
void shutdownLoggerTelemetry()
|
|
316
349
|
.then(() => closeLogger())
|
|
317
350
|
.finally(() => process.exit(1));
|
package/dist/qq/client.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import WebSocket from "ws";
|
|
2
2
|
import { createLogger } from "../logger.js";
|
|
3
|
+
import { jitteredDelay, SLOW_PROBE_MS } from "../shared/reconnect.js";
|
|
3
4
|
const log = createLogger("QQ");
|
|
4
5
|
const TOKEN_URL = "https://bots.qq.com/app/getAppAccessToken";
|
|
5
6
|
const API_BASE = "https://api.sgroup.qq.com";
|
|
@@ -271,7 +272,12 @@ async function connectWebSocket(config, handler) {
|
|
|
271
272
|
sessionId = null;
|
|
272
273
|
seq = null;
|
|
273
274
|
}
|
|
274
|
-
const
|
|
275
|
+
const isFatalClose = code === 4004 || code === 4006 || code === 4007;
|
|
276
|
+
const baseDelay = RECONNECT_DELAYS_MS[Math.min(reconnectAttempt, RECONNECT_DELAYS_MS.length - 1)];
|
|
277
|
+
const delay = isFatalClose ? jitteredDelay(SLOW_PROBE_MS) : jitteredDelay(baseDelay);
|
|
278
|
+
if (isFatalClose) {
|
|
279
|
+
log.warn(`QQ 致命关闭码 ${code}(鉴权失败),转慢探测(每 ${Math.round(SLOW_PROBE_MS / 1000)}s 一次)`);
|
|
280
|
+
}
|
|
275
281
|
reconnectAttempt += 1;
|
|
276
282
|
reconnectTimer = setTimeout(() => {
|
|
277
283
|
if (currentConfig && currentHandler) {
|
|
@@ -80,17 +80,18 @@ export class RequestQueue {
|
|
|
80
80
|
finally {
|
|
81
81
|
this.runningControllers.delete(key);
|
|
82
82
|
const q = this.queues.get(key);
|
|
83
|
-
if (!q)
|
|
84
|
-
return;
|
|
85
|
-
const next = q.tasks.shift();
|
|
86
|
-
if (next) {
|
|
87
|
-
setImmediate(() => this.run(key, next.prompt, next.execute).catch((err) => {
|
|
88
|
-
log.error(`Unhandled error in next task execution for ${key}:`, err);
|
|
89
|
-
}));
|
|
90
|
-
}
|
|
83
|
+
if (!q) { /* queue already cleared */ }
|
|
91
84
|
else {
|
|
92
|
-
|
|
93
|
-
|
|
85
|
+
const next = q.tasks.shift();
|
|
86
|
+
if (next) {
|
|
87
|
+
setImmediate(() => this.run(key, next.prompt, next.execute).catch((err) => {
|
|
88
|
+
log.error(`Unhandled error in next task execution for ${key}:`, err);
|
|
89
|
+
}));
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
q.running = false;
|
|
93
|
+
this.queues.delete(key);
|
|
94
|
+
}
|
|
94
95
|
}
|
|
95
96
|
}
|
|
96
97
|
}
|
package/dist/shared/ai-task.d.ts
CHANGED
|
@@ -42,5 +42,19 @@ export interface TaskRunState {
|
|
|
42
42
|
startedAt: number;
|
|
43
43
|
/** AI 工具标识,用于动态显示工具名称。 */
|
|
44
44
|
toolId: string;
|
|
45
|
+
/**
|
|
46
|
+
* 进程退出(shutdown / 崩溃)时,用于为仍在运行的任务补发一条终态遥测事件,
|
|
47
|
+
* 避免 `ai.task.start` 没有对应的 complete/error(遥测里的 `miss`)。
|
|
48
|
+
* 已哈希(与 ai.task.* emit 处一致),不含原始 userId/msgId。
|
|
49
|
+
*/
|
|
50
|
+
taskKey: string;
|
|
51
|
+
platform: string;
|
|
52
|
+
/** 已哈希的 userId,与 ai.task.start/complete/error 中的 userKey 一致。 */
|
|
53
|
+
userKey: string;
|
|
45
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* 将 AI/CLI 错误文本归类为一个稳定的 errorType 字符串,用于遥测聚合。
|
|
57
|
+
* 分类基于线上真实错误签名(见 logs/r2-events),新增分支时同步更新测试。
|
|
58
|
+
*/
|
|
59
|
+
export declare function classifyErrorType(error: string): string;
|
|
46
60
|
export declare function runAITask(deps: TaskDeps, ctx: TaskContext, prompt: string, toolAdapter: ToolAdapter, platformAdapter: TaskAdapter): Promise<void>;
|
package/dist/shared/ai-task.js
CHANGED
|
@@ -10,25 +10,65 @@ const log = createLogger('AITask');
|
|
|
10
10
|
function isUsageLimitError(error) {
|
|
11
11
|
return /usage limit/i.test(error) || /try again at\s+\d{1,2}:\d{2}\s*(AM|PM)/i.test(error);
|
|
12
12
|
}
|
|
13
|
-
|
|
13
|
+
/**
|
|
14
|
+
* 将 AI/CLI 错误文本归类为一个稳定的 errorType 字符串,用于遥测聚合。
|
|
15
|
+
* 分类基于线上真实错误签名(见 logs/r2-events),新增分支时同步更新测试。
|
|
16
|
+
*/
|
|
17
|
+
export function classifyErrorType(error) {
|
|
14
18
|
const s = error.toLowerCase();
|
|
15
19
|
if (s.includes('aborted'))
|
|
16
20
|
return 'aborted';
|
|
21
|
+
// 空输出:流正常结束但无内容(claude-sdk-adapter 的兜底消息)
|
|
22
|
+
if (s.includes('无输出') || s.includes('响应异常结束') || s.includes('empty output')) {
|
|
23
|
+
return 'empty_output';
|
|
24
|
+
}
|
|
17
25
|
if (isUsageLimitError(error) || s.includes('rate limit') || s.includes('quota'))
|
|
18
26
|
return 'limit';
|
|
19
|
-
|
|
27
|
+
// 鉴权:凭据无效或需要登录(含中文「登录」/「login」)
|
|
28
|
+
if (s.includes('invalid api key') ||
|
|
29
|
+
s.includes('unauthorized') ||
|
|
30
|
+
s.includes('401') ||
|
|
31
|
+
s.includes('need to log in') ||
|
|
32
|
+
s.includes('need to login') ||
|
|
33
|
+
s.includes('需要登录') ||
|
|
34
|
+
s.includes('需要先登录') ||
|
|
35
|
+
s.includes('log in required') ||
|
|
36
|
+
s.includes('login required')) {
|
|
20
37
|
return 'auth';
|
|
38
|
+
}
|
|
21
39
|
if (s.includes('model') && (s.includes('not support') || s.includes('not found') || s.includes('invalid'))) {
|
|
22
40
|
return 'model';
|
|
23
41
|
}
|
|
24
|
-
|
|
42
|
+
// 安装/配置缺失:二进制或可执行文件缺失、环境变量缺失、token 未配置
|
|
43
|
+
if (s.includes('native cli binary') ||
|
|
44
|
+
s.includes('executable not found') ||
|
|
45
|
+
(s.includes('binary') && s.includes('not found')) ||
|
|
46
|
+
s.includes('missing environment variable') ||
|
|
47
|
+
s.includes('token data is not available')) {
|
|
48
|
+
return 'setup';
|
|
49
|
+
}
|
|
50
|
+
// 会话失效:找不到会话/对话,或会话过期/损坏
|
|
51
|
+
if (s.includes('no conversation found') ||
|
|
52
|
+
s.includes('session not found') ||
|
|
53
|
+
(s.includes('session') && (s.includes('expired') || s.includes('corrupt')))) {
|
|
54
|
+
return 'session';
|
|
55
|
+
}
|
|
56
|
+
// 进程退出或被信号终止
|
|
57
|
+
if (s.includes('process exited') ||
|
|
58
|
+
s.includes('exit code') ||
|
|
59
|
+
s.includes('exited with code') ||
|
|
60
|
+
s.includes('terminated by signal') ||
|
|
61
|
+
s.includes('sigkill')) {
|
|
25
62
|
return 'process';
|
|
63
|
+
}
|
|
64
|
+
// 网络:超时/连接重置/DNS/网络请求失败(含中文「网络」)
|
|
26
65
|
if (s.includes('timeout') ||
|
|
27
66
|
s.includes('etimedout') ||
|
|
28
67
|
s.includes('econnreset') ||
|
|
29
68
|
s.includes('enotfound') ||
|
|
30
69
|
s.includes('eai_again') ||
|
|
31
|
-
s.includes('network')
|
|
70
|
+
s.includes('network') ||
|
|
71
|
+
s.includes('网络')) {
|
|
32
72
|
return 'network';
|
|
33
73
|
}
|
|
34
74
|
return 'unknown';
|
|
@@ -115,7 +155,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
115
155
|
log.info(`[AITask] Starting: userId=${ctx.userId}, initialSessionId=${currentSessionId ?? 'new'}, prompt="${prompt.slice(0, 50)}..."`);
|
|
116
156
|
emitStructuredEvent('AITask', 'ai.task.start', {
|
|
117
157
|
platform: ctx.platform,
|
|
118
|
-
taskKey: ctx.taskKey,
|
|
158
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
119
159
|
userKey: hashUserId(ctx.userId),
|
|
120
160
|
toolId: aiCommand,
|
|
121
161
|
});
|
|
@@ -179,7 +219,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
179
219
|
return;
|
|
180
220
|
emitStructuredEvent('AITask', 'ai.task.complete', {
|
|
181
221
|
platform: ctx.platform,
|
|
182
|
-
taskKey: ctx.taskKey,
|
|
222
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
183
223
|
userKey: hashUserId(ctx.userId),
|
|
184
224
|
toolId: aiCommand,
|
|
185
225
|
durationMs: result.durationMs,
|
|
@@ -242,7 +282,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
242
282
|
log.error(`Task error for user ${ctx.userId}: ${error}`);
|
|
243
283
|
emitStructuredEvent('AITask', 'ai.task.error', {
|
|
244
284
|
platform: ctx.platform,
|
|
245
|
-
taskKey: ctx.taskKey,
|
|
285
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
246
286
|
userKey: hashUserId(ctx.userId),
|
|
247
287
|
toolId: aiCommand,
|
|
248
288
|
durationMs: Date.now() - taskState.startedAt,
|
|
@@ -290,13 +330,18 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
290
330
|
if (!settled) {
|
|
291
331
|
emitStructuredEvent('AITask', 'ai.task.error', {
|
|
292
332
|
platform: ctx.platform,
|
|
293
|
-
taskKey: ctx.taskKey,
|
|
333
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
294
334
|
userKey: hashUserId(ctx.userId),
|
|
295
335
|
toolId: aiCommand,
|
|
296
336
|
durationMs: Date.now() - taskState.startedAt,
|
|
297
337
|
errorSnippet: 'aborted',
|
|
298
338
|
errorType: 'aborted',
|
|
299
339
|
});
|
|
340
|
+
// 用户取消(/new、/resume、队列超时、stale 清理):把「思考中…」占位卡片编辑为终态,
|
|
341
|
+
// 避免卡片卡在转圈。停按钮路径会先 settle() 再 abort,settled=true 时此处跳过,不会双发。
|
|
342
|
+
void platformAdapter.sendError('⏹️ 已取消').catch(() => {
|
|
343
|
+
/* 占位卡片可能已被删除或流过期,编辑失败可忽略 */
|
|
344
|
+
});
|
|
300
345
|
}
|
|
301
346
|
activeHandle?.abort();
|
|
302
347
|
cleanup();
|
|
@@ -307,6 +352,9 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
307
352
|
settle,
|
|
308
353
|
startedAt: Date.now(),
|
|
309
354
|
toolId: aiCommand,
|
|
355
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
356
|
+
platform: ctx.platform,
|
|
357
|
+
userKey: hashUserId(ctx.userId),
|
|
310
358
|
};
|
|
311
359
|
try {
|
|
312
360
|
startRun();
|
|
@@ -318,7 +366,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
318
366
|
log.error(`[AITask] Synchronous error in startRun: ${err}`);
|
|
319
367
|
emitStructuredEvent('AITask', 'ai.task.error', {
|
|
320
368
|
platform: ctx.platform,
|
|
321
|
-
taskKey: ctx.taskKey,
|
|
369
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
322
370
|
userKey: hashUserId(ctx.userId),
|
|
323
371
|
toolId: aiCommand,
|
|
324
372
|
durationMs: 0,
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 跨平台子进程树终止:用于 Codex / CodeBuddy CLI 子进程。
|
|
3
|
+
*
|
|
4
|
+
* 在 Unix 上子进程需以 `detached: true` 启动,使其成为独立进程组的组长;
|
|
5
|
+
* 这样 `process.kill(-pid)` 才能同时命中 CLI 及其派生的孙进程(MCP server、
|
|
6
|
+
* shell 工具调用等)。Windows 不支持负 PID 信号,改用 `taskkill /T /F`
|
|
7
|
+
* 按 PID 递归终止整棵进程树。
|
|
8
|
+
*
|
|
9
|
+
* 进程退出(shutdown / 崩溃)时,`destroyAllLiveChildren()` 会强制清理所有
|
|
10
|
+
* 仍在运行的被追踪子进程,避免僵尸 / 孤儿进程。
|
|
11
|
+
*/
|
|
12
|
+
import { type ChildProcess } from 'node:child_process';
|
|
13
|
+
/** 追踪一个已 spawn 的子进程,使其在关停时能被强制终止。 */
|
|
14
|
+
export declare function trackChild(child: ChildProcess): ChildProcess;
|
|
15
|
+
/**
|
|
16
|
+
* 终止子进程及其所有后代。默认优雅:先 SIGTERM,`graceMs` 后升级为 SIGKILL。
|
|
17
|
+
* 传 `force: true` 则立即 SIGKILL(用于关停)。
|
|
18
|
+
*/
|
|
19
|
+
export declare function killProcessTree(child: ChildProcess, opts?: {
|
|
20
|
+
force?: boolean;
|
|
21
|
+
graceMs?: number;
|
|
22
|
+
}): void;
|
|
23
|
+
/** 强制终止所有仍在运行的被追踪子进程。关停 / 进程退出时调用。 */
|
|
24
|
+
export declare function destroyAllLiveChildren(): void;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 跨平台子进程树终止:用于 Codex / CodeBuddy CLI 子进程。
|
|
3
|
+
*
|
|
4
|
+
* 在 Unix 上子进程需以 `detached: true` 启动,使其成为独立进程组的组长;
|
|
5
|
+
* 这样 `process.kill(-pid)` 才能同时命中 CLI 及其派生的孙进程(MCP server、
|
|
6
|
+
* shell 工具调用等)。Windows 不支持负 PID 信号,改用 `taskkill /T /F`
|
|
7
|
+
* 按 PID 递归终止整棵进程树。
|
|
8
|
+
*
|
|
9
|
+
* 进程退出(shutdown / 崩溃)时,`destroyAllLiveChildren()` 会强制清理所有
|
|
10
|
+
* 仍在运行的被追踪子进程,避免僵尸 / 孤儿进程。
|
|
11
|
+
*/
|
|
12
|
+
import { spawn } from 'node:child_process';
|
|
13
|
+
import { createLogger } from '../logger.js';
|
|
14
|
+
const log = createLogger('ProcessKill');
|
|
15
|
+
/** 当前被追踪的存活子进程,供关停时强制清理。 */
|
|
16
|
+
const liveChildren = new Set();
|
|
17
|
+
/** 追踪一个已 spawn 的子进程,使其在关停时能被强制终止。 */
|
|
18
|
+
export function trackChild(child) {
|
|
19
|
+
liveChildren.add(child);
|
|
20
|
+
child.once('close', () => liveChildren.delete(child));
|
|
21
|
+
return child;
|
|
22
|
+
}
|
|
23
|
+
/** 向整棵进程树发送信号:Unix 用负 PID(整组),Windows 用 taskkill /T /F。 */
|
|
24
|
+
function signalTree(child, signal) {
|
|
25
|
+
const pid = child.pid;
|
|
26
|
+
if (pid == null)
|
|
27
|
+
return;
|
|
28
|
+
try {
|
|
29
|
+
if (process.platform === 'win32') {
|
|
30
|
+
// Windows 没有「优雅 SIGTERM」语义,统一 /T(递归) /F(强制)
|
|
31
|
+
const tk = spawn('taskkill', ['/pid', String(pid), '/T', '/F'], {
|
|
32
|
+
windowsHide: true,
|
|
33
|
+
stdio: 'ignore',
|
|
34
|
+
});
|
|
35
|
+
tk.unref();
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
process.kill(-pid, signal);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
// ESRCH(已退出)或未 detached(无独立进程组)——退化为直接信号
|
|
43
|
+
try {
|
|
44
|
+
child.kill(signal);
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
/* 进程已退出,忽略 */
|
|
48
|
+
}
|
|
49
|
+
log.debug(`signalTree(${signal}) for pid=${pid} 退化为直杀: ${err.message}`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* 终止子进程及其所有后代。默认优雅:先 SIGTERM,`graceMs` 后升级为 SIGKILL。
|
|
54
|
+
* 传 `force: true` 则立即 SIGKILL(用于关停)。
|
|
55
|
+
*/
|
|
56
|
+
export function killProcessTree(child, opts = {}) {
|
|
57
|
+
const { force = false, graceMs = 5000 } = opts;
|
|
58
|
+
if (child.pid == null)
|
|
59
|
+
return;
|
|
60
|
+
if (force) {
|
|
61
|
+
signalTree(child, 'SIGKILL');
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
signalTree(child, 'SIGTERM');
|
|
65
|
+
const escalate = setTimeout(() => signalTree(child, 'SIGKILL'), graceMs);
|
|
66
|
+
escalate.unref();
|
|
67
|
+
}
|
|
68
|
+
/** 强制终止所有仍在运行的被追踪子进程。关停 / 进程退出时调用。 */
|
|
69
|
+
export function destroyAllLiveChildren() {
|
|
70
|
+
for (const child of [...liveChildren]) {
|
|
71
|
+
try {
|
|
72
|
+
signalTree(child, 'SIGKILL');
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
/* best effort */
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
liveChildren.clear();
|
|
79
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 重连韧性共享工具:抖动(jitter)+ 致命错误慢探测。
|
|
3
|
+
*
|
|
4
|
+
* 各平台长连接断开后各自重连,原本退避固定无抖动 → 车队重启会锁步重连风暴;
|
|
5
|
+
* 且致命(鉴权)错误也紧密/无限重试, hammer 网关。
|
|
6
|
+
*
|
|
7
|
+
* 策略(已与产品确认):
|
|
8
|
+
* - 所有退避加 ±30% 抖动,避免锁步。
|
|
9
|
+
* - 鉴权类致命错误转为**慢探测**(每 5 分钟一次 + 显眼告警):
|
|
10
|
+
* 既不紧密 hammer,也不永久断连(尊重 WeWork「避免永久断连」意图)。
|
|
11
|
+
*/
|
|
12
|
+
/** 慢探测间隔:致命错误后每 5 分钟探测一次。 */
|
|
13
|
+
export declare const SLOW_PROBE_MS: number;
|
|
14
|
+
/**
|
|
15
|
+
* 在 baseMs 上叠加 ±frac 的抖动,避免车队重启锁步重连。
|
|
16
|
+
* jitteredDelay(1000, 0.3) ∈ [700, 1300]。
|
|
17
|
+
*/
|
|
18
|
+
export declare function jitteredDelay(baseMs: number, frac?: number): number;
|
|
19
|
+
/**
|
|
20
|
+
* 判断是否为「致命/不可重试」错误(凭据无效、鉴权失败)。
|
|
21
|
+
* 这类错误紧密重试无意义,应转慢探测。
|
|
22
|
+
*/
|
|
23
|
+
export declare function isFatalReconnectError(err: unknown): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* 计算重连延迟:致命慢探测模式下用 SLOW_PROBE_MS,否则用 jittered(base)。
|
|
26
|
+
* `fatal` 由调用方维护(致命时置 true,成功连上后置 false)。
|
|
27
|
+
*/
|
|
28
|
+
export declare function reconnectDelay(baseMs: number, fatal: boolean): number;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 重连韧性共享工具:抖动(jitter)+ 致命错误慢探测。
|
|
3
|
+
*
|
|
4
|
+
* 各平台长连接断开后各自重连,原本退避固定无抖动 → 车队重启会锁步重连风暴;
|
|
5
|
+
* 且致命(鉴权)错误也紧密/无限重试, hammer 网关。
|
|
6
|
+
*
|
|
7
|
+
* 策略(已与产品确认):
|
|
8
|
+
* - 所有退避加 ±30% 抖动,避免锁步。
|
|
9
|
+
* - 鉴权类致命错误转为**慢探测**(每 5 分钟一次 + 显眼告警):
|
|
10
|
+
* 既不紧密 hammer,也不永久断连(尊重 WeWork「避免永久断连」意图)。
|
|
11
|
+
*/
|
|
12
|
+
import { createLogger } from '../logger.js';
|
|
13
|
+
const log = createLogger('Reconnect');
|
|
14
|
+
/** 慢探测间隔:致命错误后每 5 分钟探测一次。 */
|
|
15
|
+
export const SLOW_PROBE_MS = 5 * 60_000;
|
|
16
|
+
/**
|
|
17
|
+
* 在 baseMs 上叠加 ±frac 的抖动,避免车队重启锁步重连。
|
|
18
|
+
* jitteredDelay(1000, 0.3) ∈ [700, 1300]。
|
|
19
|
+
*/
|
|
20
|
+
export function jitteredDelay(baseMs, frac = 0.3) {
|
|
21
|
+
if (baseMs <= 0)
|
|
22
|
+
return 0;
|
|
23
|
+
const jitter = baseMs * frac * (Math.random() * 2 - 1);
|
|
24
|
+
return Math.max(0, Math.round(baseMs + jitter));
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* 判断是否为「致命/不可重试」错误(凭据无效、鉴权失败)。
|
|
28
|
+
* 这类错误紧密重试无意义,应转慢探测。
|
|
29
|
+
*/
|
|
30
|
+
export function isFatalReconnectError(err) {
|
|
31
|
+
const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
|
32
|
+
if (msg.includes('401') || msg.includes('403'))
|
|
33
|
+
return true;
|
|
34
|
+
if (msg.includes('unauthorized') || msg.includes('forbidden'))
|
|
35
|
+
return true;
|
|
36
|
+
if (msg.includes('invalid signature'))
|
|
37
|
+
return true;
|
|
38
|
+
if (msg.includes('not subscribed'))
|
|
39
|
+
return true;
|
|
40
|
+
if (msg.includes('请重新登录') || msg.includes('需要重新登录') || msg.includes('token 已过期'))
|
|
41
|
+
return true;
|
|
42
|
+
if (msg.includes('invalid') && /token|secret|credential|appid|app_id|corp/.test(msg))
|
|
43
|
+
return true;
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* 计算重连延迟:致命慢探测模式下用 SLOW_PROBE_MS,否则用 jittered(base)。
|
|
48
|
+
* `fatal` 由调用方维护(致命时置 true,成功连上后置 false)。
|
|
49
|
+
*/
|
|
50
|
+
export function reconnectDelay(baseMs, fatal) {
|
|
51
|
+
if (fatal) {
|
|
52
|
+
log.warn(`致命错误,转慢探测(${Math.round(SLOW_PROBE_MS / 1000)}s 一次)`);
|
|
53
|
+
return jitteredDelay(SLOW_PROBE_MS);
|
|
54
|
+
}
|
|
55
|
+
return jitteredDelay(baseMs);
|
|
56
|
+
}
|
|
@@ -6,3 +6,19 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import type { TaskRunState } from './ai-task.js';
|
|
8
8
|
export declare function startTaskCleanup(runningTasks: Map<string, TaskRunState>): () => void;
|
|
9
|
+
/**
|
|
10
|
+
* 在进程退出(优雅关闭 / 崩溃)路径上,为仍在运行的任务补发一条终态遥测事件。
|
|
11
|
+
*
|
|
12
|
+
* `runningTasks` 中存在的任务代表「已发出 ai.task.start 但尚未走到 complete/error」
|
|
13
|
+
* 的在途请求。正常流程会在 handle-ai-request 的 extraCleanup 里删除已结算任务,
|
|
14
|
+
* 因此进入这里时剩下的就是真正被进程退出打断的任务。
|
|
15
|
+
*
|
|
16
|
+
* 与用户主动 `/new`、队列超时触发的 `aborted` 区分,统一用 `interrupted` 标记。
|
|
17
|
+
* 补发后立即调用 state.settle() 置 settled=true:优雅关闭路径随后仍会调用
|
|
18
|
+
* handle.abort()(释放底层资源),但因已 settled,abort 不会再补发一条 aborted,
|
|
19
|
+
* 避免对同一任务重复计数。
|
|
20
|
+
*
|
|
21
|
+
* 该函数必须同步执行,且应在遥测刷盘(shutdownLoggerTelemetry)之前调用,
|
|
22
|
+
* 以保证补发的事件能进入上传队列。
|
|
23
|
+
*/
|
|
24
|
+
export declare function emitInterruptedTerminals(runningTasks: Map<string, TaskRunState>): void;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Tasks older than 30 minutes are aborted and removed from the running-tasks
|
|
5
5
|
* map so they never accumulate indefinitely.
|
|
6
6
|
*/
|
|
7
|
-
import { createLogger } from '../logger.js';
|
|
7
|
+
import { createLogger, emitStructuredEvent } from '../logger.js';
|
|
8
8
|
const log = createLogger('TaskCleanup');
|
|
9
9
|
const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
|
|
10
10
|
const STALE_THRESHOLD_MS = 30 * 60 * 1000; // 30 minutes
|
|
@@ -28,3 +28,36 @@ export function startTaskCleanup(runningTasks) {
|
|
|
28
28
|
timer.unref();
|
|
29
29
|
return () => clearInterval(timer);
|
|
30
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* 在进程退出(优雅关闭 / 崩溃)路径上,为仍在运行的任务补发一条终态遥测事件。
|
|
33
|
+
*
|
|
34
|
+
* `runningTasks` 中存在的任务代表「已发出 ai.task.start 但尚未走到 complete/error」
|
|
35
|
+
* 的在途请求。正常流程会在 handle-ai-request 的 extraCleanup 里删除已结算任务,
|
|
36
|
+
* 因此进入这里时剩下的就是真正被进程退出打断的任务。
|
|
37
|
+
*
|
|
38
|
+
* 与用户主动 `/new`、队列超时触发的 `aborted` 区分,统一用 `interrupted` 标记。
|
|
39
|
+
* 补发后立即调用 state.settle() 置 settled=true:优雅关闭路径随后仍会调用
|
|
40
|
+
* handle.abort()(释放底层资源),但因已 settled,abort 不会再补发一条 aborted,
|
|
41
|
+
* 避免对同一任务重复计数。
|
|
42
|
+
*
|
|
43
|
+
* 该函数必须同步执行,且应在遥测刷盘(shutdownLoggerTelemetry)之前调用,
|
|
44
|
+
* 以保证补发的事件能进入上传队列。
|
|
45
|
+
*/
|
|
46
|
+
export function emitInterruptedTerminals(runningTasks) {
|
|
47
|
+
if (runningTasks.size === 0)
|
|
48
|
+
return;
|
|
49
|
+
const now = Date.now();
|
|
50
|
+
for (const state of runningTasks.values()) {
|
|
51
|
+
emitStructuredEvent('AITask', 'ai.task.error', {
|
|
52
|
+
platform: state.platform,
|
|
53
|
+
taskKey: state.taskKey,
|
|
54
|
+
userKey: state.userKey,
|
|
55
|
+
toolId: state.toolId,
|
|
56
|
+
durationMs: now - state.startedAt,
|
|
57
|
+
errorSnippet: 'interrupted',
|
|
58
|
+
errorType: 'interrupted',
|
|
59
|
+
});
|
|
60
|
+
// 标记已结算,使随后 shutdown 的 abort() 跳过重复的 aborted 事件
|
|
61
|
+
state.settle();
|
|
62
|
+
}
|
|
63
|
+
}
|
package/dist/telegram/client.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Telegraf } from "telegraf";
|
|
2
2
|
import { createLogger } from "../logger.js";
|
|
3
|
+
import { isFatalReconnectError, jitteredDelay } from "../shared/reconnect.js";
|
|
3
4
|
const log = createLogger("Telegram");
|
|
4
5
|
let bot;
|
|
5
6
|
let botUsername;
|
|
@@ -32,8 +33,13 @@ export async function initTelegram(config, setupHandlers) {
|
|
|
32
33
|
catch {
|
|
33
34
|
/* ignore */
|
|
34
35
|
}
|
|
36
|
+
// 致命错误(token 无效等):不再重试,避免烧满 10 次
|
|
37
|
+
if (isFatalReconnectError(err)) {
|
|
38
|
+
log.error("Telegram 致命错误,停止重连(请检查 bot token):", err);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
35
41
|
const maxAttempts = 10;
|
|
36
|
-
const delayMs = Math.min(5000 * attempt, 60000);
|
|
42
|
+
const delayMs = jitteredDelay(Math.min(5000 * attempt, 60000));
|
|
37
43
|
if (attempt < maxAttempts) {
|
|
38
44
|
log.info(`Telegram reconnect in ${Math.round(delayMs / 1000)}s (attempt ${attempt}/${maxAttempts})`);
|
|
39
45
|
await new Promise((r) => setTimeout(r, delayMs));
|
|
@@ -190,7 +190,7 @@ export async function shutdownTelemetryUpload() {
|
|
|
190
190
|
uploadEnabled = false;
|
|
191
191
|
endpoint = undefined;
|
|
192
192
|
bearer = undefined;
|
|
193
|
-
|
|
193
|
+
const lines = pending;
|
|
194
194
|
while (lines.length > 0) {
|
|
195
195
|
const batch = lines.splice(0, BATCH_MAX_LINES);
|
|
196
196
|
try {
|
package/dist/wework/client.js
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
import { WebSocket } from 'ws';
|
|
11
11
|
import { randomBytes } from 'node:crypto';
|
|
12
12
|
import { createLogger } from '../logger.js';
|
|
13
|
+
import { jitteredDelay, SLOW_PROBE_MS, isFatalReconnectError } from '../shared/reconnect.js';
|
|
13
14
|
const log = createLogger('WeWork');
|
|
14
15
|
const DEFAULT_WS_URL = 'wss://openws.work.weixin.qq.com';
|
|
15
16
|
const HEARTBEAT_INTERVAL = 30000; // 30秒
|
|
@@ -383,6 +384,8 @@ function stopHeartbeat() {
|
|
|
383
384
|
* Schedule reconnection attempt
|
|
384
385
|
* 超过 MAX_RECONNECT_ATTEMPTS 后自动重置计数器继续重试,避免永久断连
|
|
385
386
|
*/
|
|
387
|
+
/** 致命(鉴权)错误后转慢探测,避免紧密 hammer 网关 */
|
|
388
|
+
let fatalSlowProbe = false;
|
|
386
389
|
function scheduleReconnect() {
|
|
387
390
|
if (isStopping || !shouldReconnect) {
|
|
388
391
|
return;
|
|
@@ -395,18 +398,27 @@ function scheduleReconnect() {
|
|
|
395
398
|
log.warn(`Max reconnect attempts (${MAX_RECONNECT_ATTEMPTS}) reached, resetting counter and retrying at lower frequency`);
|
|
396
399
|
reconnectAttempts = 0;
|
|
397
400
|
}
|
|
398
|
-
// 逐步增加间隔,5s → 7.5s → 11s → ... 最大 60s
|
|
399
|
-
const backoff =
|
|
400
|
-
|
|
401
|
+
// 逐步增加间隔,5s → 7.5s → 11s → ... 最大 60s;致命(鉴权)错误转慢探测
|
|
402
|
+
const backoff = fatalSlowProbe
|
|
403
|
+
? SLOW_PROBE_MS
|
|
404
|
+
: Math.min(5000 * Math.pow(1.5, Math.floor(reconnectAttempts / 5)), 60000);
|
|
405
|
+
const interval = jitteredDelay(Math.round(backoff));
|
|
401
406
|
reconnectTimer = setTimeout(async () => {
|
|
402
407
|
reconnectTimer = null;
|
|
403
408
|
reconnectAttempts++;
|
|
404
|
-
log.info(`Reconnecting... Attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS} (interval: ${interval}ms)`);
|
|
409
|
+
log.info(`Reconnecting... Attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS} (interval: ${interval}ms${fatalSlowProbe ? ', slow-probe' : ''})`);
|
|
405
410
|
try {
|
|
406
411
|
await connectWebSocket();
|
|
412
|
+
fatalSlowProbe = false; // 连上后恢复正常退避
|
|
407
413
|
}
|
|
408
414
|
catch (err) {
|
|
409
|
-
|
|
415
|
+
if (isFatalReconnectError(err)) {
|
|
416
|
+
fatalSlowProbe = true;
|
|
417
|
+
log.warn('WeWork 致命错误(鉴权失败),转慢探测:', err);
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
log.error('Reconnection failed:', err);
|
|
421
|
+
}
|
|
410
422
|
}
|
|
411
423
|
}, interval);
|
|
412
424
|
}
|
|
@@ -213,6 +213,9 @@ export function setupWeWorkHandlers(config, sessionManager) {
|
|
|
213
213
|
const state = ctx.runningTasks.get(taskKey);
|
|
214
214
|
if (state) {
|
|
215
215
|
log.warn(`[SAFETY_TIMEOUT] Task ${taskKey} exceeded ${WEWORK_TASK_SAFETY_TIMEOUT_MS}ms, aborting`);
|
|
216
|
+
// 先 settle 再 abort:settled=true 后 abort() 跳过自身的 sendError,由这里的 sendTextReply
|
|
217
|
+
// 兜底(企微流式消息可能已过期,文本兜底更稳),避免双重终态消息。
|
|
218
|
+
state.settle();
|
|
216
219
|
state.handle.abort();
|
|
217
220
|
ctx.runningTasks.delete(taskKey);
|
|
218
221
|
sendTextReply(chatId, `AI 处理超时(${Math.round(WEWORK_TASK_SAFETY_TIMEOUT_MS / 1000)}s),已自动取消。请重试。`, senderCtx.reqId).catch(() => { });
|
|
@@ -12,6 +12,10 @@ export interface CentrifugeClientConfig {
|
|
|
12
12
|
userId: string;
|
|
13
13
|
httpBaseUrl?: string;
|
|
14
14
|
httpAccessToken?: string;
|
|
15
|
+
/** 返回当前有效的 access token(已刷新)。无则用 httpAccessToken 兜底。 */
|
|
16
|
+
getAccessToken?: () => string;
|
|
17
|
+
/** 单飞刷新 token;HTTP 401 时调用,刷新后 getAccessToken 返回新 token。 */
|
|
18
|
+
refreshToken?: () => Promise<void>;
|
|
15
19
|
workspaceSessionId?: string;
|
|
16
20
|
/**
|
|
17
21
|
* Called before sending a WeChat KF reply to update the channel's channelId
|
|
@@ -226,20 +226,50 @@ export class WorkBuddyCentrifugeClient {
|
|
|
226
226
|
};
|
|
227
227
|
const url = `${this.config.httpBaseUrl}/v2/backgroundagent/wecom/local-proxy/receive`;
|
|
228
228
|
log.debug(`${this.logPrefix} HTTP COPILOT_RESPONSE → ${url} chatId=${sessionId} msgLen=${message.length}`);
|
|
229
|
-
// Retry COPILOT_RESPONSE
|
|
229
|
+
// Retry COPILOT_RESPONSE with exponential backoff for rate limits (95002)
|
|
230
|
+
const MAX_RETRIES = 5;
|
|
231
|
+
const INITIAL_BACKOFF_MS = 2000;
|
|
230
232
|
let sent = false;
|
|
231
|
-
|
|
233
|
+
let backoffMs = INITIAL_BACKOFF_MS;
|
|
234
|
+
let refreshedOn401 = false;
|
|
235
|
+
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
|
236
|
+
const token = this.config.getAccessToken?.() ?? this.config.httpAccessToken ?? '';
|
|
232
237
|
try {
|
|
233
238
|
const res = await fetch(url, {
|
|
234
239
|
method: 'POST',
|
|
235
240
|
headers: {
|
|
236
241
|
'Content-Type': 'application/json',
|
|
237
|
-
Authorization: `Bearer ${
|
|
242
|
+
Authorization: `Bearer ${token}`,
|
|
238
243
|
},
|
|
239
244
|
body: JSON.stringify(httpPayload),
|
|
240
245
|
signal: AbortSignal.timeout(30_000),
|
|
241
246
|
});
|
|
242
247
|
const body = await res.text().catch(() => '');
|
|
248
|
+
// Check for WeChat KF rate limit (95002)
|
|
249
|
+
if (body.includes('errcode=95002') || body.includes('"errcode":95002')) {
|
|
250
|
+
if (attempt < MAX_RETRIES) {
|
|
251
|
+
log.warn(`${this.logPrefix} HTTP COPILOT_RESPONSE rate limited (95002), attempt ${attempt}/${MAX_RETRIES}, retrying in ${backoffMs}ms`);
|
|
252
|
+
await new Promise((r) => setTimeout(r, backoffMs));
|
|
253
|
+
backoffMs = Math.min(backoffMs * 2, 60_000); // Cap at 60s
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
else {
|
|
257
|
+
log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE rate limited after ${MAX_RETRIES} attempts, giving up`);
|
|
258
|
+
break;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
// 401: access token 过期 → 单飞刷新后重试一次
|
|
262
|
+
if (res.status === 401 && !refreshedOn401 && this.config.refreshToken) {
|
|
263
|
+
refreshedOn401 = true;
|
|
264
|
+
try {
|
|
265
|
+
await this.config.refreshToken();
|
|
266
|
+
log.info(`${this.logPrefix} Token refreshed on 401, retrying COPILOT_RESPONSE`);
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
catch (refreshErr) {
|
|
270
|
+
log.error(`${this.logPrefix} Token refresh failed on 401:`, refreshErr);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
243
273
|
if (!res.ok) {
|
|
244
274
|
log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE failed: ${res.status} ${body.substring(0, 300)}`);
|
|
245
275
|
}
|
|
@@ -250,17 +280,18 @@ export class WorkBuddyCentrifugeClient {
|
|
|
250
280
|
break;
|
|
251
281
|
}
|
|
252
282
|
catch (err) {
|
|
253
|
-
if (attempt <
|
|
254
|
-
log.warn(`${this.logPrefix} HTTP COPILOT_RESPONSE attempt ${attempt} failed, retrying in
|
|
255
|
-
await new Promise((r) => setTimeout(r,
|
|
283
|
+
if (attempt < MAX_RETRIES) {
|
|
284
|
+
log.warn(`${this.logPrefix} HTTP COPILOT_RESPONSE attempt ${attempt} failed, retrying in ${backoffMs}ms:`, err);
|
|
285
|
+
await new Promise((r) => setTimeout(r, backoffMs));
|
|
286
|
+
backoffMs = Math.min(backoffMs * 2, 60_000);
|
|
256
287
|
}
|
|
257
288
|
else {
|
|
258
|
-
log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE error after
|
|
289
|
+
log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE error after ${MAX_RETRIES} attempts:`, err);
|
|
259
290
|
}
|
|
260
291
|
}
|
|
261
292
|
}
|
|
262
293
|
if (!sent) {
|
|
263
|
-
this.enqueuePendingReply(url, httpPayload
|
|
294
|
+
this.enqueuePendingReply(url, httpPayload);
|
|
264
295
|
}
|
|
265
296
|
// Release the heartbeat lock so the periodic registration can resume
|
|
266
297
|
this.config.releaseChannelLockFn?.();
|
|
@@ -372,7 +403,7 @@ export class WorkBuddyCentrifugeClient {
|
|
|
372
403
|
/**
|
|
373
404
|
* Enqueue a failed reply for later delivery
|
|
374
405
|
*/
|
|
375
|
-
enqueuePendingReply(url, payload
|
|
406
|
+
enqueuePendingReply(url, payload) {
|
|
376
407
|
// Evict expired entries
|
|
377
408
|
const now = Date.now();
|
|
378
409
|
this.pendingReplies = this.pendingReplies.filter((r) => now - r.addedAt < PENDING_REPLY_TTL_MS);
|
|
@@ -380,7 +411,7 @@ export class WorkBuddyCentrifugeClient {
|
|
|
380
411
|
const evicted = this.pendingReplies.shift();
|
|
381
412
|
log.warn(`${this.logPrefix} Pending replies full, evicting oldest (msgId=${evicted?.payload.msgId})`);
|
|
382
413
|
}
|
|
383
|
-
this.pendingReplies.push({ url, payload,
|
|
414
|
+
this.pendingReplies.push({ url, payload, addedAt: now });
|
|
384
415
|
log.warn(`${this.logPrefix} Queued pending reply (queue=${this.pendingReplies.length}, msgId=${payload.msgId})`);
|
|
385
416
|
}
|
|
386
417
|
/**
|
|
@@ -398,27 +429,44 @@ export class WorkBuddyCentrifugeClient {
|
|
|
398
429
|
log.info(`${this.logPrefix} Flushing ${toSend.length} pending reply(ies)`);
|
|
399
430
|
}
|
|
400
431
|
for (const reply of toSend) {
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
432
|
+
// 用当前(可能已刷新的)token;401 时单飞刷新后重试一次
|
|
433
|
+
let refreshedOn401 = false;
|
|
434
|
+
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
435
|
+
const token = this.config.getAccessToken?.() ?? this.config.httpAccessToken ?? '';
|
|
436
|
+
try {
|
|
437
|
+
const res = await fetch(reply.url, {
|
|
438
|
+
method: 'POST',
|
|
439
|
+
headers: {
|
|
440
|
+
'Content-Type': 'application/json',
|
|
441
|
+
Authorization: `Bearer ${token}`,
|
|
442
|
+
},
|
|
443
|
+
body: JSON.stringify(reply.payload),
|
|
444
|
+
signal: AbortSignal.timeout(30_000),
|
|
445
|
+
});
|
|
446
|
+
if (res.status === 401 && !refreshedOn401 && this.config.refreshToken) {
|
|
447
|
+
refreshedOn401 = true;
|
|
448
|
+
try {
|
|
449
|
+
await this.config.refreshToken();
|
|
450
|
+
continue;
|
|
451
|
+
}
|
|
452
|
+
catch {
|
|
453
|
+
/* 刷新失败,落到下面正常判定 */
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
const body = await res.text().catch(() => '');
|
|
457
|
+
if (res.ok) {
|
|
458
|
+
log.info(`${this.logPrefix} Flushed pending reply ok: msgId=${reply.payload.msgId}`);
|
|
459
|
+
}
|
|
460
|
+
else {
|
|
461
|
+
log.error(`${this.logPrefix} Flushed pending reply failed: ${res.status} ${body.substring(0, 200)}`);
|
|
462
|
+
}
|
|
463
|
+
break;
|
|
414
464
|
}
|
|
415
|
-
|
|
416
|
-
log.error(`${this.logPrefix} Flushed pending reply
|
|
465
|
+
catch (err) {
|
|
466
|
+
log.error(`${this.logPrefix} Flushed pending reply error: msgId=${reply.payload.msgId}`, err);
|
|
467
|
+
break;
|
|
417
468
|
}
|
|
418
469
|
}
|
|
419
|
-
catch (err) {
|
|
420
|
-
log.error(`${this.logPrefix} Flushed pending reply error: msgId=${reply.payload.msgId}`, err);
|
|
421
|
-
}
|
|
422
470
|
}
|
|
423
471
|
this.flushing = false;
|
|
424
472
|
}
|
package/dist/workbuddy/client.js
CHANGED
|
@@ -9,6 +9,7 @@ import { existsSync, mkdirSync } from 'node:fs';
|
|
|
9
9
|
import { join } from 'node:path';
|
|
10
10
|
import { hostname, homedir } from 'node:os';
|
|
11
11
|
import { createLogger } from '../logger.js';
|
|
12
|
+
import { jitteredDelay, SLOW_PROBE_MS, isFatalReconnectError } from '../shared/reconnect.js';
|
|
12
13
|
import { WorkBuddyOAuth } from './oauth.js';
|
|
13
14
|
import { WorkBuddyCentrifugeClient } from './centrifuge-client.js';
|
|
14
15
|
const log = createLogger('WorkBuddy');
|
|
@@ -23,8 +24,11 @@ let stateChangeHandler = null;
|
|
|
23
24
|
let heartbeatTimer = null;
|
|
24
25
|
let reconnectTimer = null;
|
|
25
26
|
let reconnectAttempt = 0;
|
|
27
|
+
let fatalSlowProbe = false;
|
|
26
28
|
let stopped = false;
|
|
27
29
|
let platformConfig = null;
|
|
30
|
+
/** 单飞刷新 token 的在途 Promise,避免并发 401 重复刷新 */
|
|
31
|
+
let refreshInFlight = null;
|
|
28
32
|
export function getChannelState() {
|
|
29
33
|
return channelState;
|
|
30
34
|
}
|
|
@@ -78,6 +82,10 @@ async function connect() {
|
|
|
78
82
|
}
|
|
79
83
|
catch (err) {
|
|
80
84
|
log.error('Host workspace registration failed:', err);
|
|
85
|
+
if (isFatalReconnectError(err)) {
|
|
86
|
+
fatalSlowProbe = true;
|
|
87
|
+
log.warn('WorkBuddy 致命错误(鉴权失败),转慢探测(token 可能已过期):', err);
|
|
88
|
+
}
|
|
81
89
|
scheduleReconnect();
|
|
82
90
|
return;
|
|
83
91
|
}
|
|
@@ -120,6 +128,8 @@ async function connect() {
|
|
|
120
128
|
userId: pc.userId ?? '',
|
|
121
129
|
httpBaseUrl: baseUrl,
|
|
122
130
|
httpAccessToken: pc.accessToken ?? '',
|
|
131
|
+
getAccessToken: () => oauthClient?.accessToken ?? pc.accessToken ?? '',
|
|
132
|
+
refreshToken: refreshWorkBuddyToken,
|
|
123
133
|
workspaceSessionId,
|
|
124
134
|
registerChannelFn,
|
|
125
135
|
releaseChannelLockFn,
|
|
@@ -128,6 +138,7 @@ async function connect() {
|
|
|
128
138
|
log.info('WorkBuddy Centrifuge connected');
|
|
129
139
|
log.info(`WeChat KF sessionId: ${workspaceSessionId}`);
|
|
130
140
|
reconnectAttempt = 0;
|
|
141
|
+
fatalSlowProbe = false;
|
|
131
142
|
updateState('connected');
|
|
132
143
|
// Step 2: Register Claw workspace to get WeChat KF routing channel + sessionId
|
|
133
144
|
oauth.registerWorkspace({
|
|
@@ -234,9 +245,10 @@ function scheduleReconnect() {
|
|
|
234
245
|
clearTimeout(reconnectTimer);
|
|
235
246
|
reconnectTimer = null;
|
|
236
247
|
}
|
|
237
|
-
const
|
|
248
|
+
const baseDelay = RECONNECT_DELAYS_MS[Math.min(reconnectAttempt, RECONNECT_DELAYS_MS.length - 1)];
|
|
249
|
+
const delay = fatalSlowProbe ? jitteredDelay(SLOW_PROBE_MS) : jitteredDelay(baseDelay);
|
|
238
250
|
reconnectAttempt++;
|
|
239
|
-
log.info(`Reconnecting in ${delay}ms (attempt ${reconnectAttempt})...`);
|
|
251
|
+
log.info(`Reconnecting in ${delay}ms (attempt ${reconnectAttempt})${fatalSlowProbe ? ' [slow-probe]' : ''}...`);
|
|
240
252
|
reconnectTimer = setTimeout(async () => {
|
|
241
253
|
reconnectTimer = null;
|
|
242
254
|
if (stopped)
|
|
@@ -261,6 +273,31 @@ export function getCentrifugeClient() {
|
|
|
261
273
|
export function getOAuth() {
|
|
262
274
|
return oauthClient;
|
|
263
275
|
}
|
|
276
|
+
/**
|
|
277
|
+
* 单飞刷新 WorkBuddy access token:并发 401 只触发一次刷新,复用同一 Promise。
|
|
278
|
+
* 刷新成功后 oauthClient.accessToken 原地更新,所有 HTTP 调用(含心跳)自动用新 token。
|
|
279
|
+
*/
|
|
280
|
+
async function refreshWorkBuddyToken() {
|
|
281
|
+
if (!oauthClient)
|
|
282
|
+
return;
|
|
283
|
+
if (refreshInFlight)
|
|
284
|
+
return refreshInFlight;
|
|
285
|
+
refreshInFlight = (async () => {
|
|
286
|
+
try {
|
|
287
|
+
log.info('Refreshing WorkBuddy access token...');
|
|
288
|
+
await oauthClient.refreshTokenAuth();
|
|
289
|
+
log.info('WorkBuddy access token refreshed');
|
|
290
|
+
}
|
|
291
|
+
catch (err) {
|
|
292
|
+
log.error('WorkBuddy token refresh failed:', err);
|
|
293
|
+
throw err;
|
|
294
|
+
}
|
|
295
|
+
finally {
|
|
296
|
+
refreshInFlight = null;
|
|
297
|
+
}
|
|
298
|
+
})();
|
|
299
|
+
return refreshInFlight;
|
|
300
|
+
}
|
|
264
301
|
export function stopWorkBuddy() {
|
|
265
302
|
log.info('Stopping WorkBuddy client...');
|
|
266
303
|
stopped = true;
|