@wu529778790/open-im 1.10.9-beta.3 → 1.10.9-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -49,6 +49,8 @@ loadUserPluginSettings();
49
49
  // 存储所有活跃的 SDKSession 对象,key 为 sessionId
50
50
  // 使用 Map 而不是 Set,因为我们需要通过 sessionId 获取 session
51
51
  const activeSessions = new Map();
52
+ // 记录每个 session 创建/恢复时的 workDir,防止跨 workDir 复用已固定 cwd 的子进程
53
+ const sessionWorkDirs = new Map();
52
54
  // 存储正在进行的流式迭代器,用于中断
53
55
  const activeStreams = new Set();
54
56
  // 空闲会话清理:跟踪最后使用时间,定期清除超时会话
@@ -92,34 +94,53 @@ const cleanupInterval = setInterval(() => {
92
94
  activeSessions.delete(id);
93
95
  }
94
96
  sessionLastUsed.delete(id);
97
+ sessionWorkDirs.delete(id);
95
98
  log.info(`Cleaned up idle session (unused ${Math.round((now - lastUsed) / 60000)}min): ${id}`);
96
99
  }
97
100
  }
98
101
  }, CLEANUP_INTERVAL_MS);
99
102
  cleanupInterval.unref(); // 不阻止进程退出
100
103
  /**
101
- * Serializes process.chdir() calls across concurrent users.
104
+ * 串行化进程级 process.chdir() —— 同一时刻仅一个 chdir 生效。
102
105
  *
103
- * process.chdir() is a process-wide global side effect — only one chdir can
104
- * be active at a time. The SDK's createSession/resumeSession do not accept a
105
- * `cwd` parameter, so we must chdir before calling them. This mutex ensures
106
- * concurrent requests don't race on the working directory.
106
+ * SDK V2 createSession/resumeSession 不接受 cwd 参数;且 send()/stream()
107
+ * 会以「调用时的 process.cwd()」派生 Claude Code 子进程。必须用互斥锁串行化
108
+ * 「整个 turn cwd 切换」,否则并发多用户会让工具跑错目录。
107
109
  *
108
- * **TODO:** Remove this mutex entirely once @anthropic-ai/claude-agent-sdk
109
- * supports a `cwd` option in createSession/resumeSession. Track upstream:
110
+ * **TODO:** SDK 支持 cwd 选项后移除此锁。upstream:
110
111
  * https://github.com/anthropics/claude-agent-sdk/issues
111
112
  */
112
113
  let chdirMutex = Promise.resolve();
113
114
  function withChdirMutex(fn) {
114
115
  const previous = chdirMutex;
115
- let resolve;
116
- chdirMutex = new Promise((r) => { resolve = r; });
117
- return previous.then(() => {
116
+ let release;
117
+ chdirMutex = new Promise((r) => { release = r; });
118
+ return previous.then(async () => {
118
119
  try {
119
- return fn();
120
+ return await fn();
120
121
  }
121
122
  finally {
122
- resolve();
123
+ release();
124
+ }
125
+ });
126
+ }
127
+ /**
128
+ * 在持有全局 chdir 互斥锁期间,把进程 cwd 切到 workDir 执行 fn,结束后恢复。
129
+ * 用于包裹 session.send()+stream(),确保子进程在正确 workDir 派生。
130
+ */
131
+ function runWithWorkDir(workDir, fn) {
132
+ return withChdirMutex(async () => {
133
+ const originalCwd = process.cwd();
134
+ if (workDir && workDir !== originalCwd) {
135
+ process.chdir(workDir);
136
+ }
137
+ try {
138
+ return await fn();
139
+ }
140
+ finally {
141
+ if (workDir && workDir !== originalCwd) {
142
+ process.chdir(originalCwd);
143
+ }
123
144
  }
124
145
  });
125
146
  }
@@ -201,18 +222,21 @@ async function getOrCreateSession(sessionId, workDir, model, permissionMode, onS
201
222
  process.chdir(workDir);
202
223
  }
203
224
  if (sessionId) {
204
- // 优先复用内存中已有的 SDKSession,避免每次都启动新进程
225
+ // 优先复用内存中已有的 SDKSession,避免每次都启动新进程。
226
+ // 仅当 workDir 与创建时一致才复用:否则子进程 cwd 已固定在旧目录,
227
+ // 需走 resume 重新派生(在当前 workDir 启动新子进程)。
205
228
  const existing = activeSessions.get(sessionId);
206
- if (existing) {
229
+ if (existing && sessionWorkDirs.get(sessionId) === workDir) {
207
230
  log.info(`Reusing existing in-memory session: ${sessionId}`);
208
231
  sessionLastUsed.set(sessionId, Date.now());
209
232
  return { session: existing, sessionId };
210
233
  }
211
- // 内存中没有,尝试通过 resume 恢复(会启动新 CLI 进程)
234
+ // 内存中没有(或 workDir 变了),尝试通过 resume 恢复(会启动新 CLI 进程)
212
235
  try {
213
236
  log.info(`Attempting to resume session: ${sessionId}`);
214
237
  session = unstable_v2_resumeSession(sessionId, sessionOptions);
215
238
  activeSessions.set(sessionId, session);
239
+ sessionWorkDirs.set(sessionId, workDir);
216
240
  sessionLastUsed.set(sessionId, Date.now());
217
241
  log.info(`Successfully resumed session: ${sessionId}`);
218
242
  return { session, sessionId };
@@ -228,6 +252,7 @@ async function getOrCreateSession(sessionId, workDir, model, permissionMode, onS
228
252
  // 暂时返回 undefined,稍后在 init 消息中获取
229
253
  const tempId = `pending-${++sessionSeq}`;
230
254
  activeSessions.set(tempId, session);
255
+ sessionWorkDirs.set(tempId, workDir);
231
256
  sessionLastUsed.set(tempId, Date.now());
232
257
  log.info(`Created new session (tempId: ${tempId})`);
233
258
  return { session, sessionId: tempId, wasReused: false };
@@ -267,6 +292,7 @@ export class ClaudeSDKAdapter {
267
292
  }
268
293
  activeSessions.clear();
269
294
  sessionLastUsed.clear();
295
+ sessionWorkDirs.clear();
270
296
  }
271
297
  /**
272
298
  * Remove a specific session from the in-memory cache and close it.
@@ -281,6 +307,7 @@ export class ClaudeSDKAdapter {
281
307
  catch { /* ignore */ }
282
308
  activeSessions.delete(sessionId);
283
309
  sessionLastUsed.delete(sessionId);
310
+ sessionWorkDirs.delete(sessionId);
284
311
  log.info(`Explicitly removed session: ${sessionId}`);
285
312
  }
286
313
  }
@@ -320,12 +347,16 @@ export class ClaudeSDKAdapter {
320
347
  }
321
348
  runningSessions.add(returnedId);
322
349
  trackedRunningId = returnedId;
323
- // 发送用户消息
324
- await session.send(prompt);
325
- // 获取响应流
326
- const stream = session.stream();
327
- currentStream = stream;
328
- activeStreams.add(stream);
350
+ // 在持有 chdir 锁期间完成 send() + stream() 获取:SDK V2 会以当前
351
+ // process.cwd() 派生 Claude Code 子进程,必须保证此刻 cwd 为 workDir。
352
+ // 锁串行化后,并发多用户的子进程不会在错误的目录派生。
353
+ const stream = await runWithWorkDir(workDir, async () => {
354
+ await session.send(prompt);
355
+ const s = session.stream();
356
+ currentStream = s;
357
+ activeStreams.add(s);
358
+ return s;
359
+ });
329
360
  let accumulated = '';
330
361
  let accumulatedThinking = '';
331
362
  const toolStats = {};
@@ -348,13 +379,19 @@ export class ClaudeSDKAdapter {
348
379
  // 更新 sessionId 映射
349
380
  // 清理 pending 临时 ID(actualSessionId 尚未赋值时用 pendingTempId)
350
381
  const idToClean = actualSessionId ?? pendingTempId;
382
+ const inheritedWorkDir = idToClean ? sessionWorkDirs.get(idToClean) : undefined;
351
383
  if (idToClean?.startsWith('pending-')) {
352
384
  activeSessions.delete(idToClean);
353
385
  }
354
386
  activeSessions.set(newSessionId, session);
387
+ if (inheritedWorkDir !== undefined) {
388
+ sessionWorkDirs.set(newSessionId, inheritedWorkDir);
389
+ }
355
390
  sessionLastUsed.set(newSessionId, Date.now());
356
- if (idToClean)
391
+ if (idToClean) {
357
392
  sessionLastUsed.delete(idToClean);
393
+ sessionWorkDirs.delete(idToClean);
394
+ }
358
395
  // 更新 runningSessions:移除旧 ID,添加新 ID
359
396
  if (idToClean)
360
397
  runningSessions.delete(idToClean);
@@ -3,6 +3,7 @@ import { ClaudeSDKAdapter, configureClaudeSdkSessionIdle } from './claude-sdk-ad
3
3
  import { CodexAdapter } from './codex-adapter.js';
4
4
  import { CodeBuddyAdapter } from './codebuddy-adapter.js';
5
5
  import { createLogger } from '../logger.js';
6
+ import { destroyAllLiveChildren } from '../shared/process-kill.js';
6
7
  const log = createLogger('Registry');
7
8
  const adapters = new Map();
8
9
  export function initAdapters(config) {
@@ -30,5 +31,7 @@ export function getAdapter(aiCommand) {
30
31
  }
31
32
  export function cleanupAdapters() {
32
33
  ClaudeSDKAdapter.destroy();
34
+ // 强制终止仍在运行的 CLI 子进程(Codex/CodeBuddy),避免僵尸 / 孤儿
35
+ destroyAllLiveChildren();
33
36
  adapters.clear();
34
37
  }
@@ -3,6 +3,7 @@ import { accessSync, constants } from 'node:fs';
3
3
  import { isAbsolute, join } from 'node:path';
4
4
  import { createLogger } from '../logger.js';
5
5
  import { processEnvForNonClaudeCliChild } from '../config/file-io.js';
6
+ import { killProcessTree, trackChild } from '../shared/process-kill.js';
6
7
  const log = createLogger('CodeBuddyCli');
7
8
  export function buildCodeBuddyArgs(prompt, sessionId, options) {
8
9
  const args = ['--print', '--output-format', 'stream-json'];
@@ -200,10 +201,14 @@ export function runCodeBuddy(cliPath, prompt, sessionId, workDir, callbacks, opt
200
201
  log.info(`Spawning CodeBuddy CLI: path=${normalizedCliPath}, cwd=${workDir}, session=${sessionId ?? 'new'}, args=${args.join(' ')}`);
201
202
  const child = spawn(spawnCmd, spawnArgs, {
202
203
  cwd: workDir,
204
+ // Unix 上放入独立进程组,使 abort/关停能用 process.kill(-pid) 杀掉整棵子树(含 MCP/shell 孙进程)
205
+ detached: process.platform !== 'win32',
203
206
  stdio: ['ignore', 'pipe', 'pipe'],
204
207
  env,
205
208
  windowsHide: process.platform === 'win32',
206
209
  });
210
+ trackChild(child);
211
+ let cliTimeoutHandle = null;
207
212
  let accumulated = '';
208
213
  let accumulatedThinking = '';
209
214
  let completed = false;
@@ -311,6 +316,10 @@ export function runCodeBuddy(cliPath, prompt, sessionId, workDir, callbacks, opt
311
316
  }
312
317
  });
313
318
  child.on('close', (code) => {
319
+ if (cliTimeoutHandle) {
320
+ clearTimeout(cliTimeoutHandle);
321
+ cliTimeoutHandle = null;
322
+ }
314
323
  if (completed)
315
324
  return;
316
325
  if (stdoutState.buffer.trim()) {
@@ -343,16 +352,36 @@ export function runCodeBuddy(cliPath, prompt, sessionId, workDir, callbacks, opt
343
352
  });
344
353
  });
345
354
  child.on('error', (err) => {
355
+ if (cliTimeoutHandle) {
356
+ clearTimeout(cliTimeoutHandle);
357
+ cliTimeoutHandle = null;
358
+ }
346
359
  if (completed)
347
360
  return;
348
361
  completed = true;
349
362
  callbacks.onError(`Failed to start CodeBuddy CLI: ${err.message}`);
350
363
  });
364
+ // 墙钟超时:防止 CLI 挂死永久占用用户队列槽
365
+ const cliTimeoutMs = Number(process.env.OPEN_IM_CLI_TIMEOUT_MS) || 30 * 60 * 1000;
366
+ cliTimeoutHandle = setTimeout(() => {
367
+ if (completed)
368
+ return;
369
+ log.warn(`CodeBuddy CLI 超过 ${cliTimeoutMs}ms,强制终止 (pid=${child.pid})`);
370
+ completed = true;
371
+ killProcessTree(child);
372
+ callbacks.onError(`CodeBuddy CLI 运行超时(${Math.round(cliTimeoutMs / 1000)}s),已终止。请重试。`);
373
+ }, cliTimeoutMs);
374
+ cliTimeoutHandle.unref();
351
375
  return {
352
376
  abort: () => {
377
+ if (completed)
378
+ return;
353
379
  completed = true;
354
- if (!child.killed)
355
- child.kill('SIGTERM');
380
+ if (cliTimeoutHandle) {
381
+ clearTimeout(cliTimeoutHandle);
382
+ cliTimeoutHandle = null;
383
+ }
384
+ killProcessTree(child);
356
385
  },
357
386
  };
358
387
  }
@@ -7,6 +7,7 @@ import { dirname, join } from 'node:path';
7
7
  import { createInterface } from 'node:readline';
8
8
  import { createLogger } from '../logger.js';
9
9
  import { processEnvForNonClaudeCliChild } from '../config/file-io.js';
10
+ import { killProcessTree, trackChild } from '../shared/process-kill.js';
10
11
  const log = createLogger('CodexCli');
11
12
  const windowsCodexLaunchCache = new Map();
12
13
  const SUPPORTED_IMAGE_EXTENSIONS = new Set([
@@ -201,10 +202,13 @@ export function runCodex(cliPath, prompt, sessionId, workDir, callbacks, options
201
202
  : args;
202
203
  const child = spawn(spawnCmd, spawnArgs, {
203
204
  cwd: workDir,
205
+ // Unix 上放入独立进程组,使 abort/关停能用 process.kill(-pid) 杀掉整棵子树(含 MCP/shell 孙进程)
206
+ detached: process.platform !== 'win32',
204
207
  stdio: ['pipe', 'pipe', 'pipe'],
205
208
  env,
206
209
  windowsHide: process.platform === 'win32',
207
210
  });
211
+ trackChild(child);
208
212
  child.stdin?.write(prompt);
209
213
  child.stdin?.end();
210
214
  let accumulated = '';
@@ -327,7 +331,12 @@ export function runCodex(cliPath, prompt, sessionId, workDir, callbacks, options
327
331
  let exitCode = null;
328
332
  let rlClosed = false;
329
333
  let childClosed = false;
334
+ let cliTimeoutHandle = null;
330
335
  const finalize = () => {
336
+ if (cliTimeoutHandle) {
337
+ clearTimeout(cliTimeoutHandle);
338
+ cliTimeoutHandle = null;
339
+ }
331
340
  if (!rlClosed || !childClosed)
332
341
  return;
333
342
  if (completed)
@@ -387,12 +396,29 @@ export function runCodex(cliPath, prompt, sessionId, workDir, callbacks, options
387
396
  childClosed = true;
388
397
  finalize();
389
398
  });
399
+ // 墙钟超时:防止 CLI 挂死(网络卡住、工具死循环等)永久占用用户队列槽
400
+ const cliTimeoutMs = Number(process.env.OPEN_IM_CLI_TIMEOUT_MS) || 30 * 60 * 1000;
401
+ cliTimeoutHandle = setTimeout(() => {
402
+ if (completed)
403
+ return;
404
+ log.warn(`Codex CLI 超过 ${cliTimeoutMs}ms,强制终止 (pid=${child.pid})`);
405
+ completed = true;
406
+ rl.close();
407
+ killProcessTree(child);
408
+ callbacks.onError(`Codex CLI 运行超时(${Math.round(cliTimeoutMs / 1000)}s),已终止。请重试。`);
409
+ }, cliTimeoutMs);
410
+ cliTimeoutHandle.unref();
390
411
  return {
391
412
  abort: () => {
413
+ if (completed)
414
+ return;
392
415
  completed = true;
416
+ if (cliTimeoutHandle) {
417
+ clearTimeout(cliTimeoutHandle);
418
+ cliTimeoutHandle = null;
419
+ }
393
420
  rl.close();
394
- if (!child.killed)
395
- child.kill('SIGTERM');
421
+ killProcessTree(child);
396
422
  },
397
423
  };
398
424
  }
@@ -1,5 +1,6 @@
1
1
  import { DWClient, TOPIC_ROBOT } from 'dingtalk-stream';
2
2
  import { createLogger } from '../logger.js';
3
+ import { jitteredDelay } from '../shared/reconnect.js';
3
4
  import { registerSessionWebhook as registerWebhook, clearWebhooks, sendText as webhookSendText, sendMarkdown as webhookSendMarkdown, downloadRobotMessageFile as webhookDownloadFile, sendProactiveText as webhookSendProactive, } from './webhook.js';
4
5
  import { prepareStreamingCard as cardPrepare, updateStreamingCard as cardUpdate, finishStreamingCard as cardFinish, createAndDeliverCard as cardCreateAndDeliver, updateCardInstance as cardUpdateInstance, sendRobotInteractiveCard as cardSendInteractive, updateRobotInteractiveCard as cardUpdateInteractive, clearUnionIdCache, } from './streaming-card.js';
5
6
  const log = createLogger('DingTalk');
@@ -111,7 +112,7 @@ export async function initDingTalk(cfg, eventHandler) {
111
112
  lastErr = err;
112
113
  if (is429(err) && attempt < maxTries) {
113
114
  log.warn(`DingTalk gateway 429 (attempt ${attempt}/${maxTries}), retrying in ${retryDelayMs / 1000}s...`);
114
- await new Promise((r) => setTimeout(r, retryDelayMs));
115
+ await new Promise((r) => setTimeout(r, jitteredDelay(retryDelayMs)));
115
116
  continue;
116
117
  }
117
118
  throw new Error(formatDingTalkInitError(err));
@@ -163,7 +163,7 @@ export function setupDingTalkHandlers(config, sessionManager) {
163
163
  sessionManager,
164
164
  sender: { sendTextReply, sendDirectorySelection },
165
165
  });
166
- const { accessControl, requestQueue, runningTasks, commandHandler } = ctx;
166
+ const { accessControl, requestQueue, runningTasks } = ctx;
167
167
  // DingTalk-specific sender callbacks for the factory
168
168
  const dingtalkSender = {
169
169
  sendThinkingMessage: async (chatId, replyToMessageId, toolId) => {
package/dist/index.js CHANGED
@@ -27,11 +27,13 @@ import { sendTextReply as sendWorkBuddyTextReply } from "./workbuddy/message-sen
27
27
  import { initAdapters, cleanupAdapters } from "./adapters/registry.js";
28
28
  import { SessionManager } from "./session/session-manager.js";
29
29
  import { loadActiveChats, getActiveChatId, flushActiveChats, } from "./shared/active-chats.js";
30
+ import { destroyAllLiveChildren } from "./shared/process-kill.js";
30
31
  import { initLogger, createLogger, closeLogger, emitStructuredEvent, shutdownLoggerTelemetry, } from "./logger.js";
31
32
  import { APP_HOME, SHUTDOWN_PORT } from "./constants.js";
32
33
  import { createRequire } from "node:module";
33
34
  import { escapePathForMarkdown } from "./shared/utils.js";
34
35
  import { applyOpenImGitCoauthorToProcessEnv } from "./shared/git-coauthor.js";
36
+ import { emitInterruptedTerminals } from "./shared/task-cleanup.js";
35
37
  const require = createRequire(import.meta.url);
36
38
  const { version: APP_VERSION } = require("../package.json");
37
39
  const log = createLogger("Main");
@@ -262,6 +264,15 @@ export async function main() {
262
264
  const uptimeSec = Math.floor((Date.now() - startedAt) / 1000);
263
265
  const m = Math.floor(uptimeSec / 60);
264
266
  const shutdownMsg = buildShutdownMessage(m);
267
+ // 在任何异步等待之前,先为仍在运行的任务补发终态遥测事件。
268
+ // 这样即使后续 await(通知发送、platform.stop)期间进程被第二次信号杀死,
269
+ // ai.task.start 也有对应的 interrupted 终态,避免遥测里的 miss。
270
+ for (const platform of successfulPlatforms) {
271
+ const handle = activeHandles.get(platform);
272
+ if (handle?.runningTasks && handle.runningTasks.size > 0) {
273
+ emitInterruptedTerminals(handle.runningTasks);
274
+ }
275
+ }
265
276
  // Send notification only to successfully initialized platforms
266
277
  for (const platform of successfulPlatforms) {
267
278
  await sendLifecycleNotification(platform, shutdownMsg).catch((err) => {
@@ -299,6 +310,16 @@ export async function main() {
299
310
  };
300
311
  process.on("SIGINT", () => shutdown().catch(() => process.exit(1)));
301
312
  process.on("SIGTERM", () => shutdown().catch(() => process.exit(1)));
313
+ // 兜底:进程退出(含异常路径,如未捕获异常 / SIGKILL)时强制清理 CLI 子进程,
314
+ // 避免僵尸 / 孤儿。正常 shutdown 已在 cleanupAdapters() 里清理过。
315
+ process.on("exit", () => {
316
+ try {
317
+ destroyAllLiveChildren();
318
+ }
319
+ catch {
320
+ /* 退出期间 best effort */
321
+ }
322
+ });
302
323
  // Global error handlers to prevent unhandled crashes
303
324
  process.on("unhandledRejection", (reason) => {
304
325
  log.error("Unhandled Promise rejection (this indicates a bug — the affected request may hang without a response):", reason);
@@ -312,6 +333,18 @@ export async function main() {
312
333
  return;
313
334
  }
314
335
  log.error("Uncaught exception (process will exit):", err);
336
+ // 进程即将因未捕获异常退出:补发在途任务的终态,避免 miss
337
+ for (const platform of successfulPlatforms) {
338
+ const handle = activeHandles.get(platform);
339
+ if (handle?.runningTasks && handle.runningTasks.size > 0) {
340
+ try {
341
+ emitInterruptedTerminals(handle.runningTasks);
342
+ }
343
+ catch {
344
+ /* best effort:不能因遥测失败再次抛出 */
345
+ }
346
+ }
347
+ }
315
348
  void shutdownLoggerTelemetry()
316
349
  .then(() => closeLogger())
317
350
  .finally(() => process.exit(1));
package/dist/qq/client.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import WebSocket from "ws";
2
2
  import { createLogger } from "../logger.js";
3
+ import { jitteredDelay, SLOW_PROBE_MS } from "../shared/reconnect.js";
3
4
  const log = createLogger("QQ");
4
5
  const TOKEN_URL = "https://bots.qq.com/app/getAppAccessToken";
5
6
  const API_BASE = "https://api.sgroup.qq.com";
@@ -271,7 +272,12 @@ async function connectWebSocket(config, handler) {
271
272
  sessionId = null;
272
273
  seq = null;
273
274
  }
274
- const delay = RECONNECT_DELAYS_MS[Math.min(reconnectAttempt, RECONNECT_DELAYS_MS.length - 1)];
275
+ const isFatalClose = code === 4004 || code === 4006 || code === 4007;
276
+ const baseDelay = RECONNECT_DELAYS_MS[Math.min(reconnectAttempt, RECONNECT_DELAYS_MS.length - 1)];
277
+ const delay = isFatalClose ? jitteredDelay(SLOW_PROBE_MS) : jitteredDelay(baseDelay);
278
+ if (isFatalClose) {
279
+ log.warn(`QQ 致命关闭码 ${code}(鉴权失败),转慢探测(每 ${Math.round(SLOW_PROBE_MS / 1000)}s 一次)`);
280
+ }
275
281
  reconnectAttempt += 1;
276
282
  reconnectTimer = setTimeout(() => {
277
283
  if (currentConfig && currentHandler) {
@@ -80,17 +80,18 @@ export class RequestQueue {
80
80
  finally {
81
81
  this.runningControllers.delete(key);
82
82
  const q = this.queues.get(key);
83
- if (!q)
84
- return;
85
- const next = q.tasks.shift();
86
- if (next) {
87
- setImmediate(() => this.run(key, next.prompt, next.execute).catch((err) => {
88
- log.error(`Unhandled error in next task execution for ${key}:`, err);
89
- }));
90
- }
83
+ if (!q) { /* queue already cleared */ }
91
84
  else {
92
- q.running = false;
93
- this.queues.delete(key);
85
+ const next = q.tasks.shift();
86
+ if (next) {
87
+ setImmediate(() => this.run(key, next.prompt, next.execute).catch((err) => {
88
+ log.error(`Unhandled error in next task execution for ${key}:`, err);
89
+ }));
90
+ }
91
+ else {
92
+ q.running = false;
93
+ this.queues.delete(key);
94
+ }
94
95
  }
95
96
  }
96
97
  }
@@ -42,5 +42,19 @@ export interface TaskRunState {
42
42
  startedAt: number;
43
43
  /** AI 工具标识,用于动态显示工具名称。 */
44
44
  toolId: string;
45
+ /**
46
+ * 进程退出(shutdown / 崩溃)时,用于为仍在运行的任务补发一条终态遥测事件,
47
+ * 避免 `ai.task.start` 没有对应的 complete/error(遥测里的 `miss`)。
48
+ * 已哈希(与 ai.task.* emit 处一致),不含原始 userId/msgId。
49
+ */
50
+ taskKey: string;
51
+ platform: string;
52
+ /** 已哈希的 userId,与 ai.task.start/complete/error 中的 userKey 一致。 */
53
+ userKey: string;
45
54
  }
55
+ /**
56
+ * 将 AI/CLI 错误文本归类为一个稳定的 errorType 字符串,用于遥测聚合。
57
+ * 分类基于线上真实错误签名(见 logs/r2-events),新增分支时同步更新测试。
58
+ */
59
+ export declare function classifyErrorType(error: string): string;
46
60
  export declare function runAITask(deps: TaskDeps, ctx: TaskContext, prompt: string, toolAdapter: ToolAdapter, platformAdapter: TaskAdapter): Promise<void>;
@@ -10,25 +10,65 @@ const log = createLogger('AITask');
10
10
  function isUsageLimitError(error) {
11
11
  return /usage limit/i.test(error) || /try again at\s+\d{1,2}:\d{2}\s*(AM|PM)/i.test(error);
12
12
  }
13
- function classifyErrorType(error) {
13
+ /**
14
+ * 将 AI/CLI 错误文本归类为一个稳定的 errorType 字符串,用于遥测聚合。
15
+ * 分类基于线上真实错误签名(见 logs/r2-events),新增分支时同步更新测试。
16
+ */
17
+ export function classifyErrorType(error) {
14
18
  const s = error.toLowerCase();
15
19
  if (s.includes('aborted'))
16
20
  return 'aborted';
21
+ // 空输出:流正常结束但无内容(claude-sdk-adapter 的兜底消息)
22
+ if (s.includes('无输出') || s.includes('响应异常结束') || s.includes('empty output')) {
23
+ return 'empty_output';
24
+ }
17
25
  if (isUsageLimitError(error) || s.includes('rate limit') || s.includes('quota'))
18
26
  return 'limit';
19
- if (s.includes('invalid api key') || s.includes('unauthorized') || s.includes('401'))
27
+ // 鉴权:凭据无效或需要登录(含中文「登录」/「login」)
28
+ if (s.includes('invalid api key') ||
29
+ s.includes('unauthorized') ||
30
+ s.includes('401') ||
31
+ s.includes('need to log in') ||
32
+ s.includes('need to login') ||
33
+ s.includes('需要登录') ||
34
+ s.includes('需要先登录') ||
35
+ s.includes('log in required') ||
36
+ s.includes('login required')) {
20
37
  return 'auth';
38
+ }
21
39
  if (s.includes('model') && (s.includes('not support') || s.includes('not found') || s.includes('invalid'))) {
22
40
  return 'model';
23
41
  }
24
- if (s.includes('process exited') || s.includes('exit code'))
42
+ // 安装/配置缺失:二进制或可执行文件缺失、环境变量缺失、token 未配置
43
+ if (s.includes('native cli binary') ||
44
+ s.includes('executable not found') ||
45
+ (s.includes('binary') && s.includes('not found')) ||
46
+ s.includes('missing environment variable') ||
47
+ s.includes('token data is not available')) {
48
+ return 'setup';
49
+ }
50
+ // 会话失效:找不到会话/对话,或会话过期/损坏
51
+ if (s.includes('no conversation found') ||
52
+ s.includes('session not found') ||
53
+ (s.includes('session') && (s.includes('expired') || s.includes('corrupt')))) {
54
+ return 'session';
55
+ }
56
+ // 进程退出或被信号终止
57
+ if (s.includes('process exited') ||
58
+ s.includes('exit code') ||
59
+ s.includes('exited with code') ||
60
+ s.includes('terminated by signal') ||
61
+ s.includes('sigkill')) {
25
62
  return 'process';
63
+ }
64
+ // 网络:超时/连接重置/DNS/网络请求失败(含中文「网络」)
26
65
  if (s.includes('timeout') ||
27
66
  s.includes('etimedout') ||
28
67
  s.includes('econnreset') ||
29
68
  s.includes('enotfound') ||
30
69
  s.includes('eai_again') ||
31
- s.includes('network')) {
70
+ s.includes('network') ||
71
+ s.includes('网络')) {
32
72
  return 'network';
33
73
  }
34
74
  return 'unknown';
@@ -115,7 +155,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
115
155
  log.info(`[AITask] Starting: userId=${ctx.userId}, initialSessionId=${currentSessionId ?? 'new'}, prompt="${prompt.slice(0, 50)}..."`);
116
156
  emitStructuredEvent('AITask', 'ai.task.start', {
117
157
  platform: ctx.platform,
118
- taskKey: ctx.taskKey,
158
+ taskKey: hashUserId(ctx.taskKey),
119
159
  userKey: hashUserId(ctx.userId),
120
160
  toolId: aiCommand,
121
161
  });
@@ -179,7 +219,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
179
219
  return;
180
220
  emitStructuredEvent('AITask', 'ai.task.complete', {
181
221
  platform: ctx.platform,
182
- taskKey: ctx.taskKey,
222
+ taskKey: hashUserId(ctx.taskKey),
183
223
  userKey: hashUserId(ctx.userId),
184
224
  toolId: aiCommand,
185
225
  durationMs: result.durationMs,
@@ -242,7 +282,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
242
282
  log.error(`Task error for user ${ctx.userId}: ${error}`);
243
283
  emitStructuredEvent('AITask', 'ai.task.error', {
244
284
  platform: ctx.platform,
245
- taskKey: ctx.taskKey,
285
+ taskKey: hashUserId(ctx.taskKey),
246
286
  userKey: hashUserId(ctx.userId),
247
287
  toolId: aiCommand,
248
288
  durationMs: Date.now() - taskState.startedAt,
@@ -290,13 +330,18 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
290
330
  if (!settled) {
291
331
  emitStructuredEvent('AITask', 'ai.task.error', {
292
332
  platform: ctx.platform,
293
- taskKey: ctx.taskKey,
333
+ taskKey: hashUserId(ctx.taskKey),
294
334
  userKey: hashUserId(ctx.userId),
295
335
  toolId: aiCommand,
296
336
  durationMs: Date.now() - taskState.startedAt,
297
337
  errorSnippet: 'aborted',
298
338
  errorType: 'aborted',
299
339
  });
340
+ // 用户取消(/new、/resume、队列超时、stale 清理):把「思考中…」占位卡片编辑为终态,
341
+ // 避免卡片卡在转圈。停按钮路径会先 settle() 再 abort,settled=true 时此处跳过,不会双发。
342
+ void platformAdapter.sendError('⏹️ 已取消').catch(() => {
343
+ /* 占位卡片可能已被删除或流过期,编辑失败可忽略 */
344
+ });
300
345
  }
301
346
  activeHandle?.abort();
302
347
  cleanup();
@@ -307,6 +352,9 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
307
352
  settle,
308
353
  startedAt: Date.now(),
309
354
  toolId: aiCommand,
355
+ taskKey: hashUserId(ctx.taskKey),
356
+ platform: ctx.platform,
357
+ userKey: hashUserId(ctx.userId),
310
358
  };
311
359
  try {
312
360
  startRun();
@@ -318,7 +366,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
318
366
  log.error(`[AITask] Synchronous error in startRun: ${err}`);
319
367
  emitStructuredEvent('AITask', 'ai.task.error', {
320
368
  platform: ctx.platform,
321
- taskKey: ctx.taskKey,
369
+ taskKey: hashUserId(ctx.taskKey),
322
370
  userKey: hashUserId(ctx.userId),
323
371
  toolId: aiCommand,
324
372
  durationMs: 0,
@@ -0,0 +1,24 @@
1
+ /**
2
+ * 跨平台子进程树终止:用于 Codex / CodeBuddy CLI 子进程。
3
+ *
4
+ * 在 Unix 上子进程需以 `detached: true` 启动,使其成为独立进程组的组长;
5
+ * 这样 `process.kill(-pid)` 才能同时命中 CLI 及其派生的孙进程(MCP server、
6
+ * shell 工具调用等)。Windows 不支持负 PID 信号,改用 `taskkill /T /F`
7
+ * 按 PID 递归终止整棵进程树。
8
+ *
9
+ * 进程退出(shutdown / 崩溃)时,`destroyAllLiveChildren()` 会强制清理所有
10
+ * 仍在运行的被追踪子进程,避免僵尸 / 孤儿进程。
11
+ */
12
+ import { type ChildProcess } from 'node:child_process';
13
+ /** 追踪一个已 spawn 的子进程,使其在关停时能被强制终止。 */
14
+ export declare function trackChild(child: ChildProcess): ChildProcess;
15
+ /**
16
+ * 终止子进程及其所有后代。默认优雅:先 SIGTERM,`graceMs` 后升级为 SIGKILL。
17
+ * 传 `force: true` 则立即 SIGKILL(用于关停)。
18
+ */
19
+ export declare function killProcessTree(child: ChildProcess, opts?: {
20
+ force?: boolean;
21
+ graceMs?: number;
22
+ }): void;
23
+ /** 强制终止所有仍在运行的被追踪子进程。关停 / 进程退出时调用。 */
24
+ export declare function destroyAllLiveChildren(): void;
@@ -0,0 +1,79 @@
1
+ /**
2
+ * 跨平台子进程树终止:用于 Codex / CodeBuddy CLI 子进程。
3
+ *
4
+ * 在 Unix 上子进程需以 `detached: true` 启动,使其成为独立进程组的组长;
5
+ * 这样 `process.kill(-pid)` 才能同时命中 CLI 及其派生的孙进程(MCP server、
6
+ * shell 工具调用等)。Windows 不支持负 PID 信号,改用 `taskkill /T /F`
7
+ * 按 PID 递归终止整棵进程树。
8
+ *
9
+ * 进程退出(shutdown / 崩溃)时,`destroyAllLiveChildren()` 会强制清理所有
10
+ * 仍在运行的被追踪子进程,避免僵尸 / 孤儿进程。
11
+ */
12
+ import { spawn } from 'node:child_process';
13
+ import { createLogger } from '../logger.js';
14
+ const log = createLogger('ProcessKill');
15
+ /** 当前被追踪的存活子进程,供关停时强制清理。 */
16
+ const liveChildren = new Set();
17
+ /** 追踪一个已 spawn 的子进程,使其在关停时能被强制终止。 */
18
+ export function trackChild(child) {
19
+ liveChildren.add(child);
20
+ child.once('close', () => liveChildren.delete(child));
21
+ return child;
22
+ }
23
+ /** 向整棵进程树发送信号:Unix 用负 PID(整组),Windows 用 taskkill /T /F。 */
24
+ function signalTree(child, signal) {
25
+ const pid = child.pid;
26
+ if (pid == null)
27
+ return;
28
+ try {
29
+ if (process.platform === 'win32') {
30
+ // Windows 没有「优雅 SIGTERM」语义,统一 /T(递归) /F(强制)
31
+ const tk = spawn('taskkill', ['/pid', String(pid), '/T', '/F'], {
32
+ windowsHide: true,
33
+ stdio: 'ignore',
34
+ });
35
+ tk.unref();
36
+ }
37
+ else {
38
+ process.kill(-pid, signal);
39
+ }
40
+ }
41
+ catch (err) {
42
+ // ESRCH(已退出)或未 detached(无独立进程组)——退化为直接信号
43
+ try {
44
+ child.kill(signal);
45
+ }
46
+ catch {
47
+ /* 进程已退出,忽略 */
48
+ }
49
+ log.debug(`signalTree(${signal}) for pid=${pid} 退化为直杀: ${err.message}`);
50
+ }
51
+ }
52
+ /**
53
+ * 终止子进程及其所有后代。默认优雅:先 SIGTERM,`graceMs` 后升级为 SIGKILL。
54
+ * 传 `force: true` 则立即 SIGKILL(用于关停)。
55
+ */
56
+ export function killProcessTree(child, opts = {}) {
57
+ const { force = false, graceMs = 5000 } = opts;
58
+ if (child.pid == null)
59
+ return;
60
+ if (force) {
61
+ signalTree(child, 'SIGKILL');
62
+ return;
63
+ }
64
+ signalTree(child, 'SIGTERM');
65
+ const escalate = setTimeout(() => signalTree(child, 'SIGKILL'), graceMs);
66
+ escalate.unref();
67
+ }
68
+ /** 强制终止所有仍在运行的被追踪子进程。关停 / 进程退出时调用。 */
69
+ export function destroyAllLiveChildren() {
70
+ for (const child of [...liveChildren]) {
71
+ try {
72
+ signalTree(child, 'SIGKILL');
73
+ }
74
+ catch {
75
+ /* best effort */
76
+ }
77
+ }
78
+ liveChildren.clear();
79
+ }
@@ -0,0 +1,28 @@
1
+ /**
2
+ * 重连韧性共享工具:抖动(jitter)+ 致命错误慢探测。
3
+ *
4
+ * 各平台长连接断开后各自重连,原本退避固定无抖动 → 车队重启会锁步重连风暴;
5
+ * 且致命(鉴权)错误也紧密/无限重试, hammer 网关。
6
+ *
7
+ * 策略(已与产品确认):
8
+ * - 所有退避加 ±30% 抖动,避免锁步。
9
+ * - 鉴权类致命错误转为**慢探测**(每 5 分钟一次 + 显眼告警):
10
+ * 既不紧密 hammer,也不永久断连(尊重 WeWork「避免永久断连」意图)。
11
+ */
12
+ /** 慢探测间隔:致命错误后每 5 分钟探测一次。 */
13
+ export declare const SLOW_PROBE_MS: number;
14
+ /**
15
+ * 在 baseMs 上叠加 ±frac 的抖动,避免车队重启锁步重连。
16
+ * jitteredDelay(1000, 0.3) ∈ [700, 1300]。
17
+ */
18
+ export declare function jitteredDelay(baseMs: number, frac?: number): number;
19
+ /**
20
+ * 判断是否为「致命/不可重试」错误(凭据无效、鉴权失败)。
21
+ * 这类错误紧密重试无意义,应转慢探测。
22
+ */
23
+ export declare function isFatalReconnectError(err: unknown): boolean;
24
+ /**
25
+ * 计算重连延迟:致命慢探测模式下用 SLOW_PROBE_MS,否则用 jittered(base)。
26
+ * `fatal` 由调用方维护(致命时置 true,成功连上后置 false)。
27
+ */
28
+ export declare function reconnectDelay(baseMs: number, fatal: boolean): number;
@@ -0,0 +1,56 @@
1
+ /**
2
+ * 重连韧性共享工具:抖动(jitter)+ 致命错误慢探测。
3
+ *
4
+ * 各平台长连接断开后各自重连,原本退避固定无抖动 → 车队重启会锁步重连风暴;
5
+ * 且致命(鉴权)错误也紧密/无限重试, hammer 网关。
6
+ *
7
+ * 策略(已与产品确认):
8
+ * - 所有退避加 ±30% 抖动,避免锁步。
9
+ * - 鉴权类致命错误转为**慢探测**(每 5 分钟一次 + 显眼告警):
10
+ * 既不紧密 hammer,也不永久断连(尊重 WeWork「避免永久断连」意图)。
11
+ */
12
+ import { createLogger } from '../logger.js';
13
+ const log = createLogger('Reconnect');
14
+ /** 慢探测间隔:致命错误后每 5 分钟探测一次。 */
15
+ export const SLOW_PROBE_MS = 5 * 60_000;
16
+ /**
17
+ * 在 baseMs 上叠加 ±frac 的抖动,避免车队重启锁步重连。
18
+ * jitteredDelay(1000, 0.3) ∈ [700, 1300]。
19
+ */
20
+ export function jitteredDelay(baseMs, frac = 0.3) {
21
+ if (baseMs <= 0)
22
+ return 0;
23
+ const jitter = baseMs * frac * (Math.random() * 2 - 1);
24
+ return Math.max(0, Math.round(baseMs + jitter));
25
+ }
26
+ /**
27
+ * 判断是否为「致命/不可重试」错误(凭据无效、鉴权失败)。
28
+ * 这类错误紧密重试无意义,应转慢探测。
29
+ */
30
+ export function isFatalReconnectError(err) {
31
+ const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
32
+ if (msg.includes('401') || msg.includes('403'))
33
+ return true;
34
+ if (msg.includes('unauthorized') || msg.includes('forbidden'))
35
+ return true;
36
+ if (msg.includes('invalid signature'))
37
+ return true;
38
+ if (msg.includes('not subscribed'))
39
+ return true;
40
+ if (msg.includes('请重新登录') || msg.includes('需要重新登录') || msg.includes('token 已过期'))
41
+ return true;
42
+ if (msg.includes('invalid') && /token|secret|credential|appid|app_id|corp/.test(msg))
43
+ return true;
44
+ return false;
45
+ }
46
+ /**
47
+ * 计算重连延迟:致命慢探测模式下用 SLOW_PROBE_MS,否则用 jittered(base)。
48
+ * `fatal` 由调用方维护(致命时置 true,成功连上后置 false)。
49
+ */
50
+ export function reconnectDelay(baseMs, fatal) {
51
+ if (fatal) {
52
+ log.warn(`致命错误,转慢探测(${Math.round(SLOW_PROBE_MS / 1000)}s 一次)`);
53
+ return jitteredDelay(SLOW_PROBE_MS);
54
+ }
55
+ return jitteredDelay(baseMs);
56
+ }
@@ -6,3 +6,19 @@
6
6
  */
7
7
  import type { TaskRunState } from './ai-task.js';
8
8
  export declare function startTaskCleanup(runningTasks: Map<string, TaskRunState>): () => void;
9
+ /**
10
+ * 在进程退出(优雅关闭 / 崩溃)路径上,为仍在运行的任务补发一条终态遥测事件。
11
+ *
12
+ * `runningTasks` 中存在的任务代表「已发出 ai.task.start 但尚未走到 complete/error」
13
+ * 的在途请求。正常流程会在 handle-ai-request 的 extraCleanup 里删除已结算任务,
14
+ * 因此进入这里时剩下的就是真正被进程退出打断的任务。
15
+ *
16
+ * 与用户主动 `/new`、队列超时触发的 `aborted` 区分,统一用 `interrupted` 标记。
17
+ * 补发后立即调用 state.settle() 置 settled=true:优雅关闭路径随后仍会调用
18
+ * handle.abort()(释放底层资源),但因已 settled,abort 不会再补发一条 aborted,
19
+ * 避免对同一任务重复计数。
20
+ *
21
+ * 该函数必须同步执行,且应在遥测刷盘(shutdownLoggerTelemetry)之前调用,
22
+ * 以保证补发的事件能进入上传队列。
23
+ */
24
+ export declare function emitInterruptedTerminals(runningTasks: Map<string, TaskRunState>): void;
@@ -4,7 +4,7 @@
4
4
  * Tasks older than 30 minutes are aborted and removed from the running-tasks
5
5
  * map so they never accumulate indefinitely.
6
6
  */
7
- import { createLogger } from '../logger.js';
7
+ import { createLogger, emitStructuredEvent } from '../logger.js';
8
8
  const log = createLogger('TaskCleanup');
9
9
  const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
10
10
  const STALE_THRESHOLD_MS = 30 * 60 * 1000; // 30 minutes
@@ -28,3 +28,36 @@ export function startTaskCleanup(runningTasks) {
28
28
  timer.unref();
29
29
  return () => clearInterval(timer);
30
30
  }
31
+ /**
32
+ * 在进程退出(优雅关闭 / 崩溃)路径上,为仍在运行的任务补发一条终态遥测事件。
33
+ *
34
+ * `runningTasks` 中存在的任务代表「已发出 ai.task.start 但尚未走到 complete/error」
35
+ * 的在途请求。正常流程会在 handle-ai-request 的 extraCleanup 里删除已结算任务,
36
+ * 因此进入这里时剩下的就是真正被进程退出打断的任务。
37
+ *
38
+ * 与用户主动 `/new`、队列超时触发的 `aborted` 区分,统一用 `interrupted` 标记。
39
+ * 补发后立即调用 state.settle() 置 settled=true:优雅关闭路径随后仍会调用
40
+ * handle.abort()(释放底层资源),但因已 settled,abort 不会再补发一条 aborted,
41
+ * 避免对同一任务重复计数。
42
+ *
43
+ * 该函数必须同步执行,且应在遥测刷盘(shutdownLoggerTelemetry)之前调用,
44
+ * 以保证补发的事件能进入上传队列。
45
+ */
46
+ export function emitInterruptedTerminals(runningTasks) {
47
+ if (runningTasks.size === 0)
48
+ return;
49
+ const now = Date.now();
50
+ for (const state of runningTasks.values()) {
51
+ emitStructuredEvent('AITask', 'ai.task.error', {
52
+ platform: state.platform,
53
+ taskKey: state.taskKey,
54
+ userKey: state.userKey,
55
+ toolId: state.toolId,
56
+ durationMs: now - state.startedAt,
57
+ errorSnippet: 'interrupted',
58
+ errorType: 'interrupted',
59
+ });
60
+ // 标记已结算,使随后 shutdown 的 abort() 跳过重复的 aborted 事件
61
+ state.settle();
62
+ }
63
+ }
@@ -1,5 +1,6 @@
1
1
  import { Telegraf } from "telegraf";
2
2
  import { createLogger } from "../logger.js";
3
+ import { isFatalReconnectError, jitteredDelay } from "../shared/reconnect.js";
3
4
  const log = createLogger("Telegram");
4
5
  let bot;
5
6
  let botUsername;
@@ -32,8 +33,13 @@ export async function initTelegram(config, setupHandlers) {
32
33
  catch {
33
34
  /* ignore */
34
35
  }
36
+ // 致命错误(token 无效等):不再重试,避免烧满 10 次
37
+ if (isFatalReconnectError(err)) {
38
+ log.error("Telegram 致命错误,停止重连(请检查 bot token):", err);
39
+ return;
40
+ }
35
41
  const maxAttempts = 10;
36
- const delayMs = Math.min(5000 * attempt, 60000);
42
+ const delayMs = jitteredDelay(Math.min(5000 * attempt, 60000));
37
43
  if (attempt < maxAttempts) {
38
44
  log.info(`Telegram reconnect in ${Math.round(delayMs / 1000)}s (attempt ${attempt}/${maxAttempts})`);
39
45
  await new Promise((r) => setTimeout(r, delayMs));
@@ -190,7 +190,7 @@ export async function shutdownTelemetryUpload() {
190
190
  uploadEnabled = false;
191
191
  endpoint = undefined;
192
192
  bearer = undefined;
193
- let lines = pending;
193
+ const lines = pending;
194
194
  while (lines.length > 0) {
195
195
  const batch = lines.splice(0, BATCH_MAX_LINES);
196
196
  try {
@@ -10,6 +10,7 @@
10
10
  import { WebSocket } from 'ws';
11
11
  import { randomBytes } from 'node:crypto';
12
12
  import { createLogger } from '../logger.js';
13
+ import { jitteredDelay, SLOW_PROBE_MS, isFatalReconnectError } from '../shared/reconnect.js';
13
14
  const log = createLogger('WeWork');
14
15
  const DEFAULT_WS_URL = 'wss://openws.work.weixin.qq.com';
15
16
  const HEARTBEAT_INTERVAL = 30000; // 30秒
@@ -383,6 +384,8 @@ function stopHeartbeat() {
383
384
  * Schedule reconnection attempt
384
385
  * 超过 MAX_RECONNECT_ATTEMPTS 后自动重置计数器继续重试,避免永久断连
385
386
  */
387
+ /** 致命(鉴权)错误后转慢探测,避免紧密 hammer 网关 */
388
+ let fatalSlowProbe = false;
386
389
  function scheduleReconnect() {
387
390
  if (isStopping || !shouldReconnect) {
388
391
  return;
@@ -395,18 +398,27 @@ function scheduleReconnect() {
395
398
  log.warn(`Max reconnect attempts (${MAX_RECONNECT_ATTEMPTS}) reached, resetting counter and retrying at lower frequency`);
396
399
  reconnectAttempts = 0;
397
400
  }
398
- // 逐步增加间隔,5s → 7.5s → 11s → ... 最大 60s
399
- const backoff = Math.min(5000 * Math.pow(1.5, Math.floor(reconnectAttempts / 5)), 60000);
400
- const interval = Math.round(backoff);
401
+ // 逐步增加间隔,5s → 7.5s → 11s → ... 最大 60s;致命(鉴权)错误转慢探测
402
+ const backoff = fatalSlowProbe
403
+ ? SLOW_PROBE_MS
404
+ : Math.min(5000 * Math.pow(1.5, Math.floor(reconnectAttempts / 5)), 60000);
405
+ const interval = jitteredDelay(Math.round(backoff));
401
406
  reconnectTimer = setTimeout(async () => {
402
407
  reconnectTimer = null;
403
408
  reconnectAttempts++;
404
- log.info(`Reconnecting... Attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS} (interval: ${interval}ms)`);
409
+ log.info(`Reconnecting... Attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS} (interval: ${interval}ms${fatalSlowProbe ? ', slow-probe' : ''})`);
405
410
  try {
406
411
  await connectWebSocket();
412
+ fatalSlowProbe = false; // 连上后恢复正常退避
407
413
  }
408
414
  catch (err) {
409
- log.error('Reconnection failed:', err);
415
+ if (isFatalReconnectError(err)) {
416
+ fatalSlowProbe = true;
417
+ log.warn('WeWork 致命错误(鉴权失败),转慢探测:', err);
418
+ }
419
+ else {
420
+ log.error('Reconnection failed:', err);
421
+ }
410
422
  }
411
423
  }, interval);
412
424
  }
@@ -213,6 +213,9 @@ export function setupWeWorkHandlers(config, sessionManager) {
213
213
  const state = ctx.runningTasks.get(taskKey);
214
214
  if (state) {
215
215
  log.warn(`[SAFETY_TIMEOUT] Task ${taskKey} exceeded ${WEWORK_TASK_SAFETY_TIMEOUT_MS}ms, aborting`);
216
+ // 先 settle 再 abort:settled=true 后 abort() 跳过自身的 sendError,由这里的 sendTextReply
217
+ // 兜底(企微流式消息可能已过期,文本兜底更稳),避免双重终态消息。
218
+ state.settle();
216
219
  state.handle.abort();
217
220
  ctx.runningTasks.delete(taskKey);
218
221
  sendTextReply(chatId, `AI 处理超时(${Math.round(WEWORK_TASK_SAFETY_TIMEOUT_MS / 1000)}s),已自动取消。请重试。`, senderCtx.reqId).catch(() => { });
@@ -12,6 +12,10 @@ export interface CentrifugeClientConfig {
12
12
  userId: string;
13
13
  httpBaseUrl?: string;
14
14
  httpAccessToken?: string;
15
+ /** 返回当前有效的 access token(已刷新)。无则用 httpAccessToken 兜底。 */
16
+ getAccessToken?: () => string;
17
+ /** 单飞刷新 token;HTTP 401 时调用,刷新后 getAccessToken 返回新 token。 */
18
+ refreshToken?: () => Promise<void>;
15
19
  workspaceSessionId?: string;
16
20
  /**
17
21
  * Called before sending a WeChat KF reply to update the channel's channelId
@@ -226,20 +226,50 @@ export class WorkBuddyCentrifugeClient {
226
226
  };
227
227
  const url = `${this.config.httpBaseUrl}/v2/backgroundagent/wecom/local-proxy/receive`;
228
228
  log.debug(`${this.logPrefix} HTTP COPILOT_RESPONSE → ${url} chatId=${sessionId} msgLen=${message.length}`);
229
- // Retry COPILOT_RESPONSE up to 3 times on network failure
229
+ // Retry COPILOT_RESPONSE with exponential backoff for rate limits (95002)
230
+ const MAX_RETRIES = 5;
231
+ const INITIAL_BACKOFF_MS = 2000;
230
232
  let sent = false;
231
- for (let attempt = 1; attempt <= 3; attempt++) {
233
+ let backoffMs = INITIAL_BACKOFF_MS;
234
+ let refreshedOn401 = false;
235
+ for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
236
+ const token = this.config.getAccessToken?.() ?? this.config.httpAccessToken ?? '';
232
237
  try {
233
238
  const res = await fetch(url, {
234
239
  method: 'POST',
235
240
  headers: {
236
241
  'Content-Type': 'application/json',
237
- Authorization: `Bearer ${this.config.httpAccessToken}`,
242
+ Authorization: `Bearer ${token}`,
238
243
  },
239
244
  body: JSON.stringify(httpPayload),
240
245
  signal: AbortSignal.timeout(30_000),
241
246
  });
242
247
  const body = await res.text().catch(() => '');
248
+ // Check for WeChat KF rate limit (95002)
249
+ if (body.includes('errcode=95002') || body.includes('"errcode":95002')) {
250
+ if (attempt < MAX_RETRIES) {
251
+ log.warn(`${this.logPrefix} HTTP COPILOT_RESPONSE rate limited (95002), attempt ${attempt}/${MAX_RETRIES}, retrying in ${backoffMs}ms`);
252
+ await new Promise((r) => setTimeout(r, backoffMs));
253
+ backoffMs = Math.min(backoffMs * 2, 60_000); // Cap at 60s
254
+ continue;
255
+ }
256
+ else {
257
+ log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE rate limited after ${MAX_RETRIES} attempts, giving up`);
258
+ break;
259
+ }
260
+ }
261
+ // 401: access token 过期 → 单飞刷新后重试一次
262
+ if (res.status === 401 && !refreshedOn401 && this.config.refreshToken) {
263
+ refreshedOn401 = true;
264
+ try {
265
+ await this.config.refreshToken();
266
+ log.info(`${this.logPrefix} Token refreshed on 401, retrying COPILOT_RESPONSE`);
267
+ continue;
268
+ }
269
+ catch (refreshErr) {
270
+ log.error(`${this.logPrefix} Token refresh failed on 401:`, refreshErr);
271
+ }
272
+ }
243
273
  if (!res.ok) {
244
274
  log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE failed: ${res.status} ${body.substring(0, 300)}`);
245
275
  }
@@ -250,17 +280,18 @@ export class WorkBuddyCentrifugeClient {
250
280
  break;
251
281
  }
252
282
  catch (err) {
253
- if (attempt < 3) {
254
- log.warn(`${this.logPrefix} HTTP COPILOT_RESPONSE attempt ${attempt} failed, retrying in 2s:`, err);
255
- await new Promise((r) => setTimeout(r, 2000));
283
+ if (attempt < MAX_RETRIES) {
284
+ log.warn(`${this.logPrefix} HTTP COPILOT_RESPONSE attempt ${attempt} failed, retrying in ${backoffMs}ms:`, err);
285
+ await new Promise((r) => setTimeout(r, backoffMs));
286
+ backoffMs = Math.min(backoffMs * 2, 60_000);
256
287
  }
257
288
  else {
258
- log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE error after 3 attempts:`, err);
289
+ log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE error after ${MAX_RETRIES} attempts:`, err);
259
290
  }
260
291
  }
261
292
  }
262
293
  if (!sent) {
263
- this.enqueuePendingReply(url, httpPayload, this.config.httpAccessToken);
294
+ this.enqueuePendingReply(url, httpPayload);
264
295
  }
265
296
  // Release the heartbeat lock so the periodic registration can resume
266
297
  this.config.releaseChannelLockFn?.();
@@ -372,7 +403,7 @@ export class WorkBuddyCentrifugeClient {
372
403
  /**
373
404
  * Enqueue a failed reply for later delivery
374
405
  */
375
- enqueuePendingReply(url, payload, accessToken) {
406
+ enqueuePendingReply(url, payload) {
376
407
  // Evict expired entries
377
408
  const now = Date.now();
378
409
  this.pendingReplies = this.pendingReplies.filter((r) => now - r.addedAt < PENDING_REPLY_TTL_MS);
@@ -380,7 +411,7 @@ export class WorkBuddyCentrifugeClient {
380
411
  const evicted = this.pendingReplies.shift();
381
412
  log.warn(`${this.logPrefix} Pending replies full, evicting oldest (msgId=${evicted?.payload.msgId})`);
382
413
  }
383
- this.pendingReplies.push({ url, payload, accessToken, addedAt: now });
414
+ this.pendingReplies.push({ url, payload, addedAt: now });
384
415
  log.warn(`${this.logPrefix} Queued pending reply (queue=${this.pendingReplies.length}, msgId=${payload.msgId})`);
385
416
  }
386
417
  /**
@@ -398,27 +429,44 @@ export class WorkBuddyCentrifugeClient {
398
429
  log.info(`${this.logPrefix} Flushing ${toSend.length} pending reply(ies)`);
399
430
  }
400
431
  for (const reply of toSend) {
401
- try {
402
- const res = await fetch(reply.url, {
403
- method: 'POST',
404
- headers: {
405
- 'Content-Type': 'application/json',
406
- Authorization: `Bearer ${reply.accessToken}`,
407
- },
408
- body: JSON.stringify(reply.payload),
409
- signal: AbortSignal.timeout(30_000),
410
- });
411
- const body = await res.text().catch(() => '');
412
- if (res.ok) {
413
- log.info(`${this.logPrefix} Flushed pending reply ok: msgId=${reply.payload.msgId}`);
432
+ // 用当前(可能已刷新的)token;401 时单飞刷新后重试一次
433
+ let refreshedOn401 = false;
434
+ for (let attempt = 1; attempt <= 2; attempt++) {
435
+ const token = this.config.getAccessToken?.() ?? this.config.httpAccessToken ?? '';
436
+ try {
437
+ const res = await fetch(reply.url, {
438
+ method: 'POST',
439
+ headers: {
440
+ 'Content-Type': 'application/json',
441
+ Authorization: `Bearer ${token}`,
442
+ },
443
+ body: JSON.stringify(reply.payload),
444
+ signal: AbortSignal.timeout(30_000),
445
+ });
446
+ if (res.status === 401 && !refreshedOn401 && this.config.refreshToken) {
447
+ refreshedOn401 = true;
448
+ try {
449
+ await this.config.refreshToken();
450
+ continue;
451
+ }
452
+ catch {
453
+ /* 刷新失败,落到下面正常判定 */
454
+ }
455
+ }
456
+ const body = await res.text().catch(() => '');
457
+ if (res.ok) {
458
+ log.info(`${this.logPrefix} Flushed pending reply ok: msgId=${reply.payload.msgId}`);
459
+ }
460
+ else {
461
+ log.error(`${this.logPrefix} Flushed pending reply failed: ${res.status} ${body.substring(0, 200)}`);
462
+ }
463
+ break;
414
464
  }
415
- else {
416
- log.error(`${this.logPrefix} Flushed pending reply failed: ${res.status} ${body.substring(0, 200)}`);
465
+ catch (err) {
466
+ log.error(`${this.logPrefix} Flushed pending reply error: msgId=${reply.payload.msgId}`, err);
467
+ break;
417
468
  }
418
469
  }
419
- catch (err) {
420
- log.error(`${this.logPrefix} Flushed pending reply error: msgId=${reply.payload.msgId}`, err);
421
- }
422
470
  }
423
471
  this.flushing = false;
424
472
  }
@@ -9,6 +9,7 @@ import { existsSync, mkdirSync } from 'node:fs';
9
9
  import { join } from 'node:path';
10
10
  import { hostname, homedir } from 'node:os';
11
11
  import { createLogger } from '../logger.js';
12
+ import { jitteredDelay, SLOW_PROBE_MS, isFatalReconnectError } from '../shared/reconnect.js';
12
13
  import { WorkBuddyOAuth } from './oauth.js';
13
14
  import { WorkBuddyCentrifugeClient } from './centrifuge-client.js';
14
15
  const log = createLogger('WorkBuddy');
@@ -23,8 +24,11 @@ let stateChangeHandler = null;
23
24
  let heartbeatTimer = null;
24
25
  let reconnectTimer = null;
25
26
  let reconnectAttempt = 0;
27
+ let fatalSlowProbe = false;
26
28
  let stopped = false;
27
29
  let platformConfig = null;
30
+ /** 单飞刷新 token 的在途 Promise,避免并发 401 重复刷新 */
31
+ let refreshInFlight = null;
28
32
  export function getChannelState() {
29
33
  return channelState;
30
34
  }
@@ -78,6 +82,10 @@ async function connect() {
78
82
  }
79
83
  catch (err) {
80
84
  log.error('Host workspace registration failed:', err);
85
+ if (isFatalReconnectError(err)) {
86
+ fatalSlowProbe = true;
87
+ log.warn('WorkBuddy 致命错误(鉴权失败),转慢探测(token 可能已过期):', err);
88
+ }
81
89
  scheduleReconnect();
82
90
  return;
83
91
  }
@@ -120,6 +128,8 @@ async function connect() {
120
128
  userId: pc.userId ?? '',
121
129
  httpBaseUrl: baseUrl,
122
130
  httpAccessToken: pc.accessToken ?? '',
131
+ getAccessToken: () => oauthClient?.accessToken ?? pc.accessToken ?? '',
132
+ refreshToken: refreshWorkBuddyToken,
123
133
  workspaceSessionId,
124
134
  registerChannelFn,
125
135
  releaseChannelLockFn,
@@ -128,6 +138,7 @@ async function connect() {
128
138
  log.info('WorkBuddy Centrifuge connected');
129
139
  log.info(`WeChat KF sessionId: ${workspaceSessionId}`);
130
140
  reconnectAttempt = 0;
141
+ fatalSlowProbe = false;
131
142
  updateState('connected');
132
143
  // Step 2: Register Claw workspace to get WeChat KF routing channel + sessionId
133
144
  oauth.registerWorkspace({
@@ -234,9 +245,10 @@ function scheduleReconnect() {
234
245
  clearTimeout(reconnectTimer);
235
246
  reconnectTimer = null;
236
247
  }
237
- const delay = RECONNECT_DELAYS_MS[Math.min(reconnectAttempt, RECONNECT_DELAYS_MS.length - 1)];
248
+ const baseDelay = RECONNECT_DELAYS_MS[Math.min(reconnectAttempt, RECONNECT_DELAYS_MS.length - 1)];
249
+ const delay = fatalSlowProbe ? jitteredDelay(SLOW_PROBE_MS) : jitteredDelay(baseDelay);
238
250
  reconnectAttempt++;
239
- log.info(`Reconnecting in ${delay}ms (attempt ${reconnectAttempt})...`);
251
+ log.info(`Reconnecting in ${delay}ms (attempt ${reconnectAttempt})${fatalSlowProbe ? ' [slow-probe]' : ''}...`);
240
252
  reconnectTimer = setTimeout(async () => {
241
253
  reconnectTimer = null;
242
254
  if (stopped)
@@ -261,6 +273,31 @@ export function getCentrifugeClient() {
261
273
  export function getOAuth() {
262
274
  return oauthClient;
263
275
  }
276
+ /**
277
+ * 单飞刷新 WorkBuddy access token:并发 401 只触发一次刷新,复用同一 Promise。
278
+ * 刷新成功后 oauthClient.accessToken 原地更新,所有 HTTP 调用(含心跳)自动用新 token。
279
+ */
280
+ async function refreshWorkBuddyToken() {
281
+ if (!oauthClient)
282
+ return;
283
+ if (refreshInFlight)
284
+ return refreshInFlight;
285
+ refreshInFlight = (async () => {
286
+ try {
287
+ log.info('Refreshing WorkBuddy access token...');
288
+ await oauthClient.refreshTokenAuth();
289
+ log.info('WorkBuddy access token refreshed');
290
+ }
291
+ catch (err) {
292
+ log.error('WorkBuddy token refresh failed:', err);
293
+ throw err;
294
+ }
295
+ finally {
296
+ refreshInFlight = null;
297
+ }
298
+ })();
299
+ return refreshInFlight;
300
+ }
264
301
  export function stopWorkBuddy() {
265
302
  log.info('Stopping WorkBuddy client...');
266
303
  stopped = true;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@wu529778790/open-im",
3
- "version": "1.10.9-beta.3",
3
+ "version": "1.10.9-beta.4",
4
4
  "description": "Multi-platform IM bridge for AI CLI tools (Claude, Codex, CodeBuddy)",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",