@wu529778790/open-im 1.10.9-beta.2 → 1.10.9-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -62
- package/README.zh-CN.md +43 -62
- package/dist/adapters/claude-sdk-adapter.d.ts +13 -0
- package/dist/adapters/claude-sdk-adapter.js +221 -23
- package/dist/adapters/registry.js +3 -0
- package/dist/channels/capabilities.js +5 -0
- package/dist/clawbot/client.d.ts +14 -0
- package/dist/clawbot/client.js +299 -0
- package/dist/clawbot/event-handler.d.ts +12 -0
- package/dist/clawbot/event-handler.js +85 -0
- package/dist/clawbot/message-sender.d.ts +18 -0
- package/dist/clawbot/message-sender.js +109 -0
- package/dist/clawbot/qr-login.d.ts +33 -0
- package/dist/clawbot/qr-login.js +120 -0
- package/dist/clawbot/types.d.ts +111 -0
- package/dist/clawbot/types.js +7 -0
- package/dist/codebuddy/cli-runner.js +31 -2
- package/dist/codex/cli-runner.js +28 -2
- package/dist/config/file-io.d.ts +6 -3
- package/dist/config/file-io.js +12 -7
- package/dist/config/types.d.ts +17 -1
- package/dist/config-web-page-i18n.d.ts +24 -2
- package/dist/config-web-page-i18n.js +24 -2
- package/dist/config-web.js +79 -0
- package/dist/config.d.ts +1 -1
- package/dist/config.js +38 -2
- package/dist/constants.d.ts +6 -0
- package/dist/constants.js +6 -0
- package/dist/dingtalk/client.js +2 -1
- package/dist/dingtalk/event-handler.js +1 -1
- package/dist/index.js +50 -0
- package/dist/qq/client.js +7 -1
- package/dist/queue/request-queue.js +11 -10
- package/dist/setup.js +131 -3
- package/dist/shared/active-chats.d.ts +2 -2
- package/dist/shared/ai-task.d.ts +14 -0
- package/dist/shared/ai-task.js +57 -9
- package/dist/shared/process-kill.d.ts +24 -0
- package/dist/shared/process-kill.js +79 -0
- package/dist/shared/reconnect.d.ts +28 -0
- package/dist/shared/reconnect.js +56 -0
- package/dist/shared/task-cleanup.d.ts +16 -0
- package/dist/shared/task-cleanup.js +34 -1
- package/dist/telegram/client.js +7 -1
- package/dist/telemetry/telemetry-upload.js +1 -1
- package/dist/wework/client.js +17 -5
- package/dist/wework/event-handler.js +3 -0
- package/dist/workbuddy/centrifuge-client.d.ts +4 -0
- package/dist/workbuddy/centrifuge-client.js +76 -28
- package/dist/workbuddy/client.js +39 -2
- package/package.json +1 -1
- package/web/dist/assets/index-BaLTMeeF.js +57 -0
- package/web/dist/index.html +1 -1
- package/dist/config/credentials.d.ts +0 -19
- package/dist/config/credentials.js +0 -36
- package/web/dist/assets/index-B-oVSMUp.js +0 -57
package/dist/shared/ai-task.js
CHANGED
|
@@ -10,25 +10,65 @@ const log = createLogger('AITask');
|
|
|
10
10
|
function isUsageLimitError(error) {
|
|
11
11
|
return /usage limit/i.test(error) || /try again at\s+\d{1,2}:\d{2}\s*(AM|PM)/i.test(error);
|
|
12
12
|
}
|
|
13
|
-
|
|
13
|
+
/**
|
|
14
|
+
* 将 AI/CLI 错误文本归类为一个稳定的 errorType 字符串,用于遥测聚合。
|
|
15
|
+
* 分类基于线上真实错误签名(见 logs/r2-events),新增分支时同步更新测试。
|
|
16
|
+
*/
|
|
17
|
+
export function classifyErrorType(error) {
|
|
14
18
|
const s = error.toLowerCase();
|
|
15
19
|
if (s.includes('aborted'))
|
|
16
20
|
return 'aborted';
|
|
21
|
+
// 空输出:流正常结束但无内容(claude-sdk-adapter 的兜底消息)
|
|
22
|
+
if (s.includes('无输出') || s.includes('响应异常结束') || s.includes('empty output')) {
|
|
23
|
+
return 'empty_output';
|
|
24
|
+
}
|
|
17
25
|
if (isUsageLimitError(error) || s.includes('rate limit') || s.includes('quota'))
|
|
18
26
|
return 'limit';
|
|
19
|
-
|
|
27
|
+
// 鉴权:凭据无效或需要登录(含中文「登录」/「login」)
|
|
28
|
+
if (s.includes('invalid api key') ||
|
|
29
|
+
s.includes('unauthorized') ||
|
|
30
|
+
s.includes('401') ||
|
|
31
|
+
s.includes('need to log in') ||
|
|
32
|
+
s.includes('need to login') ||
|
|
33
|
+
s.includes('需要登录') ||
|
|
34
|
+
s.includes('需要先登录') ||
|
|
35
|
+
s.includes('log in required') ||
|
|
36
|
+
s.includes('login required')) {
|
|
20
37
|
return 'auth';
|
|
38
|
+
}
|
|
21
39
|
if (s.includes('model') && (s.includes('not support') || s.includes('not found') || s.includes('invalid'))) {
|
|
22
40
|
return 'model';
|
|
23
41
|
}
|
|
24
|
-
|
|
42
|
+
// 安装/配置缺失:二进制或可执行文件缺失、环境变量缺失、token 未配置
|
|
43
|
+
if (s.includes('native cli binary') ||
|
|
44
|
+
s.includes('executable not found') ||
|
|
45
|
+
(s.includes('binary') && s.includes('not found')) ||
|
|
46
|
+
s.includes('missing environment variable') ||
|
|
47
|
+
s.includes('token data is not available')) {
|
|
48
|
+
return 'setup';
|
|
49
|
+
}
|
|
50
|
+
// 会话失效:找不到会话/对话,或会话过期/损坏
|
|
51
|
+
if (s.includes('no conversation found') ||
|
|
52
|
+
s.includes('session not found') ||
|
|
53
|
+
(s.includes('session') && (s.includes('expired') || s.includes('corrupt')))) {
|
|
54
|
+
return 'session';
|
|
55
|
+
}
|
|
56
|
+
// 进程退出或被信号终止
|
|
57
|
+
if (s.includes('process exited') ||
|
|
58
|
+
s.includes('exit code') ||
|
|
59
|
+
s.includes('exited with code') ||
|
|
60
|
+
s.includes('terminated by signal') ||
|
|
61
|
+
s.includes('sigkill')) {
|
|
25
62
|
return 'process';
|
|
63
|
+
}
|
|
64
|
+
// 网络:超时/连接重置/DNS/网络请求失败(含中文「网络」)
|
|
26
65
|
if (s.includes('timeout') ||
|
|
27
66
|
s.includes('etimedout') ||
|
|
28
67
|
s.includes('econnreset') ||
|
|
29
68
|
s.includes('enotfound') ||
|
|
30
69
|
s.includes('eai_again') ||
|
|
31
|
-
s.includes('network')
|
|
70
|
+
s.includes('network') ||
|
|
71
|
+
s.includes('网络')) {
|
|
32
72
|
return 'network';
|
|
33
73
|
}
|
|
34
74
|
return 'unknown';
|
|
@@ -115,7 +155,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
115
155
|
log.info(`[AITask] Starting: userId=${ctx.userId}, initialSessionId=${currentSessionId ?? 'new'}, prompt="${prompt.slice(0, 50)}..."`);
|
|
116
156
|
emitStructuredEvent('AITask', 'ai.task.start', {
|
|
117
157
|
platform: ctx.platform,
|
|
118
|
-
taskKey: ctx.taskKey,
|
|
158
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
119
159
|
userKey: hashUserId(ctx.userId),
|
|
120
160
|
toolId: aiCommand,
|
|
121
161
|
});
|
|
@@ -179,7 +219,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
179
219
|
return;
|
|
180
220
|
emitStructuredEvent('AITask', 'ai.task.complete', {
|
|
181
221
|
platform: ctx.platform,
|
|
182
|
-
taskKey: ctx.taskKey,
|
|
222
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
183
223
|
userKey: hashUserId(ctx.userId),
|
|
184
224
|
toolId: aiCommand,
|
|
185
225
|
durationMs: result.durationMs,
|
|
@@ -242,7 +282,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
242
282
|
log.error(`Task error for user ${ctx.userId}: ${error}`);
|
|
243
283
|
emitStructuredEvent('AITask', 'ai.task.error', {
|
|
244
284
|
platform: ctx.platform,
|
|
245
|
-
taskKey: ctx.taskKey,
|
|
285
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
246
286
|
userKey: hashUserId(ctx.userId),
|
|
247
287
|
toolId: aiCommand,
|
|
248
288
|
durationMs: Date.now() - taskState.startedAt,
|
|
@@ -290,13 +330,18 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
290
330
|
if (!settled) {
|
|
291
331
|
emitStructuredEvent('AITask', 'ai.task.error', {
|
|
292
332
|
platform: ctx.platform,
|
|
293
|
-
taskKey: ctx.taskKey,
|
|
333
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
294
334
|
userKey: hashUserId(ctx.userId),
|
|
295
335
|
toolId: aiCommand,
|
|
296
336
|
durationMs: Date.now() - taskState.startedAt,
|
|
297
337
|
errorSnippet: 'aborted',
|
|
298
338
|
errorType: 'aborted',
|
|
299
339
|
});
|
|
340
|
+
// 用户取消(/new、/resume、队列超时、stale 清理):把「思考中…」占位卡片编辑为终态,
|
|
341
|
+
// 避免卡片卡在转圈。停按钮路径会先 settle() 再 abort,settled=true 时此处跳过,不会双发。
|
|
342
|
+
void platformAdapter.sendError('⏹️ 已取消').catch(() => {
|
|
343
|
+
/* 占位卡片可能已被删除或流过期,编辑失败可忽略 */
|
|
344
|
+
});
|
|
300
345
|
}
|
|
301
346
|
activeHandle?.abort();
|
|
302
347
|
cleanup();
|
|
@@ -307,6 +352,9 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
307
352
|
settle,
|
|
308
353
|
startedAt: Date.now(),
|
|
309
354
|
toolId: aiCommand,
|
|
355
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
356
|
+
platform: ctx.platform,
|
|
357
|
+
userKey: hashUserId(ctx.userId),
|
|
310
358
|
};
|
|
311
359
|
try {
|
|
312
360
|
startRun();
|
|
@@ -318,7 +366,7 @@ export function runAITask(deps, ctx, prompt, toolAdapter, platformAdapter) {
|
|
|
318
366
|
log.error(`[AITask] Synchronous error in startRun: ${err}`);
|
|
319
367
|
emitStructuredEvent('AITask', 'ai.task.error', {
|
|
320
368
|
platform: ctx.platform,
|
|
321
|
-
taskKey: ctx.taskKey,
|
|
369
|
+
taskKey: hashUserId(ctx.taskKey),
|
|
322
370
|
userKey: hashUserId(ctx.userId),
|
|
323
371
|
toolId: aiCommand,
|
|
324
372
|
durationMs: 0,
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 跨平台子进程树终止:用于 Codex / CodeBuddy CLI 子进程。
|
|
3
|
+
*
|
|
4
|
+
* 在 Unix 上子进程需以 `detached: true` 启动,使其成为独立进程组的组长;
|
|
5
|
+
* 这样 `process.kill(-pid)` 才能同时命中 CLI 及其派生的孙进程(MCP server、
|
|
6
|
+
* shell 工具调用等)。Windows 不支持负 PID 信号,改用 `taskkill /T /F`
|
|
7
|
+
* 按 PID 递归终止整棵进程树。
|
|
8
|
+
*
|
|
9
|
+
* 进程退出(shutdown / 崩溃)时,`destroyAllLiveChildren()` 会强制清理所有
|
|
10
|
+
* 仍在运行的被追踪子进程,避免僵尸 / 孤儿进程。
|
|
11
|
+
*/
|
|
12
|
+
import { type ChildProcess } from 'node:child_process';
|
|
13
|
+
/** 追踪一个已 spawn 的子进程,使其在关停时能被强制终止。 */
|
|
14
|
+
export declare function trackChild(child: ChildProcess): ChildProcess;
|
|
15
|
+
/**
|
|
16
|
+
* 终止子进程及其所有后代。默认优雅:先 SIGTERM,`graceMs` 后升级为 SIGKILL。
|
|
17
|
+
* 传 `force: true` 则立即 SIGKILL(用于关停)。
|
|
18
|
+
*/
|
|
19
|
+
export declare function killProcessTree(child: ChildProcess, opts?: {
|
|
20
|
+
force?: boolean;
|
|
21
|
+
graceMs?: number;
|
|
22
|
+
}): void;
|
|
23
|
+
/** 强制终止所有仍在运行的被追踪子进程。关停 / 进程退出时调用。 */
|
|
24
|
+
export declare function destroyAllLiveChildren(): void;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 跨平台子进程树终止:用于 Codex / CodeBuddy CLI 子进程。
|
|
3
|
+
*
|
|
4
|
+
* 在 Unix 上子进程需以 `detached: true` 启动,使其成为独立进程组的组长;
|
|
5
|
+
* 这样 `process.kill(-pid)` 才能同时命中 CLI 及其派生的孙进程(MCP server、
|
|
6
|
+
* shell 工具调用等)。Windows 不支持负 PID 信号,改用 `taskkill /T /F`
|
|
7
|
+
* 按 PID 递归终止整棵进程树。
|
|
8
|
+
*
|
|
9
|
+
* 进程退出(shutdown / 崩溃)时,`destroyAllLiveChildren()` 会强制清理所有
|
|
10
|
+
* 仍在运行的被追踪子进程,避免僵尸 / 孤儿进程。
|
|
11
|
+
*/
|
|
12
|
+
import { spawn } from 'node:child_process';
|
|
13
|
+
import { createLogger } from '../logger.js';
|
|
14
|
+
const log = createLogger('ProcessKill');
|
|
15
|
+
/** 当前被追踪的存活子进程,供关停时强制清理。 */
|
|
16
|
+
const liveChildren = new Set();
|
|
17
|
+
/** 追踪一个已 spawn 的子进程,使其在关停时能被强制终止。 */
|
|
18
|
+
export function trackChild(child) {
|
|
19
|
+
liveChildren.add(child);
|
|
20
|
+
child.once('close', () => liveChildren.delete(child));
|
|
21
|
+
return child;
|
|
22
|
+
}
|
|
23
|
+
/** 向整棵进程树发送信号:Unix 用负 PID(整组),Windows 用 taskkill /T /F。 */
|
|
24
|
+
function signalTree(child, signal) {
|
|
25
|
+
const pid = child.pid;
|
|
26
|
+
if (pid == null)
|
|
27
|
+
return;
|
|
28
|
+
try {
|
|
29
|
+
if (process.platform === 'win32') {
|
|
30
|
+
// Windows 没有「优雅 SIGTERM」语义,统一 /T(递归) /F(强制)
|
|
31
|
+
const tk = spawn('taskkill', ['/pid', String(pid), '/T', '/F'], {
|
|
32
|
+
windowsHide: true,
|
|
33
|
+
stdio: 'ignore',
|
|
34
|
+
});
|
|
35
|
+
tk.unref();
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
process.kill(-pid, signal);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
// ESRCH(已退出)或未 detached(无独立进程组)——退化为直接信号
|
|
43
|
+
try {
|
|
44
|
+
child.kill(signal);
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
/* 进程已退出,忽略 */
|
|
48
|
+
}
|
|
49
|
+
log.debug(`signalTree(${signal}) for pid=${pid} 退化为直杀: ${err.message}`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* 终止子进程及其所有后代。默认优雅:先 SIGTERM,`graceMs` 后升级为 SIGKILL。
|
|
54
|
+
* 传 `force: true` 则立即 SIGKILL(用于关停)。
|
|
55
|
+
*/
|
|
56
|
+
export function killProcessTree(child, opts = {}) {
|
|
57
|
+
const { force = false, graceMs = 5000 } = opts;
|
|
58
|
+
if (child.pid == null)
|
|
59
|
+
return;
|
|
60
|
+
if (force) {
|
|
61
|
+
signalTree(child, 'SIGKILL');
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
signalTree(child, 'SIGTERM');
|
|
65
|
+
const escalate = setTimeout(() => signalTree(child, 'SIGKILL'), graceMs);
|
|
66
|
+
escalate.unref();
|
|
67
|
+
}
|
|
68
|
+
/** 强制终止所有仍在运行的被追踪子进程。关停 / 进程退出时调用。 */
|
|
69
|
+
export function destroyAllLiveChildren() {
|
|
70
|
+
for (const child of [...liveChildren]) {
|
|
71
|
+
try {
|
|
72
|
+
signalTree(child, 'SIGKILL');
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
/* best effort */
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
liveChildren.clear();
|
|
79
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 重连韧性共享工具:抖动(jitter)+ 致命错误慢探测。
|
|
3
|
+
*
|
|
4
|
+
* 各平台长连接断开后各自重连,原本退避固定无抖动 → 车队重启会锁步重连风暴;
|
|
5
|
+
* 且致命(鉴权)错误也紧密/无限重试, hammer 网关。
|
|
6
|
+
*
|
|
7
|
+
* 策略(已与产品确认):
|
|
8
|
+
* - 所有退避加 ±30% 抖动,避免锁步。
|
|
9
|
+
* - 鉴权类致命错误转为**慢探测**(每 5 分钟一次 + 显眼告警):
|
|
10
|
+
* 既不紧密 hammer,也不永久断连(尊重 WeWork「避免永久断连」意图)。
|
|
11
|
+
*/
|
|
12
|
+
/** 慢探测间隔:致命错误后每 5 分钟探测一次。 */
|
|
13
|
+
export declare const SLOW_PROBE_MS: number;
|
|
14
|
+
/**
|
|
15
|
+
* 在 baseMs 上叠加 ±frac 的抖动,避免车队重启锁步重连。
|
|
16
|
+
* jitteredDelay(1000, 0.3) ∈ [700, 1300]。
|
|
17
|
+
*/
|
|
18
|
+
export declare function jitteredDelay(baseMs: number, frac?: number): number;
|
|
19
|
+
/**
|
|
20
|
+
* 判断是否为「致命/不可重试」错误(凭据无效、鉴权失败)。
|
|
21
|
+
* 这类错误紧密重试无意义,应转慢探测。
|
|
22
|
+
*/
|
|
23
|
+
export declare function isFatalReconnectError(err: unknown): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* 计算重连延迟:致命慢探测模式下用 SLOW_PROBE_MS,否则用 jittered(base)。
|
|
26
|
+
* `fatal` 由调用方维护(致命时置 true,成功连上后置 false)。
|
|
27
|
+
*/
|
|
28
|
+
export declare function reconnectDelay(baseMs: number, fatal: boolean): number;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 重连韧性共享工具:抖动(jitter)+ 致命错误慢探测。
|
|
3
|
+
*
|
|
4
|
+
* 各平台长连接断开后各自重连,原本退避固定无抖动 → 车队重启会锁步重连风暴;
|
|
5
|
+
* 且致命(鉴权)错误也紧密/无限重试, hammer 网关。
|
|
6
|
+
*
|
|
7
|
+
* 策略(已与产品确认):
|
|
8
|
+
* - 所有退避加 ±30% 抖动,避免锁步。
|
|
9
|
+
* - 鉴权类致命错误转为**慢探测**(每 5 分钟一次 + 显眼告警):
|
|
10
|
+
* 既不紧密 hammer,也不永久断连(尊重 WeWork「避免永久断连」意图)。
|
|
11
|
+
*/
|
|
12
|
+
import { createLogger } from '../logger.js';
|
|
13
|
+
const log = createLogger('Reconnect');
|
|
14
|
+
/** 慢探测间隔:致命错误后每 5 分钟探测一次。 */
|
|
15
|
+
export const SLOW_PROBE_MS = 5 * 60_000;
|
|
16
|
+
/**
|
|
17
|
+
* 在 baseMs 上叠加 ±frac 的抖动,避免车队重启锁步重连。
|
|
18
|
+
* jitteredDelay(1000, 0.3) ∈ [700, 1300]。
|
|
19
|
+
*/
|
|
20
|
+
export function jitteredDelay(baseMs, frac = 0.3) {
|
|
21
|
+
if (baseMs <= 0)
|
|
22
|
+
return 0;
|
|
23
|
+
const jitter = baseMs * frac * (Math.random() * 2 - 1);
|
|
24
|
+
return Math.max(0, Math.round(baseMs + jitter));
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* 判断是否为「致命/不可重试」错误(凭据无效、鉴权失败)。
|
|
28
|
+
* 这类错误紧密重试无意义,应转慢探测。
|
|
29
|
+
*/
|
|
30
|
+
export function isFatalReconnectError(err) {
|
|
31
|
+
const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
|
32
|
+
if (msg.includes('401') || msg.includes('403'))
|
|
33
|
+
return true;
|
|
34
|
+
if (msg.includes('unauthorized') || msg.includes('forbidden'))
|
|
35
|
+
return true;
|
|
36
|
+
if (msg.includes('invalid signature'))
|
|
37
|
+
return true;
|
|
38
|
+
if (msg.includes('not subscribed'))
|
|
39
|
+
return true;
|
|
40
|
+
if (msg.includes('请重新登录') || msg.includes('需要重新登录') || msg.includes('token 已过期'))
|
|
41
|
+
return true;
|
|
42
|
+
if (msg.includes('invalid') && /token|secret|credential|appid|app_id|corp/.test(msg))
|
|
43
|
+
return true;
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* 计算重连延迟:致命慢探测模式下用 SLOW_PROBE_MS,否则用 jittered(base)。
|
|
48
|
+
* `fatal` 由调用方维护(致命时置 true,成功连上后置 false)。
|
|
49
|
+
*/
|
|
50
|
+
export function reconnectDelay(baseMs, fatal) {
|
|
51
|
+
if (fatal) {
|
|
52
|
+
log.warn(`致命错误,转慢探测(${Math.round(SLOW_PROBE_MS / 1000)}s 一次)`);
|
|
53
|
+
return jitteredDelay(SLOW_PROBE_MS);
|
|
54
|
+
}
|
|
55
|
+
return jitteredDelay(baseMs);
|
|
56
|
+
}
|
|
@@ -6,3 +6,19 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import type { TaskRunState } from './ai-task.js';
|
|
8
8
|
export declare function startTaskCleanup(runningTasks: Map<string, TaskRunState>): () => void;
|
|
9
|
+
/**
|
|
10
|
+
* 在进程退出(优雅关闭 / 崩溃)路径上,为仍在运行的任务补发一条终态遥测事件。
|
|
11
|
+
*
|
|
12
|
+
* `runningTasks` 中存在的任务代表「已发出 ai.task.start 但尚未走到 complete/error」
|
|
13
|
+
* 的在途请求。正常流程会在 handle-ai-request 的 extraCleanup 里删除已结算任务,
|
|
14
|
+
* 因此进入这里时剩下的就是真正被进程退出打断的任务。
|
|
15
|
+
*
|
|
16
|
+
* 与用户主动 `/new`、队列超时触发的 `aborted` 区分,统一用 `interrupted` 标记。
|
|
17
|
+
* 补发后立即调用 state.settle() 置 settled=true:优雅关闭路径随后仍会调用
|
|
18
|
+
* handle.abort()(释放底层资源),但因已 settled,abort 不会再补发一条 aborted,
|
|
19
|
+
* 避免对同一任务重复计数。
|
|
20
|
+
*
|
|
21
|
+
* 该函数必须同步执行,且应在遥测刷盘(shutdownLoggerTelemetry)之前调用,
|
|
22
|
+
* 以保证补发的事件能进入上传队列。
|
|
23
|
+
*/
|
|
24
|
+
export declare function emitInterruptedTerminals(runningTasks: Map<string, TaskRunState>): void;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Tasks older than 30 minutes are aborted and removed from the running-tasks
|
|
5
5
|
* map so they never accumulate indefinitely.
|
|
6
6
|
*/
|
|
7
|
-
import { createLogger } from '../logger.js';
|
|
7
|
+
import { createLogger, emitStructuredEvent } from '../logger.js';
|
|
8
8
|
const log = createLogger('TaskCleanup');
|
|
9
9
|
const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
|
|
10
10
|
const STALE_THRESHOLD_MS = 30 * 60 * 1000; // 30 minutes
|
|
@@ -28,3 +28,36 @@ export function startTaskCleanup(runningTasks) {
|
|
|
28
28
|
timer.unref();
|
|
29
29
|
return () => clearInterval(timer);
|
|
30
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* 在进程退出(优雅关闭 / 崩溃)路径上,为仍在运行的任务补发一条终态遥测事件。
|
|
33
|
+
*
|
|
34
|
+
* `runningTasks` 中存在的任务代表「已发出 ai.task.start 但尚未走到 complete/error」
|
|
35
|
+
* 的在途请求。正常流程会在 handle-ai-request 的 extraCleanup 里删除已结算任务,
|
|
36
|
+
* 因此进入这里时剩下的就是真正被进程退出打断的任务。
|
|
37
|
+
*
|
|
38
|
+
* 与用户主动 `/new`、队列超时触发的 `aborted` 区分,统一用 `interrupted` 标记。
|
|
39
|
+
* 补发后立即调用 state.settle() 置 settled=true:优雅关闭路径随后仍会调用
|
|
40
|
+
* handle.abort()(释放底层资源),但因已 settled,abort 不会再补发一条 aborted,
|
|
41
|
+
* 避免对同一任务重复计数。
|
|
42
|
+
*
|
|
43
|
+
* 该函数必须同步执行,且应在遥测刷盘(shutdownLoggerTelemetry)之前调用,
|
|
44
|
+
* 以保证补发的事件能进入上传队列。
|
|
45
|
+
*/
|
|
46
|
+
export function emitInterruptedTerminals(runningTasks) {
|
|
47
|
+
if (runningTasks.size === 0)
|
|
48
|
+
return;
|
|
49
|
+
const now = Date.now();
|
|
50
|
+
for (const state of runningTasks.values()) {
|
|
51
|
+
emitStructuredEvent('AITask', 'ai.task.error', {
|
|
52
|
+
platform: state.platform,
|
|
53
|
+
taskKey: state.taskKey,
|
|
54
|
+
userKey: state.userKey,
|
|
55
|
+
toolId: state.toolId,
|
|
56
|
+
durationMs: now - state.startedAt,
|
|
57
|
+
errorSnippet: 'interrupted',
|
|
58
|
+
errorType: 'interrupted',
|
|
59
|
+
});
|
|
60
|
+
// 标记已结算,使随后 shutdown 的 abort() 跳过重复的 aborted 事件
|
|
61
|
+
state.settle();
|
|
62
|
+
}
|
|
63
|
+
}
|
package/dist/telegram/client.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Telegraf } from "telegraf";
|
|
2
2
|
import { createLogger } from "../logger.js";
|
|
3
|
+
import { isFatalReconnectError, jitteredDelay } from "../shared/reconnect.js";
|
|
3
4
|
const log = createLogger("Telegram");
|
|
4
5
|
let bot;
|
|
5
6
|
let botUsername;
|
|
@@ -32,8 +33,13 @@ export async function initTelegram(config, setupHandlers) {
|
|
|
32
33
|
catch {
|
|
33
34
|
/* ignore */
|
|
34
35
|
}
|
|
36
|
+
// 致命错误(token 无效等):不再重试,避免烧满 10 次
|
|
37
|
+
if (isFatalReconnectError(err)) {
|
|
38
|
+
log.error("Telegram 致命错误,停止重连(请检查 bot token):", err);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
35
41
|
const maxAttempts = 10;
|
|
36
|
-
const delayMs = Math.min(5000 * attempt, 60000);
|
|
42
|
+
const delayMs = jitteredDelay(Math.min(5000 * attempt, 60000));
|
|
37
43
|
if (attempt < maxAttempts) {
|
|
38
44
|
log.info(`Telegram reconnect in ${Math.round(delayMs / 1000)}s (attempt ${attempt}/${maxAttempts})`);
|
|
39
45
|
await new Promise((r) => setTimeout(r, delayMs));
|
|
@@ -190,7 +190,7 @@ export async function shutdownTelemetryUpload() {
|
|
|
190
190
|
uploadEnabled = false;
|
|
191
191
|
endpoint = undefined;
|
|
192
192
|
bearer = undefined;
|
|
193
|
-
|
|
193
|
+
const lines = pending;
|
|
194
194
|
while (lines.length > 0) {
|
|
195
195
|
const batch = lines.splice(0, BATCH_MAX_LINES);
|
|
196
196
|
try {
|
package/dist/wework/client.js
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
import { WebSocket } from 'ws';
|
|
11
11
|
import { randomBytes } from 'node:crypto';
|
|
12
12
|
import { createLogger } from '../logger.js';
|
|
13
|
+
import { jitteredDelay, SLOW_PROBE_MS, isFatalReconnectError } from '../shared/reconnect.js';
|
|
13
14
|
const log = createLogger('WeWork');
|
|
14
15
|
const DEFAULT_WS_URL = 'wss://openws.work.weixin.qq.com';
|
|
15
16
|
const HEARTBEAT_INTERVAL = 30000; // 30秒
|
|
@@ -383,6 +384,8 @@ function stopHeartbeat() {
|
|
|
383
384
|
* Schedule reconnection attempt
|
|
384
385
|
* 超过 MAX_RECONNECT_ATTEMPTS 后自动重置计数器继续重试,避免永久断连
|
|
385
386
|
*/
|
|
387
|
+
/** 致命(鉴权)错误后转慢探测,避免紧密 hammer 网关 */
|
|
388
|
+
let fatalSlowProbe = false;
|
|
386
389
|
function scheduleReconnect() {
|
|
387
390
|
if (isStopping || !shouldReconnect) {
|
|
388
391
|
return;
|
|
@@ -395,18 +398,27 @@ function scheduleReconnect() {
|
|
|
395
398
|
log.warn(`Max reconnect attempts (${MAX_RECONNECT_ATTEMPTS}) reached, resetting counter and retrying at lower frequency`);
|
|
396
399
|
reconnectAttempts = 0;
|
|
397
400
|
}
|
|
398
|
-
// 逐步增加间隔,5s → 7.5s → 11s → ... 最大 60s
|
|
399
|
-
const backoff =
|
|
400
|
-
|
|
401
|
+
// 逐步增加间隔,5s → 7.5s → 11s → ... 最大 60s;致命(鉴权)错误转慢探测
|
|
402
|
+
const backoff = fatalSlowProbe
|
|
403
|
+
? SLOW_PROBE_MS
|
|
404
|
+
: Math.min(5000 * Math.pow(1.5, Math.floor(reconnectAttempts / 5)), 60000);
|
|
405
|
+
const interval = jitteredDelay(Math.round(backoff));
|
|
401
406
|
reconnectTimer = setTimeout(async () => {
|
|
402
407
|
reconnectTimer = null;
|
|
403
408
|
reconnectAttempts++;
|
|
404
|
-
log.info(`Reconnecting... Attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS} (interval: ${interval}ms)`);
|
|
409
|
+
log.info(`Reconnecting... Attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS} (interval: ${interval}ms${fatalSlowProbe ? ', slow-probe' : ''})`);
|
|
405
410
|
try {
|
|
406
411
|
await connectWebSocket();
|
|
412
|
+
fatalSlowProbe = false; // 连上后恢复正常退避
|
|
407
413
|
}
|
|
408
414
|
catch (err) {
|
|
409
|
-
|
|
415
|
+
if (isFatalReconnectError(err)) {
|
|
416
|
+
fatalSlowProbe = true;
|
|
417
|
+
log.warn('WeWork 致命错误(鉴权失败),转慢探测:', err);
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
log.error('Reconnection failed:', err);
|
|
421
|
+
}
|
|
410
422
|
}
|
|
411
423
|
}, interval);
|
|
412
424
|
}
|
|
@@ -213,6 +213,9 @@ export function setupWeWorkHandlers(config, sessionManager) {
|
|
|
213
213
|
const state = ctx.runningTasks.get(taskKey);
|
|
214
214
|
if (state) {
|
|
215
215
|
log.warn(`[SAFETY_TIMEOUT] Task ${taskKey} exceeded ${WEWORK_TASK_SAFETY_TIMEOUT_MS}ms, aborting`);
|
|
216
|
+
// 先 settle 再 abort:settled=true 后 abort() 跳过自身的 sendError,由这里的 sendTextReply
|
|
217
|
+
// 兜底(企微流式消息可能已过期,文本兜底更稳),避免双重终态消息。
|
|
218
|
+
state.settle();
|
|
216
219
|
state.handle.abort();
|
|
217
220
|
ctx.runningTasks.delete(taskKey);
|
|
218
221
|
sendTextReply(chatId, `AI 处理超时(${Math.round(WEWORK_TASK_SAFETY_TIMEOUT_MS / 1000)}s),已自动取消。请重试。`, senderCtx.reqId).catch(() => { });
|
|
@@ -12,6 +12,10 @@ export interface CentrifugeClientConfig {
|
|
|
12
12
|
userId: string;
|
|
13
13
|
httpBaseUrl?: string;
|
|
14
14
|
httpAccessToken?: string;
|
|
15
|
+
/** 返回当前有效的 access token(已刷新)。无则用 httpAccessToken 兜底。 */
|
|
16
|
+
getAccessToken?: () => string;
|
|
17
|
+
/** 单飞刷新 token;HTTP 401 时调用,刷新后 getAccessToken 返回新 token。 */
|
|
18
|
+
refreshToken?: () => Promise<void>;
|
|
15
19
|
workspaceSessionId?: string;
|
|
16
20
|
/**
|
|
17
21
|
* Called before sending a WeChat KF reply to update the channel's channelId
|
|
@@ -226,20 +226,50 @@ export class WorkBuddyCentrifugeClient {
|
|
|
226
226
|
};
|
|
227
227
|
const url = `${this.config.httpBaseUrl}/v2/backgroundagent/wecom/local-proxy/receive`;
|
|
228
228
|
log.debug(`${this.logPrefix} HTTP COPILOT_RESPONSE → ${url} chatId=${sessionId} msgLen=${message.length}`);
|
|
229
|
-
// Retry COPILOT_RESPONSE
|
|
229
|
+
// Retry COPILOT_RESPONSE with exponential backoff for rate limits (95002)
|
|
230
|
+
const MAX_RETRIES = 5;
|
|
231
|
+
const INITIAL_BACKOFF_MS = 2000;
|
|
230
232
|
let sent = false;
|
|
231
|
-
|
|
233
|
+
let backoffMs = INITIAL_BACKOFF_MS;
|
|
234
|
+
let refreshedOn401 = false;
|
|
235
|
+
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
|
236
|
+
const token = this.config.getAccessToken?.() ?? this.config.httpAccessToken ?? '';
|
|
232
237
|
try {
|
|
233
238
|
const res = await fetch(url, {
|
|
234
239
|
method: 'POST',
|
|
235
240
|
headers: {
|
|
236
241
|
'Content-Type': 'application/json',
|
|
237
|
-
Authorization: `Bearer ${
|
|
242
|
+
Authorization: `Bearer ${token}`,
|
|
238
243
|
},
|
|
239
244
|
body: JSON.stringify(httpPayload),
|
|
240
245
|
signal: AbortSignal.timeout(30_000),
|
|
241
246
|
});
|
|
242
247
|
const body = await res.text().catch(() => '');
|
|
248
|
+
// Check for WeChat KF rate limit (95002)
|
|
249
|
+
if (body.includes('errcode=95002') || body.includes('"errcode":95002')) {
|
|
250
|
+
if (attempt < MAX_RETRIES) {
|
|
251
|
+
log.warn(`${this.logPrefix} HTTP COPILOT_RESPONSE rate limited (95002), attempt ${attempt}/${MAX_RETRIES}, retrying in ${backoffMs}ms`);
|
|
252
|
+
await new Promise((r) => setTimeout(r, backoffMs));
|
|
253
|
+
backoffMs = Math.min(backoffMs * 2, 60_000); // Cap at 60s
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
else {
|
|
257
|
+
log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE rate limited after ${MAX_RETRIES} attempts, giving up`);
|
|
258
|
+
break;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
// 401: access token 过期 → 单飞刷新后重试一次
|
|
262
|
+
if (res.status === 401 && !refreshedOn401 && this.config.refreshToken) {
|
|
263
|
+
refreshedOn401 = true;
|
|
264
|
+
try {
|
|
265
|
+
await this.config.refreshToken();
|
|
266
|
+
log.info(`${this.logPrefix} Token refreshed on 401, retrying COPILOT_RESPONSE`);
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
catch (refreshErr) {
|
|
270
|
+
log.error(`${this.logPrefix} Token refresh failed on 401:`, refreshErr);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
243
273
|
if (!res.ok) {
|
|
244
274
|
log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE failed: ${res.status} ${body.substring(0, 300)}`);
|
|
245
275
|
}
|
|
@@ -250,17 +280,18 @@ export class WorkBuddyCentrifugeClient {
|
|
|
250
280
|
break;
|
|
251
281
|
}
|
|
252
282
|
catch (err) {
|
|
253
|
-
if (attempt <
|
|
254
|
-
log.warn(`${this.logPrefix} HTTP COPILOT_RESPONSE attempt ${attempt} failed, retrying in
|
|
255
|
-
await new Promise((r) => setTimeout(r,
|
|
283
|
+
if (attempt < MAX_RETRIES) {
|
|
284
|
+
log.warn(`${this.logPrefix} HTTP COPILOT_RESPONSE attempt ${attempt} failed, retrying in ${backoffMs}ms:`, err);
|
|
285
|
+
await new Promise((r) => setTimeout(r, backoffMs));
|
|
286
|
+
backoffMs = Math.min(backoffMs * 2, 60_000);
|
|
256
287
|
}
|
|
257
288
|
else {
|
|
258
|
-
log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE error after
|
|
289
|
+
log.error(`${this.logPrefix} HTTP COPILOT_RESPONSE error after ${MAX_RETRIES} attempts:`, err);
|
|
259
290
|
}
|
|
260
291
|
}
|
|
261
292
|
}
|
|
262
293
|
if (!sent) {
|
|
263
|
-
this.enqueuePendingReply(url, httpPayload
|
|
294
|
+
this.enqueuePendingReply(url, httpPayload);
|
|
264
295
|
}
|
|
265
296
|
// Release the heartbeat lock so the periodic registration can resume
|
|
266
297
|
this.config.releaseChannelLockFn?.();
|
|
@@ -372,7 +403,7 @@ export class WorkBuddyCentrifugeClient {
|
|
|
372
403
|
/**
|
|
373
404
|
* Enqueue a failed reply for later delivery
|
|
374
405
|
*/
|
|
375
|
-
enqueuePendingReply(url, payload
|
|
406
|
+
enqueuePendingReply(url, payload) {
|
|
376
407
|
// Evict expired entries
|
|
377
408
|
const now = Date.now();
|
|
378
409
|
this.pendingReplies = this.pendingReplies.filter((r) => now - r.addedAt < PENDING_REPLY_TTL_MS);
|
|
@@ -380,7 +411,7 @@ export class WorkBuddyCentrifugeClient {
|
|
|
380
411
|
const evicted = this.pendingReplies.shift();
|
|
381
412
|
log.warn(`${this.logPrefix} Pending replies full, evicting oldest (msgId=${evicted?.payload.msgId})`);
|
|
382
413
|
}
|
|
383
|
-
this.pendingReplies.push({ url, payload,
|
|
414
|
+
this.pendingReplies.push({ url, payload, addedAt: now });
|
|
384
415
|
log.warn(`${this.logPrefix} Queued pending reply (queue=${this.pendingReplies.length}, msgId=${payload.msgId})`);
|
|
385
416
|
}
|
|
386
417
|
/**
|
|
@@ -398,27 +429,44 @@ export class WorkBuddyCentrifugeClient {
|
|
|
398
429
|
log.info(`${this.logPrefix} Flushing ${toSend.length} pending reply(ies)`);
|
|
399
430
|
}
|
|
400
431
|
for (const reply of toSend) {
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
432
|
+
// 用当前(可能已刷新的)token;401 时单飞刷新后重试一次
|
|
433
|
+
let refreshedOn401 = false;
|
|
434
|
+
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
435
|
+
const token = this.config.getAccessToken?.() ?? this.config.httpAccessToken ?? '';
|
|
436
|
+
try {
|
|
437
|
+
const res = await fetch(reply.url, {
|
|
438
|
+
method: 'POST',
|
|
439
|
+
headers: {
|
|
440
|
+
'Content-Type': 'application/json',
|
|
441
|
+
Authorization: `Bearer ${token}`,
|
|
442
|
+
},
|
|
443
|
+
body: JSON.stringify(reply.payload),
|
|
444
|
+
signal: AbortSignal.timeout(30_000),
|
|
445
|
+
});
|
|
446
|
+
if (res.status === 401 && !refreshedOn401 && this.config.refreshToken) {
|
|
447
|
+
refreshedOn401 = true;
|
|
448
|
+
try {
|
|
449
|
+
await this.config.refreshToken();
|
|
450
|
+
continue;
|
|
451
|
+
}
|
|
452
|
+
catch {
|
|
453
|
+
/* 刷新失败,落到下面正常判定 */
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
const body = await res.text().catch(() => '');
|
|
457
|
+
if (res.ok) {
|
|
458
|
+
log.info(`${this.logPrefix} Flushed pending reply ok: msgId=${reply.payload.msgId}`);
|
|
459
|
+
}
|
|
460
|
+
else {
|
|
461
|
+
log.error(`${this.logPrefix} Flushed pending reply failed: ${res.status} ${body.substring(0, 200)}`);
|
|
462
|
+
}
|
|
463
|
+
break;
|
|
414
464
|
}
|
|
415
|
-
|
|
416
|
-
log.error(`${this.logPrefix} Flushed pending reply
|
|
465
|
+
catch (err) {
|
|
466
|
+
log.error(`${this.logPrefix} Flushed pending reply error: msgId=${reply.payload.msgId}`, err);
|
|
467
|
+
break;
|
|
417
468
|
}
|
|
418
469
|
}
|
|
419
|
-
catch (err) {
|
|
420
|
-
log.error(`${this.logPrefix} Flushed pending reply error: msgId=${reply.payload.msgId}`, err);
|
|
421
|
-
}
|
|
422
470
|
}
|
|
423
471
|
this.flushing = false;
|
|
424
472
|
}
|