openclaw-sc 5.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +13 -0
- package/INSTALL.md +13 -0
- package/LICENSE +233 -0
- package/README.md +123 -0
- package/SECURITY.md +27 -0
- package/index.js +3479 -0
- package/lib/code-review-shared.js +164 -0
- package/lib/config.js +164 -0
- package/lib/constants.js +438 -0
- package/lib/decomposer.js +896 -0
- package/lib/dialog-recall.js +389 -0
- package/lib/env.js +59 -0
- package/lib/level-rules.js +72 -0
- package/lib/log-manager.js +258 -0
- package/lib/preemption.js +278 -0
- package/lib/priority-calc.js +134 -0
- package/lib/prompt-injection.js +748 -0
- package/lib/route-evidence.js +701 -0
- package/lib/shared-fs.js +244 -0
- package/lib/steward-rules.js +648 -0
- package/lib/task-center.js +602 -0
- package/lib/task-chain.js +713 -0
- package/lib/task-checker.js +317 -0
- package/lib/task-profiles.js +255 -0
- package/lib/tcell.js +411 -0
- package/lib/tool-auto-discover.js +522 -0
- package/lib/tool-enrich-subagent.js +375 -0
- package/lib/tool-handlers.js +178 -0
- package/lib/tool-selector.js +459 -0
- package/openclaw.plugin.json +55 -0
- package/package.json +63 -0
- package/security.js +141 -0
- package/tools/bridge.js +3288 -0
- package/tools/dashboard/tk-dashboard.py +262 -0
- package/tools/hippocampus-multi-search.js +1038 -0
- package/tools/hippocampus-sqlite.js +738 -0
- package/tools/hippocampus-store.js +262 -0
- package/tools/mcp-tools.config.json +653 -0
- package/tools/pipeline-engine.js +667 -0
- package/tools/sidecar/_check_tools.py +34 -0
- package/tools/sidecar/sidecar-server.cjs +1360 -0
- package/tools/sidecar/subagent-runner.cjs +1471 -0
- package/tools/system-tools.js +619 -0
- package/vector/README.md +100 -0
- package/vector/usearch-bridge.js +639 -0
- package/vector/usearch-http.js +74 -0
- package/vector/usearch-serve.js +156 -0
- package/workers/worker.js +1419 -0
package/index.js
ADDED
|
@@ -0,0 +1,3479 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 🦞 sc v5.38.0 — 纯MCP架构,统一配置化
|
|
3
|
+
* Worker池 + GPU加速 + MCP工具集,权限由 tools/mcp-tools.config.json 管控
|
|
4
|
+
* 版本历史详见 CHANGELOG.md
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { Worker } from "worker_threads";
|
|
8
|
+
import { spawn } from "child_process"; // 仅异步spawn
|
|
9
|
+
import { freemem, homedir } from "os";
|
|
10
|
+
import { join, dirname } from "path";
|
|
11
|
+
import { fileURLToPath } from "url";
|
|
12
|
+
import crypto from "crypto";
|
|
13
|
+
import { readFile, writeFile, mkdir, readdir, unlink, stat } from "fs/promises";
|
|
14
|
+
import { statSync, existsSync, readdirSync } from "fs";
|
|
15
|
+
import { validatePath } from "./security.js";
|
|
16
|
+
import {
|
|
17
|
+
PHYSICAL_CORES,
|
|
18
|
+
CORE_RESERVED_IDLE,
|
|
19
|
+
CORE_RESERVED_MIN,
|
|
20
|
+
MIN_WORKERS,
|
|
21
|
+
MAX_WORKERS,
|
|
22
|
+
SCALE_UP_THRESHOLD,
|
|
23
|
+
HEARTBEAT_MS,
|
|
24
|
+
TASK_TIMEOUT_MS,
|
|
25
|
+
IDLE_TERMINATE_MS,
|
|
26
|
+
STATS_CACHE_TTL,
|
|
27
|
+
MEM_CACHE_TTL,
|
|
28
|
+
FILE_EDIT_LOCK_TIMEOUT,
|
|
29
|
+
RECOVERY_COOLDOWN_MS,
|
|
30
|
+
EMERGENCY_MODE_SUPPRESSION_MS,
|
|
31
|
+
RATE_LIMIT_PER_SEC,
|
|
32
|
+
RATE_BLOCK_DURATION_MS,
|
|
33
|
+
MAX_ACTIVE_SPAWNS,
|
|
34
|
+
SPAWN_HISTORY_WINDOW_MS,
|
|
35
|
+
MAX_SPAWN_PER_WINDOW,
|
|
36
|
+
ROUTE_GUARD_VIOLATION_DECAY_MS,
|
|
37
|
+
CACHE_DISABLED,
|
|
38
|
+
TASK_TIMEOUT_MAP,
|
|
39
|
+
GRACE_PER_EXTRA_60S,
|
|
40
|
+
MAX_GRACE_MULTIPLIER,
|
|
41
|
+
SESSION_KEEP_RECENT,
|
|
42
|
+
SESSION_DIR,
|
|
43
|
+
SHARED_DIR,
|
|
44
|
+
WARMED_MODELS_MAX,
|
|
45
|
+
RESTRICTED_TOOLS,
|
|
46
|
+
TOOL_ROUTE_MAP,
|
|
47
|
+
CORE_TOOLS,
|
|
48
|
+
ROUTE_CACHE_TTL_MS,
|
|
49
|
+
ROUTE_CACHE_MAX,
|
|
50
|
+
MAX_WORKER_AGE,
|
|
51
|
+
MAX_WORKER_TASKS,
|
|
52
|
+
ROLLING_RESTART_BATCH_RATIO,
|
|
53
|
+
ROLLING_RESTART_CHECK_INTERVAL_MS,
|
|
54
|
+
WORKER_REPLACEMENT_LOG,
|
|
55
|
+
TASK_CATEGORY,
|
|
56
|
+
TASK_CATEGORY_MAP,
|
|
57
|
+
EXPLORATORY_CHECKPOINT_INTERVAL_MS,
|
|
58
|
+
ROLE_CONFIG,
|
|
59
|
+
TASK_TYPE_ROLE_MAP,
|
|
60
|
+
getInitialRoleDistribution,
|
|
61
|
+
getAdaptiveRoleDistribution,
|
|
62
|
+
PREEMPT_QUEUE_THRESHOLD,
|
|
63
|
+
PREEMPT_CHECK_INTERVAL_MS,
|
|
64
|
+
TRAFFIC_PATTERNS_DIR,
|
|
65
|
+
TRAFFIC_WINDOW_MIN,
|
|
66
|
+
METABOLIC_WEIGHTS,
|
|
67
|
+
METABOLIC_SMOOTH_POINTS,
|
|
68
|
+
METABOLIC_RATE_MIN,
|
|
69
|
+
METABOLIC_RATE_MAX,
|
|
70
|
+
METABOLIC_RATE_ADJUST_STEP,
|
|
71
|
+
METABOLIC_RATE_DEFAULT,
|
|
72
|
+
USER_ACTIVE_TIMEOUT_MS,
|
|
73
|
+
DIALOG_DIR,
|
|
74
|
+
MCP_PORT,
|
|
75
|
+
} from './lib/constants.js';
|
|
76
|
+
|
|
77
|
+
// traffic-predictor.js (Cerebellum/小脑) — 被动扩缩容已够用,不需要预测,已移除
|
|
78
|
+
import {
|
|
79
|
+
ensureSharedDir,
|
|
80
|
+
sanitizeTaskName,
|
|
81
|
+
writeSharedResult,
|
|
82
|
+
readSharedResult,
|
|
83
|
+
cleanupSharedDir,
|
|
84
|
+
cleanupOldSessions,
|
|
85
|
+
cleanupTaskStates,
|
|
86
|
+
ensurePreemptDir,
|
|
87
|
+
writePreemptState,
|
|
88
|
+
readPreemptState,
|
|
89
|
+
clearPreemptState,
|
|
90
|
+
} from './lib/shared-fs.js';
|
|
91
|
+
import {
|
|
92
|
+
handleCoreStats,
|
|
93
|
+
handleCoreImageBatch,
|
|
94
|
+
} from './lib/tool-handlers.js';
|
|
95
|
+
// dialog-recall.js — 仅 bridge.js 使用,index.js 不需要
|
|
96
|
+
// decomposer.js — 仅 bridge.js / worker.js 使用,index.js 不需要
|
|
97
|
+
// pipeline-engine.js — 仅 bridge.js 使用,index.js 不需要
|
|
98
|
+
import { register as registerToolDiscover } from './lib/tool-auto-discover.js';
|
|
99
|
+
// system-tools.js — 仅 bridge.js 使用,index.js 不需要
|
|
100
|
+
// dashboard已删除(管理员2026-05-31要求移除)
|
|
101
|
+
import {
|
|
102
|
+
recordRouteDecision,
|
|
103
|
+
update用Evidence,
|
|
104
|
+
queryRouteDecisions,
|
|
105
|
+
getRouteStats as getRouteEvidenceStats,
|
|
106
|
+
initRouteEvidence,
|
|
107
|
+
} from './lib/route-evidence.js';
|
|
108
|
+
// config.js getDefaultChatModel — 仅 bridge.js 使用,index.js 不需要
|
|
109
|
+
// code-review-shared.js — 仅 bridge.js 使用,index.js 不需要
|
|
110
|
+
import {
|
|
111
|
+
detectChain,
|
|
112
|
+
getChainRoute,
|
|
113
|
+
enrichRouteWithChain,
|
|
114
|
+
executeDetectedChain,
|
|
115
|
+
readChain,
|
|
116
|
+
cleanupChainLogs,
|
|
117
|
+
listChains,
|
|
118
|
+
} from './lib/task-chain.js';
|
|
119
|
+
import { evaluatePreemption, savePreemptState } from './lib/preemption.js';
|
|
120
|
+
import { tcell } from './lib/tcell.js';
|
|
121
|
+
// task-profiles.js core_pickSubagentModel — 仅 bridge.js 使用,index.js 不需要
|
|
122
|
+
import {
|
|
123
|
+
initLogger,
|
|
124
|
+
logWorker,
|
|
125
|
+
logError,
|
|
126
|
+
logAccess,
|
|
127
|
+
logWorkerEvent,
|
|
128
|
+
stopHippocampusFlush,
|
|
129
|
+
getLogger,
|
|
130
|
+
} from './lib/log-manager.js';
|
|
131
|
+
// ⛔ route-audit 已物理删除 (2026-06-13)
|
|
132
|
+
// import { startGoedelMetaWorker, stopGoedelMetaWorker, loadKeywordWeights } from './lib/route-audit.js';
|
|
133
|
+
// import { auditRecentRoutes, startRouteAuditor, stopRouteAuditor } from './lib/route-auditor.js';
|
|
134
|
+
import {
|
|
135
|
+
StewardGuard,
|
|
136
|
+
TOOL_TIERS as STEWARD_TOOL_TIERS,
|
|
137
|
+
sanitizeToolParams,
|
|
138
|
+
buildAutoRedirectTask,
|
|
139
|
+
isAutoRedirectEnabled,
|
|
140
|
+
} from './lib/steward-rules.js';
|
|
141
|
+
import {
|
|
142
|
+
core_createTask,
|
|
143
|
+
core_reportResult,
|
|
144
|
+
core_collectResults,
|
|
145
|
+
saveCheckpoint,
|
|
146
|
+
readCheckpoint,
|
|
147
|
+
cleanupCheckpoints,
|
|
148
|
+
} from './lib/task-center.js';
|
|
149
|
+
import { register as registerEnrichSubagent } from './lib/tool-enrich-subagent.js';
|
|
150
|
+
|
|
151
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
152
|
+
const __dirname = dirname(__filename);
|
|
153
|
+
|
|
154
|
+
// ====== Checkpoint 目录(由 task-center.js 管理)======
|
|
155
|
+
|
|
156
|
+
async function safeReadJson(filePath) {
|
|
157
|
+
try { return JSON.parse(await readFile(filePath, "utf-8")); } catch { return null; }
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// dynamic cores分配:资源调度强度精细平滑控制,取代离散4档模式
|
|
161
|
+
// 资源调度强度高→更多Worker,资源调度强度低→更少Worker(让出CPU给主线程/聊天)
|
|
162
|
+
function getDynamicMaxWorkers() {
|
|
163
|
+
try {
|
|
164
|
+
const meta = getMetabolicRateConfig();
|
|
165
|
+
const stats = pool?.getStats?.();
|
|
166
|
+
if (!stats) return PHYSICAL_CORES - CORE_RESERVED_MIN;
|
|
167
|
+
const queueDepth = stats.queueDepth || 0;
|
|
168
|
+
|
|
169
|
+
// 从资源调度强度平滑插值得到当前Worker上限
|
|
170
|
+
// 资源调度强度0.2→maxWorkers=4, 0.5→8, 0.8→14, 1.0→20
|
|
171
|
+
let maxW = meta.maxWorkers;
|
|
172
|
+
|
|
173
|
+
// 队列深度额外加成:队列深→增加Worker(在资源调度强度基础上临时补强)
|
|
174
|
+
if (queueDepth >= 20) {
|
|
175
|
+
// 深度排队:最多加6个
|
|
176
|
+
maxW = Math.min(PHYSICAL_CORES, maxW + 6);
|
|
177
|
+
} else if (queueDepth >= 10) {
|
|
178
|
+
// 排队:最多加3个
|
|
179
|
+
maxW = Math.min(PHYSICAL_CORES, maxW + 3);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// 上限不超过物理核心数
|
|
183
|
+
return Math.min(maxW, PHYSICAL_CORES);
|
|
184
|
+
} catch {
|
|
185
|
+
return PHYSICAL_CORES - CORE_RESERVED_MIN;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// 根据内存水位 + 资源调度强度平滑插值 + Worker队列深度动态调整缓存TTL
|
|
190
|
+
// ⚠️ 注意:只能调 getMemoryLevel()(直接)和 pool.getStats()(直接),不能调 getCachedMemoryLevel()/getCachedStats()(有缓存会循环依赖)
|
|
191
|
+
function getDynamicCacheTTL() {
|
|
192
|
+
const meta = getMetabolicRateConfig();
|
|
193
|
+
|
|
194
|
+
// 近战斗模式(≥0.9):缓存禁用
|
|
195
|
+
if (meta.cacheDisabled) {
|
|
196
|
+
return { stats: CACHE_DISABLED, mem: CACHE_DISABLED };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const mem = getMemoryLevel();
|
|
200
|
+
const level = mem.level;
|
|
201
|
+
|
|
202
|
+
// 熔断:不缓存
|
|
203
|
+
if (level === 'meltdown') return { stats: CACHE_DISABLED, mem: CACHE_DISABLED };
|
|
204
|
+
|
|
205
|
+
// 根据内存水位得到基础TTL
|
|
206
|
+
let base;
|
|
207
|
+
switch(level) {
|
|
208
|
+
case 'green': base = { stats: 30000, mem: 15000 }; break;
|
|
209
|
+
case 'yellow': base = { stats: 10000, mem: 5000 }; break;
|
|
210
|
+
case 'red': base = { stats: 3000, mem: 2000 }; break;
|
|
211
|
+
default: base = { stats: 10000, mem: 5000 };
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// 资源调度强度连续插值调整:cacheMult 在锚点间平滑变化
|
|
215
|
+
// rest(0.2)→×3.0, balanced(0.5)→×1.0, focus(0.8)→×0.5, battle(1.0)→×0(禁用)
|
|
216
|
+
const cacheMult = meta.cacheMult;
|
|
217
|
+
if (cacheMult !== 1) {
|
|
218
|
+
base = {
|
|
219
|
+
stats: Math.max(1000, Math.round(base.stats * cacheMult)),
|
|
220
|
+
mem: Math.max(500, Math.round(base.mem * cacheMult)),
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// 根据队列深度调整:空闲涨、繁忙降
|
|
225
|
+
// 用 pool.getStats() 直接拿队列深度(跳过缓存绕开循环依赖)
|
|
226
|
+
try {
|
|
227
|
+
const stats = pool?.getStats?.();
|
|
228
|
+
if (stats) {
|
|
229
|
+
const queueDepth = stats.queueDepth || 0;
|
|
230
|
+
|
|
231
|
+
if (queueDepth === 0 && level === 'green') {
|
|
232
|
+
// 空闲:队列空 + 内存充足 → 缓存再加倍
|
|
233
|
+
return { stats: base.stats * 2, mem: base.mem * 2 };
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if (queueDepth > 3 && (level === 'yellow' || level === 'red')) {
|
|
237
|
+
// 繁忙:队列深 + 内存紧张 → 缓存减半(不低于最小值)
|
|
238
|
+
return {
|
|
239
|
+
stats: Math.max(2000, Math.floor(base.stats / 2)),
|
|
240
|
+
mem: Math.max(1000, Math.floor(base.mem / 2)),
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
} catch {
|
|
245
|
+
// 出错时走基础TTL,不中断
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
return base;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
let cachedStats = null;
|
|
252
|
+
let cachedStatsTime = 0;
|
|
253
|
+
|
|
254
|
+
let cachedMemLevel = null;
|
|
255
|
+
let cachedMemTime = 0;
|
|
256
|
+
|
|
257
|
+
// ====== dual-mode状态 ======
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
// ====== ⛽ dashboard进程ID(模块级变量,跨activate/shutdown共享) ======
|
|
262
|
+
let dashboardPid = null;
|
|
263
|
+
|
|
264
|
+
// ====== 🧬 资源调度精细调节系统 ======
|
|
265
|
+
// 资源调度强度 ∈ [0.2, 1.0] 连续值,控制调度密度/缓存强度/Worker数
|
|
266
|
+
// 0.2=休息 0.5=均衡 0.8=专注 1.0=战斗 — 之间线性插值,无离散档位
|
|
267
|
+
let _metabolicRate = METABOLIC_RATE_DEFAULT;
|
|
268
|
+
let _metabolicTimer = null;
|
|
269
|
+
let _nu = 0.5; // 系统活动度 ν ∈ [0.2, 1.0](4参数加权归一化)
|
|
270
|
+
let _nuBias = 0; // 中庸校正偏置 [-0.05, 0.05]
|
|
271
|
+
let _jointLoss = 0; // 当前联合损失值
|
|
272
|
+
|
|
273
|
+
// 用户活跃度缓存
|
|
274
|
+
let _lastUserActive = true;
|
|
275
|
+
let _lastUserCheckTime = 0;
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* 检测用户是否活跃(近20min有消息)
|
|
279
|
+
* 通过检查对话日记文件最后修改时间判断
|
|
280
|
+
* 缓存1min防I/O抖动
|
|
281
|
+
*/
|
|
282
|
+
function checkUserActive() {
|
|
283
|
+
const now = Date.now();
|
|
284
|
+
// 缓存1min
|
|
285
|
+
if (now - _lastUserCheckTime < 60000) {
|
|
286
|
+
return _lastUserActive;
|
|
287
|
+
}
|
|
288
|
+
_lastUserCheckTime = now;
|
|
289
|
+
try {
|
|
290
|
+
// 取今天的对话日记
|
|
291
|
+
const today = new Date();
|
|
292
|
+
const yyyy = today.getFullYear();
|
|
293
|
+
const mm = String(today.getMonth() + 1).padStart(2, '0');
|
|
294
|
+
const dd = String(today.getDate()).padStart(2, '0');
|
|
295
|
+
const dialogPath = join(DIALOG_DIR, `${yyyy}-${mm}-${dd}.md`);
|
|
296
|
+
if (!existsSync(dialogPath)) {
|
|
297
|
+
// 没有今天的日记 → 不活跃
|
|
298
|
+
_lastUserActive = false;
|
|
299
|
+
return false;
|
|
300
|
+
}
|
|
301
|
+
const mtime = statSync(dialogPath).mtimeMs;
|
|
302
|
+
const elapsed = now - mtime;
|
|
303
|
+
_lastUserActive = elapsed < USER_ACTIVE_TIMEOUT_MS;
|
|
304
|
+
return _lastUserActive;
|
|
305
|
+
} catch {
|
|
306
|
+
// 无法读取时保守返回true(不降低资源调度强度)
|
|
307
|
+
return true;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function getMetabolicRate() {
|
|
312
|
+
return _metabolicRate;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function setMetabolicRate(val) {
|
|
316
|
+
const clamped = Math.max(METABOLIC_RATE_MIN, Math.min(METABOLIC_RATE_MAX, parseFloat(val) || METABOLIC_RATE_DEFAULT));
|
|
317
|
+
const rounded = Math.round(clamped * 100) / 100;
|
|
318
|
+
const prev = _metabolicRate;
|
|
319
|
+
_metabolicRate = rounded;
|
|
320
|
+
// TODO: 移除调试日志 console.log(`[sc] 🧬 metabolic rate: ${prev.toFixed(2)} → ${rounded.toFixed(2)}`);
|
|
321
|
+
return { rate: rounded, prev };
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* 线性插值:在 METABOLIC_SMOOTH_POINTS 中找到相邻两点,按比例插值
|
|
326
|
+
* edges界外取端点值
|
|
327
|
+
*/
|
|
328
|
+
function interpolateMetabolicValue(rate, key) {
|
|
329
|
+
const points = METABOLIC_SMOOTH_POINTS;
|
|
330
|
+
if (!points || points.length === 0) return null;
|
|
331
|
+
const sorted = [...points].sort((a, b) => a.rate - b.rate);
|
|
332
|
+
|
|
333
|
+
// 低于下界取第一个点
|
|
334
|
+
if (rate <= sorted[0].rate) return sorted[0][key];
|
|
335
|
+
// 高于上界取最后一个点
|
|
336
|
+
if (rate >= sorted[sorted.length - 1].rate) return sorted[sorted.length - 1][key];
|
|
337
|
+
|
|
338
|
+
// 找到相邻区间
|
|
339
|
+
for (let i = 0; i < sorted.length - 1; i++) {
|
|
340
|
+
if (rate >= sorted[i].rate && rate <= sorted[i + 1].rate) {
|
|
341
|
+
const lo = sorted[i];
|
|
342
|
+
const hi = sorted[i + 1];
|
|
343
|
+
const t = (rate - lo.rate) / (hi.rate - lo.rate); // [0, 1]
|
|
344
|
+
|
|
345
|
+
// cacheDisabled 是布尔值,用阈值判定
|
|
346
|
+
if (key === 'cacheDisabled') {
|
|
347
|
+
// rate ≥ 0.9 才完全禁用缓存(接近战斗模式)
|
|
348
|
+
return rate >= 0.9;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
const loVal = lo[key];
|
|
352
|
+
const hiVal = hi[key];
|
|
353
|
+
|
|
354
|
+
// 数值插值
|
|
355
|
+
if (typeof loVal === 'number' && typeof hiVal === 'number') {
|
|
356
|
+
const val = loVal + (hiVal - loVal) * t;
|
|
357
|
+
// 整数取整,小数保留两位
|
|
358
|
+
if (Number.isInteger(loVal) && Number.isInteger(hiVal)) {
|
|
359
|
+
return Math.round(val);
|
|
360
|
+
}
|
|
361
|
+
return Math.round(val * 100) / 100;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// 其他类型取最近
|
|
365
|
+
return t < 0.5 ? loVal : hiVal;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
return sorted[sorted.length - 1][key];
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* 根据当前资源调度强度返回连续插值配置
|
|
374
|
+
* 取代旧的离散4档模式,任意资源调度强度值在锚点间线性插值
|
|
375
|
+
*/
|
|
376
|
+
function getMetabolicRateConfig() {
|
|
377
|
+
const rate = _metabolicRate;
|
|
378
|
+
const nu = _nu ?? 0.5;
|
|
379
|
+
|
|
380
|
+
return {
|
|
381
|
+
rate,
|
|
382
|
+
nu,
|
|
383
|
+
jointLoss: _jointLoss ?? 0, // 🆕 联合损失值
|
|
384
|
+
label: `smooth:${rate.toFixed(2)}`,
|
|
385
|
+
minWorkers: interpolateMetabolicValue(rate, 'minWorkers'),
|
|
386
|
+
maxWorkers: interpolateMetabolicValue(rate, 'maxWorkers'),
|
|
387
|
+
cacheMult: interpolateMetabolicValue(rate, 'cacheMult'),
|
|
388
|
+
cacheDisabled: interpolateMetabolicValue(rate, 'cacheDisabled'),
|
|
389
|
+
rateMult: interpolateMetabolicValue(rate, 'rateMult'),
|
|
390
|
+
autoWarmup: rate < 0.35, // 低资源调度强度时开启自动warmup
|
|
391
|
+
};
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* 根据工具名和任务级别计算最终超时(毫s)
|
|
396
|
+
* 整合 ν 轴缩放 × 任务复杂度因子
|
|
397
|
+
* ν=0 → ×2(最宽松休息模式),ν=0.5 → ×1.5(均衡),ν=1.0 → ×1.0(最紧缩战斗模式)
|
|
398
|
+
*
|
|
399
|
+
* @param {string} toolName - 工具名(从 TASK_TIMEOUT_MAP 取 kill 基准值)
|
|
400
|
+
* @param {string} [level='L3'] - 任务 L 级别 (L0-L7)
|
|
401
|
+
* @returns {number} 超时毫s数
|
|
402
|
+
*/
|
|
403
|
+
function calcTimeout(toolName, level = 'L3') {
|
|
404
|
+
// 1. 基准超时(s)
|
|
405
|
+
const baseEntry = TASK_TIMEOUT_MAP[toolName] || TASK_TIMEOUT_MAP.default;
|
|
406
|
+
const baseSeconds = baseEntry.kill;
|
|
407
|
+
|
|
408
|
+
// 2. ν 缩放因子:ν=0→×2(最宽松), ν=0.5→×1.5(均衡), ν=1.0→×1.0(最紧缩)
|
|
409
|
+
const meta = getMetabolicRateConfig();
|
|
410
|
+
const nuScale = 2 - meta.nu;
|
|
411
|
+
|
|
412
|
+
// 3. 级别复杂度因子
|
|
413
|
+
const levelNum = parseInt((level || 'L3').replace('L', ''), 10);
|
|
414
|
+
const complexityMult = levelNum <= 1 ? 0.8
|
|
415
|
+
: levelNum <= 3 ? 1.0
|
|
416
|
+
: levelNum <= 5 ? 1.5
|
|
417
|
+
: 2.0;
|
|
418
|
+
|
|
419
|
+
// 4. 计算并兜底
|
|
420
|
+
const timeoutMs = Math.round(baseSeconds * 1000 * nuScale * complexityMult);
|
|
421
|
+
if (!timeoutMs || timeoutMs <= 0 || !Number.isFinite(timeoutMs)) {
|
|
422
|
+
return TASK_TIMEOUT_MAP.default.kill * 1000;
|
|
423
|
+
}
|
|
424
|
+
return timeoutMs;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// ====== 中庸联合损失配置 ======
|
|
428
|
+
const JOINT_LOSS_CONFIG = {
|
|
429
|
+
alpha: 0.4, // metabolic rate偏离"中"的惩罚权重
|
|
430
|
+
beta: 0.3, // 超时偏离平均值的惩罚权重
|
|
431
|
+
gamma: 0.3, // 耦合违反惩罚权重
|
|
432
|
+
threshold: 0.15, // 触发 ν_bias 注入的损失阈值
|
|
433
|
+
biasStep: 0.02, // 每次调节的 ν_bias 微调步长
|
|
434
|
+
biasClamp: 0.05, // ν_bias 限幅范围 [-0.05, 0.05]
|
|
435
|
+
};
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* 计算中庸联合损失 L(η, τ)
|
|
439
|
+
*
|
|
440
|
+
* Loss = α(η-0.5)² + β(τ-τ_avg)² + γ×couplingTerm
|
|
441
|
+
*
|
|
442
|
+
* 三个约束项:
|
|
443
|
+
* 1. η 偏离 0.5 的惩罚 —— 不让metabolic rate偏极端
|
|
444
|
+
* 2. τ 偏离 avgTimeout 的惩罚 —— 不让超时偏极端
|
|
445
|
+
* 3. 耦合项 = |Δη|×|Δτ| —— 两者同时大幅变化则惩罚
|
|
446
|
+
*
|
|
447
|
+
* @param {number} nu - 当前 ν 值
|
|
448
|
+
* @param {number} rate - 当前metabolic rate η
|
|
449
|
+
* @returns {number} 联合损失值 [0, ~1]
|
|
450
|
+
*/
|
|
451
|
+
function calcJointLoss(nu, rate) {
|
|
452
|
+
const cfg = JOINT_LOSS_CONFIG;
|
|
453
|
+
|
|
454
|
+
// 项 1: metabolic rate偏离"中"(0.5)的程度
|
|
455
|
+
const etaDev = (rate - 0.5) ** 2;
|
|
456
|
+
|
|
457
|
+
// 项 2: 超时偏离系统平均超时的程度
|
|
458
|
+
// 从最近60s超时记录估算平均超时偏离
|
|
459
|
+
const recentTimeouts = failContextBuffer.getRecent(60000)
|
|
460
|
+
.filter(e => e.errorCode === 'TIMEOUT');
|
|
461
|
+
const avgTimeoutDev = recentTimeouts.length > 0
|
|
462
|
+
? Math.min(1, recentTimeouts.length / 10)
|
|
463
|
+
: 0;
|
|
464
|
+
|
|
465
|
+
// 项 3: 耦合违反程度 — η↑时τ是否真的↓?
|
|
466
|
+
// 期望超时基数 = (2 - nu),实际超时率若偏高但 ν 偏低,说明耦合失效
|
|
467
|
+
const tauBase = 2 - nu;
|
|
468
|
+
const tauActual = recentTimeouts.length > 0
|
|
469
|
+
? Math.max(0.5, 1 - recentTimeouts.length / 20)
|
|
470
|
+
: tauBase;
|
|
471
|
+
const couplingTerm = (tauBase - tauActual) ** 2;
|
|
472
|
+
|
|
473
|
+
// 综合损失
|
|
474
|
+
const loss = cfg.alpha * etaDev + cfg.beta * avgTimeoutDev + cfg.gamma * couplingTerm;
|
|
475
|
+
return Math.min(1, loss);
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/**
|
|
479
|
+
* 计算各任务类型超时stats(供 core_stats 展示)
|
|
480
|
+
* 从 failContextBuffer 中取最近5min超时记录,按 tool 分组stats
|
|
481
|
+
*
|
|
482
|
+
* @returns {object} 按工具名分组的超时stats { toolName: { count, avgTimeoutMs, timeoutRate }, total: { count, tools } }
|
|
483
|
+
*/
|
|
484
|
+
// 🔧 BUGFIX: _lastMetabolicStep 前置声明
|
|
485
|
+
let _lastMetabolicStep = 0;
|
|
486
|
+
|
|
487
|
+
function computeTimeoutStats() {
|
|
488
|
+
const recent5m = failContextBuffer.getRecent(300000)
|
|
489
|
+
.filter(e => e.errorCode === 'TIMEOUT');
|
|
490
|
+
|
|
491
|
+
// 按工具分组
|
|
492
|
+
const byTool = {};
|
|
493
|
+
for (const entry of recent5m) {
|
|
494
|
+
if (!byTool[entry.tool]) {
|
|
495
|
+
byTool[entry.tool] = { count: 0, tool: entry.tool };
|
|
496
|
+
}
|
|
497
|
+
byTool[entry.tool].count++;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// 计算总超时数
|
|
501
|
+
const totalTimeouts = recent5m.length;
|
|
502
|
+
|
|
503
|
+
// 估算总任务数(从 pool stats获取最近5min的任务分发量)
|
|
504
|
+
// 使用池缓存stats中的 tasksProcessed 差值
|
|
505
|
+
const now = Date.now();
|
|
506
|
+
// 从 failContextBuffer 的 TIMEOUT 以外的记录估算总任务量
|
|
507
|
+
const allRecent = failContextBuffer.getRecent(300000);
|
|
508
|
+
const totalAbnormalEvents = allRecent.length; // ⚠️ 仅失败/异常事件,非总任务数
|
|
509
|
+
const timeoutRate = totalAbnormalEvents > 0 ? +(totalTimeouts / totalAbnormalEvents).toFixed(4) : 0;
|
|
510
|
+
|
|
511
|
+
const _totalTasks = totalAbnormalEvents; // 异常事件数作为总任务数的代理
|
|
512
|
+
return {
|
|
513
|
+
perTool: Object.values(byTool).map(t => ({
|
|
514
|
+
tool: t.tool,
|
|
515
|
+
timeoutCount: t.count,
|
|
516
|
+
timeoutRate: _totalTasks > 0 ? +(t.count / _totalTasks).toFixed(4) : 0,
|
|
517
|
+
})),
|
|
518
|
+
total: {
|
|
519
|
+
timeoutCount: totalTimeouts,
|
|
520
|
+
totalTasks: _totalTasks,
|
|
521
|
+
timeoutRate,
|
|
522
|
+
},
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/**
|
|
527
|
+
* 资源调度强度精细自动调节:基于4个实时负载指标每15s微调±0.05
|
|
528
|
+
* 权重:内存水位(0.40) + 队列深度趋势(0.28) + 心跳失败率(0.17) + 用户活跃(0.15)
|
|
529
|
+
* ν = 4参数加权评分归一化到 [0.2, 1.0]
|
|
530
|
+
*/
|
|
531
|
+
function autoRegulateMetabolicRate() {
|
|
532
|
+
try {
|
|
533
|
+
const mem = getCachedMemoryLevel();
|
|
534
|
+
const stats = getCachedStats();
|
|
535
|
+
|
|
536
|
+
// 1. 内存水位评分 [0,1]:freeGB越高越接近0(降资源调度强度),越低越接近1(升资源调度强度)
|
|
537
|
+
const freeGB = mem.freeGB || 8;
|
|
538
|
+
const memScore = Math.max(0, Math.min(1, 1 - freeGB / 16));
|
|
539
|
+
|
|
540
|
+
// 2. 队列深度趋势 [0,1]:队列深→高资源调度强度
|
|
541
|
+
const qDepth = stats.queueDepth || 0;
|
|
542
|
+
const qScore = Math.max(0, Math.min(1, qDepth / 30));
|
|
543
|
+
|
|
544
|
+
// 3. 心跳失败率 [0,1]:从failContextBuffer中取最近30s的心跳失败
|
|
545
|
+
const recent30s = failContextBuffer.getRecent(60000);
|
|
546
|
+
const hbFails = recent30s.filter(e => e.tool === 'ping' || e.errorCode === 'HEARTBEAT_FAIL').length;
|
|
547
|
+
const hbScore = Math.max(0, Math.min(1, hbFails / 5));
|
|
548
|
+
|
|
549
|
+
// 4. 用户活跃评分 [0,1]
|
|
550
|
+
// 用户在聊天 → userScore低(0) → 降资源调度强度(让出CPU给聊天体验)
|
|
551
|
+
// 用户不在 → userScore高(1) → 升资源调度强度(更激进后台处理)
|
|
552
|
+
const userActive = checkUserActive();
|
|
553
|
+
const userScore = userActive ? 0 : 1;
|
|
554
|
+
|
|
555
|
+
// 加权综合评分 ν [0,1](4参数,总和=1.0,已移除timeout权重)
|
|
556
|
+
const w = METABOLIC_WEIGHTS;
|
|
557
|
+
const weightedScore = memScore * w.memory + qScore * w.queue + hbScore * w.heartbeat + userScore * w.userActive;
|
|
558
|
+
|
|
559
|
+
// ν = 加权评分归一化到 [0.2, 1.0] + 中庸偏置
|
|
560
|
+
const rawNu = METABOLIC_RATE_MIN + weightedScore * (METABOLIC_RATE_MAX - METABOLIC_RATE_MIN);
|
|
561
|
+
_nu = Math.round(Math.max(0, Math.min(1, rawNu + _nuBias)) * 100) / 100;
|
|
562
|
+
|
|
563
|
+
// 目标资源调度强度 = ν(4参数 ν 直接驱动资源调度强度)
|
|
564
|
+
const targetRate = _nu;
|
|
565
|
+
|
|
566
|
+
// 当前与目标的差值,步长不超过±0.05
|
|
567
|
+
const diff = targetRate - _metabolicRate;
|
|
568
|
+
const step = Math.max(-METABOLIC_RATE_ADJUST_STEP, Math.min(METABOLIC_RATE_ADJUST_STEP, diff));
|
|
569
|
+
|
|
570
|
+
if (Math.abs(step) >= 0.01 || step !== _lastMetabolicStep) {
|
|
571
|
+
const newRate = Math.round((_metabolicRate + step) * 100) / 100;
|
|
572
|
+
if (newRate !== _metabolicRate) {
|
|
573
|
+
setMetabolicRate(newRate);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
_lastMetabolicStep = step;
|
|
577
|
+
|
|
578
|
+
// 7. 计算联合损失,注入 ν_bias(中庸校正)
|
|
579
|
+
_jointLoss = calcJointLoss(_nu, _metabolicRate);
|
|
580
|
+
if (_jointLoss > JOINT_LOSS_CONFIG.threshold) {
|
|
581
|
+
// 当metabolic rate偏高(>0.7)且ν偏高(>0.6) → 往低拉;
|
|
582
|
+
// 当metabolic rate偏低(<0.3)且ν偏低(<0.4) → 往高拉
|
|
583
|
+
const correction = (_metabolicRate > 0.7 && _nu > 0.6) ? -1
|
|
584
|
+
: (_metabolicRate < 0.3 && _nu < 0.4) ? 1
|
|
585
|
+
: 0;
|
|
586
|
+
_nuBias += correction * JOINT_LOSS_CONFIG.biasStep;
|
|
587
|
+
_nuBias = Math.max(-JOINT_LOSS_CONFIG.biasClamp, Math.min(JOINT_LOSS_CONFIG.biasClamp, _nuBias));
|
|
588
|
+
}
|
|
589
|
+
} catch (err) {
|
|
590
|
+
console.warn(`[sc] ⚠️ metabolic rate自动调节异常: ${err.message}`);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
/**
|
|
595
|
+
* 启动资源调度自动调节定时器 (每15s)
|
|
596
|
+
*/
|
|
597
|
+
function startMetabolicAutoRegulation() {
|
|
598
|
+
if (_metabolicTimer) clearInterval(_metabolicTimer);
|
|
599
|
+
_metabolicTimer = setInterval(() => autoRegulateMetabolicRate(), 15000);
|
|
600
|
+
// TODO: 移除调试日志 console.log('[sc] 🧬 metabolic rate自动调节started (每15s)');
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* 停止资源调度自动调节定时器
|
|
605
|
+
*/
|
|
606
|
+
function stopMetabolicAutoRegulation() {
|
|
607
|
+
if (_metabolicTimer) {
|
|
608
|
+
clearInterval(_metabolicTimer);
|
|
609
|
+
_metabolicTimer = null;
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
// ====== 双通道痛觉(A-delta 快通道 + C纤维 慢通道)======
|
|
614
|
+
|
|
615
|
+
/**
|
|
616
|
+
* 环形缓冲区:记录失败上下文 {tool, errorCode, paramHash?, timestamp}
|
|
617
|
+
* 容量200,满则覆盖最旧记录
|
|
618
|
+
*/
|
|
619
|
+
class RingBuffer {
|
|
620
|
+
constructor(capacity) {
|
|
621
|
+
this.capacity = capacity;
|
|
622
|
+
this.buffer = new Array(capacity);
|
|
623
|
+
this.cursor = 0;
|
|
624
|
+
this.filled = false;
|
|
625
|
+
}
|
|
626
|
+
push(item) {
|
|
627
|
+
this.buffer[this.cursor] = { ...item, timestamp: item.timestamp || Date.now() };
|
|
628
|
+
this.cursor = (this.cursor + 1) % this.capacity;
|
|
629
|
+
if (!this.filled && this.cursor === 0) this.filled = true;
|
|
630
|
+
}
|
|
631
|
+
/** 返回按时间降序排列的所有记录 */
|
|
632
|
+
getAll() {
|
|
633
|
+
const items = this.filled
|
|
634
|
+
? [...this.buffer.slice(this.cursor), ...this.buffer.slice(0, this.cursor)]
|
|
635
|
+
: this.buffer.slice(0, this.cursor);
|
|
636
|
+
return items.filter(Boolean).sort((a, b) => b.timestamp - a.timestamp);
|
|
637
|
+
}
|
|
638
|
+
/** 返回指定毫s窗口内的记录 */
|
|
639
|
+
getRecent(ms) {
|
|
640
|
+
const cutoff = Date.now() - ms;
|
|
641
|
+
return this.getAll().filter(item => item.timestamp >= cutoff);
|
|
642
|
+
}
|
|
643
|
+
/** 清空 */
|
|
644
|
+
clear() { this.buffer.fill(null); this.cursor = 0; this.filled = false; }
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
// 失败上下文环形缓冲(容量200)
|
|
648
|
+
const failContextBuffer = new RingBuffer(200);
|
|
649
|
+
|
|
650
|
+
// 快速通道(A-delta)诊断缓存
|
|
651
|
+
const fastChannelDiag = {
|
|
652
|
+
pending: [], // {tool, paramHash, firstSeen, lastSeen, count, diagnostic}
|
|
653
|
+
};
|
|
654
|
+
|
|
655
|
+
// 慢速通道(C纤维)状态
|
|
656
|
+
const slowChannelState = {
|
|
657
|
+
lastAggregation: Date.now(),
|
|
658
|
+
timer: null,
|
|
659
|
+
status: 'normal', // 'normal' | 'watch' | 'escalating' | 'critical'
|
|
660
|
+
report: { trend: 'stable', hotTools: [], failureRate: 0 },
|
|
661
|
+
};
|
|
662
|
+
|
|
663
|
+
// 计算参数指纹(工具名+关键参数摘要 -> 定位同一个操作模式)
|
|
664
|
+
function computeParamHash(params) {
|
|
665
|
+
if (!params) return '';
|
|
666
|
+
try {
|
|
667
|
+
const keys = Object.keys(params).sort();
|
|
668
|
+
const parts = keys.map(k => {
|
|
669
|
+
const v = params[k];
|
|
670
|
+
if (typeof v === 'string') return `${k}:${v.slice(0, 20)}`;
|
|
671
|
+
if (Array.isArray(v)) return `${k}:arr(${v.length})`;
|
|
672
|
+
return `${k}:${String(v).slice(0, 20)}`;
|
|
673
|
+
});
|
|
674
|
+
return parts.join('|');
|
|
675
|
+
} catch { return ''; }
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
/**
|
|
679
|
+
* 推送失败上下文到环形缓冲区 + 快速通道诊断
|
|
680
|
+
* @param {string} tool - 工具名
|
|
681
|
+
* @param {string} errorCode - 错误码
|
|
682
|
+
* @param {object} [params] - 调用参数(用于计算 paramHash)
|
|
683
|
+
* @returns {{diagnostic: string|null, channel: string}} - 诊断信息, 'fast' | 'none'
|
|
684
|
+
*/
|
|
685
|
+
function pushFailContext(tool, errorCode, params) {
|
|
686
|
+
const paramHash = computeParamHash(params);
|
|
687
|
+
const entry = { tool, errorCode, paramHash, timestamp: Date.now() };
|
|
688
|
+
failContextBuffer.push(entry);
|
|
689
|
+
|
|
690
|
+
// 快速通道检查:同 tool + 同 errorCode + 同 paramHash 在30s内≥2次 → 输出诊断
|
|
691
|
+
const recent30s = failContextBuffer.getRecent(60000);
|
|
692
|
+
const samePattern = recent30s.filter(
|
|
693
|
+
e => e.tool === tool && e.errorCode === errorCode && e.paramHash === paramHash
|
|
694
|
+
);
|
|
695
|
+
|
|
696
|
+
if (samePattern.length >= 2) {
|
|
697
|
+
const diagnostic = `[sc] ⚡ A-delta快通道诊断: 工具 "${tool}" (${errorCode}) 在60s内连续失败${samePattern.length}次,参数模式="${paramHash.slice(0, 40)}"。建议: 检查该工具参数是否正确,或考虑调整调用方式。`;
|
|
698
|
+
fastChannelDiag.pending.push({
|
|
699
|
+
tool, paramHash, firstSeen: samePattern[samePattern.length - 1].timestamp,
|
|
700
|
+
lastSeen: Date.now(), count: samePattern.length, diagnostic,
|
|
701
|
+
});
|
|
702
|
+
// 快速通道只保留最近10诊断
|
|
703
|
+
if (fastChannelDiag.pending.length > 10) fastChannelDiag.pending.shift();
|
|
704
|
+
console.warn(diagnostic);
|
|
705
|
+
return { diagnostic, channel: 'fast' };
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
return { diagnostic: null, channel: 'none' };
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
/**
|
|
712
|
+
* 慢速通道(C纤维)聚合分析:检测失败率趋势
|
|
713
|
+
* @param {number} [windowMs] - 分析窗口(默认5min)
|
|
714
|
+
* @returns {{shouldDegrade: boolean, report: object}}
|
|
715
|
+
*/
|
|
716
|
+
function slowChannelAggregate(windowMs) {
|
|
717
|
+
windowMs = windowMs || 5 * 60 * 1000;
|
|
718
|
+
const recent = failContextBuffer.getRecent(windowMs);
|
|
719
|
+
|
|
720
|
+
if (recent.length === 0) {
|
|
721
|
+
slowChannelState.status = 'normal';
|
|
722
|
+
slowChannelState.report = { trend: 'stable', hotTools: [], failureRate: 0, totalFails: 0, windowSec: windowMs / 1000 };
|
|
723
|
+
return { shouldDegrade: false, report: slowChannelState.report };
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
// 按工具分组计数
|
|
727
|
+
const toolCounts = {};
|
|
728
|
+
for (const e of recent) {
|
|
729
|
+
toolCounts[e.tool] = (toolCounts[e.tool] || 0) + 1;
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
// 按错误码分组
|
|
733
|
+
const errorCounts = {};
|
|
734
|
+
for (const e of recent) {
|
|
735
|
+
const key = `${e.tool}:${e.errorCode}`;
|
|
736
|
+
errorCounts[key] = (errorCounts[key] || 0) + 1;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
const totalFails = recent.length;
|
|
740
|
+
const hotTools = Object.entries(toolCounts)
|
|
741
|
+
.sort((a, b) => b[1] - a[1])
|
|
742
|
+
.slice(0, 5)
|
|
743
|
+
.map(([tool, count]) => ({ tool, count, share: (count / totalFails * 100).toFixed(1) + '%' }));
|
|
744
|
+
|
|
745
|
+
const failureRate = totalFails / (windowMs / 1000); // 次/s
|
|
746
|
+
|
|
747
|
+
// 判断趋势等级
|
|
748
|
+
let status = 'normal';
|
|
749
|
+
let shouldDegrade = false;
|
|
750
|
+
|
|
751
|
+
if (failureRate > 0.3 || totalFails >= 15) {
|
|
752
|
+
// 失败率>0.3次/s 或 5min内>=15次失败 → 紧急
|
|
753
|
+
status = 'critical';
|
|
754
|
+
shouldDegrade = true;
|
|
755
|
+
} else if (failureRate > 0.15 || totalFails >= 8) {
|
|
756
|
+
// 失败率>0.15次/s 或 >=8次 → 升级中
|
|
757
|
+
status = 'escalating';
|
|
758
|
+
} else if (failureRate > 0.05 || totalFails >= 3) {
|
|
759
|
+
// 低水平但持续失败 → 观察
|
|
760
|
+
status = 'watch';
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
slowChannelState.status = status;
|
|
764
|
+
slowChannelState.report = {
|
|
765
|
+
trend: status,
|
|
766
|
+
hotTools,
|
|
767
|
+
failureRate: parseFloat(failureRate.toFixed(4)),
|
|
768
|
+
totalFails,
|
|
769
|
+
windowSec: windowMs / 1000,
|
|
770
|
+
errorBreakdown: errorCounts,
|
|
771
|
+
};
|
|
772
|
+
|
|
773
|
+
if (status !== 'normal') {
|
|
774
|
+
const msg = `[sc] 🔬 慢通道聚合: ${totalFails}次失败/${(windowMs/1000).toFixed(0)}s,率=${failureRate.toFixed(4)}次/s,状态=${status}` +
|
|
775
|
+
`,热点工具: ${hotTools.map(t => `${t.tool}(${t.count}次)`).join(', ')}`;
|
|
776
|
+
if (shouldDegrade) {
|
|
777
|
+
console.warn(`[sc] 🚨 慢通道判定: 失败率飙升,建议降级 - ${msg}`);
|
|
778
|
+
} else {
|
|
779
|
+
console.warn(`[sc] ⚠️ 慢通道告警: ${msg}`);
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
return { shouldDegrade, report: slowChannelState.report };
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
/**
|
|
787
|
+
* 获取双通道状态摘要(供 core_stats 等工具展示)
|
|
788
|
+
*/
|
|
789
|
+
function getDualChannelStatus() {
|
|
790
|
+
const bufferStatus = failContextBuffer.getAll().slice(0, 5).map(e => ({
|
|
791
|
+
tool: e.tool,
|
|
792
|
+
errorCode: e.errorCode,
|
|
793
|
+
ago: ((Date.now() - e.timestamp) / 1000).toFixed(0) + 's',
|
|
794
|
+
}));
|
|
795
|
+
return {
|
|
796
|
+
fastChannel: {
|
|
797
|
+
activeDiagnostics: fastChannelDiag.pending.length,
|
|
798
|
+
recent: fastChannelDiag.pending.slice(-3).map(d => ({
|
|
799
|
+
tool: d.tool, count: d.count, paramHash: d.paramHash.slice(0, 30),
|
|
800
|
+
ago: ((Date.now() - d.lastSeen) / 1000).toFixed(0) + 's',
|
|
801
|
+
})),
|
|
802
|
+
},
|
|
803
|
+
slowChannel: {
|
|
804
|
+
status: slowChannelState.status,
|
|
805
|
+
trend: slowChannelState.report.trend,
|
|
806
|
+
failureRate: slowChannelState.report.failureRate,
|
|
807
|
+
totalFails: slowChannelState.report.totalFails,
|
|
808
|
+
hotTools: slowChannelState.report.hotTools,
|
|
809
|
+
},
|
|
810
|
+
bufferRecent: bufferStatus,
|
|
811
|
+
};
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
// ====== 内存水位 ======
|
|
819
|
+
function getMemoryLevel() {
|
|
820
|
+
const freeGB = freemem() / 1024 / 1024 / 1024;
|
|
821
|
+
const heapUsedMB = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
|
|
822
|
+
if (freeGB > 8) return { level: 'green', freeGB, heapUsedMB, action: 'normal' };
|
|
823
|
+
if (freeGB >= 4) return { level: 'yellow', freeGB, heapUsedMB, action: 'reduce' };
|
|
824
|
+
if (freeGB >= 2) return { level: 'red', freeGB, heapUsedMB, action: 'block_subagent' };
|
|
825
|
+
return { level: 'meltdown', freeGB, heapUsedMB, action: 'block_all' };
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
// ====== 双树路由最后结果(供 before_tool_call 矛盾检测)======
|
|
829
|
+
const lastRouteTask = { recommendedTool: null, confidence: 0, bigTreeResult: null, timestamp: 0 };
|
|
830
|
+
// ====== enrichment layer:给任务描述加工具推荐说明 ======
|
|
831
|
+
/**
|
|
832
|
+
* 我(主agent)调 core_routeTask 后复制 enrichedTask 到 sessions_spawn 的 task 参数
|
|
833
|
+
* 这样子agent拿到的prompt自带工具推荐,不用自己瞎猜
|
|
834
|
+
*
|
|
835
|
+
* 注:tool-enrich-subagent.js 里也有一个同名的 buildEnrichedTask,
|
|
836
|
+
* 那个版本更完整支持结构化推荐列表和话术enrichment layer格式。本函数是精简版,
|
|
837
|
+
* 只接收单个 decision 字符串。两个互不冲突,各有各的调用链。
|
|
838
|
+
*/
|
|
839
|
+
function buildEnrichedTask(taskDesc, decision, confidence, toolNote) {
|
|
840
|
+
if (!taskDesc || !decision) return taskDesc;
|
|
841
|
+
const confidencePct = (confidence * 100).toFixed(0);
|
|
842
|
+
const noteClean = toolNote ? toolNote.replace(/[🔴🟢🟡🛡️⚡🧠🌊📌✅❌🔥💡🫡🤡💚🦞🏛️🧬💾🔐🧠🖼️🔧🩺⚙️🌡️🚨ℹ️⚠️☀️]/g, '').trim().substring(0, 60) : '';
|
|
843
|
+
const noteSuffix = noteClean ? `(${noteClean})` : '';
|
|
844
|
+
return `${taskDesc}\n\n📌 推荐工具:${decision}(置信度${confidencePct}%)${noteSuffix}`;
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
// ====== 工具调用计数器 ======
|
|
848
|
+
let toolCallCounter = 0;
|
|
849
|
+
const TOOL_HISTORY_MAX = 15;
|
|
850
|
+
|
|
851
|
+
// ====== 工具速率限制(BUG-9) ======
|
|
852
|
+
const toolCallRate = {};
|
|
853
|
+
const rateBlockedUntil = {};
|
|
854
|
+
|
|
855
|
+
// ====== 工具速率限制检查函数(BUG-9) ======
|
|
856
|
+
/**
|
|
857
|
+
* 检查指定工具是否超过速率限制(>RATE_LIMIT_PER_SEC次/s)
|
|
858
|
+
* 超过则熔断RATE_BLOCK_DURATION_MS毫s
|
|
859
|
+
* @param {string} toolName - 工具名
|
|
860
|
+
* @param {boolean} [skip=false] - 为true时跳过限流检查
|
|
861
|
+
* @throws {Error} 如果被限流
|
|
862
|
+
*/
|
|
863
|
+
function checkRateLimit(toolName) {
|
|
864
|
+
const now = Date.now();
|
|
865
|
+
|
|
866
|
+
// 检查是否正处于熔断期
|
|
867
|
+
const blockedUntil = rateBlockedUntil[toolName];
|
|
868
|
+
if (blockedUntil && now < blockedUntil) {
|
|
869
|
+
const remainingSec = Math.ceil((blockedUntil - now) / 1000);
|
|
870
|
+
throw new Error(`[ratelimit] 工具 ${toolName} 被限流中(还差 ${remainingSec} s自动恢复),请稍后retry`);
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
// 熔断期已过,清理状态
|
|
874
|
+
if (blockedUntil && now >= blockedUntil) {
|
|
875
|
+
delete rateBlockedUntil[toolName];
|
|
876
|
+
delete toolCallRate[toolName];
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
// 初始化调用记录
|
|
880
|
+
if (!toolCallRate[toolName]) {
|
|
881
|
+
toolCallRate[toolName] = [];
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
// 清理1s窗口外的记录
|
|
885
|
+
const windowStart = now - 1000;
|
|
886
|
+
toolCallRate[toolName] = toolCallRate[toolName].filter(t => t > windowStart);
|
|
887
|
+
|
|
888
|
+
// 资源调度强度精细平滑集成:rateMult在锚点间线性插值
|
|
889
|
+
// rest(0.2)→×0.5, balanced(0.5)→×1.0, focus(0.8)→×1.5, battle(1.0)→×2.0
|
|
890
|
+
const meta = getMetabolicRateConfig();
|
|
891
|
+
const effectiveRateLimit = Math.max(1, Math.round(RATE_LIMIT_PER_SEC * meta.rateMult));
|
|
892
|
+
|
|
893
|
+
// 超限 → 熔断(熔断时间随资源调度强度平滑变化)
|
|
894
|
+
// 资源调度强度越高→熔断时间越短(对高负载容忍更高)
|
|
895
|
+
if (toolCallRate[toolName].length >= effectiveRateLimit) {
|
|
896
|
+
// 基于资源调度强度的平滑熔断时间系数:资源调度强度0.2→×2.0, 0.5→×1.0, 0.8→×0.67, 1.0→×0.5
|
|
897
|
+
const blockMult = Math.max(0.5, Math.min(2.0, 2.0 - meta.rate * 1.5));
|
|
898
|
+
const blockDuration = Math.max(5000, Math.round(RATE_BLOCK_DURATION_MS * blockMult));
|
|
899
|
+
rateBlockedUntil[toolName] = now + blockDuration;
|
|
900
|
+
throw new Error(`[ratelimit] 工具 ${toolName} 调用过频(>${effectiveRateLimit}次/s),已熔断 ${blockDuration/1000} s`);
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
// 记录本次调用
|
|
904
|
+
toolCallRate[toolName].push(now);
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
// ====== 子agent spawn 计数(BUG-8) ======
|
|
908
|
+
const activeSpawns = new Map(); // toolCallTime → toolName 记录
|
|
909
|
+
let activeSpawnCount = 0;
|
|
910
|
+
|
|
911
|
+
// ====== 基于sessions.json的活跃子agent计数(带缓存)======
|
|
912
|
+
let cachedActiveSubagentCount = -1;
|
|
913
|
+
let cachedActiveSubagentTime = 0;
|
|
914
|
+
const ACTIVE_SUBAGENT_CACHE_TTL_MS = 2000; // 2s缓存
|
|
915
|
+
|
|
916
|
+
/**
|
|
917
|
+
* 从 sessions.json 读取真实活跃子agent数
|
|
918
|
+
* @returns {Promise<number>} 活跃子agent数, -1表示无法读取
|
|
919
|
+
*/
|
|
920
|
+
async function countActiveSubagentsFromFile() {
|
|
921
|
+
const now = Date.now();
|
|
922
|
+
if (cachedActiveSubagentCount >= 0 && (now - cachedActiveSubagentTime) < ACTIVE_SUBAGENT_CACHE_TTL_MS) {
|
|
923
|
+
return cachedActiveSubagentCount;
|
|
924
|
+
}
|
|
925
|
+
try {
|
|
926
|
+
const { readFile } = await import('fs/promises');
|
|
927
|
+
// 先尝试新版 sessions/ 目录下的 sessions.json,再回退旧版 sessions.json
|
|
928
|
+
// 设计原因:OpenClaw 新版将 session 数据从根目录 sessions.json 移入 sessions/sessions.json
|
|
929
|
+
const SESSION_PATH_NEW = join(SESSION_DIR, 'sessions.json');
|
|
930
|
+
const SESSION_PATH_OLD = SESSION_DIR + '.json';
|
|
931
|
+
const raw = await readFile(existsSync(SESSION_PATH_NEW) ? SESSION_PATH_NEW : SESSION_PATH_OLD, 'utf-8');
|
|
932
|
+
const data = JSON.parse(raw);
|
|
933
|
+
let count = 0;
|
|
934
|
+
for (const key of Object.keys(data)) {
|
|
935
|
+
if (key.includes(':subagent:')) count++;
|
|
936
|
+
}
|
|
937
|
+
cachedActiveSubagentCount = count;
|
|
938
|
+
cachedActiveSubagentTime = now;
|
|
939
|
+
return count;
|
|
940
|
+
} catch {
|
|
941
|
+
// 文件不存在或解析失败 → 不清洗缓存,降级为无法计数
|
|
942
|
+
cachedActiveSubagentCount = -1;
|
|
943
|
+
return -1;
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
// ====== 工具路由守卫 ======
|
|
948
|
+
// 主agent不应直接调用的工具列表(应由子agent执行)
|
|
949
|
+
// TOOL_ROUTE_MAP imported from ./lib/constants.js
|
|
950
|
+
|
|
951
|
+
// 通行证系统已删除(v5.38.0)—— StewardGuard autoRedirect 替代了通行证机制
|
|
952
|
+
// routeGuardState 只保留计数器,不需要 passes Map
|
|
953
|
+
const routeGuardState = {
|
|
954
|
+
consecutiveViolations: 0,
|
|
955
|
+
lastViolationTool: null,
|
|
956
|
+
violationCount: {},
|
|
957
|
+
totalInterceptions: 0,
|
|
958
|
+
lastViolationTime: 0,
|
|
959
|
+
};
|
|
960
|
+
|
|
961
|
+
// ====== 子agent监控状态 ======
|
|
962
|
+
// 🧠 设计决策:子agent监控间隔90s。不小于60s避免每次监控
|
|
963
|
+
// 都挤占主线程(读sessions.json是同步IO),不大于120s以免
|
|
964
|
+
// 卡死子agent太久才发现。卡死阈值120s给对方留30s缓冲。
|
|
965
|
+
// 自己间隔90s意味着卡死30s后就能检测到卡了2min以上。
|
|
966
|
+
let monitorState = {
|
|
967
|
+
active: false,
|
|
968
|
+
timer: null,
|
|
969
|
+
intervalSeconds: 90,
|
|
970
|
+
maxStalledSeconds: 120,
|
|
971
|
+
autoKill: false,
|
|
972
|
+
alertOnly: true,
|
|
973
|
+
lastSnapshot: null,
|
|
974
|
+
stalledHistory: new Map(), // sessionId → { firstStalledAt, alertSent }
|
|
975
|
+
};
|
|
976
|
+
|
|
977
|
+
// ====== 监控通知路径(杉哥通知文件)======
|
|
978
|
+
// 让主agent在收到子agent完成事件后能读到异步告警
|
|
979
|
+
const MONITOR_ALERT_FILE = join(SHARED_DIR, 'monitor-alert.json');
|
|
980
|
+
// 🔧 BUGFIX: 从DIALOG_DIR移出防污染
|
|
981
|
+
const MONITOR_NOTIFY_LOG = join(SHARED_DIR, 'monitor-notifications.md');
|
|
982
|
+
// 已通知过的告警sessionId集合(防跨轮次重复通知)
|
|
983
|
+
const notifiedAlertIds = new Set();
|
|
984
|
+
|
|
985
|
+
function getMonitorState() {
|
|
986
|
+
return {
|
|
987
|
+
active: monitorState.active,
|
|
988
|
+
intervalSeconds: monitorState.intervalSeconds,
|
|
989
|
+
maxStalledSeconds: monitorState.maxStalledSeconds,
|
|
990
|
+
autoKill: monitorState.autoKill,
|
|
991
|
+
alertOnly: monitorState.alertOnly,
|
|
992
|
+
trackedSessions: monitorState.lastSnapshot?.subagents?.length || 0,
|
|
993
|
+
};
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
// ====== 任务类型差异化超时阈值 ======
|
|
997
|
+
// TASK_TIMEOUT_MAP imported from ./lib/constants.js
|
|
998
|
+
|
|
999
|
+
// ====== 阶梯式宽限常量 ======
|
|
1000
|
+
// GRACE_PER_EXTRA_60S, MAX_GRACE_MULTIPLIER imported from ./lib/constants.js
|
|
1001
|
+
|
|
1002
|
+
// ====== 路由缓存自适应加权常量 ======
|
|
1003
|
+
const SYNAPSE_HALF_LIFE_MS = 3600000; // 时间衰减半周期1小时
|
|
1004
|
+
const SYNAPSE_WEIGHT_HOT_THRESHOLD = 50; // 热点阈值:weight>50得2倍TTL
|
|
1005
|
+
|
|
1006
|
+
// ====== 路由热缓存(路由缓存自适应加权版) ======
|
|
1007
|
+
const routeCache = new Map();
|
|
1008
|
+
// ROUTE_CACHE_TTL_MS, ROUTE_CACHE_MAX imported from ./lib/constants.js
|
|
1009
|
+
|
|
1010
|
+
// ====== 缓存加权权重计算 ======
|
|
1011
|
+
function calculateSynapticWeight(entry) {
|
|
1012
|
+
const elapsed = Date.now() - entry.lastAccess;
|
|
1013
|
+
const timeDecay = Math.exp(-elapsed / SYNAPSE_HALF_LIFE_MS * Math.LN2);
|
|
1014
|
+
return Math.max(1, Math.round(entry.accessCount * (1 + timeDecay)));
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
function updateEntryWeight(entry) {
|
|
1018
|
+
const prevWeight = entry.weight;
|
|
1019
|
+
entry.weight = calculateSynapticWeight(entry);
|
|
1020
|
+
if (entry.weight > prevWeight && prevWeight >= 1) {
|
|
1021
|
+
entry.hot = true; // 权重持续增长=hot path
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
function evictLowestWeightEntry() {
|
|
1026
|
+
let lowestWeight = Infinity;
|
|
1027
|
+
let lowestKey = undefined;
|
|
1028
|
+
for (const [key, entry] of routeCache) {
|
|
1029
|
+
updateEntryWeight(entry);
|
|
1030
|
+
if (entry.weight < lowestWeight) {
|
|
1031
|
+
lowestWeight = entry.weight;
|
|
1032
|
+
lowestKey = key;
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
if (lowestKey !== undefined) routeCache.delete(lowestKey);
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
// ====== 路由缓存辅助函数(路由缓存自适应加权版) ======
|
|
1039
|
+
function routeCacheSet(key, value) {
|
|
1040
|
+
const existing = routeCache.get(key);
|
|
1041
|
+
if (existing) {
|
|
1042
|
+
// 更新已有目:保留访问历史和权重
|
|
1043
|
+
existing.result = value.result;
|
|
1044
|
+
existing.timestamp = Date.now();
|
|
1045
|
+
existing.accessCount++;
|
|
1046
|
+
existing.lastAccess = Date.now();
|
|
1047
|
+
updateEntryWeight(existing);
|
|
1048
|
+
} else {
|
|
1049
|
+
// 新目:初始化权重
|
|
1050
|
+
const newEntry = {
|
|
1051
|
+
result: value.result,
|
|
1052
|
+
timestamp: Date.now(),
|
|
1053
|
+
accessCount: 1,
|
|
1054
|
+
lastAccess: Date.now(),
|
|
1055
|
+
weight: 1,
|
|
1056
|
+
hot: false
|
|
1057
|
+
};
|
|
1058
|
+
routeCache.set(key, newEntry);
|
|
1059
|
+
}
|
|
1060
|
+
// 淘汰:按最低权重驱逐(而非LRU插入顺序)
|
|
1061
|
+
if (routeCache.size > ROUTE_CACHE_MAX) {
|
|
1062
|
+
evictLowestWeightEntry();
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
// 提取公共函数:创建大树异步校对Promise
|
|
1067
|
+
function createBigTreePromise(quickResult, taskDesc) {
|
|
1068
|
+
lastRouteTask.recommendedTool = quickResult.tool;
|
|
1069
|
+
lastRouteTask.confidence = quickResult.confidence;
|
|
1070
|
+
lastRouteTask.bigTreeResult = null;
|
|
1071
|
+
lastRouteTask.timestamp = Date.now();
|
|
1072
|
+
|
|
1073
|
+
return Promise.all([
|
|
1074
|
+
pool.exec({ type: 'route-system', text: taskDesc }, 'low').catch(() => null),
|
|
1075
|
+
pool.exec({ type: 'route-intent', text: taskDesc }, 'low').catch(() => null),
|
|
1076
|
+
pool.exec({ type: 'route-capability', text: taskDesc }, 'low').catch(() => null),
|
|
1077
|
+
]).then(([system, intent, capability]) => {
|
|
1078
|
+
return pool.exec({
|
|
1079
|
+
type: 'route-strategy',
|
|
1080
|
+
quick: quickResult,
|
|
1081
|
+
system: system || { loadLevel: 'green', details: {} },
|
|
1082
|
+
intent: intent || { taskType: '查询类', intent: '', scope: 'local', dangerDetected: false, constraints: [] },
|
|
1083
|
+
capability: capability || { directTools: [], compoundTools: [], fallback: 'subagent', gaps: [] },
|
|
1084
|
+
}, 'low').catch(() => null);
|
|
1085
|
+
}).then((result) => {
|
|
1086
|
+
lastRouteTask.bigTreeResult = result;
|
|
1087
|
+
if (result && result.strategy === 'block') {
|
|
1088
|
+
// TODO: 移除调试日志 console.log(`[sc] 大树校对发现冲突,建议拦截: recommended=${quickResult.tool} risk=${result.risk}`);
|
|
1089
|
+
}
|
|
1090
|
+
}).catch((err) => { console.warn('[sc] 大树异步校对失败:', err?.message); });
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
// SHARED_DIR, SESSION_DIR, SESSION_KEEP_RECENT imported from ./lib/constants.js
|
|
1094
|
+
// ensureSharedDir, sanitizeTaskName, writeSharedResult, readSharedResult,
|
|
1095
|
+
// cleanupSharedDir, cleanupOldSessions - all imported from ./lib/shared-fs.js
|
|
1096
|
+
|
|
1097
|
+
// ====== 紧急停止 ======
|
|
1098
|
+
async function cpuAbort() {
|
|
1099
|
+
console.warn('[sc] 🚨 紧急停止: 终止所有Worker');
|
|
1100
|
+
|
|
1101
|
+
// 1. 设关闭标志
|
|
1102
|
+
isShuttingDown = true;
|
|
1103
|
+
// 2. 终止所有Worker线程
|
|
1104
|
+
for (const entry of workers) {
|
|
1105
|
+
try {
|
|
1106
|
+
if (entry.worker && !entry.terminating) {
|
|
1107
|
+
entry.terminating = true;
|
|
1108
|
+
await entry.worker.terminate();
|
|
1109
|
+
}
|
|
1110
|
+
} catch {}
|
|
1111
|
+
}
|
|
1112
|
+
workers.length = 0;
|
|
1113
|
+
// 3. 清空队列
|
|
1114
|
+
taskQueues.high = [];
|
|
1115
|
+
taskQueues.normal = [];
|
|
1116
|
+
taskQueues.low = [];
|
|
1117
|
+
// 4. 拒绝所有 pendingJobs
|
|
1118
|
+
for (const [jobId, job] of pendingJobs) {
|
|
1119
|
+
try { job.reject(new Error('[sc] 🚨 被用户手动终止')); } catch {}
|
|
1120
|
+
}
|
|
1121
|
+
pendingJobs.clear();
|
|
1122
|
+
// 5. 关闭心跳和伸缩定时器
|
|
1123
|
+
if (heartbeatTimer) clearInterval(heartbeatTimer);
|
|
1124
|
+
if (scaleTimer) clearInterval(scaleTimer);
|
|
1125
|
+
// 6. 清理共享文件
|
|
1126
|
+
try {
|
|
1127
|
+
const { readdir, unlink } = await import('fs/promises');
|
|
1128
|
+
const files = await readdir(SHARED_DIR).catch(() => []);
|
|
1129
|
+
for (const f of files) {
|
|
1130
|
+
await unlink(join(SHARED_DIR, f)).catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
1131
|
+
}
|
|
1132
|
+
} catch {}
|
|
1133
|
+
// 7. 清理 suppressionTimer
|
|
1134
|
+
|
|
1135
|
+
// 8. 清掉路由缓存,防矛盾检测拿到脏数据
|
|
1136
|
+
lastRouteTask.timestamp = 0;
|
|
1137
|
+
// 9. 恢复Worker池(杀完再重建,不是永久死亡)
|
|
1138
|
+
isShuttingDown = false;
|
|
1139
|
+
try {
|
|
1140
|
+
pool.restart(MIN_WORKERS);
|
|
1141
|
+
readyPromise = Promise.resolve(true);
|
|
1142
|
+
warmupDone = false;
|
|
1143
|
+
// 异步warmup模型缓存
|
|
1144
|
+
pool.warmup().catch(() => {});
|
|
1145
|
+
// TODO: 移除调试日志 console.log(`[sc] ✅ cpuAbort 后恢复: ${MIN_WORKERS} 个 Worker started`);
|
|
1146
|
+
} catch (e) {
|
|
1147
|
+
console.error('[sc] ❌ cpuAbort 后恢复失败:', e.message);
|
|
1148
|
+
}
|
|
1149
|
+
console.warn('[sc] 🚨 cpuAbort 完成');
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
// ====== 全局 ID 生成器 ======
|
|
1153
|
+
let nextWorkerId = 0;
|
|
1154
|
+
function genWorkerId() { return nextWorkerId++; }
|
|
1155
|
+
|
|
1156
|
+
// ====== 状态 ======
|
|
1157
|
+
const workers = [];
|
|
1158
|
+
const pendingJobs = new Map();
|
|
1159
|
+
const taskQueues = { high: [], normal: [], low: [] };
|
|
1160
|
+
|
|
1161
|
+
// ====== 队列老化阈值: 低优先级任务排队超过 15 s自动提升一级,防止饿死 ======
|
|
1162
|
+
const AGING_THRESHOLD_MS = 15000;
|
|
1163
|
+
|
|
1164
|
+
let isShuttingDown = false;
|
|
1165
|
+
let warmupDone = false;
|
|
1166
|
+
// ====== 模型warmup LRU 缓存(防膨胀) ======
|
|
1167
|
+
// key=modelId, value={insertedAt, lastAccess}
|
|
1168
|
+
// WARMED_MODELS_MAX imported from ./lib/constants.js
|
|
1169
|
+
const warmedModels = new Map();
|
|
1170
|
+
|
|
1171
|
+
/** 检查模型是否已warmup,并标记为最近使用(LRU touch) */
|
|
1172
|
+
function isModelWarmed(modelId) {
|
|
1173
|
+
if (warmedModels.has(modelId)) {
|
|
1174
|
+
// Touch: 更新最后访问时间,确保 LRU 淘汰准确
|
|
1175
|
+
warmedModels.set(modelId, {
|
|
1176
|
+
insertedAt: warmedModels.get(modelId).insertedAt,
|
|
1177
|
+
lastAccess: Date.now()
|
|
1178
|
+
});
|
|
1179
|
+
return true;
|
|
1180
|
+
}
|
|
1181
|
+
return false;
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
/** 标记模型已warmup,超过上限时淘汰最近最少使用的目 */
|
|
1185
|
+
function markModelWarmed(modelId) {
|
|
1186
|
+
const now = Date.now();
|
|
1187
|
+
warmedModels.set(modelId, { insertedAt: now, lastAccess: now });
|
|
1188
|
+
if (warmedModels.size > WARMED_MODELS_MAX) {
|
|
1189
|
+
// LRU淘汰: 找最近最少访问(lastAccess 最小的)
|
|
1190
|
+
let lruKey = null;
|
|
1191
|
+
let lruTime = Infinity;
|
|
1192
|
+
for (const [k, v] of warmedModels) {
|
|
1193
|
+
if (v.lastAccess < lruTime) {
|
|
1194
|
+
lruTime = v.lastAccess;
|
|
1195
|
+
lruKey = k;
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
if (lruKey) warmedModels.delete(lruKey);
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
let heartbeatTimer = null;
|
|
1202
|
+
let scaleTimer = null;
|
|
1203
|
+
|
|
1204
|
+
let readyPromise;
|
|
1205
|
+
|
|
1206
|
+
// ====== 子agent监控核心逻辑 ======
|
|
1207
|
+
|
|
1208
|
+
/**
|
|
1209
|
+
* 从 sessions.json 读取所有会话数据
|
|
1210
|
+
*/
|
|
1211
|
+
/**
|
|
1212
|
+
* 读取 sessions.json — 支持新旧两种路径格式
|
|
1213
|
+
*
|
|
1214
|
+
* OpenClaw 新版将 sessions.json 移入了 sessions/ 目录下
|
|
1215
|
+
* (~/.openclaw/agents/main/sessions/sessions.json),
|
|
1216
|
+
* 旧版在 sessions 目录同级(~/.openclaw/agents/main/sessions.json)。
|
|
1217
|
+
* 此处优先读新版路径,失败时自动降级到旧版路径,
|
|
1218
|
+
* 避免后续 OpenClaw 版本升级/回滚导致路径失效。
|
|
1219
|
+
*/
|
|
1220
|
+
async function readSessionsJson() {
|
|
1221
|
+
const { readFile, access } = await import('fs/promises');
|
|
1222
|
+
const _paths = [
|
|
1223
|
+
join(SESSION_DIR, 'sessions.json'), // 新版路径(OpenClaw 2026.5+)
|
|
1224
|
+
SESSION_DIR + '.json' // 旧版路径(兼容降级)
|
|
1225
|
+
];
|
|
1226
|
+
for (const _p of _paths) {
|
|
1227
|
+
try {
|
|
1228
|
+
await access(_p);
|
|
1229
|
+
const raw = await readFile(_p, 'utf-8');
|
|
1230
|
+
return JSON.parse(raw);
|
|
1231
|
+
} catch { /* 路径不存在,试下一个 */ }
|
|
1232
|
+
}
|
|
1233
|
+
return null;
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
/**
|
|
1237
|
+
* 写回 sessions.json (用于强杀)
|
|
1238
|
+
*/
|
|
1239
|
+
async function writeSessionsJson(data) {
|
|
1240
|
+
const { writeFile, mkdir: mkd } = await import('fs/promises');
|
|
1241
|
+
const tmpPath = SESSION_DIR + '.json.tmp';
|
|
1242
|
+
try {
|
|
1243
|
+
await writeFile(tmpPath, JSON.stringify(data), 'utf-8');
|
|
1244
|
+
const { rename } = await import('fs/promises');
|
|
1245
|
+
await rename(tmpPath, SESSION_DIR + '.json');
|
|
1246
|
+
// 🔧 v5.31.0: 同时写新版 sessions/sessions.json(OpenClaw 2026.5+ 路径)
|
|
1247
|
+
try {
|
|
1248
|
+
await mkd(SESSION_DIR, { recursive: true });
|
|
1249
|
+
await writeFile(join(SESSION_DIR, 'sessions.json'), JSON.stringify(data), 'utf-8');
|
|
1250
|
+
} catch { /* 新版路径写入失败不阻塞 */ }
|
|
1251
|
+
return true;
|
|
1252
|
+
} catch {
|
|
1253
|
+
return false;
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
/**
|
|
1258
|
+
* 获取当前所有子agent的快照
|
|
1259
|
+
* @param {number} maxStalledSeconds - 卡死判定阈值(s)
|
|
1260
|
+
* @returns {{ subagents: Array, stalled: Array, total: number, alive: number }}
|
|
1261
|
+
*/
|
|
1262
|
+
async function snapshotSubagents(maxStalledSeconds) {
|
|
1263
|
+
const data = await readSessionsJson();
|
|
1264
|
+
if (!data) {
|
|
1265
|
+
return { subagents: [], stalled: [], total: 0, alive: 0, error: '无法读取会话数据' };
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
const now = Date.now();
|
|
1269
|
+
const thresholdMs = (maxStalledSeconds || 120) * 1000;
|
|
1270
|
+
const subagents = [];
|
|
1271
|
+
const stalled = [];
|
|
1272
|
+
let aliveCount = 0;
|
|
1273
|
+
|
|
1274
|
+
for (const [key, session] of Object.entries(data)) {
|
|
1275
|
+
if (!key.includes(':subagent:')) continue;
|
|
1276
|
+
|
|
1277
|
+
const updatedAt = session.updatedAt || 0;
|
|
1278
|
+
const sessionStartedAt = session.sessionStartedAt || updatedAt;
|
|
1279
|
+
const ageMs = now - updatedAt;
|
|
1280
|
+
const runtimeMs = now - sessionStartedAt;
|
|
1281
|
+
const isStalled = ageMs > thresholdMs;
|
|
1282
|
+
const kind = session.subagentRole || (key.includes('spawn-child') ? 'spawn-child' : 'direct');
|
|
1283
|
+
|
|
1284
|
+
// 🩺 判断卡死vs超时:
|
|
1285
|
+
// 卡死 = session状态标示进程已死(done/failed/timeout) → 需要清理session
|
|
1286
|
+
// 超时 = 进程可能还活着但没响应(lastUpdate>阈值) → 发告警,不杀
|
|
1287
|
+
const sessionStatus = session.status || (session.abortedLastRun ? 'aborted' : null);
|
|
1288
|
+
const isStuck = isStalled && (sessionStatus === 'done' || sessionStatus === 'failed' || sessionStatus === 'timeout' || sessionStatus === 'aborted');
|
|
1289
|
+
|
|
1290
|
+
const entry = {
|
|
1291
|
+
key,
|
|
1292
|
+
sessionId: session.sessionId,
|
|
1293
|
+
spawnedBy: session.spawnedBy || 'unknown',
|
|
1294
|
+
ageSec: Math.round(ageMs / 1000),
|
|
1295
|
+
runtimeSec: Math.round(runtimeMs / 1000),
|
|
1296
|
+
updatedAt,
|
|
1297
|
+
startedAt: sessionStartedAt,
|
|
1298
|
+
model: session.model || 'unknown',
|
|
1299
|
+
thinkingLevel: session.thinkingLevel || 'unknown',
|
|
1300
|
+
kind,
|
|
1301
|
+
stalled: isStalled,
|
|
1302
|
+
stuck: isStuck, // true=卡死(进程已死), false=超时(进程可能还活着)
|
|
1303
|
+
sessionStatus, // 原始status值,供日志使用
|
|
1304
|
+
};
|
|
1305
|
+
|
|
1306
|
+
subagents.push(entry);
|
|
1307
|
+
if (!isStalled) aliveCount++;
|
|
1308
|
+
if (isStalled) stalled.push(entry);
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
// 按 age 降序排序(最老的在前)
|
|
1312
|
+
subagents.sort((a, b) => b.ageSec - a.ageSec);
|
|
1313
|
+
stalled.sort((a, b) => b.ageSec - a.ageSec);
|
|
1314
|
+
|
|
1315
|
+
return { subagents, stalled, total: subagents.length, alive: aliveCount };
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
|
|
1319
|
+
/**
|
|
1320
|
+
* 强杀指定的子agent
|
|
1321
|
+
* @param {string} sessionId - 要杀的 sessionId
|
|
1322
|
+
* @param {string} sessionKey - 要杀的 session key
|
|
1323
|
+
* @returns {{ success: boolean, message: string }}
|
|
1324
|
+
*/
|
|
1325
|
+
async function killSubagent(sessionId, sessionKey) {
|
|
1326
|
+
try {
|
|
1327
|
+
// ⚡ 强杀前自动保存 checkpoint
|
|
1328
|
+
try { await saveCheckpoint(sessionId, { taskName: sessionKey }); } catch {}
|
|
1329
|
+
|
|
1330
|
+
const data = await readSessionsJson();
|
|
1331
|
+
if (!data) return { success: false, message: '无法读取会话数据' };
|
|
1332
|
+
|
|
1333
|
+
let removed = false;
|
|
1334
|
+
let removedKey = null;
|
|
1335
|
+
|
|
1336
|
+
// 优先按 sessionId 匹配
|
|
1337
|
+
for (const [key, session] of Object.entries(data)) {
|
|
1338
|
+
if (session.sessionId === sessionId) {
|
|
1339
|
+
delete data[key];
|
|
1340
|
+
removed = true;
|
|
1341
|
+
removedKey = key;
|
|
1342
|
+
break;
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
// 其次按 sessionKey 匹配
|
|
1347
|
+
if (!removed && sessionKey && data[sessionKey]) {
|
|
1348
|
+
delete data[sessionKey];
|
|
1349
|
+
removed = true;
|
|
1350
|
+
removedKey = sessionKey;
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
if (!removed) {
|
|
1354
|
+
return { success: false, message: `未找到 sessionId=${sessionId}` };
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
const wrote = await writeSessionsJson(data);
|
|
1358
|
+
if (wrote) {
|
|
1359
|
+
// TODO: 移除调试日志 console.log(`[sc] ? 已强杀子agent: ${removedKey}`);
|
|
1360
|
+
return { success: true, message: `已强杀子agent: ${removedKey}`, killedKey: removedKey };
|
|
1361
|
+
}
|
|
1362
|
+
return { success: false, message: '写入会话文件失败' };
|
|
1363
|
+
} catch (err) {
|
|
1364
|
+
return { success: false, message: `强杀失败: ${err.message}` };
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
/**
|
|
1369
|
+
* 将监控告警推送到杉哥可见的通知文件和对话日志
|
|
1370
|
+
* @param {Array} alerts - 本轮超时提醒列表
|
|
1371
|
+
* @param {Array} killed - 保留参数(已弃用,不再强杀),传空数组
|
|
1372
|
+
*/
|
|
1373
|
+
async function tryWriteMonitorNotification(alerts, killed) {
|
|
1374
|
+
if (!alerts.length) return;
|
|
1375
|
+
|
|
1376
|
+
try {
|
|
1377
|
+
const now = new Date();
|
|
1378
|
+
const ts = now.toISOString().replace('T', ' ').substring(0, 19);
|
|
1379
|
+
const { appendFile, writeFile } = await import('fs/promises');
|
|
1380
|
+
|
|
1381
|
+
// 过滤已有通知的告警sessionId(跨轮次去重)
|
|
1382
|
+
const newAlerts = alerts.filter(a => !notifiedAlertIds.has(a.sessionId));
|
|
1383
|
+
for (const a of newAlerts) notifiedAlertIds.add(a.sessionId);
|
|
1384
|
+
|
|
1385
|
+
if (!newAlerts.length) return;
|
|
1386
|
+
|
|
1387
|
+
// 构造人可读通知文本
|
|
1388
|
+
let msg = `\n\n---\n⏰ **子agent超时提醒** (${ts})\n`;
|
|
1389
|
+
if (newAlerts.length > 0) {
|
|
1390
|
+
msg += `⚠️ 超时通知 (${newAlerts.length}个):\n`;
|
|
1391
|
+
for (const a of newAlerts) {
|
|
1392
|
+
const levelLabel = a.action === 'warning_3' ? '三级(10min)' : a.action === 'warning_2' ? '二级(5min)' : a.action === 'warning_1' ? '一级(2min)' : a.action;
|
|
1393
|
+
msg += ` - ${a.key || a.sessionId} 已卡死${a.ageSec}s (${levelLabel}) — 未强杀,持续关注\n`;
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
// 1️. 写入对话日志目录下的监控通知文件(主agent可读)
|
|
1398
|
+
await appendFile(MONITOR_NOTIFY_LOG, msg, 'utf-8').catch(() => {});
|
|
1399
|
+
|
|
1400
|
+
// 2️. 写入结构化JSON(latest only,供程序化读取)
|
|
1401
|
+
await writeFile(MONITOR_ALERT_FILE, JSON.stringify({
|
|
1402
|
+
timestamp: ts,
|
|
1403
|
+
alerts: newAlerts,
|
|
1404
|
+
killed: [],
|
|
1405
|
+
summary: `子agent超时提醒: ${newAlerts.length}个(三级分级,未强杀)`,
|
|
1406
|
+
}, null, 2), 'utf-8').catch(() => {});
|
|
1407
|
+
|
|
1408
|
+
// TODO: 移除调试日志 console.log(`[sc] \u{1F4E2} 子agent超时提醒: ${newAlerts.length}个(未强杀)`);
|
|
1409
|
+
} catch (err) {
|
|
1410
|
+
console.warn(`[sc] \u26A0\uFE0F 写入超时提醒通知失败: ${err.message}`);
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
/**
|
|
1415
|
+
* 执行一次完整的监控轮次
|
|
1416
|
+
*/
|
|
1417
|
+
async function runMonitorCycle() {
|
|
1418
|
+
const maxSec = monitorState.maxStalledSeconds;
|
|
1419
|
+
const snapshot = await snapshotSubagents(maxSec);
|
|
1420
|
+
monitorState.lastSnapshot = snapshot;
|
|
1421
|
+
|
|
1422
|
+
const alerts = [];
|
|
1423
|
+
const killed = [];
|
|
1424
|
+
|
|
1425
|
+
if (snapshot.stalled.length > 0) {
|
|
1426
|
+
for (const stalled of snapshot.stalled) {
|
|
1427
|
+
const sid = stalled.sessionId;
|
|
1428
|
+
|
|
1429
|
+
// ════════════════════════════════════════
|
|
1430
|
+
// 💀 卡死处理:进程已死(done/failed/timeout状态)
|
|
1431
|
+
// 直接 killSubagent 清理session,不写通知
|
|
1432
|
+
// ════════════════════════════════════════
|
|
1433
|
+
if (stalled.stuck) {
|
|
1434
|
+
killed.push({ sessionId: sid, key: stalled.key, ageSec: stalled.ageSec, runtimeSec: stalled.runtimeSec, action: 'stuck_kill', reason: `session状态=${stalled.sessionStatus}` });
|
|
1435
|
+
const killResult = await killSubagent(sid, stalled.key).catch(err => ({ success: false, message: err.message }));
|
|
1436
|
+
if (killResult.success) {
|
|
1437
|
+
// TODO: 移除调试日志 console.log(`[sc] 💀 卡死子agent已强杀: ${stalled.key || sid} (${stalled.sessionStatus}, 已卡死${stalled.ageSec}s)`);
|
|
1438
|
+
} else {
|
|
1439
|
+
console.warn(`[sc] ⚠️ 卡死子agent强杀失败: ${sid} - ${killResult.message}`);
|
|
1440
|
+
}
|
|
1441
|
+
// 卡死不写通知,从stalledHistory移除
|
|
1442
|
+
monitorState.stalledHistory.delete(sid);
|
|
1443
|
+
continue;
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
// ════════════════════════════════════════
|
|
1447
|
+
// ⏰ 超时处理:进程可能还活着但超过120s没反应
|
|
1448
|
+
// 写alert文件供主agent读,不强杀
|
|
1449
|
+
// ════════════════════════════════════════
|
|
1450
|
+
const prev = monitorState.stalledHistory.get(sid);
|
|
1451
|
+
|
|
1452
|
+
if (!prev) {
|
|
1453
|
+
// 首次超时检测
|
|
1454
|
+
monitorState.stalledHistory.set(sid, { firstStalledAt: Date.now(), alertSent: false });
|
|
1455
|
+
alerts.push({ sessionId: sid, key: stalled.key, ageSec: stalled.ageSec, runtimeSec: stalled.runtimeSec, action: 'detected' });
|
|
1456
|
+
continue;
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
if (!prev.alertSent) {
|
|
1460
|
+
prev.alertSent = true;
|
|
1461
|
+
alerts.push({ sessionId: sid, key: stalled.key, ageSec: stalled.ageSec, runtimeSec: stalled.runtimeSec, action: 'alert' });
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
// 多级超时提醒
|
|
1465
|
+
const stalledDurationSec = stalled.ageSec;
|
|
1466
|
+
if (stalledDurationSec >= 600) {
|
|
1467
|
+
// 三级警告:超600s(10min)
|
|
1468
|
+
const alertKey = `warning-3-${sid}`;
|
|
1469
|
+
if (!notifiedAlertIds.has(alertKey)) {
|
|
1470
|
+
notifiedAlertIds.add(alertKey);
|
|
1471
|
+
alerts.push({ sessionId: sid, key: stalled.key, ageSec: stalled.ageSec, runtimeSec: stalled.runtimeSec, action: 'warning_3', message: '⏰ 子agent超600s(10min),请人工关注(未强杀)' });
|
|
1472
|
+
}
|
|
1473
|
+
} else if (stalledDurationSec >= 300) {
|
|
1474
|
+
// 二级警告:超300s(5min)
|
|
1475
|
+
const alertKey = `warning-2-${sid}`;
|
|
1476
|
+
if (!notifiedAlertIds.has(alertKey)) {
|
|
1477
|
+
notifiedAlertIds.add(alertKey);
|
|
1478
|
+
alerts.push({ sessionId: sid, key: stalled.key, ageSec: stalled.ageSec, runtimeSec: stalled.runtimeSec, action: 'warning_2', message: '⏰ 子agent超300s(5min),持续卡死中(未强杀)' });
|
|
1479
|
+
}
|
|
1480
|
+
} else if (stalledDurationSec >= 120) {
|
|
1481
|
+
// 一级提醒:超120s(2min)
|
|
1482
|
+
const alertKey = `warning-1-${sid}`;
|
|
1483
|
+
if (!notifiedAlertIds.has(alertKey)) {
|
|
1484
|
+
notifiedAlertIds.add(alertKey);
|
|
1485
|
+
alerts.push({ sessionId: sid, key: stalled.key, ageSec: stalled.ageSec, runtimeSec: stalled.runtimeSec, action: 'warning_1', message: '⏰ 子agent超120s(2min),已发提醒(未强杀)' });
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
// 清理已恢复的子agent记录
|
|
1491
|
+
const aliveIds = new Set(snapshot.subagents.filter(s => !s.stalled).map(s => s.sessionId));
|
|
1492
|
+
for (const [sid] of monitorState.stalledHistory) {
|
|
1493
|
+
if (!aliveIds.has(sid)) {
|
|
1494
|
+
monitorState.stalledHistory.delete(sid);
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
// ====== 系统状态报告:汇总本次轮询所有状态,写一份完整报告到 shared/tasks/ ======
|
|
1500
|
+
try {
|
|
1501
|
+
const poolStats = pool.getStats();
|
|
1502
|
+
const mem = getMemoryLevel();
|
|
1503
|
+
const aliveCount = snapshot.subagents ? snapshot.subagents.filter(s => !s.terminated).length : 0;
|
|
1504
|
+
const totalCount = snapshot.subagents ? snapshot.subagents.length : 0;
|
|
1505
|
+
const longRunning = snapshot.subagents ? snapshot.subagents.filter(s => !s.terminated && s.runtimeSec > 120) : [];
|
|
1506
|
+
const stuckItems = killed.filter(a => a.action === 'stuck_kill');
|
|
1507
|
+
const timeoutItems = alerts.filter(a => a.action.startsWith('warning'));
|
|
1508
|
+
|
|
1509
|
+
const ts = new Date().toISOString().replace('T', ' ').substring(0, 19);
|
|
1510
|
+
|
|
1511
|
+
let output = `📊 系统状态报告 (${ts})\n`;
|
|
1512
|
+
output += `━━━━━━━━━━━━━━━━━━━━━━━━━━\n`;
|
|
1513
|
+
|
|
1514
|
+
// 小弟概览
|
|
1515
|
+
output += `\n👥 子agent: 共${totalCount}个 | 活跃${aliveCount}个`;
|
|
1516
|
+
if (longRunning.length > 0) output += ` | 超时${longRunning.length}个`;
|
|
1517
|
+
if (stuckItems.length > 0) output += ` | 💀卡死强杀${stuckItems.length}个`;
|
|
1518
|
+
if (timeoutItems.length > 0) output += ` | ⏰超时告警${timeoutItems.length}个`;
|
|
1519
|
+
output += `\n`;
|
|
1520
|
+
|
|
1521
|
+
// Worker池状态
|
|
1522
|
+
output += `\n⚙️ Worker池: 共${poolStats.total}个 | 忙碌${poolStats.busy}个 | 队列${poolStats.queueDepth}个 | 飞行中${poolStats.inFlight}个\n`;
|
|
1523
|
+
|
|
1524
|
+
// 内存 & 紧急模式
|
|
1525
|
+
const memIcons = { green: '🟢', yellow: '🟡', red: '🔴', meltdown: '💥' };
|
|
1526
|
+
output += `\n💾 内存: ${memIcons[mem.level] || '❓'} ${mem.level} | 空闲${mem.freeGB.toFixed(1)}GB | 堆${mem.heapUsedMB}MB\n`;
|
|
1527
|
+
output += `🚨 紧急模式: ${emergency ? '🔴 已触发' : '🟢 正常'}\n`;
|
|
1528
|
+
|
|
1529
|
+
// 超2min的小弟明细
|
|
1530
|
+
if (longRunning.length > 0) {
|
|
1531
|
+
output += `\n⏰ 跑超2min的小弟:\n`;
|
|
1532
|
+
for (const s of longRunning) {
|
|
1533
|
+
output += ` - ${s.key || s.taskName || s.sessionId} 已跑${s.runtimeSec}s\n`;
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
// 💀 卡死强杀明细
|
|
1538
|
+
if (stuckItems.length > 0) {
|
|
1539
|
+
output += `\n💀 卡死强杀:\n`;
|
|
1540
|
+
for (const a of stuckItems) {
|
|
1541
|
+
output += ` - ${a.key || a.sessionId} ${a.reason||'进程已死'},已卡死${a.ageSec}s,已强杀清理\n`;
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
// ⏰ 超时告警明细(不杀,只告警)
|
|
1546
|
+
const stallWarnings = alerts.filter(a => a.action.startsWith('warning'));
|
|
1547
|
+
if (stallWarnings.length > 0) {
|
|
1548
|
+
output += `\n⏰ 超时告警(不杀,只告警):\n`;
|
|
1549
|
+
for (const a of stallWarnings) {
|
|
1550
|
+
const levelMap = { warning_3: '三级(10min)', warning_2: '二级(5min)', warning_1: '一级(2min)' };
|
|
1551
|
+
output += ` - ${a.key || a.sessionId} 卡死${a.ageSec}s (已跑${a.runtimeSec}s) — ${levelMap[a.action] || a.action}提醒\n`;
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
const reportId = `monitor-report-${Date.now()}`;
|
|
1556
|
+
writeSharedResult(reportId, { status: 'info', output }).catch(() => {});
|
|
1557
|
+
|
|
1558
|
+
// ====== ? 用户通知:检测到异常时通知杉哥 ======
|
|
1559
|
+
// ⏰ 通知所有超时提醒(warning_1/warning_2/warning_3)
|
|
1560
|
+
// 💀 卡死强杀事件不通知(直接杀,不写文件)
|
|
1561
|
+
const userAlerts = alerts.filter(a => a.action.startsWith('warning'));
|
|
1562
|
+
if (userAlerts.length > 0) {
|
|
1563
|
+
tryWriteMonitorNotification(userAlerts, []);
|
|
1564
|
+
}
|
|
1565
|
+
} catch (err) {
|
|
1566
|
+
console.warn(`[sc] ⚠️ 生成监控报告失败: ${err.message}`);
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
return { snapshot, alerts, killed };
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1572
|
+
/**
|
|
1573
|
+
* 启动后台监控循环
|
|
1574
|
+
*/
|
|
1575
|
+
function startMonitorBackground() {
|
|
1576
|
+
if (monitorState.timer) {
|
|
1577
|
+
clearInterval(monitorState.timer);
|
|
1578
|
+
monitorState.timer = null;
|
|
1579
|
+
}
|
|
1580
|
+
monitorState.active = true;
|
|
1581
|
+
|
|
1582
|
+
// 立即跑一轮
|
|
1583
|
+
runMonitorCycle().catch(err => {
|
|
1584
|
+
console.warn(`[sc] ⚠️ 监控首轮失败: ${err.message}`);
|
|
1585
|
+
});
|
|
1586
|
+
|
|
1587
|
+
monitorState.timer = setInterval(() => {
|
|
1588
|
+
if (!monitorState.active) {
|
|
1589
|
+
clearInterval(monitorState.timer);
|
|
1590
|
+
monitorState.timer = null;
|
|
1591
|
+
return;
|
|
1592
|
+
}
|
|
1593
|
+
runMonitorCycle().catch(err => {
|
|
1594
|
+
console.warn(`[sc] ⚠️ 监控轮询失败: ${err.message}`);
|
|
1595
|
+
});
|
|
1596
|
+
}, monitorState.intervalSeconds * 1000);
|
|
1597
|
+
|
|
1598
|
+
// TODO: 移除调试日志 console.log(`[sc] ? 子agent监控started (interval=${monitorState.intervalSeconds}s, 阈值=${monitorState.maxStalledSeconds}s, 💀卡死强杀+⏰超时告警)`);
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
/**
|
|
1602
|
+
* 停止后台监控循环
|
|
1603
|
+
*/
|
|
1604
|
+
function stopMonitorBackground() {
|
|
1605
|
+
monitorState.active = false;
|
|
1606
|
+
if (monitorState.timer) {
|
|
1607
|
+
clearInterval(monitorState.timer);
|
|
1608
|
+
monitorState.timer = null;
|
|
1609
|
+
}
|
|
1610
|
+
monitorState.stalledHistory.clear();
|
|
1611
|
+
notifiedAlertIds.clear();
|
|
1612
|
+
// TODO: 移除调试日志 console.log('[sc] ? 子agent监控已停止');
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
// ====== Worker 替换日志 ======
|
|
1616
|
+
async function logWorkerReplacement(workerId, reason, detail = '') {
|
|
1617
|
+
try {
|
|
1618
|
+
const { appendFile } = await import('fs/promises');
|
|
1619
|
+
const ts = new Date().toISOString();
|
|
1620
|
+
const line = `[${ts}] Worker-${workerId} 替换: ${reason}${detail ? ' | ' + detail : ''}\n`;
|
|
1621
|
+
await appendFile(WORKER_REPLACEMENT_LOG, line, 'utf-8');
|
|
1622
|
+
} catch (err) {
|
|
1623
|
+
console.warn(`[sc] ⚠️ 写入替换日志失败: ${err.message}`);
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
// ====== Worker 池 ======
|
|
1628
|
+
class CpuWorkerPool {
|
|
1629
|
+
constructor() {
|
|
1630
|
+
// 初始化角色计数
|
|
1631
|
+
// 从 ROLE_CONFIG 动态初始化所有角色计数(含扩增角色)
|
|
1632
|
+
this._roleCounts = Object.fromEntries(Object.keys(ROLE_CONFIG).map(k => [k, 0]));
|
|
1633
|
+
this._preemptCheckTimer = null;
|
|
1634
|
+
this._preemptedCount = 0;
|
|
1635
|
+
|
|
1636
|
+
this._initPool(MIN_WORKERS);
|
|
1637
|
+
this._startHeartbeat();
|
|
1638
|
+
this._startScaleTimer();
|
|
1639
|
+
this._startRollingRestartTimer();
|
|
1640
|
+
this._readyTimer = null;
|
|
1641
|
+
|
|
1642
|
+
// 确保 preempt 目录ready
|
|
1643
|
+
ensurePreemptDir().catch(() => {});
|
|
1644
|
+
readyPromise = new Promise((resolve) => {
|
|
1645
|
+
const READY_TIMEOUT_MS = 30000;
|
|
1646
|
+
const timeout = setTimeout(() => {
|
|
1647
|
+
console.warn(`[sc] ⏱ Worker 池 ${READY_TIMEOUT_MS/1000}s 超时,降级运行(alive=${workers.filter(w => w.alive).length}/${MIN_WORKERS})`);
|
|
1648
|
+
resolve(false);
|
|
1649
|
+
}, READY_TIMEOUT_MS);
|
|
1650
|
+
const check = () => {
|
|
1651
|
+
if (isShuttingDown) { clearTimeout(timeout); resolve(false); return; }
|
|
1652
|
+
if (workers.filter(w => w.alive).length >= MIN_WORKERS) {
|
|
1653
|
+
clearTimeout(timeout);
|
|
1654
|
+
resolve(true);
|
|
1655
|
+
// 池ready后自动清理旧session + 孤儿任务元数据
|
|
1656
|
+
cleanupOldSessions().catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
1657
|
+
cleanupTaskStates().catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
1658
|
+
} else {
|
|
1659
|
+
this._readyTimer = setTimeout(check, 100);
|
|
1660
|
+
}
|
|
1661
|
+
};
|
|
1662
|
+
check();
|
|
1663
|
+
});
|
|
1664
|
+
// TODO: 移除调试日志 console.log(`[sc] worker pool started:MIN=${MIN_WORKERS} MAX=${MAX_WORKERS}(动态)`);
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
_initPool(count) {
|
|
1668
|
+
const workerPath = join(__dirname, "workers", "worker.js");
|
|
1669
|
+
const roleDistribution = getInitialRoleDistribution();
|
|
1670
|
+
const roleKeys = ["scanner", "compute", "router", "stemcell"];
|
|
1671
|
+
|
|
1672
|
+
// 先按最小配置分配
|
|
1673
|
+
for (const role of roleKeys) {
|
|
1674
|
+
const target = roleDistribution[role];
|
|
1675
|
+
for (let i = 0; i < target; i++) {
|
|
1676
|
+
this._spawnWorker(workerPath, role);
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
// 剩余Worker作为backupWorker(兜底角色)
|
|
1681
|
+
// 🧬 设计决策: backupWorker 是保底角色,任一角色(scanner/compute/router)全崩时自动分化填补
|
|
1682
|
+
// 不是多余复杂度,是专业分工的核心设计。见 known-design-decisions.md → Worker 池设计
|
|
1683
|
+
const spawned = workers.length;
|
|
1684
|
+
for (let i = spawned; i < count; i++) {
|
|
1685
|
+
this._spawnWorker(workerPath, "stemcell");
|
|
1686
|
+
}
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1689
|
+
_spawnWorker(workerPath, role) {
|
|
1690
|
+
role = role || "stemcell";
|
|
1691
|
+
try {
|
|
1692
|
+
const actualId = genWorkerId();
|
|
1693
|
+
const w = new Worker(workerPath, { workerData: { id: actualId } });
|
|
1694
|
+
const entry = {
|
|
1695
|
+
id: actualId,
|
|
1696
|
+
role,
|
|
1697
|
+
worker: w,
|
|
1698
|
+
busy: false,
|
|
1699
|
+
idleSince: Date.now(),
|
|
1700
|
+
alive: true,
|
|
1701
|
+
crashCount: 0,
|
|
1702
|
+
lastCrash: 0,
|
|
1703
|
+
currentJobId: null,
|
|
1704
|
+
currentJobStartTime: null,
|
|
1705
|
+
hbPending: false,
|
|
1706
|
+
terminating: false,
|
|
1707
|
+
startTime: Date.now(),
|
|
1708
|
+
completedTasks: 0,
|
|
1709
|
+
partialResult: null,
|
|
1710
|
+
};
|
|
1711
|
+
|
|
1712
|
+
this._roleCounts[role] = (this._roleCounts[role] || 0) + 1;
|
|
1713
|
+
|
|
1714
|
+
w.on("message", (msg) => this._onWorkerMessage(entry, msg));
|
|
1715
|
+
w.on("error", (err) => this._onWorkerCrash(entry, err, workerPath, "crash"));
|
|
1716
|
+
w.on("exit", (code) => this._onWorkerCrash(entry, new Error(`Worker exited with code ${code}`), workerPath, "exit"));
|
|
1717
|
+
|
|
1718
|
+
workers.push(entry);
|
|
1719
|
+
// 日志: Worker 创建事件
|
|
1720
|
+
const lg = global.__sansanLogger;
|
|
1721
|
+
if (lg) lg.logWorkerEvent(actualId, 'SPAWN', `role=${role} 已创建 (池中共${workers.length}个)`).catch(() => {});
|
|
1722
|
+
return entry;
|
|
1723
|
+
} catch (e) {
|
|
1724
|
+
console.error("[sc] spawnWorker 失败:", e.message);
|
|
1725
|
+
return null;
|
|
1726
|
+
}
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
/**
|
|
1730
|
+
* 获取任务类型对应的角色
|
|
1731
|
+
* @param {string} taskType - 任务类型
|
|
1732
|
+
* @returns {string} 角色名: scanner/compute/router/stemcell
|
|
1733
|
+
*/
|
|
1734
|
+
_getTaskRole(taskType) {
|
|
1735
|
+
return TASK_TYPE_ROLE_MAP[taskType] || 'stemcell';
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
/**
|
|
1739
|
+
* 找最适合处理该任务的空闲Worker
|
|
1740
|
+
* 优先匹配角色,然后stemcell兜底
|
|
1741
|
+
* @param {string} role - 目标角色
|
|
1742
|
+
* @returns {object|null} Worker entry
|
|
1743
|
+
*/
|
|
1744
|
+
_findAvailableWorker(role) {
|
|
1745
|
+
// 第一优先:同角色空闲Worker
|
|
1746
|
+
const roleWorkers = workers.filter(w => w.alive && !w.busy && !w.hbPending && !w.terminating && w.role === role);
|
|
1747
|
+
if (roleWorkers.length > 0) return roleWorkers.sort((a, b) => (a.idleSince || 0) - (b.idleSince || 0))[0];
|
|
1748
|
+
|
|
1749
|
+
// 第二优先:stemcell 兜底
|
|
1750
|
+
if (role !== 'router') {
|
|
1751
|
+
const stemcells = workers.filter(w => w.alive && !w.busy && !w.hbPending && !w.terminating && w.role === 'stemcell');
|
|
1752
|
+
if (stemcells.length > 0) return stemcells.sort((a, b) => (a.idleSince || 0) - (b.idleSince || 0))[0];
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
// 第三优先:其他角色空闲Worker跨角色帮忙(弹性转岗:所有角色都能转)
|
|
1756
|
+
// 谁闲着就拉谁干活,不挑角色
|
|
1757
|
+
const anyIdle = workers.filter(w => w.alive && !w.busy && !w.hbPending && !w.terminating);
|
|
1758
|
+
if (anyIdle.length > 0) {
|
|
1759
|
+
const picked = anyIdle.sort((a, b) => (a.idleSince || 0) - (b.idleSince || 0))[0];
|
|
1760
|
+
// 标记这位Worker借调到了别的角色
|
|
1761
|
+
picked._borrowedRole = role;
|
|
1762
|
+
return picked;
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
return null;
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
/**
|
|
1769
|
+
* 高优任务抢占触发
|
|
1770
|
+
* 评估所有活跃任务,抢占最低优先级分的任务
|
|
1771
|
+
* 抢占成功后:序列化状态到 shared/preempt/,将任务重新加入队列头部等待恢复
|
|
1772
|
+
* @returns {Promise<boolean>} 是否成功preempt
|
|
1773
|
+
*/
|
|
1774
|
+
async _triggerPreemption() {
|
|
1775
|
+
if (isShuttingDown) return false;
|
|
1776
|
+
|
|
1777
|
+
const aliveBusy = workers.filter(w => w.alive && w.busy && !w.terminating && w.currentJobId);
|
|
1778
|
+
const totalQueued = Object.values(taskQueues).reduce((a, b) => a + b.length, 0);
|
|
1779
|
+
|
|
1780
|
+
// 阈值:所有Worker忙碌 + 队列 > 5
|
|
1781
|
+
if (aliveBusy.length < workers.filter(w => w.alive && !w.terminating).length || totalQueued <= PREEMPT_QUEUE_THRESHOLD) {
|
|
1782
|
+
return false;
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
// 用独立的 evaluatePreemption 函数计算优先级分
|
|
1786
|
+
const evalResult = evaluatePreemption(
|
|
1787
|
+
{ getStats: () => this.getStats() },
|
|
1788
|
+
taskQueues,
|
|
1789
|
+
workers
|
|
1790
|
+
);
|
|
1791
|
+
|
|
1792
|
+
if (!evalResult.shouldPreempt || !evalResult.target) {
|
|
1793
|
+
return false;
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
const target = evalResult.target;
|
|
1797
|
+
const we = target.workerEntry;
|
|
1798
|
+
|
|
1799
|
+
// 标记Worker已被preempt
|
|
1800
|
+
we._preempted = true;
|
|
1801
|
+
|
|
1802
|
+
// TODO: 移除调试日志 console.log('[sc] 资源抢占: Worker ' + we.id + '(' + we.role + ') 任务 ' + target.task.type + ' 被抢占 (优先级分=' + target.score.toFixed(3) + ')');
|
|
1803
|
+
|
|
1804
|
+
// 1. 序列化状态
|
|
1805
|
+
const stateSaved = await savePreemptState(target);
|
|
1806
|
+
if (!stateSaved) {
|
|
1807
|
+
delete we._preempted;
|
|
1808
|
+
console.warn('[sc] preempt状态保存失败,取消preempt');
|
|
1809
|
+
return false;
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
// 2. 从 pendingJobs 摘除
|
|
1813
|
+
const jobId = target.jobId;
|
|
1814
|
+
const pending = pendingJobs.get(jobId);
|
|
1815
|
+
if (pending) {
|
|
1816
|
+
clearTimeout(pending.timeout);
|
|
1817
|
+
pendingJobs.delete(jobId);
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
// 3. 将任务重新加入队列头部
|
|
1821
|
+
const recoveryTask = {
|
|
1822
|
+
...target.task,
|
|
1823
|
+
_preempted: true,
|
|
1824
|
+
_preemptedJobId: jobId,
|
|
1825
|
+
_preemptedAt: Date.now(),
|
|
1826
|
+
_originalRole: we.role,
|
|
1827
|
+
};
|
|
1828
|
+
|
|
1829
|
+
taskQueues.high.unshift({
|
|
1830
|
+
task: recoveryTask,
|
|
1831
|
+
resolve: target.pendingJob.resolve,
|
|
1832
|
+
reject: target.pendingJob.reject,
|
|
1833
|
+
});
|
|
1834
|
+
|
|
1835
|
+
// 4. 清理Worker状态
|
|
1836
|
+
we.busy = false;
|
|
1837
|
+
we.currentJobId = null;
|
|
1838
|
+
we.currentJobStartTime = null;
|
|
1839
|
+
we.idleSince = Date.now();
|
|
1840
|
+
delete we._preempted;
|
|
1841
|
+
|
|
1842
|
+
this._preemptedCount++;
|
|
1843
|
+
|
|
1844
|
+
// 5. 立即处理队列
|
|
1845
|
+
setImmediate(() => this._processQueue());
|
|
1846
|
+
|
|
1847
|
+
return true;
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1850
|
+
_onWorkerCrash(entry, err, workerPath, reason = "crash") {
|
|
1851
|
+
if (entry.killedByScaleDown) {
|
|
1852
|
+
// alive=false 已由scale down逻辑设置,但 pendingJobs 仍需回收
|
|
1853
|
+
const toReject = [];
|
|
1854
|
+
for (const [jobId, p] of pendingJobs) {
|
|
1855
|
+
if (p.workerId === entry.id) {
|
|
1856
|
+
clearTimeout(p.timeout);
|
|
1857
|
+
pendingJobs.delete(jobId);
|
|
1858
|
+
toReject.push(p);
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
for (const p of toReject) {
|
|
1862
|
+
p.reject(new Error(`[sc] Worker ${entry.id} 被scale down`));
|
|
1863
|
+
}
|
|
1864
|
+
const idx = workers.indexOf(entry);
|
|
1865
|
+
if (idx >= 0) workers.splice(idx, 1);
|
|
1866
|
+
return;
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
if (entry.isHeartbeatKill) {
|
|
1870
|
+
delete entry.isHeartbeatKill;
|
|
1871
|
+
entry.alive = false;
|
|
1872
|
+
entry.busy = false;
|
|
1873
|
+
entry.currentJobId = null;
|
|
1874
|
+
// BUG-FIX: 心跳强杀Worker时回收该Worker的pendingJobs
|
|
1875
|
+
const toRejectHb = [];
|
|
1876
|
+
for (const [jobId, p] of pendingJobs) {
|
|
1877
|
+
if (p.workerId === entry.id) {
|
|
1878
|
+
clearTimeout(p.timeout);
|
|
1879
|
+
pendingJobs.delete(jobId);
|
|
1880
|
+
toRejectHb.push(p);
|
|
1881
|
+
}
|
|
1882
|
+
}
|
|
1883
|
+
for (const p of toRejectHb) {
|
|
1884
|
+
p.reject(new Error(`[sc] Worker ${entry.id} 心跳超时被强杀`));
|
|
1885
|
+
}
|
|
1886
|
+
const idx = workers.indexOf(entry);
|
|
1887
|
+
if (idx >= 0) workers.splice(idx, 1);
|
|
1888
|
+
if (!isShuttingDown) {
|
|
1889
|
+
setTimeout(() => {
|
|
1890
|
+
if (isShuttingDown) return;
|
|
1891
|
+
let aliveCount = workers.filter(w => w.alive && !w.terminating).length;
|
|
1892
|
+
if (aliveCount < getDynamicMaxWorkers()) {
|
|
1893
|
+
const e = this._spawnWorker(workerPath);
|
|
1894
|
+
if (e) aliveCount++;
|
|
1895
|
+
}
|
|
1896
|
+
while (aliveCount < MIN_WORKERS) {
|
|
1897
|
+
const e = this._spawnWorker(workerPath);
|
|
1898
|
+
if (!e) break;
|
|
1899
|
+
aliveCount++;
|
|
1900
|
+
}
|
|
1901
|
+
this._processQueue();
|
|
1902
|
+
}, 0);
|
|
1903
|
+
}
|
|
1904
|
+
return;
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
if (!entry.alive) return;
|
|
1908
|
+
|
|
1909
|
+
if (reason === "crash") console.error(`[sc] Worker ${entry.id} 崩溃:`, err.message);
|
|
1910
|
+
else console.warn(`[sc] Worker ${entry.id} 离线 (${reason}): ${err.message}`);
|
|
1911
|
+
// 日志: Worker 崩溃事件
|
|
1912
|
+
const lg = global.__sansanLogger;
|
|
1913
|
+
if (lg) lg.logWorkerEvent(entry.id, 'CRASH', `${reason}: ${err.message.substring(0, 100)}`).catch(() => {});
|
|
1914
|
+
entry.alive = false;
|
|
1915
|
+
entry.busy = false;
|
|
1916
|
+
const deadIdx = workers.indexOf(entry);
|
|
1917
|
+
if (deadIdx >= 0) workers.splice(deadIdx, 1);
|
|
1918
|
+
entry.currentJobId = null;
|
|
1919
|
+
entry.terminating = false;
|
|
1920
|
+
|
|
1921
|
+
// 清理探索型 checkpoint 定时器(Worker崩溃)
|
|
1922
|
+
if (entry._exploratoryTimer) {
|
|
1923
|
+
clearInterval(entry._exploratoryTimer);
|
|
1924
|
+
entry._exploratoryTimer = null;
|
|
1925
|
+
}
|
|
1926
|
+
|
|
1927
|
+
if (isShuttingDown) return;
|
|
1928
|
+
|
|
1929
|
+
const toReject = [];
|
|
1930
|
+
for (const [jobId, p] of pendingJobs) {
|
|
1931
|
+
if (p.workerId === entry.id) {
|
|
1932
|
+
clearTimeout(p.timeout);
|
|
1933
|
+
pendingJobs.delete(jobId);
|
|
1934
|
+
toReject.push(p);
|
|
1935
|
+
}
|
|
1936
|
+
}
|
|
1937
|
+
for (const p of toReject) {
|
|
1938
|
+
p.reject(new Error(`[sc] Worker ${entry.id} 已崩溃/退出`));
|
|
1939
|
+
}
|
|
1940
|
+
|
|
1941
|
+
this._processQueue();
|
|
1942
|
+
|
|
1943
|
+
const now = Date.now();
|
|
1944
|
+
// # FIXED: V2 - 改用基于时间的衰减:30min内连续崩溃才累加,超时自动归零
|
|
1945
|
+
if (now - entry.lastCrash < 30000) {
|
|
1946
|
+
entry.crashCount = Math.min(entry.crashCount + 1, 6); // 上限6次避免无限
|
|
1947
|
+
} else {
|
|
1948
|
+
entry.crashCount = Math.min(1, Math.max(0, entry.crashCount - 2)); // 衰减2级
|
|
1949
|
+
}
|
|
1950
|
+
entry.lastCrash = now;
|
|
1951
|
+
|
|
1952
|
+
const delay = Math.min(1000 * Math.pow(2, entry.crashCount - 1), 30000);
|
|
1953
|
+
// TODO: 移除调试日志 console.log(`[sc] Worker ${entry.id} 第 ${entry.crashCount} 次崩溃,${delay}ms 后重启`);
|
|
1954
|
+
|
|
1955
|
+
// # FIXED: V2 - 新Worker不从旧Worker继承 crashCount/lastCrash,指数退避可衰减
|
|
1956
|
+
setTimeout(() => {
|
|
1957
|
+
if (isShuttingDown) return;
|
|
1958
|
+
|
|
1959
|
+
let aliveCount = workers.filter(w => w.alive && !w.terminating).length;
|
|
1960
|
+
|
|
1961
|
+
if (aliveCount < getDynamicMaxWorkers()) {
|
|
1962
|
+
const newEntry = this._spawnWorker(workerPath);
|
|
1963
|
+
if (newEntry) {
|
|
1964
|
+
// 新Worker从0开始计数,旧crashCount仅用于指数退避delay计算
|
|
1965
|
+
newEntry.crashCount = 0;
|
|
1966
|
+
newEntry.lastCrash = 0;
|
|
1967
|
+
aliveCount++;
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
while (aliveCount < MIN_WORKERS) {
|
|
1972
|
+
const e = this._spawnWorker(workerPath);
|
|
1973
|
+
if (e) aliveCount++;
|
|
1974
|
+
else break;
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
this._processQueue();
|
|
1978
|
+
}, delay);
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
_startScaleTimer() {
|
|
1982
|
+
// 🧬 设计决策: 5s间隔刚好平衡响应速度和开销
|
|
1983
|
+
// 扩容延迟不超过5s,且每次调用开销<1ms
|
|
1984
|
+
// 见 known-design-decisions.md → 魔法数字
|
|
1985
|
+
scaleTimer = setInterval(() => this._autoScale(), 5000);
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
_autoScale() {
|
|
1989
|
+
if (isShuttingDown) return;
|
|
1990
|
+
|
|
1991
|
+
// 内存红灯时主动scale down
|
|
1992
|
+
const memLevel = getCachedMemoryLevel();
|
|
1993
|
+
if (memLevel.level === 'red' || memLevel.level === 'meltdown') {
|
|
1994
|
+
// 强制缩到 MIN_WORKERS
|
|
1995
|
+
const current = workers;
|
|
1996
|
+
let removed = 0;
|
|
1997
|
+
for (let i = current.length - 1; i >= 0 && current.length > MIN_WORKERS; i--) {
|
|
1998
|
+
const entry = current[i];
|
|
1999
|
+
if (!entry.busy && !entry.hbPending) {
|
|
2000
|
+
entry.killedByScaleDown = true;
|
|
2001
|
+
entry.terminating = true;
|
|
2002
|
+
entry.alive = false;
|
|
2003
|
+
entry.worker.terminate().catch(() => {});
|
|
2004
|
+
current.splice(i, 1);
|
|
2005
|
+
removed++;
|
|
2006
|
+
}
|
|
2007
|
+
}
|
|
2008
|
+
if (removed > 0) {
|
|
2009
|
+
const nuLog = getMetabolicRateConfig().nu;
|
|
2010
|
+
// TODO: 移除调试日志 console.log(`[sc] 内存${memLevel.level} 强制scale down -${removed} → ${current.filter(w => w.alive).length}/${getDynamicMaxWorkers()} (ν=${nuLog.toFixed(2)})`);
|
|
2011
|
+
}
|
|
2012
|
+
return;
|
|
2013
|
+
}
|
|
2014
|
+
|
|
2015
|
+
const alive = workers.filter(w => w.alive);
|
|
2016
|
+
const activeCount = alive.filter(w => !w.terminating).length;
|
|
2017
|
+
const totalQueued = Object.values(taskQueues).reduce((a, b) => a + b.length, 0);
|
|
2018
|
+
const queueRatio = totalQueued / Math.max(1, activeCount);
|
|
2019
|
+
|
|
2020
|
+
// 资源调度强度平滑精细控制:基于连续资源调度强度值而非离散档位
|
|
2021
|
+
const meta = getMetabolicRateConfig();
|
|
2022
|
+
const effectiveMinWorkers = meta.minWorkers;
|
|
2023
|
+
const effectiveMaxWorkers = meta.maxWorkers;
|
|
2024
|
+
|
|
2025
|
+
// 低资源调度强度(≈休息):主动scale down到最低Worker数
|
|
2026
|
+
if (meta.rate < 0.3) {
|
|
2027
|
+
const toRemove = Math.max(0, activeCount - effectiveMinWorkers);
|
|
2028
|
+
if (toRemove > 0) {
|
|
2029
|
+
const idleForRest = alive.filter(w => !w.busy && !w.hbPending && !w.terminating);
|
|
2030
|
+
const toKill = idleForRest.slice(0, toRemove);
|
|
2031
|
+
for (const e of toKill) {
|
|
2032
|
+
e.killedByScaleDown = true;
|
|
2033
|
+
e.terminating = true;
|
|
2034
|
+
e.alive = false;
|
|
2035
|
+
e.worker.terminate().catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
2036
|
+
}
|
|
2037
|
+
if (toKill.length > 0) console.log(`[sc] 💤 低metabolic ratescale down -${toKill.length} → ${activeCount - toKill.length}/${effectiveMinWorkers} (ν=${meta.nu.toFixed(2)})`);
|
|
2038
|
+
}
|
|
2039
|
+
// 低资源调度强度warmupWorker
|
|
2040
|
+
if (meta.autoWarmup) {
|
|
2041
|
+
this.warmup().catch(() => {});
|
|
2042
|
+
}
|
|
2043
|
+
return;
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
// 平滑扩容阈值:资源调度强度越高→扩容越激进(阈值越低,一次扩越多)
|
|
2047
|
+
// 资源调度强度0.3→SCALE_UP_THRESHOLD×1.0, 0.8→×0.5, 1.0→×0.3
|
|
2048
|
+
const thresholdFactor = Math.max(0.3, Math.min(1.0, 1.0 - (meta.rate - 0.3) / 0.7 * 0.7));
|
|
2049
|
+
const scaleThreshold = Math.max(0.3, SCALE_UP_THRESHOLD * thresholdFactor);
|
|
2050
|
+
|
|
2051
|
+
// 动态扩容批量:资源调度强度越高→一次扩越多
|
|
2052
|
+
// 资源调度强度0.3→扩2个, 0.5→扩2个, 0.8→扩3个, 1.0→扩4个
|
|
2053
|
+
const addBatchSize = Math.min(4, Math.max(2, Math.round(meta.rate * 4)));
|
|
2054
|
+
|
|
2055
|
+
if (queueRatio > scaleThreshold && activeCount < effectiveMaxWorkers) {
|
|
2056
|
+
const toAdd = Math.min(addBatchSize, effectiveMaxWorkers - activeCount);
|
|
2057
|
+
let actualAdded = 0;
|
|
2058
|
+
for (let i = 0; i < toAdd; i++) {
|
|
2059
|
+
if (this._spawnWorker(join(__dirname, "workers", "worker.js"))) actualAdded++;
|
|
2060
|
+
}
|
|
2061
|
+
if (actualAdded > 0) {
|
|
2062
|
+
const newTotal = activeCount + actualAdded;
|
|
2063
|
+
// TODO: 移除调试日志 console.log(`[sc] 📈 扩容 +${actualAdded} → ${newTotal}/${effectiveMaxWorkers} (metabolic rate=${meta.rate.toFixed(2)} ν=${meta.nu.toFixed(2)})`);
|
|
2064
|
+
}
|
|
2065
|
+
this._processQueue();
|
|
2066
|
+
return;
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
// 后台学习已删除(管理员2026-05-31要求移除)
|
|
2070
|
+
const idleWorkers = alive.filter(w => !w.busy && !w.hbPending && !w.terminating);
|
|
2071
|
+
|
|
2072
|
+
if (alive.length > effectiveMinWorkers) {
|
|
2073
|
+
const idle = idleWorkers.filter(w => (Date.now() - w.idleSince) > IDLE_TERMINATE_MS);
|
|
2074
|
+
const canRemove = Math.min(idle.length, alive.length - effectiveMinWorkers);
|
|
2075
|
+
if (canRemove > 0) {
|
|
2076
|
+
const toKill = idle.slice(0, canRemove);
|
|
2077
|
+
for (const e of toKill) {
|
|
2078
|
+
e.killedByScaleDown = true;
|
|
2079
|
+
e.terminating = true;
|
|
2080
|
+
e.alive = false;
|
|
2081
|
+
e.worker.terminate().catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
2082
|
+
}
|
|
2083
|
+
// TODO: 移除调试日志 console.log(`[sc] 📉 scale down -${toKill.length} (metabolic rate=${meta.rate.toFixed(2)} ν=${meta.nu.toFixed(2)})`);
|
|
2084
|
+
}
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
// Worker寿命管理: 空闲老年Worker置换(不计数在scale down中,由单独逻辑管理)
|
|
2088
|
+
this._replaceAgedWorkers().catch(err => {
|
|
2089
|
+
console.warn(`[sc] ⚠️ 自动Worker寿命置换检查失败: ${err.message}`);
|
|
2090
|
+
});
|
|
2091
|
+
|
|
2092
|
+
// 🕐 派兵进度检查(复用5s _autoScale 循环,0额外开销)
|
|
2093
|
+
this._checkSpawnProgress().catch(() => {});
|
|
2094
|
+
}
|
|
2095
|
+
|
|
2096
|
+
// ====== Worker 寿命管理(Worker寿命管理)======
|
|
2097
|
+
|
|
2098
|
+
/**
|
|
2099
|
+
* Worker寿命管理设计: Worker 6小时/2000任务寿命上限,防内存泄漏累积
|
|
2100
|
+
* 超过上限即标记为aged,由_replaceAgedWorkers()滚动置换
|
|
2101
|
+
* 设计决策见: memory/known-design-decisions.md → Worker 池设计
|
|
2102
|
+
*/
|
|
2103
|
+
_isWorkerAged(entry) {
|
|
2104
|
+
if (!entry || !entry.alive || entry.terminating) return false;
|
|
2105
|
+
const age = Date.now() - (entry.startTime || Date.now());
|
|
2106
|
+
const tasks = entry.completedTasks || 0;
|
|
2107
|
+
return age >= MAX_WORKER_AGE || tasks >= MAX_WORKER_TASKS;
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
/** Rolling Restart: replace aged idle workers, max 1/3 at a time */
|
|
2111
|
+
async _replaceAgedWorkers() {
|
|
2112
|
+
if (isShuttingDown) return;
|
|
2113
|
+
|
|
2114
|
+
const aliveWorkers = workers.filter(w => w.alive && !w.terminating);
|
|
2115
|
+
if (aliveWorkers.length <= MIN_WORKERS) return; // minimum safety net
|
|
2116
|
+
|
|
2117
|
+
// Only replace IDLE aged workers, never busy ones
|
|
2118
|
+
const agedIdle = aliveWorkers.filter(w => !w.busy && !w.hbPending && this._isWorkerAged(w));
|
|
2119
|
+
if (agedIdle.length === 0) return;
|
|
2120
|
+
|
|
2121
|
+
// Rolling Restart: max 1/3 of current live workers per batch
|
|
2122
|
+
// 🧬 设计决策: 每次最多置换1/3,保证任何时候至少2/3 Worker可用
|
|
2123
|
+
// 不是效率低,是故意保守。全部同时重启会导致服务降级
|
|
2124
|
+
const maxReplace = Math.max(1, Math.floor(aliveWorkers.length * ROLLING_RESTART_BATCH_RATIO));
|
|
2125
|
+
const toReplace = agedIdle.slice(0, maxReplace);
|
|
2126
|
+
if (toReplace.length === 0) return;
|
|
2127
|
+
|
|
2128
|
+
const workerPath = join(__dirname, "workers", "worker.js");
|
|
2129
|
+
|
|
2130
|
+
for (const entry of toReplace) {
|
|
2131
|
+
const age = Date.now() - entry.startTime;
|
|
2132
|
+
const tasks = entry.completedTasks || 0;
|
|
2133
|
+
const reason = age >= MAX_WORKER_AGE
|
|
2134
|
+
? `年龄超限(${(age / 3600000).toFixed(1)}h)`
|
|
2135
|
+
: `任务超限(${tasks}任务)`;
|
|
2136
|
+
|
|
2137
|
+
entry.killedByScaleDown = true;
|
|
2138
|
+
entry.terminating = true;
|
|
2139
|
+
entry.alive = false;
|
|
2140
|
+
entry.worker.terminate().catch(() => {});
|
|
2141
|
+
|
|
2142
|
+
const idx = workers.indexOf(entry);
|
|
2143
|
+
if (idx >= 0) workers.splice(idx, 1);
|
|
2144
|
+
|
|
2145
|
+
logWorkerReplacement(entry.id, reason, `年龄=${(age / 3600000).toFixed(1)}h, 任务=${tasks}`);
|
|
2146
|
+
// TODO: 移除调试日志 console.log(`[sc] 🔄 老年Worker ${entry.id} 优雅下线: ${reason}`);
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
// Spawn replacements
|
|
2150
|
+
for (let i = 0; i < toReplace.length; i++) {
|
|
2151
|
+
this._spawnWorker(workerPath);
|
|
2152
|
+
}
|
|
2153
|
+
|
|
2154
|
+
// TODO: 移除调试日志 console.log(`[sc] 🔄 Rolling Restart: 替换 ${toReplace.length} 个老年Worker`);
|
|
2155
|
+
}
|
|
2156
|
+
|
|
2157
|
+
/**
|
|
2158
|
+
* 🕐 派兵进度检查 — 每5s_autoScale循环调用
|
|
2159
|
+
*
|
|
2160
|
+
* 扫 shared/tasks/ 下registered的multi_aspect子agent状态文件,
|
|
2161
|
+
* 状态变更时写 shared/progress-report.json,
|
|
2162
|
+
* 主agent下一轮对话自动读到并汇报。
|
|
2163
|
+
*
|
|
2164
|
+
* 设计:
|
|
2165
|
+
* - 复用 _autoScale() 的 5 s循环,0 额外定时开销
|
|
2166
|
+
* - 轻量:只读文件元数据 + 状态比较,不调AI
|
|
2167
|
+
* - 分级汇报:5s→15s→30s→60s逐步深入
|
|
2168
|
+
*/
|
|
2169
|
+
async _checkSpawnProgress() {
|
|
2170
|
+
if (isShuttingDown) return;
|
|
2171
|
+
try {
|
|
2172
|
+
const tasksDir = join(homedir(), '.openclaw', 'workspace', 'memory', 'shared', 'tasks');
|
|
2173
|
+
const progressFile = join(homedir(), '.openclaw', 'workspace', 'memory', 'shared', 'progress-report.json');
|
|
2174
|
+
|
|
2175
|
+
// 确保目录存在
|
|
2176
|
+
try { await mkdir(join(homedir(), '.openclaw', 'workspace', 'memory', 'shared'), { recursive: true }); } catch {}
|
|
2177
|
+
|
|
2178
|
+
// 读已完成/失败/超时的任务文件
|
|
2179
|
+
let files = [];
|
|
2180
|
+
try { files = await readdir(tasksDir); } catch { return; }
|
|
2181
|
+
|
|
2182
|
+
const taskFiles = files.filter(f =>
|
|
2183
|
+
f.endsWith('.json') &&
|
|
2184
|
+
!f.endsWith('.meta.json') &&
|
|
2185
|
+
!f.endsWith('.tmp') &&
|
|
2186
|
+
!f.endsWith('.reading.json')
|
|
2187
|
+
);
|
|
2188
|
+
if (taskFiles.length === 0) return;
|
|
2189
|
+
|
|
2190
|
+
// 读上次进度
|
|
2191
|
+
let prevProgress = { checkedTasks: [], checkedAt: 0 };
|
|
2192
|
+
try {
|
|
2193
|
+
prevProgress = JSON.parse(await readFile(progressFile, 'utf-8'));
|
|
2194
|
+
} catch {}
|
|
2195
|
+
|
|
2196
|
+
const now = Date.now();
|
|
2197
|
+
const checkedSet = new Set(prevProgress.checkedTasks || []);
|
|
2198
|
+
const newDone = [];
|
|
2199
|
+
const newFailed = [];
|
|
2200
|
+
|
|
2201
|
+
for (const f of taskFiles) {
|
|
2202
|
+
if (checkedSet.has(f)) continue; // 已检查过的不重复
|
|
2203
|
+
try {
|
|
2204
|
+
const st = await stat(join(tasksDir, f));
|
|
2205
|
+
const ageSec = (now - st.mtimeMs) / 1000;
|
|
2206
|
+
if (ageSec < 3) continue; // 文件太新,等稳定
|
|
2207
|
+
|
|
2208
|
+
const raw = await readFile(join(tasksDir, f), 'utf-8');
|
|
2209
|
+
const task = JSON.parse(raw);
|
|
2210
|
+
if (task.status === 'done') newDone.push(task);
|
|
2211
|
+
else if (task.status === 'failed' || task.status === 'timeout') newFailed.push(task);
|
|
2212
|
+
} catch {}
|
|
2213
|
+
}
|
|
2214
|
+
|
|
2215
|
+
if (newDone.length === 0 && newFailed.length === 0) return;
|
|
2216
|
+
|
|
2217
|
+
// 写进度报告
|
|
2218
|
+
const report = {
|
|
2219
|
+
updatedAt: now,
|
|
2220
|
+
done: newDone.map(t => ({ taskId: t.taskId, summary: t.output?.summary || t.output?.conclusion || '已完成' })),
|
|
2221
|
+
failed: newFailed.map(t => ({ taskId: t.taskId, error: t.errors?.[0] || '未知错误' })),
|
|
2222
|
+
checkedTasks: [...checkedSet, ...newDone.map(t => t.taskId + '.json'), ...newFailed.map(t => t.taskId + '.json')],
|
|
2223
|
+
checkedAt: now,
|
|
2224
|
+
};
|
|
2225
|
+
await writeFile(progressFile, JSON.stringify(report, null, 2), 'utf-8');
|
|
2226
|
+
|
|
2227
|
+
if (newDone.length > 0) {
|
|
2228
|
+
// TODO: 移除调试日志 console.log(`[sc] 🕐 派兵进度: ${newDone.length}个完成, ${newFailed.length}个失败`);
|
|
2229
|
+
}
|
|
2230
|
+
} catch (err) {
|
|
2231
|
+
// 静默失败,不阻塞_autoScale
|
|
2232
|
+
if (getEnv('NODE_ENV') !== 'production') {
|
|
2233
|
+
console.warn(`[sc] ⚠️ _checkSpawnProgress: ${err.message}`);
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
}
|
|
2237
|
+
|
|
2238
|
+
_startRollingRestartTimer() {
|
|
2239
|
+
this._rollingRestartTimer = setInterval(() => {
|
|
2240
|
+
this._replaceAgedWorkers().catch(err => {
|
|
2241
|
+
console.warn(`[sc] ⚠️ Rolling Restart 检查失败: ${err.message}`);
|
|
2242
|
+
});
|
|
2243
|
+
}, ROLLING_RESTART_CHECK_INTERVAL_MS);
|
|
2244
|
+
// TODO: 移除调试日志 console.log(`[sc] 🔄 Rolling Restart 定时器started (interval=${ROLLING_RESTART_CHECK_INTERVAL_MS/1000}s)`);
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
_startHeartbeat() {
|
|
2248
|
+
this._hbRunning = false;
|
|
2249
|
+
heartbeatTimer = setInterval(() => this._heartbeat(), HEARTBEAT_MS);
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
async _heartbeat() {
|
|
2253
|
+
if (this._hbRunning || isShuttingDown) return;
|
|
2254
|
+
this._hbRunning = true;
|
|
2255
|
+
try {
|
|
2256
|
+
const idle = workers.filter(w => w.alive && !w.busy);
|
|
2257
|
+
await Promise.allSettled(idle.map(async (e) => {
|
|
2258
|
+
if (!e.alive || e.busy || e.hbPending || e.terminating) return;
|
|
2259
|
+
e.hbPending = true;
|
|
2260
|
+
const jobId = crypto.randomUUID();
|
|
2261
|
+
let rejectFn;
|
|
2262
|
+
const p = new Promise((res, rej) => {
|
|
2263
|
+
rejectFn = rej;
|
|
2264
|
+
const t = setTimeout(() => { pendingJobs.delete(jobId); rej(new Error("心跳超时")); }, 5000);
|
|
2265
|
+
pendingJobs.set(jobId, { resolve: res, reject: rej, timeout: t, workerId: e.id, kind: "heartbeat" });
|
|
2266
|
+
});
|
|
2267
|
+
try {
|
|
2268
|
+
e.worker.postMessage({ jobId, type: "ping" });
|
|
2269
|
+
} catch (postErr) {
|
|
2270
|
+
const rec = pendingJobs.get(jobId);
|
|
2271
|
+
if (rec) { clearTimeout(rec.timeout); pendingJobs.delete(jobId); }
|
|
2272
|
+
rejectFn(postErr);
|
|
2273
|
+
}
|
|
2274
|
+
try {
|
|
2275
|
+
await p;
|
|
2276
|
+
e.crashCount = 0;
|
|
2277
|
+
} catch {
|
|
2278
|
+
e.alive = false;
|
|
2279
|
+
e.terminating = true;
|
|
2280
|
+
e.isHeartbeatKill = true;
|
|
2281
|
+
e.worker.terminate().catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
2282
|
+
} finally {
|
|
2283
|
+
e.hbPending = false;
|
|
2284
|
+
if (e.alive && !e.terminating && !e.busy) this._processQueue();
|
|
2285
|
+
}
|
|
2286
|
+
}));
|
|
2287
|
+
} finally {
|
|
2288
|
+
this._hbRunning = false;
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
2291
|
+
|
|
2292
|
+
_onWorkerMessage(entry, msg) {
|
|
2293
|
+
if (isShuttingDown) return;
|
|
2294
|
+
const { jobId, type, data, error } = msg;
|
|
2295
|
+
if (!jobId) return;
|
|
2296
|
+
|
|
2297
|
+
const pending = pendingJobs.get(jobId);
|
|
2298
|
+
|
|
2299
|
+
if (pending?.kind === "heartbeat") {
|
|
2300
|
+
clearTimeout(pending.timeout);
|
|
2301
|
+
pendingJobs.delete(jobId);
|
|
2302
|
+
entry.crashCount = 0;
|
|
2303
|
+
if (type === "result") pending.resolve(data);
|
|
2304
|
+
else pending.reject(new Error("heartbeat failed"));
|
|
2305
|
+
return;
|
|
2306
|
+
}
|
|
2307
|
+
|
|
2308
|
+
if (entry.currentJobId !== null && jobId !== entry.currentJobId) return;
|
|
2309
|
+
|
|
2310
|
+
// 被抢占的Worker忽略旧消息
|
|
2311
|
+
if (entry._preempted) {
|
|
2312
|
+
return;
|
|
2313
|
+
}
|
|
2314
|
+
|
|
2315
|
+
// 🧠 可判定终止: 探索型任务 partialResult 实时写入(不终止任务)
|
|
2316
|
+
if (type === "partialResult") {
|
|
2317
|
+
if (data) {
|
|
2318
|
+
entry.partialResult = data;
|
|
2319
|
+
// 立即写一次 checkpoint 确保中间结果不丢
|
|
2320
|
+
const cpName = `exploratory-${data.taskType || 'unknown'}-${jobId.substring(0, 8)}`;
|
|
2321
|
+
writeSharedResult(cpName, {
|
|
2322
|
+
taskType: data.taskType || 'unknown',
|
|
2323
|
+
jobId,
|
|
2324
|
+
workerId: entry.id,
|
|
2325
|
+
status: 'running',
|
|
2326
|
+
lastHeartbeat: Date.now(),
|
|
2327
|
+
partialResult: data,
|
|
2328
|
+
}).catch(() => {});
|
|
2329
|
+
}
|
|
2330
|
+
return; // ⚡ 不清理 pendingJobs — 任务仍在运行
|
|
2331
|
+
}
|
|
2332
|
+
|
|
2333
|
+
if (!pending) {
|
|
2334
|
+
if (entry.currentJobId === null) {
|
|
2335
|
+
entry.busy = false;
|
|
2336
|
+
entry.idleSince = Date.now();
|
|
2337
|
+
}
|
|
2338
|
+
this._processQueue();
|
|
2339
|
+
return;
|
|
2340
|
+
}
|
|
2341
|
+
|
|
2342
|
+
clearTimeout(pending.timeout);
|
|
2343
|
+
pendingJobs.delete(jobId);
|
|
2344
|
+
entry.busy = false;
|
|
2345
|
+
entry.idleSince = Date.now();
|
|
2346
|
+
entry.currentJobId = null;
|
|
2347
|
+
entry.crashCount = 0;
|
|
2348
|
+
|
|
2349
|
+
// 清理探索型 checkpoint 定时器(任务完成)
|
|
2350
|
+
if (entry._exploratoryTimer) {
|
|
2351
|
+
clearInterval(entry._exploratoryTimer);
|
|
2352
|
+
entry._exploratoryTimer = null;
|
|
2353
|
+
}
|
|
2354
|
+
|
|
2355
|
+
if (type === "result") {
|
|
2356
|
+
entry.completedTasks = (entry.completedTasks || 0) + 1;
|
|
2357
|
+
pending.resolve(data);
|
|
2358
|
+
}
|
|
2359
|
+
else if (type === "error") {
|
|
2360
|
+
let errMsg = error;
|
|
2361
|
+
try { const e = JSON.parse(error); errMsg = e.message || error; } catch {}
|
|
2362
|
+
pending.reject(new Error(errMsg || "Worker task failed"));
|
|
2363
|
+
} else {
|
|
2364
|
+
pending.reject(new Error(`Unknown worker message type: ${type}`));
|
|
2365
|
+
}
|
|
2366
|
+
this._processQueue();
|
|
2367
|
+
}
|
|
2368
|
+
|
|
2369
|
+
_processQueue() {
|
|
2370
|
+
if (isShuttingDown) return;
|
|
2371
|
+
// BUG-FIX: 可重入时用setImmediateretry而非静默return,避免调度丢失
|
|
2372
|
+
if (this._reentering) {
|
|
2373
|
+
setImmediate(() => this._processQueue());
|
|
2374
|
+
return;
|
|
2375
|
+
}
|
|
2376
|
+
this._reentering = true;
|
|
2377
|
+
let _escapeCount = 0;
|
|
2378
|
+
try {
|
|
2379
|
+
for (;;) {
|
|
2380
|
+
// 逃生门:超过Worker数×2次迭代强制退出,防postMessage抛异常后busy永远无法恢复
|
|
2381
|
+
if (++_escapeCount > workers.length * 3 + 10) {
|
|
2382
|
+
console.warn('[sc] _processQueue 逃生门触发: 超过最大迭代次数');
|
|
2383
|
+
return;
|
|
2384
|
+
}
|
|
2385
|
+
|
|
2386
|
+
// ====== 三级老化防饥饿: 低优先级任务排队超过 15 s自动提升一级 ======
|
|
2387
|
+
const _agingNow = Date.now();
|
|
2388
|
+
// low → normal
|
|
2389
|
+
for (let _ai = taskQueues.low.length - 1; _ai >= 0; _ai--) {
|
|
2390
|
+
const _item = taskQueues.low[_ai];
|
|
2391
|
+
if (_item._enqueuedAt && (_agingNow - _item._enqueuedAt) > AGING_THRESHOLD_MS) {
|
|
2392
|
+
taskQueues.low.splice(_ai, 1);
|
|
2393
|
+
taskQueues.normal.push(_item);
|
|
2394
|
+
}
|
|
2395
|
+
}
|
|
2396
|
+
// normal → high
|
|
2397
|
+
for (let _ai = taskQueues.normal.length - 1; _ai >= 0; _ai--) {
|
|
2398
|
+
const _item = taskQueues.normal[_ai];
|
|
2399
|
+
if (_item._enqueuedAt && (_agingNow - _item._enqueuedAt) > AGING_THRESHOLD_MS) {
|
|
2400
|
+
taskQueues.normal.splice(_ai, 1);
|
|
2401
|
+
taskQueues.high.push(_item);
|
|
2402
|
+
}
|
|
2403
|
+
}
|
|
2404
|
+
|
|
2405
|
+
// 角色感知分发:先找可匹配的Worker
|
|
2406
|
+
let available = null;
|
|
2407
|
+
|
|
2408
|
+
for (const level of ["high", "normal", "low"]) {
|
|
2409
|
+
if (taskQueues[level].length > 0) {
|
|
2410
|
+
const candidateTask = taskQueues[level][0];
|
|
2411
|
+
const taskType = candidateTask.task?.type || "";
|
|
2412
|
+
const targetRole = this._getTaskRole(taskType);
|
|
2413
|
+
const candidateWorker = this._findAvailableWorker(targetRole);
|
|
2414
|
+
if (candidateWorker) {
|
|
2415
|
+
available = candidateWorker;
|
|
2416
|
+
break;
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
}
|
|
2420
|
+
|
|
2421
|
+
if (!available) {
|
|
2422
|
+
// 没有空闲Worker -> 高优任务抢占评估
|
|
2423
|
+
this._triggerPreemption().catch(err => {
|
|
2424
|
+
console.warn('[sc] preempt检查失败: ' + err.message);
|
|
2425
|
+
});
|
|
2426
|
+
return;
|
|
2427
|
+
}
|
|
2428
|
+
|
|
2429
|
+
let dispatched = false;
|
|
2430
|
+
try {
|
|
2431
|
+
for (const level of ["high", "normal", "low"]) {
|
|
2432
|
+
if (taskQueues[level].length > 0) {
|
|
2433
|
+
// 双重检查可用性
|
|
2434
|
+
if (!available.alive || available.terminating) continue;
|
|
2435
|
+
const next = taskQueues[level].shift();
|
|
2436
|
+
available.busy = true;
|
|
2437
|
+
this._runOnWorker(next.task, available).then(next.resolve).catch(next.reject);
|
|
2438
|
+
dispatched = true;
|
|
2439
|
+
break;
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
} catch (dispatchErr) {
|
|
2443
|
+
console.warn('[sc] _processQueue 分发异常:', dispatchErr.message);
|
|
2444
|
+
if (available) { available.busy = false; }
|
|
2445
|
+
return;
|
|
2446
|
+
}
|
|
2447
|
+
if (!dispatched) return;
|
|
2448
|
+
}
|
|
2449
|
+
} finally {
|
|
2450
|
+
this._reentering = false;
|
|
2451
|
+
}
|
|
2452
|
+
}
|
|
2453
|
+
|
|
2454
|
+
_runOnWorker(task, targetWorker) {
|
|
2455
|
+
return new Promise((resolve, reject) => {
|
|
2456
|
+
const we = targetWorker;
|
|
2457
|
+
if (!we || !we.alive || we.hbPending || we.terminating) {
|
|
2458
|
+
if (we) { we.busy = false; we.currentJobId = null; we.idleSince = Date.now(); }
|
|
2459
|
+
reject(new Error("[sc] 目标 Worker 不可用"));
|
|
2460
|
+
this._processQueue();
|
|
2461
|
+
return;
|
|
2462
|
+
}
|
|
2463
|
+
|
|
2464
|
+
we.currentJobId = null;
|
|
2465
|
+
|
|
2466
|
+
if (isShuttingDown) {
|
|
2467
|
+
we.busy = false;
|
|
2468
|
+
reject(new Error("[sc] 正在关闭"));
|
|
2469
|
+
return;
|
|
2470
|
+
}
|
|
2471
|
+
|
|
2472
|
+
const jobId = crypto.randomUUID();
|
|
2473
|
+
we.currentJobId = jobId;
|
|
2474
|
+
we.currentJobStartTime = Date.now(); // 记录任务开始时间,供preempt评估使用
|
|
2475
|
+
we.idleSince = Date.now(); // 记录任务开始时间,供监控使用
|
|
2476
|
+
|
|
2477
|
+
// 探索型任务自动写 partialResult checkpoint (每30s)
|
|
2478
|
+
if (we._exploratoryTimer) {
|
|
2479
|
+
clearInterval(we._exploratoryTimer);
|
|
2480
|
+
we._exploratoryTimer = null;
|
|
2481
|
+
}
|
|
2482
|
+
const taskType = task.type;
|
|
2483
|
+
if (TASK_CATEGORY_MAP[taskType] === TASK_CATEGORY.EXPLORATORY) {
|
|
2484
|
+
const jobIdPrefix = jobId.substring(0, 8);
|
|
2485
|
+
we._exploratoryTimer = setInterval(() => {
|
|
2486
|
+
// 🧠 可判定终止: 先向Worker请求实时进度,再写入shared/
|
|
2487
|
+
if (we.alive && !we.terminating && we.currentJobId === jobId) {
|
|
2488
|
+
try {
|
|
2489
|
+
we.worker.postMessage({ jobId, type: "requestPartialResult" });
|
|
2490
|
+
} catch {}
|
|
2491
|
+
}
|
|
2492
|
+
// 回退: 直接写入池端缓存的 partialResult (可能滞后一轮)
|
|
2493
|
+
writeSharedResult(`exploratory-${taskType}-${jobIdPrefix}`, {
|
|
2494
|
+
taskType,
|
|
2495
|
+
jobId,
|
|
2496
|
+
workerId: we.id,
|
|
2497
|
+
status: 'running',
|
|
2498
|
+
startedAt: we.idleSince,
|
|
2499
|
+
lastHeartbeat: Date.now(),
|
|
2500
|
+
partialResult: we.partialResult || null,
|
|
2501
|
+
}).catch((err) => {
|
|
2502
|
+
console.warn('[sc] ⚠️ 探索型partial结果write failed:', err?.message);
|
|
2503
|
+
});
|
|
2504
|
+
}, EXPLORATORY_CHECKPOINT_INTERVAL_MS);
|
|
2505
|
+
}
|
|
2506
|
+
// Phase 2: calcTimeout 使用 ν 轴(真实 _nu,含 ν_bias 校正)
|
|
2507
|
+
const _nu = getMetabolicRateConfig().nu;
|
|
2508
|
+
const _calcLevel = task.level || 'L3';
|
|
2509
|
+
const calculatedTimeout = calcTimeout(task.type, _calcLevel);
|
|
2510
|
+
// Phase 3: 超时计算日志(含 ν 和复杂度因子)
|
|
2511
|
+
const _timeoutLevelNum = parseInt((_calcLevel || 'L3').replace('L', ''), 10);
|
|
2512
|
+
const _timeoutComplexity = _timeoutLevelNum <= 1 ? 0.8
|
|
2513
|
+
: _timeoutLevelNum <= 3 ? 1.0
|
|
2514
|
+
: _timeoutLevelNum <= 5 ? 1.5
|
|
2515
|
+
: 2.0;
|
|
2516
|
+
// 🧠 设计决策:timeout日志不用emoji。正常超时计算是DEBUG信息,不是warn。实际超时才打warn。
|
|
2517
|
+
// 正常超时计算是DEBUG信息,已降级不刷屏
|
|
2518
|
+
const timeout = setTimeout(() => {
|
|
2519
|
+
const rec = pendingJobs.get(jobId);
|
|
2520
|
+
if (!rec) return;
|
|
2521
|
+
pendingJobs.delete(jobId);
|
|
2522
|
+
console.warn(`[sc] Worker ${we.id} 任务超时 ${(calculatedTimeout/1000).toFixed(1)}s (ν=${_nu.toFixed(2)}, level=${_calcLevel}),强制终止`);
|
|
2523
|
+
// Phase 3: 记录超时到失败上下文(供 timeoutStats 和 联合损失使用)
|
|
2524
|
+
pushFailContext(task.type, 'TIMEOUT', { level: _calcLevel });
|
|
2525
|
+
we.terminating = true;
|
|
2526
|
+
we.currentJobId = null;
|
|
2527
|
+
we.worker.terminate().catch((err) => { console.warn('[sc] 异步错误:', err?.message); }).finally(() => { we.busy = false; });
|
|
2528
|
+
// 清理探索型 checkpoint 定时器
|
|
2529
|
+
if (we._exploratoryTimer) {
|
|
2530
|
+
clearInterval(we._exploratoryTimer);
|
|
2531
|
+
we._exploratoryTimer = null;
|
|
2532
|
+
// 超时时刻写一次 checkpoint 记录超时
|
|
2533
|
+
const taskType = task.type;
|
|
2534
|
+
writeSharedResult(`exploratory-${taskType}-${jobId.substring(0, 8)}`, {
|
|
2535
|
+
taskType,
|
|
2536
|
+
jobId,
|
|
2537
|
+
workerId: we.id,
|
|
2538
|
+
startedAt: we.idleSince,
|
|
2539
|
+
status: 'timeout',
|
|
2540
|
+
lastHeartbeat: Date.now(),
|
|
2541
|
+
partialResult: we.partialResult || null,
|
|
2542
|
+
}).catch(() => {});
|
|
2543
|
+
}
|
|
2544
|
+
// LEARN-5: 超时后尝试返回部分结果
|
|
2545
|
+
if (we.partialResult) {
|
|
2546
|
+
resolve({ status: 'partial', data: we.partialResult, warning: '任务超时,返回部分结果' });
|
|
2547
|
+
} else {
|
|
2548
|
+
rec.reject(new Error(`[sc] Worker ${we.id} 超时 ${(calculatedTimeout/1000).toFixed(1)}s (ν=${_nu.toFixed(2)}, level=${_calcLevel}) 被强杀`));
|
|
2549
|
+
}
|
|
2550
|
+
}, calculatedTimeout);
|
|
2551
|
+
|
|
2552
|
+
pendingJobs.set(jobId, { resolve, reject, timeout, workerId: we.id });
|
|
2553
|
+
|
|
2554
|
+
try {
|
|
2555
|
+
we.worker.postMessage({ ...task, jobId });
|
|
2556
|
+
} catch (e) {
|
|
2557
|
+
clearTimeout(timeout);
|
|
2558
|
+
pendingJobs.delete(jobId);
|
|
2559
|
+
we.busy = false;
|
|
2560
|
+
we.currentJobId = null;
|
|
2561
|
+
reject(e);
|
|
2562
|
+
this._processQueue();
|
|
2563
|
+
}
|
|
2564
|
+
});
|
|
2565
|
+
}
|
|
2566
|
+
|
|
2567
|
+
// ====== 公开 API ======
|
|
2568
|
+
|
|
2569
|
+
exec(task, priority = "normal") {
|
|
2570
|
+
if (isShuttingDown) return Promise.reject(new Error("[sc] 正在关闭"));
|
|
2571
|
+
|
|
2572
|
+
const level = (priority === "high" || priority === "low") ? priority : "normal";
|
|
2573
|
+
const limits = { high: 50, normal: 80, low: 100 };
|
|
2574
|
+
|
|
2575
|
+
if (taskQueues[level].length >= limits[level]) {
|
|
2576
|
+
return Promise.reject(new Error(`[sc] ${level} 队列已满`));
|
|
2577
|
+
}
|
|
2578
|
+
|
|
2579
|
+
return new Promise((resolve, reject) => {
|
|
2580
|
+
taskQueues[level].push({ task, resolve, reject, _enqueuedAt: Date.now() });
|
|
2581
|
+
this._processQueue();
|
|
2582
|
+
});
|
|
2583
|
+
}
|
|
2584
|
+
|
|
2585
|
+
getStats() {
|
|
2586
|
+
const alive = workers.filter(w => w.alive);
|
|
2587
|
+
const totalQueued = Object.values(taskQueues).reduce((a, b) => a + b.length, 0);
|
|
2588
|
+
const inFlight = [...pendingJobs.values()].filter(p => p.kind !== "heartbeat").length;
|
|
2589
|
+
// 角色细粒度stats
|
|
2590
|
+
const roleBreakdown = {};
|
|
2591
|
+
for (const w of alive) {
|
|
2592
|
+
const r = w.role || "unknown";
|
|
2593
|
+
if (!roleBreakdown[r]) roleBreakdown[r] = { total: 0, busy: 0 };
|
|
2594
|
+
roleBreakdown[r].total++;
|
|
2595
|
+
if (w.busy) roleBreakdown[r].busy++;
|
|
2596
|
+
}
|
|
2597
|
+
return {
|
|
2598
|
+
total: alive.length,
|
|
2599
|
+
busy: alive.filter(w => w.busy).length,
|
|
2600
|
+
inFlight,
|
|
2601
|
+
queueDepth: totalQueued,
|
|
2602
|
+
queueHigh: taskQueues.high.length,
|
|
2603
|
+
queueNormal: taskQueues.normal.length,
|
|
2604
|
+
queueLow: taskQueues.low.length,
|
|
2605
|
+
maxWorkers: getDynamicMaxWorkers(),
|
|
2606
|
+
minWorkers: MIN_WORKERS,
|
|
2607
|
+
physicalCores: PHYSICAL_CORES,
|
|
2608
|
+
roleBreakdown,
|
|
2609
|
+
preemptedCount: this._preemptedCount,
|
|
2610
|
+
};
|
|
2611
|
+
}
|
|
2612
|
+
|
|
2613
|
+
async shutdown() {
|
|
2614
|
+
isShuttingDown = true;
|
|
2615
|
+
clearInterval(heartbeatTimer);
|
|
2616
|
+
clearInterval(scaleTimer);
|
|
2617
|
+
if (this._rollingRestartTimer) {
|
|
2618
|
+
clearInterval(this._rollingRestartTimer);
|
|
2619
|
+
this._rollingRestartTimer = null;
|
|
2620
|
+
}
|
|
2621
|
+
if (this._readyTimer) clearTimeout(this._readyTimer);
|
|
2622
|
+
|
|
2623
|
+
for (const [jobId, p] of pendingJobs) { clearTimeout(p.timeout); p.reject(new Error("[sc] 正在关闭")); }
|
|
2624
|
+
pendingJobs.clear();
|
|
2625
|
+
for (const q of Object.values(taskQueues)) {
|
|
2626
|
+
for (const item of q) {
|
|
2627
|
+
item.reject(new Error("[sc] 正在关闭"));
|
|
2628
|
+
}
|
|
2629
|
+
q.length = 0;
|
|
2630
|
+
}
|
|
2631
|
+
|
|
2632
|
+
// 老年Worker优先下线,再终止剩余Worker
|
|
2633
|
+
const sortedWorkers = [...workers].sort((a, b) => {
|
|
2634
|
+
const aAged = this._isWorkerAged(a) ? 1 : 0;
|
|
2635
|
+
const bAged = this._isWorkerAged(b) ? 1 : 0;
|
|
2636
|
+
if (aAged !== bAged) return bAged - aAged;
|
|
2637
|
+
return (a.completedTasks || 0) - (b.completedTasks || 0);
|
|
2638
|
+
});
|
|
2639
|
+
await Promise.allSettled(
|
|
2640
|
+
sortedWorkers.map(async w => { w.alive = false; try { await w.worker.terminate(); } catch {} })
|
|
2641
|
+
);
|
|
2642
|
+
workers.length = 0;
|
|
2643
|
+
// TODO: 移除调试日志 console.log("[sc] 已关闭");
|
|
2644
|
+
}
|
|
2645
|
+
|
|
2646
|
+
/**
|
|
2647
|
+
* 🔧 v5.31.0: 公开重启方法,替代 cpuAbort 直接调私有API
|
|
2648
|
+
* @param {number} count - Worker数量
|
|
2649
|
+
*/
|
|
2650
|
+
restart(count) {
|
|
2651
|
+
this._hbRunning = false;
|
|
2652
|
+
this._initPool(count);
|
|
2653
|
+
this._startHeartbeat();
|
|
2654
|
+
this._startScaleTimer();
|
|
2655
|
+
}
|
|
2656
|
+
|
|
2657
|
+
async warmup() {
|
|
2658
|
+
if (isShuttingDown) return { warmed: false, newlyWarmed: 0, warmedCount: warmedModels.size, totalTargets: 0 };
|
|
2659
|
+
// TODO: 移除调试日志 console.log("[sc] 🔥 warmup...");
|
|
2660
|
+
const cache = globalThis.__oc_embedded_auth_cache__;
|
|
2661
|
+
const cfg = await safeReadJson(join(homedir(), ".openclaw", "openclaw.json"));
|
|
2662
|
+
const targets = [];
|
|
2663
|
+
if (cfg?.models?.providers) {
|
|
2664
|
+
for (const [provider, pcfg] of Object.entries(cfg.models.providers)) {
|
|
2665
|
+
for (const m of (pcfg.models || [])) {
|
|
2666
|
+
if (m.id && !m.id.startsWith("glm")) targets.push(`${provider}/${m.id}`);
|
|
2667
|
+
}
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2670
|
+
// 如果没有配置任何模型,尝试从环境变量读取默认模型;没有也不硬编码默认值
|
|
2671
|
+
if (targets.length === 0) {
|
|
2672
|
+
const envDefault = getEnv('OPENCLAW_DEFAULT_MODEL', '');
|
|
2673
|
+
if (envDefault) {
|
|
2674
|
+
targets.push(envDefault);
|
|
2675
|
+
}
|
|
2676
|
+
// 无默认模型时不推硬编码值,由调用方处理缺失配置
|
|
2677
|
+
}
|
|
2678
|
+
const unique = [...new Set(targets)].slice(0, 20);
|
|
2679
|
+
let newlyWarmed = 0;
|
|
2680
|
+
for (const model of unique) {
|
|
2681
|
+
if (isModelWarmed(model)) continue;
|
|
2682
|
+
try {
|
|
2683
|
+
const [p, m] = model.split("/");
|
|
2684
|
+
const r = await this.exec({ type: "resolve-model", provider: p, modelId: m }, "high");
|
|
2685
|
+
if (cache && typeof cache.set === "function") cache.set(model, r);
|
|
2686
|
+
markModelWarmed(model);
|
|
2687
|
+
newlyWarmed++;
|
|
2688
|
+
} catch (err) { console.warn(` ⚠️ ${model}: ${err.message}`); }
|
|
2689
|
+
}
|
|
2690
|
+
const allWarmed = unique.every(m => isModelWarmed(m));
|
|
2691
|
+
warmupDone = allWarmed;
|
|
2692
|
+
return { warmed: allWarmed, warmedCount: warmedModels.size, newlyWarmed, totalTargets: unique.length };
|
|
2693
|
+
}
|
|
2694
|
+
|
|
2695
|
+
get ready() { return readyPromise; }
|
|
2696
|
+
}
|
|
2697
|
+
|
|
2698
|
+
// ====== 单例 ======
|
|
2699
|
+
const pool = new CpuWorkerPool();
|
|
2700
|
+
|
|
2701
|
+
// ====== 缓存包装函数(优化性能)======
|
|
2702
|
+
function getCachedStats() {
|
|
2703
|
+
const now = Date.now();
|
|
2704
|
+
const ttl = getDynamicCacheTTL().stats;
|
|
2705
|
+
if (ttl === CACHE_DISABLED) { // 不缓存,实时读取
|
|
2706
|
+
cachedStats = pool.getStats();
|
|
2707
|
+
cachedStatsTime = now;
|
|
2708
|
+
return cachedStats;
|
|
2709
|
+
}
|
|
2710
|
+
if (cachedStats && (now - cachedStatsTime < ttl)) {
|
|
2711
|
+
return cachedStats;
|
|
2712
|
+
}
|
|
2713
|
+
cachedStats = pool.getStats();
|
|
2714
|
+
cachedStatsTime = now;
|
|
2715
|
+
return cachedStats;
|
|
2716
|
+
}
|
|
2717
|
+
|
|
2718
|
+
function getCachedMemoryLevel() {
|
|
2719
|
+
const now = Date.now();
|
|
2720
|
+
const ttl = getDynamicCacheTTL().mem;
|
|
2721
|
+
if (ttl === CACHE_DISABLED) { // 不缓存,实时读取
|
|
2722
|
+
cachedMemLevel = getMemoryLevel();
|
|
2723
|
+
cachedMemTime = now;
|
|
2724
|
+
return cachedMemLevel;
|
|
2725
|
+
}
|
|
2726
|
+
if (cachedMemLevel && (now - cachedMemTime < ttl)) {
|
|
2727
|
+
return cachedMemLevel;
|
|
2728
|
+
}
|
|
2729
|
+
cachedMemLevel = getMemoryLevel();
|
|
2730
|
+
cachedMemTime = now;
|
|
2731
|
+
return cachedMemLevel;
|
|
2732
|
+
}
|
|
2733
|
+
|
|
2734
|
+
/**
|
|
2735
|
+
* 🔍 compressTaskDescription - 子agent任务描述去冗余+checksum校验
|
|
2736
|
+
*
|
|
2737
|
+
* 压缩规则:
|
|
2738
|
+
* 1. 保留头部(🏃 模式、思考行)
|
|
2739
|
+
* 2. 提取核心指令(任务、步骤、输出路径、验收标准)
|
|
2740
|
+
* 3. 移除模板尾巴(⚡ 规则段落、熔断、compaction、安全约束等)
|
|
2741
|
+
* 4. 已压缩的单行规则(⚡ 规则:...)保留
|
|
2742
|
+
* 5. 末尾添加SHA256 checksum(前100字)
|
|
2743
|
+
*
|
|
2744
|
+
* @param {string} taskStr - 原始任务描述
|
|
2745
|
+
* @returns {{ compressed: string, originalLen: number, compressedLen: number, ratio: number, checksum: string }}
|
|
2746
|
+
*/
|
|
2747
|
+
function compressTaskDescription(taskStr) {
|
|
2748
|
+
if (!taskStr || typeof taskStr !== 'string') {
|
|
2749
|
+
return { compressed: taskStr || '', originalLen: 0, compressedLen: 0, ratio: 0, checksum: '' };
|
|
2750
|
+
}
|
|
2751
|
+
if (taskStr.length < 50) {
|
|
2752
|
+
// 太短无需压缩,加校验即可
|
|
2753
|
+
const shortChecksum = crypto.createHash('sha256').update(taskStr, 'utf-8').digest('hex').substring(0, 16);
|
|
2754
|
+
const annotated = taskStr + `\n\n[checksum:${shortChecksum}]`;
|
|
2755
|
+
return { compressed: annotated, originalLen: taskStr.length, compressedLen: annotated.length, ratio: -Math.round((annotated.length - taskStr.length) / taskStr.length * 100), checksum: shortChecksum };
|
|
2756
|
+
}
|
|
2757
|
+
|
|
2758
|
+
const originalLen = taskStr.length;
|
|
2759
|
+
const lines = taskStr.split('\n');
|
|
2760
|
+
const kept = [];
|
|
2761
|
+
let skipMode = 0; // 0=normal, 1=in-rules-block, 2=in-example-block
|
|
2762
|
+
let hasHeader = false;
|
|
2763
|
+
|
|
2764
|
+
for (let i = 0; i < lines.length; i++) {
|
|
2765
|
+
const line = lines[i];
|
|
2766
|
+
const trimmed = line.trim();
|
|
2767
|
+
|
|
2768
|
+
// ---- 模式1: 检测并保留头部 ----
|
|
2769
|
+
if (trimmed.startsWith('🏃') || trimmed.startsWith('🏃') ||
|
|
2770
|
+
trimmed.match(/^模式\s*[::]/) || trimmed.match(/^思考\s*[::]/)) {
|
|
2771
|
+
kept.push(line);
|
|
2772
|
+
hasHeader = true;
|
|
2773
|
+
continue;
|
|
2774
|
+
}
|
|
2775
|
+
|
|
2776
|
+
// ---- 模式2: 检测规则段落入口 ----
|
|
2777
|
+
if (trimmed.startsWith('⚡') && (
|
|
2778
|
+
trimmed.includes('子 agent') ||
|
|
2779
|
+
trimmed.includes('运行规则') ||
|
|
2780
|
+
trimmed.includes('永久生效') ||
|
|
2781
|
+
trimmed.includes('不可省略') ||
|
|
2782
|
+
trimmed.match(/规则\s*\(/) ||
|
|
2783
|
+
trimmed.match(/模板.*运行规则/)
|
|
2784
|
+
)) {
|
|
2785
|
+
skipMode = 1;
|
|
2786
|
+
continue;
|
|
2787
|
+
}
|
|
2788
|
+
|
|
2789
|
+
// ---- 模式3: 检测示例段落入口 ----
|
|
2790
|
+
if (trimmed.startsWith('成功示例:') ||
|
|
2791
|
+
trimmed.startsWith('超时示例:') ||
|
|
2792
|
+
trimmed.startsWith('失败示例:') ||
|
|
2793
|
+
trimmed.startsWith('死命令:') ||
|
|
2794
|
+
trimmed.startsWith('结果规范:') ||
|
|
2795
|
+
trimmed.includes('直接以') && trimmed.includes('{') && trimmed.includes('结尾')) {
|
|
2796
|
+
skipMode = 2;
|
|
2797
|
+
continue;
|
|
2798
|
+
}
|
|
2799
|
+
|
|
2800
|
+
// ---- 跳过规则段落内容 ----
|
|
2801
|
+
if (skipMode === 1) {
|
|
2802
|
+
// 检测规则段落的结束标记
|
|
2803
|
+
if (trimmed === '' || trimmed.startsWith('#') || trimmed.startsWith('---') ||
|
|
2804
|
+
trimmed.startsWith('任务:') || trimmed.startsWith('步骤:') ||
|
|
2805
|
+
trimmed.startsWith('输出路径:') || trimmed.startsWith('验收标准:') ||
|
|
2806
|
+
trimmed.startsWith('结果') || trimmed.startsWith('---')) {
|
|
2807
|
+
// 空行可能是段落分割,但如果后面还是规则就继续
|
|
2808
|
+
if (trimmed === '' && i + 1 < lines.length) {
|
|
2809
|
+
const next = lines[i + 1].trim();
|
|
2810
|
+
if (next.match(/^\d+\./) || next.startsWith('三级') || next.startsWith('Tool') ||
|
|
2811
|
+
next.startsWith('安全') || next.startsWith('工具调用') || next.startsWith('禁止')) {
|
|
2812
|
+
continue;
|
|
2813
|
+
}
|
|
2814
|
+
}
|
|
2815
|
+
skipMode = 0;
|
|
2816
|
+
// 不输出空行,减少冗余
|
|
2817
|
+
if (trimmed && !trimmed.startsWith('⚡')) kept.push(line);
|
|
2818
|
+
}
|
|
2819
|
+
continue;
|
|
2820
|
+
}
|
|
2821
|
+
|
|
2822
|
+
// ---- 跳过示例段落内容 ----
|
|
2823
|
+
if (skipMode === 2) {
|
|
2824
|
+
if (trimmed === '' || trimmed.match(/^(?:任务|步骤|输出|验收|结果|\{|```)/)) {
|
|
2825
|
+
skipMode = 0;
|
|
2826
|
+
if (trimmed) kept.push(line);
|
|
2827
|
+
}
|
|
2828
|
+
continue;
|
|
2829
|
+
}
|
|
2830
|
+
|
|
2831
|
+
// ---- 模式4: 跳过已知的独立模板行 ----
|
|
2832
|
+
const skipLinePatterns = [
|
|
2833
|
+
/^⚡\s*子\s*agent\s*运行规则/i,
|
|
2834
|
+
/^⚡\s*规则(?!:)/i,
|
|
2835
|
+
/^⚡\s*子\s*agent\s*模板/i,
|
|
2836
|
+
/^⚡\s*安全约束/i,
|
|
2837
|
+
/^三级熔断/i,
|
|
2838
|
+
/^Tool Output Compaction/i,
|
|
2839
|
+
/安全约束.*永久生效/i,
|
|
2840
|
+
/工具调用合并/i,
|
|
2841
|
+
/^禁止.*browser/i,
|
|
2842
|
+
/^禁止.*message/i,
|
|
2843
|
+
/禁止使用browser/i,
|
|
2844
|
+
/禁止使用message/i,
|
|
2845
|
+
/exec.*工作区/i,
|
|
2846
|
+
/使?用前.*(?:调|core_routeTask)/i,
|
|
2847
|
+
/末尾.*MEMORY\.md/i,
|
|
2848
|
+
/结构化返回/i,
|
|
2849
|
+
/死命令:/i,
|
|
2850
|
+
/成功示例:/i,
|
|
2851
|
+
/超时示例:/i,
|
|
2852
|
+
/失败示例:/i,
|
|
2853
|
+
/非.*JSON.*崩溃/i,
|
|
2854
|
+
/\[\+\d+ more characters/i,
|
|
2855
|
+
/rerun with narrower/i,
|
|
2856
|
+
];
|
|
2857
|
+
let isSkip = false;
|
|
2858
|
+
for (const pat of skipLinePatterns) {
|
|
2859
|
+
if (pat.test(trimmed)) { isSkip = true; break; }
|
|
2860
|
+
}
|
|
2861
|
+
if (isSkip) continue;
|
|
2862
|
+
|
|
2863
|
+
// ---- 模式5: 跳过规则编号行 ----
|
|
2864
|
+
if (trimmed.match(/^\d+\.\s*(?:三级熔断|Tool Output|安全约束|禁止browser|禁止message|禁止使用|工具调用|结构化|返回接口|建议|MEMORY\.md)/i)) {
|
|
2865
|
+
continue;
|
|
2866
|
+
}
|
|
2867
|
+
|
|
2868
|
+
// ---- 模式6: 检测已压缩的规则单行,直接保留 ----
|
|
2869
|
+
if (trimmed.startsWith('⚡ 规则:') || trimmed.match(/^⚡\s*规则:/)) {
|
|
2870
|
+
kept.push(line);
|
|
2871
|
+
continue;
|
|
2872
|
+
}
|
|
2873
|
+
|
|
2874
|
+
// ---- 核心内容:保留 ----
|
|
2875
|
+
kept.push(line);
|
|
2876
|
+
}
|
|
2877
|
+
|
|
2878
|
+
// 合并连续的空行为最多一个
|
|
2879
|
+
let compressed = kept.join('\n');
|
|
2880
|
+
compressed = compressed.replace(/\n{3,}/g, '\n\n').trim();
|
|
2881
|
+
|
|
2882
|
+
const compressedLen = compressed.length;
|
|
2883
|
+
const ratio = originalLen > 0 ? Math.round((1 - compressedLen / originalLen) * 100) : 0;
|
|
2884
|
+
|
|
2885
|
+
// 生成checksum: 对整个compressed做SHA256
|
|
2886
|
+
const checksumSource = compressed;
|
|
2887
|
+
const checksum = crypto.createHash('sha256').update(checksumSource, 'utf-8').digest('hex').substring(0, 16);
|
|
2888
|
+
|
|
2889
|
+
compressed += '\n\n[checksum:' + checksum + ']';
|
|
2890
|
+
|
|
2891
|
+
return { compressed, originalLen, compressedLen, ratio, checksum };
|
|
2892
|
+
}
|
|
2893
|
+
|
|
2894
|
+
let mcpServerHandle = null;
|
|
2895
|
+
|
|
2896
|
+
// ====== OpenClaw 插件生命周期钩子 ======
|
|
2897
|
+
function register(ctx) {
|
|
2898
|
+
ctx.logger.info("sc v5.38.0 registered (dual-mode + 缓存 + 动态batchSize + dynamic cores)");
|
|
2899
|
+
|
|
2900
|
+
|
|
2901
|
+
|
|
2902
|
+
// 初始化triple-evidenceroute audit路由证据系统
|
|
2903
|
+
|
|
2904
|
+
// ⛔ route-audit 已物理删除 (2026-06-13)
|
|
2905
|
+
initRouteEvidence().then((ok) => {
|
|
2906
|
+
if (ok) ctx.logger.info('[sc] 🏛️ triple-evidence routing system ready');
|
|
2907
|
+
}).catch(err => {
|
|
2908
|
+
ctx.logger.warn(`[sc] ⚠️ triple-evidenceinit failed: ${err.message}`);
|
|
2909
|
+
});
|
|
2910
|
+
|
|
2911
|
+
// 🧠 USearch HNSW — bridge.js 按需加载,内存映射索引
|
|
2912
|
+
// 见 vector/usearch-bridge.js — 首次搜索冷加载模型,之后 ~7ms
|
|
2913
|
+
// ====== 🧬 evolution engine + 🛡️ 快速路径缓存 注册 ======
|
|
2914
|
+
|
|
2915
|
+
|
|
2916
|
+
|
|
2917
|
+
// 🧬 进化参数注入辅助函数:在所有路由结果中添加进化策略参数
|
|
2918
|
+
|
|
2919
|
+
|
|
2920
|
+
|
|
2921
|
+
// ====== 多核并行决策引擎工具 ======
|
|
2922
|
+
|
|
2923
|
+
|
|
2924
|
+
|
|
2925
|
+
// ====== triple-evidenceroute audit路由证据查询工具 ======
|
|
2926
|
+
|
|
2927
|
+
// ====== 🧠 负载模式预测状态查询 ======
|
|
2928
|
+
|
|
2929
|
+
|
|
2930
|
+
// ====== 🔗 任务链检测与执行 ======
|
|
2931
|
+
|
|
2932
|
+
|
|
2933
|
+
// ====== triple-evidenceroute audit路由证据查询工具 ======
|
|
2934
|
+
|
|
2935
|
+
|
|
2936
|
+
|
|
2937
|
+
// ====== 原有工具 ======
|
|
2938
|
+
|
|
2939
|
+
|
|
2940
|
+
|
|
2941
|
+
|
|
2942
|
+
|
|
2943
|
+
|
|
2944
|
+
|
|
2945
|
+
|
|
2946
|
+
|
|
2947
|
+
|
|
2948
|
+
|
|
2949
|
+
|
|
2950
|
+
|
|
2951
|
+
// ====== 🧠 子 agent 模型分配器 ======
|
|
2952
|
+
|
|
2953
|
+
|
|
2954
|
+
|
|
2955
|
+
|
|
2956
|
+
|
|
2957
|
+
|
|
2958
|
+
// ====== ? Checkpoint 恢复 ======
|
|
2959
|
+
|
|
2960
|
+
|
|
2961
|
+
|
|
2962
|
+
|
|
2963
|
+
|
|
2964
|
+
|
|
2965
|
+
|
|
2966
|
+
|
|
2967
|
+
|
|
2968
|
+
|
|
2969
|
+
|
|
2970
|
+
|
|
2971
|
+
|
|
2972
|
+
|
|
2973
|
+
|
|
2974
|
+
|
|
2975
|
+
|
|
2976
|
+
|
|
2977
|
+
|
|
2978
|
+
|
|
2979
|
+
|
|
2980
|
+
|
|
2981
|
+
|
|
2982
|
+
|
|
2983
|
+
|
|
2984
|
+
|
|
2985
|
+
// ====== 🧠 语义搜索(基于 embedding 模型,从 openclaw.json 读取配置)======
|
|
2986
|
+
|
|
2987
|
+
|
|
2988
|
+
// ====== 🏃 快速执行器 - 零LLM机械操作(L0级) ======
|
|
2989
|
+
|
|
2990
|
+
|
|
2991
|
+
|
|
2992
|
+
|
|
2993
|
+
|
|
2994
|
+
|
|
2995
|
+
// ====== cpu_apiQueue — API请求队列 ======
|
|
2996
|
+
|
|
2997
|
+
|
|
2998
|
+
// ====== 🦞 Task Center 工具注册 ======
|
|
2999
|
+
|
|
3000
|
+
|
|
3001
|
+
|
|
3002
|
+
|
|
3003
|
+
|
|
3004
|
+
|
|
3005
|
+
// ====== 失败原因分类器 + circuit breaker log ======
|
|
3006
|
+
// 工具自身问题(不retry,直接换路)vs 瞬时问题(允许retry)
|
|
3007
|
+
const TOOL_OWN_ISSUE_PATTERNS = [
|
|
3008
|
+
// HTTP 状态码
|
|
3009
|
+
/40[134]/i, // 400/401/403/404
|
|
3010
|
+
/50[0-9]/i, // 500-509
|
|
3011
|
+
// 工具缺失/无效
|
|
3012
|
+
/not found/i,
|
|
3013
|
+
/does not exist/i,
|
|
3014
|
+
/is not a function/i,
|
|
3015
|
+
/invalid/i,
|
|
3016
|
+
/unknown/i,
|
|
3017
|
+
/unexpected/i,
|
|
3018
|
+
/missing/i,
|
|
3019
|
+
/required/i,
|
|
3020
|
+
// 能力不足
|
|
3021
|
+
/insufficient/i,
|
|
3022
|
+
/not supported/i,
|
|
3023
|
+
/unavailable/i,
|
|
3024
|
+
/cannot/i,
|
|
3025
|
+
/refused to/i,
|
|
3026
|
+
/denied/i,
|
|
3027
|
+
/拒绝/i,
|
|
3028
|
+
/不支持/i,
|
|
3029
|
+
/不存在/i,
|
|
3030
|
+
/无效/i,
|
|
3031
|
+
// 超时(工具自身)
|
|
3032
|
+
/timed?out/i,
|
|
3033
|
+
/timeout/i,
|
|
3034
|
+
/超时/i,
|
|
3035
|
+
// 语法/解析错误
|
|
3036
|
+
/syntax/i,
|
|
3037
|
+
/parse/i,
|
|
3038
|
+
/malformed/i,
|
|
3039
|
+
];
|
|
3040
|
+
|
|
3041
|
+
const TRANSIENT_ISSUE_PATTERNS = [
|
|
3042
|
+
// 网络抖动
|
|
3043
|
+
/ECONNRESET/i,
|
|
3044
|
+
/ETIMED?OUT/i,
|
|
3045
|
+
/ECONNREFUSED/i,
|
|
3046
|
+
/ENOTFOUND/i,
|
|
3047
|
+
/EAI_AGAIN/i,
|
|
3048
|
+
/EPIPE/i,
|
|
3049
|
+
/socket/i,
|
|
3050
|
+
/network/i,
|
|
3051
|
+
/网络/i,
|
|
3052
|
+
/连接/i,
|
|
3053
|
+
// 文件锁
|
|
3054
|
+
/EBUSY/i,
|
|
3055
|
+
/EACCES/i,
|
|
3056
|
+
// 临时限流
|
|
3057
|
+
/429/i,
|
|
3058
|
+
/ratelimit/i,
|
|
3059
|
+
/rate.?limit/i,
|
|
3060
|
+
/too many/i,
|
|
3061
|
+
/throttl/i,
|
|
3062
|
+
// 临时不可用
|
|
3063
|
+
/retry again/i,
|
|
3064
|
+
/try again/i,
|
|
3065
|
+
/temporarily/i,
|
|
3066
|
+
/暂时/i,
|
|
3067
|
+
/稍后/i,
|
|
3068
|
+
/busy/i,
|
|
3069
|
+
];
|
|
3070
|
+
|
|
3071
|
+
function classifyFailure(error) {
|
|
3072
|
+
const msg = (error?.message || error?.toString() || '').toString();
|
|
3073
|
+
|
|
3074
|
+
// 优先检查瞬时模式
|
|
3075
|
+
for (const p of TRANSIENT_ISSUE_PATTERNS) {
|
|
3076
|
+
if (p.test(msg)) return { type: 'transient', reason: msg.substring(0, 120) };
|
|
3077
|
+
}
|
|
3078
|
+
|
|
3079
|
+
// 再检查工具自身问题
|
|
3080
|
+
for (const p of TOOL_OWN_ISSUE_PATTERNS) {
|
|
3081
|
+
if (p.test(msg)) return { type: 'tool_own', reason: msg.substring(0, 120) };
|
|
3082
|
+
}
|
|
3083
|
+
|
|
3084
|
+
// 默认:未知问题算瞬时,允许retry
|
|
3085
|
+
return { type: 'transient', reason: msg.substring(0, 120), classification: 'default_transient' };
|
|
3086
|
+
}
|
|
3087
|
+
|
|
3088
|
+
async function writeCircuitBreakerLog(entry) {
|
|
3089
|
+
try {
|
|
3090
|
+
const { readFile, writeFile } = await import('fs/promises');
|
|
3091
|
+
const { join: jn } = await import('path');
|
|
3092
|
+
const logPath = jn(SHARED_DIR, 'circuit-breaker-log.json');
|
|
3093
|
+
|
|
3094
|
+
let log = [];
|
|
3095
|
+
try {
|
|
3096
|
+
const raw = await readFile(logPath, 'utf-8');
|
|
3097
|
+
log = JSON.parse(raw);
|
|
3098
|
+
} catch {}
|
|
3099
|
+
|
|
3100
|
+
if (!Array.isArray(log)) log = [];
|
|
3101
|
+
|
|
3102
|
+
entry.timestamp = new Date().toISOString();
|
|
3103
|
+
log.push(entry);
|
|
3104
|
+
|
|
3105
|
+
// 保留最近 100
|
|
3106
|
+
if (log.length > 100) log = log.slice(-100);
|
|
3107
|
+
|
|
3108
|
+
await writeFile(logPath, JSON.stringify(log, null, 2), 'utf-8');
|
|
3109
|
+
} catch (err) {
|
|
3110
|
+
console.warn('[sc] ⚠️ circuit breaker logwrite failed:', err.message);
|
|
3111
|
+
}
|
|
3112
|
+
}
|
|
3113
|
+
|
|
3114
|
+
// ?? MCP Server — 立即启动
|
|
3115
|
+
// 不用 setImmediate:OpenClaw 的 bundle-mcp 在当前事件循环就查端口18790,
|
|
3116
|
+
// setImmediate 推送到下轮事件循环会导致 bundle-mcp 先拿到 ECONNREFUSED。
|
|
3117
|
+
// 防二次 register()(OpenClaw 可能在加载工具策略后再次注册插件)
|
|
3118
|
+
if (mcpServerHandle) { return; }
|
|
3119
|
+
import('./tools/bridge.js').then(bridge => {
|
|
3120
|
+
bridge.setCpuInstance({
|
|
3121
|
+
pool,
|
|
3122
|
+
getStats: () => getCachedStats(),
|
|
3123
|
+
getMemoryLevel,
|
|
3124
|
+
});
|
|
3125
|
+
bridge.startMcpServer(MCP_PORT).then(h => {
|
|
3126
|
+
mcpServerHandle = h;
|
|
3127
|
+
ctx.logger.info("sc MCP Server started (http://127.0.0.1:" + h.port + "/sse)");
|
|
3128
|
+
}).catch(e => {
|
|
3129
|
+
if (e?.code === 'EADDRINUSE') {
|
|
3130
|
+
ctx.logger.info("sc MCP Server 端口已被占用(旧实例在服务),跳过启动");
|
|
3131
|
+
} else {
|
|
3132
|
+
ctx.logger.warn("sc MCP Server start skipped: " + e.message);
|
|
3133
|
+
}
|
|
3134
|
+
});
|
|
3135
|
+
}).catch(e => {
|
|
3136
|
+
ctx.logger.warn("sc MCP Server 不可用: " + e.message);
|
|
3137
|
+
});
|
|
3138
|
+
}
|
|
3139
|
+
|
|
3140
|
+
async function activate(ctx) {
|
|
3141
|
+
ctx.logger.info("sc v5.38.0 Worker 池已激活 (" + getCachedStats().total + "/" + getCachedStats().maxWorkers + ")");
|
|
3142
|
+
pool.warmup().catch(e => ctx.logger.error("sc warmup失败: " + e.message));
|
|
3143
|
+
|
|
3144
|
+
// 🗄️ memory/shared/ 目录初始化
|
|
3145
|
+
ensureSharedDir().catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
3146
|
+
ctx.logger.info("✅ memory/shared/ 目录ready");
|
|
3147
|
+
|
|
3148
|
+
// 🦞 初始化独立日志系统 + hippocampus集成
|
|
3149
|
+
initLogger().then(logger => {
|
|
3150
|
+
global.__sansanLogger = logger;
|
|
3151
|
+
ctx.logger.info('📋 日志系统已初始化 (logs/, 轮转5MB×5, hippocampus15min)');
|
|
3152
|
+
}).catch(err => {
|
|
3153
|
+
ctx.logger.warn(`[sc] ⚠️ 日志系统init failed: ${err.message}`);
|
|
3154
|
+
});
|
|
3155
|
+
|
|
3156
|
+
|
|
3157
|
+
|
|
3158
|
+
// 🛡️ 高频任务快速路径初始化
|
|
3159
|
+
tcell.init().then(() => {
|
|
3160
|
+
ctx.logger.info('🛡️ fast-path cachestarted');
|
|
3161
|
+
const tcellStats = tcell.getStats();
|
|
3162
|
+
ctx.logger.info(`🛡️ fast-path cache: ${tcellStats.totalEntries} 缓存, 命中率 ${tcellStats.hitRate}`);
|
|
3163
|
+
}).catch(err => {
|
|
3164
|
+
ctx.logger.warn(`🛡️ fast-path cacheinit failed: ${err.message}`);
|
|
3165
|
+
});
|
|
3166
|
+
|
|
3167
|
+
// 🧹 每小时清理 >1小时未读的共享文件
|
|
3168
|
+
cleanupSharedDir(3600000).catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
3169
|
+
// 🧹 启动时清理过期 checkpoint
|
|
3170
|
+
cleanupCheckpoints().catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
3171
|
+
const cleanupTimer = setInterval(() => {
|
|
3172
|
+
cleanupSharedDir(3600000).catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
3173
|
+
cleanupCheckpoints().catch((err) => { console.warn('[sc] 异步错误:', err?.message); });
|
|
3174
|
+
cleanupChainLogs().catch((err) => { console.warn('[chain scheduler] 异步错误:', err?.message); });
|
|
3175
|
+
}, 3600000);
|
|
3176
|
+
global.__sansanCpuCleanupTimer = cleanupTimer;
|
|
3177
|
+
|
|
3178
|
+
// 🔗 任务链日志清理(集成到 cleanupTimer)
|
|
3179
|
+
// 第一小时内先执行一次
|
|
3180
|
+
cleanupChainLogs().catch(err => {
|
|
3181
|
+
console.warn('[chain scheduler] ⚠️ 初始链日志清理失败:', err.message);
|
|
3182
|
+
});
|
|
3183
|
+
|
|
3184
|
+
// 🔬 启动慢通道聚合(每5min检测失败趋势)
|
|
3185
|
+
slowChannelState.timer = setInterval(() => {
|
|
3186
|
+
slowChannelAggregate();
|
|
3187
|
+
}, 5 * 60 * 1000);
|
|
3188
|
+
ctx.logger.info("🔬 慢通道聚合started (间隔5min)");
|
|
3189
|
+
|
|
3190
|
+
// 🧬 启动资源调度自动调节(每15s)
|
|
3191
|
+
startMetabolicAutoRegulation();
|
|
3192
|
+
ctx.logger.info(`🧬 metabolic rate系统started (初始=${_metabolicRate.toFixed(2)}, 自动调节每15s)`);
|
|
3193
|
+
|
|
3194
|
+
// 🤵 初始化steward rules engine (2026-06-21: 同步,不读外部文件)
|
|
3195
|
+
StewardGuard.init();
|
|
3196
|
+
|
|
3197
|
+
|
|
3198
|
+
|
|
3199
|
+
// 🔗 每小时清理任务链日志
|
|
3200
|
+
// 已集成到 cleanupTimer 中
|
|
3201
|
+
|
|
3202
|
+
// 🧰 加载custom tools(从 tools/custom/ 扫描注册)
|
|
3203
|
+
import('./lib/tool-registry.js').then(({ loadCustomTools }) => {
|
|
3204
|
+
loadCustomTools(ctx).catch(err => {
|
|
3205
|
+
ctx.logger.warn(`[tool-registry] ⚠️ custom tools加载失败: ${err.message}`);
|
|
3206
|
+
});
|
|
3207
|
+
}).catch(err => {
|
|
3208
|
+
ctx.logger.warn(`[tool-registry] ⚠️ 加载 tool-registry 模块失败: ${err.message}`);
|
|
3209
|
+
});
|
|
3210
|
+
|
|
3211
|
+
// 🛡️ 注册enrichment layer(cpu_enrichSubagentTask) — 子agent任务话术enrichment layer
|
|
3212
|
+
try {
|
|
3213
|
+
await registerEnrichSubagent(ctx, { pool });
|
|
3214
|
+
ctx.logger.info('🛡️ cpu_enrichSubagentTask enrichment layerregistered');
|
|
3215
|
+
} catch (err) {
|
|
3216
|
+
ctx.logger.warn(`[sc] ⚠️ enrichment layercpu_enrichSubagentTask注册失败: ${err.message}`);
|
|
3217
|
+
}
|
|
3218
|
+
|
|
3219
|
+
// 🔍 注册tool auto-discover(core_toolDiscover) — 自动捕获新工具并加入推荐
|
|
3220
|
+
try {
|
|
3221
|
+
await registerToolDiscover(ctx, {});
|
|
3222
|
+
ctx.logger.info('🔍 core_toolDiscover tool auto-discoverregistered');
|
|
3223
|
+
} catch (err) {
|
|
3224
|
+
ctx.logger.warn(`[sc] ⚠️ tool auto-discovercore_toolDiscover注册失败: ${err.message}`);
|
|
3225
|
+
}
|
|
3226
|
+
|
|
3227
|
+
// 📊 dashboard自动启动:启动后1min弹窗,延迟启动防挤占启动资源
|
|
3228
|
+
// 🧠 设计决策:不跟Gateway同时启动,等Worker池和MCP稳定后再弹。
|
|
3229
|
+
// shutdown时同步杀掉dashboard进程(见 shutdown() 函数)。
|
|
3230
|
+
// dashboard自动启动,失败后每30sretry,最多5次(防MCP热重载导致启动失败)
|
|
3231
|
+
// ★ 自动探测 Python 安装路径:先遍历 Python 目录取最高版本,再兜底 PATH
|
|
3232
|
+
const findPythonExe = () => {
|
|
3233
|
+
const pythonDir = join(homedir(), 'AppData', 'Local', 'Programs', 'Python');
|
|
3234
|
+
try {
|
|
3235
|
+
const entries = readdirSync(pythonDir);
|
|
3236
|
+
// 匹配 PythonXXX 格式的版本目录,按版本号降序排列
|
|
3237
|
+
const pyDirs = entries
|
|
3238
|
+
.filter(e => /^Python\d+$/.test(e))
|
|
3239
|
+
.sort((a, b) => parseInt(b.replace('Python', ''), 10) - parseInt(a.replace('Python', ''), 10));
|
|
3240
|
+
for (const dir of pyDirs) {
|
|
3241
|
+
const exe = join(pythonDir, dir, 'pythonw.exe');
|
|
3242
|
+
if (existsSync(exe)) {
|
|
3243
|
+
ctx.logger.info(`[sc] 📊 自动探测到 Python: ${exe}`);
|
|
3244
|
+
return exe;
|
|
3245
|
+
}
|
|
3246
|
+
}
|
|
3247
|
+
} catch {}
|
|
3248
|
+
// 兜底:PATH 中找 pythonw
|
|
3249
|
+
try {
|
|
3250
|
+
spawnSync('pythonw', ['--version'], { timeout: 3000, stdio: 'pipe' });
|
|
3251
|
+
ctx.logger.info('[sc] 📊 使用 PATH 中的 pythonw');
|
|
3252
|
+
return 'pythonw';
|
|
3253
|
+
} catch {}
|
|
3254
|
+
ctx.logger.warn('[sc] ⚠️ 未找到 pythonw 可执行文件,请安装 Python');
|
|
3255
|
+
return null;
|
|
3256
|
+
};
|
|
3257
|
+
|
|
3258
|
+
/**
|
|
3259
|
+
* 启动dashboard(内部函数,可被自动计时器或手动触发调用)
|
|
3260
|
+
* 捕获 stderr 代替静默 stdio:ignore,启动5s后验证进程存活
|
|
3261
|
+
*/
|
|
3262
|
+
const launchDashboard = (attempt = 1, maxAttempts = 5) => {
|
|
3263
|
+
const pyw = findPythonExe();
|
|
3264
|
+
if (!pyw) {
|
|
3265
|
+
ctx.logger.error(`[sc] 📊 dashboard启动失败: 未找到 Python 环境`);
|
|
3266
|
+
return;
|
|
3267
|
+
}
|
|
3268
|
+
const script = join(__dirname, 'tools', 'dashboard', 'tk-dashboard.py');
|
|
3269
|
+
try {
|
|
3270
|
+
// stdio: pipe 捕获 stderr — 不再静默吞掉错误
|
|
3271
|
+
const proc = spawn(pyw, [script], { detached: true, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
3272
|
+
dashboardPid = proc.pid;
|
|
3273
|
+
|
|
3274
|
+
// 捕获 stderr — Python 运行时错误不再被吞掉
|
|
3275
|
+
let stderrBuf = '';
|
|
3276
|
+
proc.stderr.on('data', (d) => { stderrBuf += d.toString(); });
|
|
3277
|
+
proc.stderr.on('end', () => {
|
|
3278
|
+
if (stderrBuf.trim()) {
|
|
3279
|
+
ctx.logger.warn(`[sc] 📊 dashboard stderr (PID=${proc.pid}): ${stderrBuf.trim().slice(0, 500)}`);
|
|
3280
|
+
}
|
|
3281
|
+
});
|
|
3282
|
+
|
|
3283
|
+
// 捕获进程级错误(如 ENOENT — spawn 不可达)
|
|
3284
|
+
let recovered = false;
|
|
3285
|
+
proc.on('error', (err) => {
|
|
3286
|
+
ctx.logger.warn(`[sc] 📊 dashboard进程错误 (第${attempt}次): ${err.message}`);
|
|
3287
|
+
dashboardPid = null;
|
|
3288
|
+
if (!recovered && attempt < maxAttempts) {
|
|
3289
|
+
recovered = true;
|
|
3290
|
+
setTimeout(() => launchDashboard(attempt + 1, maxAttempts), 30 * 1000);
|
|
3291
|
+
}
|
|
3292
|
+
});
|
|
3293
|
+
|
|
3294
|
+
// 进程退出时清理 pid,异常退出则自动重启
|
|
3295
|
+
proc.on('exit', (code) => {
|
|
3296
|
+
ctx.logger.info(`[sc] 📊 dashboard进程已退出 (PID=${proc.pid}, code=${code})`);
|
|
3297
|
+
if (dashboardPid === proc.pid) dashboardPid = null;
|
|
3298
|
+
if (!recovered && code !== 0 && attempt < maxAttempts) {
|
|
3299
|
+
recovered = true;
|
|
3300
|
+
ctx.logger.info(`[sc] 📊 dashboard异常退出,30s后自动重启 (尝试 ${attempt+1}/${maxAttempts})`);
|
|
3301
|
+
setTimeout(() => launchDashboard(attempt + 1, maxAttempts), 30 * 1000);
|
|
3302
|
+
}
|
|
3303
|
+
});
|
|
3304
|
+
|
|
3305
|
+
ctx.logger.info(`[sc] 📊 dashboardstarted (PID=${proc.pid})`);
|
|
3306
|
+
proc.unref();
|
|
3307
|
+
|
|
3308
|
+
// 启动后 5 s验证进程是否存活(只当 exit/error 未触发retry时)
|
|
3309
|
+
setTimeout(() => {
|
|
3310
|
+
if (recovered || dashboardPid !== proc.pid) return;
|
|
3311
|
+
try {
|
|
3312
|
+
const check = spawnSync('tasklist', ['/FI', `PID eq ${proc.pid}`, '/NH'], { timeout: 3000, stdio: 'pipe', encoding: 'utf-8' });
|
|
3313
|
+
if (!check.stdout.includes(String(proc.pid))) {
|
|
3314
|
+
ctx.logger.warn(`[sc] 📊 dashboard进程 (PID=${proc.pid}) 似乎已消失,标记为retry`);
|
|
3315
|
+
if (dashboardPid === proc.pid) dashboardPid = null;
|
|
3316
|
+
if (!recovered && attempt < maxAttempts) {
|
|
3317
|
+
recovered = true;
|
|
3318
|
+
setTimeout(() => launchDashboard(attempt + 1, maxAttempts), 30 * 1000);
|
|
3319
|
+
}
|
|
3320
|
+
} else {
|
|
3321
|
+
ctx.logger.info(`[sc] ✅ dashboard进程存活确认 (PID=${proc.pid})`);
|
|
3322
|
+
}
|
|
3323
|
+
} catch (e) { /* tasklist check non-critical */ }
|
|
3324
|
+
}, 5000);
|
|
3325
|
+
|
|
3326
|
+
} catch (err) {
|
|
3327
|
+
ctx.logger.warn(`[sc] 📊 dashboard启动失败 (第${attempt}次): ${err.message}`);
|
|
3328
|
+
if (attempt < maxAttempts) {
|
|
3329
|
+
setTimeout(() => launchDashboard(attempt + 1, maxAttempts), 30 * 1000);
|
|
3330
|
+
} else {
|
|
3331
|
+
ctx.logger.error(`[sc] 📊 dashboard启动失败,已retry${maxAttempts}次,放弃。`);
|
|
3332
|
+
}
|
|
3333
|
+
}
|
|
3334
|
+
};
|
|
3335
|
+
// 自动启动:Gateway启动1min后
|
|
3336
|
+
setTimeout(() => launchDashboard(), 60 * 1000);
|
|
3337
|
+
|
|
3338
|
+
/**
|
|
3339
|
+
* 公开启动dashboard函数供外部手动触发(杉哥说"开dashboard"时调用)
|
|
3340
|
+
* 通过 globalThis 暴露给custom tools使用
|
|
3341
|
+
*/
|
|
3342
|
+
|
|
3343
|
+
|
|
3344
|
+
() => launchDashboard(1, 3);
|
|
3345
|
+
globalThis.__sansanCpu_dashboard = {
|
|
3346
|
+
launch: () => launchDashboard(1, 3),
|
|
3347
|
+
get pid() { return dashboardPid; },
|
|
3348
|
+
};
|
|
3349
|
+
// 同步 dashboardPid 到可读的 global getter
|
|
3350
|
+
Object.defineProperty(globalThis, '__sansanCpu_dashboard_pid', {
|
|
3351
|
+
get: () => dashboardPid,
|
|
3352
|
+
configurable: true,
|
|
3353
|
+
enumerable: true,
|
|
3354
|
+
});
|
|
3355
|
+
|
|
3356
|
+
// ? 自动启动子agent后台监控(防止子agent卡死不退出),启动后2min开启
|
|
3357
|
+
setTimeout(() => {
|
|
3358
|
+
startMonitorBackground();
|
|
3359
|
+
ctx.logger.info('? 子agent后台监控已自动启动 (interval=90s, 阈值=120s, 💀卡死强杀+⏰超时告警)');
|
|
3360
|
+
}, 2 * 60 * 1000);
|
|
3361
|
+
}
|
|
3362
|
+
|
|
3363
|
+
export default {
|
|
3364
|
+
register,
|
|
3365
|
+
activate,
|
|
3366
|
+
pool,
|
|
3367
|
+
exec: (task, priority) => pool.exec(task, priority),
|
|
3368
|
+
warmup: () => pool.warmup(),
|
|
3369
|
+
ready: pool.ready,
|
|
3370
|
+
getStats: () => getCachedStats(),
|
|
3371
|
+
getMemoryLevel,
|
|
3372
|
+
cleanupSharedDir,
|
|
3373
|
+
cleanupOldSessions,
|
|
3374
|
+
ensureSharedDir,
|
|
3375
|
+
writeSharedResult,
|
|
3376
|
+
readSharedResult,
|
|
3377
|
+
snapshotSubagents,
|
|
3378
|
+
killSubagent,
|
|
3379
|
+
cpuAbort,
|
|
3380
|
+
getMonitorState,
|
|
3381
|
+
compressTaskDescription,
|
|
3382
|
+
tcell,
|
|
3383
|
+
getMetabolicRate,
|
|
3384
|
+
setMetabolicRate,
|
|
3385
|
+
getMetabolicRateConfig,
|
|
3386
|
+
startMetabolicAutoRegulation,
|
|
3387
|
+
stopMetabolicAutoRegulation,
|
|
3388
|
+
shutdown: () => {
|
|
3389
|
+
// 停止资源调度自动调节
|
|
3390
|
+
stopMetabolicAutoRegulation();
|
|
3391
|
+
// 清理共享文件定时器
|
|
3392
|
+
const timer = global.__sansanCpuCleanupTimer;
|
|
3393
|
+
if (timer) { clearInterval(timer); global.__sansanCpuCleanupTimer = null; }
|
|
3394
|
+
|
|
3395
|
+
// 停止子agent监控
|
|
3396
|
+
if (monitorState.timer) {
|
|
3397
|
+
clearInterval(monitorState.timer);
|
|
3398
|
+
monitorState.timer = null;
|
|
3399
|
+
monitorState.active = false;
|
|
3400
|
+
}
|
|
3401
|
+
monitorState.stalledHistory.clear();
|
|
3402
|
+
|
|
3403
|
+
// 清理C纤维慢通道定时器
|
|
3404
|
+
if (slowChannelState.timer) {
|
|
3405
|
+
clearInterval(slowChannelState.timer);
|
|
3406
|
+
slowChannelState.timer = null;
|
|
3407
|
+
}
|
|
3408
|
+
|
|
3409
|
+
// 停止工具自动内化自动self-check
|
|
3410
|
+
|
|
3411
|
+
|
|
3412
|
+
// 杀掉dashboard进程(同步关闭,避免重启后残留旧窗口)
|
|
3413
|
+
if (dashboardPid) {
|
|
3414
|
+
try {
|
|
3415
|
+
spawnSync('taskkill', ['/PID', String(dashboardPid), '/F'], { stdio: 'ignore', timeout: 3000 });
|
|
3416
|
+
dashboardPid = null;
|
|
3417
|
+
} catch (e) { console.error("[sc] shutdown 杀dashboard进程失败:", e.message); /* 进程可能已经自己退出了 */ }
|
|
3418
|
+
}
|
|
3419
|
+
|
|
3420
|
+
if (mcpServerHandle) {
|
|
3421
|
+
try { mcpServerHandle.shutdown(); } catch {}
|
|
3422
|
+
mcpServerHandle = null;
|
|
3423
|
+
}
|
|
3424
|
+
|
|
3425
|
+
// 清理残留子进程:关闭sc后如果还有node子进程残留,强制杀掉
|
|
3426
|
+
try {
|
|
3427
|
+
const childPids = execSync(
|
|
3428
|
+
`powershell -NoProfile -Command "Get-CimInstance Win32_Process -Filter \"Name='node.exe' AND CommandLine LIKE '%sc%' AND CommandLine NOT LIKE '%sidecar-server%' AND ProcessId != ${process.pid}\" | Select-Object -ExpandProperty ProcessId"`,
|
|
3429
|
+
{ timeout: 5000, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }
|
|
3430
|
+
);
|
|
3431
|
+
if (childPids?.trim()) {
|
|
3432
|
+
const pids = childPids.trim().split('\n').filter(Boolean);
|
|
3433
|
+
for (const pid of pids) {
|
|
3434
|
+
try { spawnSync('taskkill', ['/f', '/pid', pid.trim()], { timeout: 3000 }); } catch (err) { console.error("[sc] shutdown taskkill子进程失败:", err.message); }
|
|
3435
|
+
}
|
|
3436
|
+
}
|
|
3437
|
+
} catch (err) { console.error("[sc] shutdown 查残留子进程失败:", err.message); }
|
|
3438
|
+
|
|
3439
|
+
// 🧠 停止日志系统 + hippocampus最终刷入
|
|
3440
|
+
try {
|
|
3441
|
+
const logger = global.__sansanLogger;
|
|
3442
|
+
if (logger) {
|
|
3443
|
+
stopHippocampusFlush();
|
|
3444
|
+
}
|
|
3445
|
+
} catch (err) {
|
|
3446
|
+
console.error('[sc] shutdown 日志停止异常:', err.message);
|
|
3447
|
+
}
|
|
3448
|
+
|
|
3449
|
+
return pool.shutdown();
|
|
3450
|
+
},
|
|
3451
|
+
mcpPort: MCP_PORT,
|
|
3452
|
+
};
|
|
3453
|
+
|
|
3454
|
+
// ====== 直接运行时(非OpenClaw插件模式)自动启动MCP Server ======
|
|
3455
|
+
const isDirectRun = typeof process !== 'undefined' && process.argv[1] &&
|
|
3456
|
+
(process.argv[1].replace(/\\/g, '/').endsWith('index.js') ||
|
|
3457
|
+
process.argv[1].replace(/\\/g, '/').endsWith('sc/index.js'));
|
|
3458
|
+
if (isDirectRun) {
|
|
3459
|
+
// TODO: 移除调试日志 console.log('[sc] 直接运行模式,自动启动 MCP Server...');
|
|
3460
|
+
import('./tools/bridge.js').then(bridge => {
|
|
3461
|
+
bridge.setCpuInstance({
|
|
3462
|
+
pool,
|
|
3463
|
+
getStats: () => ({}),
|
|
3464
|
+
getMemoryLevel: () => 'normal',
|
|
3465
|
+
});
|
|
3466
|
+
bridge.startMcpServer(MCP_PORT).then(h => {
|
|
3467
|
+
mcpServerHandle = h;
|
|
3468
|
+
// TODO: 移除调试日志 console.log('[sc] MCP Server started (http://127.0.0.1:' + h.port + '/sse)');
|
|
3469
|
+
}).catch(e => {
|
|
3470
|
+
if (e?.code === 'EADDRINUSE') {
|
|
3471
|
+
// TODO: 移除调试日志 console.log('[sc] MCP Server 端口已被占用(旧实例在服务),跳过启动');
|
|
3472
|
+
} else {
|
|
3473
|
+
console.warn('[sc] MCP Server start skipped:', e.message);
|
|
3474
|
+
}
|
|
3475
|
+
});
|
|
3476
|
+
}).catch(e => {
|
|
3477
|
+
console.warn('[sc] MCP Server unavailable:', e.message);
|
|
3478
|
+
});
|
|
3479
|
+
}
|