@mortiseai/stem 0.0.22 → 0.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -23
- package/bin/stem-dev.mjs +4 -0
- package/bin/stem-launcher-lib.mjs +57 -0
- package/bin/stem-supervisor-lib.mjs +167 -43
- package/bin/stem.mjs +27 -20
- package/dist/cli.mjs +1847 -930
- package/dist/daemon.mjs +1493 -613
- package/dist/mcp.mjs +1489 -609
- package/package.json +8 -2
|
@@ -2,16 +2,38 @@
|
|
|
2
2
|
// 独立成库的唯一理由:入口脚本 import 即执行,smoke 无法直接测;这里的函数
|
|
3
3
|
// 全部纯逻辑/可注入,scripts/smoke-supervisor.ts 逐项锁语义。
|
|
4
4
|
import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
5
|
-
import { dirname, join } from 'node:path'
|
|
6
|
-
import { homedir
|
|
5
|
+
import { dirname, join, resolve } from 'node:path'
|
|
6
|
+
import { homedir } from 'node:os'
|
|
7
7
|
import { spawn, spawnSync } from 'node:child_process'
|
|
8
8
|
|
|
9
9
|
// 配置类错误的专用退出码(stem-entry-cli 侧同值):见 78 不重启。
|
|
10
10
|
export const EXIT_CODE_CONFIG = 78
|
|
11
|
+
// Child completed an in-process memory cleanup, persisted resumable state, and asks the
|
|
12
|
+
// supervisor to replace the allocator/process without classifying the event as a crash.
|
|
13
|
+
export const EXIT_CODE_MEMORY_RESTART = 75
|
|
11
14
|
export const CRASH_WINDOW_MS = 60_000
|
|
12
15
|
export const CRASH_GIVE_UP_COUNT = 3
|
|
13
16
|
export const RESTART_BACKOFF_MAX_MS = 10_000
|
|
14
17
|
|
|
18
|
+
const pad2 = n => String(n).padStart(2, '0')
|
|
19
|
+
|
|
20
|
+
/** 与子进程默认日志布局一致:日期分桶,桶内按本地时分秒 + sessionId 排序。 */
|
|
21
|
+
export function buildSessionLogDir(cwd, sessionId, startedAt = new Date()) {
|
|
22
|
+
const d = startedAt instanceof Date ? startedAt : new Date(startedAt)
|
|
23
|
+
const date = `${d.getFullYear()}-${pad2(d.getMonth() + 1)}-${pad2(d.getDate())}`
|
|
24
|
+
const time = `${pad2(d.getHours())}-${pad2(d.getMinutes())}-${pad2(d.getSeconds())}`
|
|
25
|
+
return join(cwd, 'logs', date, `${time}-${sessionId}`)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Supervisor 与子进程交换会话 ID 的短生命周期文件也统一落 Stem storage。 */
|
|
29
|
+
function createSupervisorAnnounceFile(cwd) {
|
|
30
|
+
const configured = String(process.env.STEM_STORAGE_DIR ?? '').trim()
|
|
31
|
+
const storageRoot = configured ? resolve(cwd, configured) : join(cwd, 'mstem-storage')
|
|
32
|
+
const tmpRoot = join(storageRoot, 'tmp', 'supervisor')
|
|
33
|
+
mkdirSync(tmpRoot, { recursive: true })
|
|
34
|
+
return join(mkdtempSync(join(tmpRoot, 'session-')), 'session.json')
|
|
35
|
+
}
|
|
36
|
+
|
|
15
37
|
/** 从 argv 剥离 --resume/-r(supervisor 重启时按通告会话重挂)。 */
|
|
16
38
|
export function stripResumeFlag(argv) {
|
|
17
39
|
const out = []
|
|
@@ -68,15 +90,22 @@ export function restartPrompt(isZh, crashInfo) {
|
|
|
68
90
|
: `[system] The process exited abnormally (${cause}) and was restarted automatically. The conversation was restored from disk, but tool execution traces and intermediate state were lost. First read the current todo list and any task/plan documents to verify actual progress (validate completed parts with commands if needed), then continue from the unfinished items. Do not redo completed work or reply with acknowledgements only.`
|
|
69
91
|
}
|
|
70
92
|
|
|
71
|
-
|
|
72
|
-
|
|
93
|
+
export function memoryRestartPrompt(isZh) {
|
|
94
|
+
return isZh
|
|
95
|
+
? '[系统] 内存保护已完成清理并受控重启。本会话对话、todo 与任务现场已从磁盘恢复。请先核对当前 todo 和实际文件状态,然后从未完成项继续执行;不要重复已完成的工作,也不要只回复确认性文字。'
|
|
96
|
+
: '[system] Memory protection completed cleanup and a controlled restart. The conversation, todo, and task state were restored from disk. Verify the current todo and actual files, then continue from the unfinished work. Do not redo completed work or reply with an acknowledgement only.'
|
|
97
|
+
}
|
|
73
98
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
99
|
+
// 注入队列的去重前缀:文案带崩溃原因(逐次可变),不能再按全串相等去重。
|
|
100
|
+
const RESTART_PROMPT_PREFIXES = [
|
|
101
|
+
'[系统] 进程异常退出',
|
|
102
|
+
'[系统] 进程因异常',
|
|
103
|
+
'[系统] 内存保护已完成清理并受控重启',
|
|
104
|
+
'[system] The process',
|
|
105
|
+
'[system] Memory protection completed cleanup and a controlled restart',
|
|
106
|
+
]
|
|
107
|
+
|
|
108
|
+
function injectResumePrompt(queuePath, prompt) {
|
|
80
109
|
let existing = []
|
|
81
110
|
try {
|
|
82
111
|
const parsed = JSON.parse(readFileSync(queuePath, 'utf8'))
|
|
@@ -95,8 +124,20 @@ export function injectRestartPrompt(queuePath, isZh, crashInfo) {
|
|
|
95
124
|
} catch { return false }
|
|
96
125
|
}
|
|
97
126
|
|
|
127
|
+
/** 注入续跑 prompt 到队列 sidecar(userPromptQueueStore 格式;resume 分支自动
|
|
128
|
+
* 再水合、经 idle-gate 依次执行)。cleanExit: true 让 resume 端静默续跑而非弹
|
|
129
|
+
* y/n 确认 — 无人值守是看护的前提;连崩风暴由退避与 3 次放弃兜底。既有排队
|
|
130
|
+
* prompt 保留在后(按前缀去重防多次崩溃堆叠)。 */
|
|
131
|
+
export function injectRestartPrompt(queuePath, isZh, crashInfo) {
|
|
132
|
+
return injectResumePrompt(queuePath, restartPrompt(isZh, crashInfo))
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function injectMemoryRestartPrompt(queuePath, isZh) {
|
|
136
|
+
return injectResumePrompt(queuePath, memoryRestartPrompt(isZh))
|
|
137
|
+
}
|
|
138
|
+
|
|
98
139
|
/** 崩溃标记(2026-08-13 事故:崩溃现场零证据,连退出信号都没记)。
|
|
99
|
-
*
|
|
140
|
+
* 落日期/时间会话目录的 crash.json(与该会话的 supervisor.log /
|
|
100
141
|
* debug 日志 / 堆快照同目录归档;此前散在 queue sidecar 目录难找易丢)。
|
|
101
142
|
* crashes 数组保最近 20 条,下次排查直接看这里而不用翻 supervisor stderr。
|
|
102
143
|
* 写失败静默(标记是诊断增强,不能反过来阻断重启)。 */
|
|
@@ -162,8 +203,8 @@ export function stripModelFlag(argv) {
|
|
|
162
203
|
}
|
|
163
204
|
|
|
164
205
|
/** 崩溃堆快照清理:跨给定目录扫描 Heap.*.heapsnapshot,按 mtime 只保最新
|
|
165
|
-
* keep 份(快照 GB 级,防 24h 连跑占满磁盘)
|
|
166
|
-
*
|
|
206
|
+
* keep 份(快照 GB 级,防 24h 连跑占满磁盘)。快照按会话归档进日期目录,
|
|
207
|
+
* 清理从"cwd 内只留 1 份"改为"cwd + logs/ + 各日期/会话目录
|
|
167
208
|
* 全局保最新 3 份"— 老会话的取证快照不再被下一次崩溃立即冲掉。 */
|
|
168
209
|
export function cleanupHeapSnapshots(dirs, keep = 3) {
|
|
169
210
|
const found = []
|
|
@@ -182,43 +223,54 @@ export function cleanupHeapSnapshots(dirs, keep = 3) {
|
|
|
182
223
|
}
|
|
183
224
|
}
|
|
184
225
|
|
|
185
|
-
/**
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
*
|
|
226
|
+
/** 遗留平铺会话日志归位:旧 debug 日志平铺在 logs/ 顶层
|
|
227
|
+
* (`<YYYY-MM-DD_HH-MM>_<sessionId>.txt`)。按文件名迁入
|
|
228
|
+
* `logs/<YYYY-MM-DD>/<HH-MM-00-sessionId>/`,并改成新的时间前缀文件名。
|
|
229
|
+
* supervisor.log / 无关文件不动;搬失败(占用中)留待下次。 */
|
|
189
230
|
export function relocateLegacySessionLogs(logsDir) {
|
|
190
231
|
let entries
|
|
191
232
|
try { entries = readdirSync(logsDir, { withFileTypes: true }) } catch { return [] }
|
|
192
233
|
const moved = []
|
|
193
234
|
for (const e of entries) {
|
|
194
235
|
if (!e.isFile()) continue
|
|
195
|
-
const m =
|
|
236
|
+
const m = /^(\d{4}-\d{2}-\d{2})_(\d{2})-(\d{2})_([A-Za-z0-9-]+)\.txt$/.exec(e.name)
|
|
196
237
|
if (!m) continue
|
|
197
238
|
try {
|
|
198
|
-
const
|
|
239
|
+
const name = `${m[2]}-${m[3]}-00-${m[4]}`
|
|
240
|
+
const dest = join(logsDir, m[1], name)
|
|
199
241
|
mkdirSync(dest, { recursive: true })
|
|
200
|
-
|
|
201
|
-
|
|
242
|
+
const target = join(dest, `${name}.txt`)
|
|
243
|
+
renameSync(join(logsDir, e.name), target)
|
|
244
|
+
moved.push(target)
|
|
202
245
|
} catch { /* 占用中/权限失败,下次启动再收 */ }
|
|
203
246
|
}
|
|
204
247
|
return moved
|
|
205
248
|
}
|
|
206
249
|
|
|
207
250
|
/** 堆快照可能出现的目录:cwd(V8 默认 / --diagnostic-dir 建目录失败回落)、
|
|
208
|
-
* logs/(--diagnostic-dir 落点)、logs/
|
|
251
|
+
* logs/(--diagnostic-dir 落点)、logs/ 下日期与会话归档目录。 */
|
|
209
252
|
export function heapSnapshotDirs(cwd) {
|
|
210
253
|
const logsDir = join(cwd, 'logs')
|
|
211
254
|
const dirs = [cwd, logsDir]
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
255
|
+
const pending = [logsDir]
|
|
256
|
+
// 只跟随真实目录,不跟符号链接;加上限避免异常目录树拖住启动。
|
|
257
|
+
while (pending.length > 0 && dirs.length < 10_000) {
|
|
258
|
+
const parent = pending.pop()
|
|
259
|
+
let entries
|
|
260
|
+
try { entries = readdirSync(parent, { withFileTypes: true }) } catch { continue }
|
|
261
|
+
for (const e of entries) {
|
|
262
|
+
if (!e.isDirectory()) continue
|
|
263
|
+
const child = join(parent, e.name)
|
|
264
|
+
dirs.push(child)
|
|
265
|
+
pending.push(child)
|
|
266
|
+
if (dirs.length >= 10_000) break
|
|
215
267
|
}
|
|
216
|
-
}
|
|
268
|
+
}
|
|
217
269
|
return dirs
|
|
218
270
|
}
|
|
219
271
|
|
|
220
272
|
/** 崩溃堆快照按会话归档:把散落在 fromDirs(cwd / logs/ 顶层)的
|
|
221
|
-
* Heap.*.heapsnapshot
|
|
273
|
+
* Heap.*.heapsnapshot 搬进日期/时间会话目录,与该会话的 crash.json /
|
|
222
274
|
* supervisor.log 放一起。搬失败(占用中/跨设备)留在原地由清理兜底。 */
|
|
223
275
|
export function relocateHeapSnapshots(fromDirs, destDir) {
|
|
224
276
|
const moved = []
|
|
@@ -295,13 +347,18 @@ export function attachDiagnosticReport(markerDir, reportPath) {
|
|
|
295
347
|
} catch { return false }
|
|
296
348
|
}
|
|
297
349
|
|
|
298
|
-
function defaultSpawnChild(args, env) {
|
|
350
|
+
function defaultSpawnChild(args, env, launch = {}) {
|
|
299
351
|
// stderr 走 pipe 而非 inherit:原生崩溃转储(V8 fatal / abort)绕过 JS 层拦截
|
|
300
352
|
// 直写 fd 2,inherit 时直接打上 TUI 且父进程零捕获(2026-08-17 事故)。pipe 后
|
|
301
353
|
// 实时透传回终端(行为不变)+ 环形缓冲留尾部作取证。TUI 渲染门只看 stdout/stdin
|
|
302
354
|
// 的 isTTY,stderr 非 TTY 的影响仅限着色(exitCleanup 的 hint 着色已改跟 stdout)。
|
|
303
355
|
const tail = createStderrTail()
|
|
304
|
-
const
|
|
356
|
+
const command = launch.command || process.execPath
|
|
357
|
+
const child = spawn(command, args, {
|
|
358
|
+
stdio: ['inherit', 'inherit', 'pipe'],
|
|
359
|
+
env,
|
|
360
|
+
cwd: launch.cwd,
|
|
361
|
+
})
|
|
305
362
|
child.stderr?.on('data', chunk => {
|
|
306
363
|
tail.push(chunk)
|
|
307
364
|
try { process.stderr.write(chunk) } catch { /* 终端已关闭等,透传失败无害 */ }
|
|
@@ -319,14 +376,19 @@ function defaultSpawnChild(args, env) {
|
|
|
319
376
|
child.on('exit', (code, signal) => {
|
|
320
377
|
setTimeout(() => resolve({ code, signal }), 2000)
|
|
321
378
|
})
|
|
322
|
-
child.on('error',
|
|
379
|
+
child.on('error', error => {
|
|
380
|
+
const message = `stem: failed to start child command ${command}: ${error.message}\n`
|
|
381
|
+
tail.push(message)
|
|
382
|
+
try { process.stderr.write(message) } catch { /* terminal already closed */ }
|
|
383
|
+
resolve({ code: EXIT_CODE_CONFIG, signal: null })
|
|
384
|
+
})
|
|
323
385
|
}),
|
|
324
386
|
}
|
|
325
387
|
}
|
|
326
388
|
|
|
327
389
|
/**
|
|
328
390
|
* 看护主循环。overrides 供 smoke 注入:
|
|
329
|
-
* spawnChild(args, env) → {pid, exited: Promise<{code, signal}>}
|
|
391
|
+
* spawnChild(args, env, {command, cwd}) → {pid, exited: Promise<{code, signal}>}
|
|
330
392
|
* sleep(ms) / log(text) / now() / cwd
|
|
331
393
|
* 返回最终退出码。
|
|
332
394
|
*/
|
|
@@ -334,14 +396,17 @@ export async function runSupervisor(distEntry, argv, options = {}) {
|
|
|
334
396
|
const {
|
|
335
397
|
isZh = true,
|
|
336
398
|
nodeFlags = [],
|
|
399
|
+
childCommand = process.execPath,
|
|
400
|
+
childPrefixArgs = null,
|
|
401
|
+
childCwd = undefined,
|
|
337
402
|
spawnChild = defaultSpawnChild,
|
|
338
403
|
sleep = ms => new Promise(r => setTimeout(r, ms)),
|
|
339
404
|
log = text => { try { process.stderr.write(`\x1b[2;38;2;156;163;175m${text}\x1b[0m\n`) } catch { /* ignore */ } },
|
|
340
405
|
now = Date.now,
|
|
341
406
|
cwd = process.cwd(),
|
|
342
|
-
announceFile =
|
|
407
|
+
announceFile = createSupervisorAnnounceFile(cwd),
|
|
343
408
|
// supervisor 自身诊断的持久化落点(2026-08-17 事故:崩溃信息只写终端,滚屏即丢)。
|
|
344
|
-
// 缺省 null = 动态解析:会话已知 →
|
|
409
|
+
// 缺省 null = 动态解析:会话已知 → 日期/时间会话目录/supervisor.log,
|
|
345
410
|
// 未知(死在首次通告前且无 --resume)→ logs/supervisor.log 顶层兜底。
|
|
346
411
|
// 显式传入(smoke 注入)则固定用该路径。
|
|
347
412
|
supervisorLogPath = null,
|
|
@@ -357,14 +422,21 @@ export async function runSupervisor(distEntry, argv, options = {}) {
|
|
|
357
422
|
},
|
|
358
423
|
installSignalHandlers = true,
|
|
359
424
|
} = options
|
|
425
|
+
const launchPrefixArgs = Array.isArray(childPrefixArgs)
|
|
426
|
+
? [...childPrefixArgs]
|
|
427
|
+
: [...nodeFlags, distEntry]
|
|
428
|
+
const supervisorStartedAt = new Date(now())
|
|
360
429
|
|
|
361
430
|
// log() 只写终端;logLine() 同时 append 到 supervisor.log(ISO 时间戳)—
|
|
362
431
|
// 崩溃路径一律用 logLine,子进程日志闸门(STEM_DEBUG)关着时这里仍有据可查。
|
|
363
|
-
// 路径每次写时解析:
|
|
432
|
+
// 路径每次写时解析:activeSessionLogDir 随子进程通告更新,崩溃日志落到
|
|
433
|
+
// 子进程 debug 日志所在的同一个日期/时间会话目录。
|
|
364
434
|
const supervisorLogFile = () => {
|
|
365
435
|
if (supervisorLogPath) return supervisorLogPath
|
|
366
|
-
return
|
|
367
|
-
? join(
|
|
436
|
+
return activeSessionLogDir
|
|
437
|
+
? join(activeSessionLogDir, 'supervisor.log')
|
|
438
|
+
: resumeId
|
|
439
|
+
? join(buildSessionLogDir(cwd, resumeId, supervisorStartedAt), 'supervisor.log')
|
|
368
440
|
: join(cwd, 'logs', 'supervisor.log')
|
|
369
441
|
}
|
|
370
442
|
const logFileOnly = text => {
|
|
@@ -383,7 +455,9 @@ export async function runSupervisor(distEntry, argv, options = {}) {
|
|
|
383
455
|
try { stemVersion = JSON.parse(readFileSync(join(dirname(distEntry), '..', 'package.json'), 'utf8')).version ?? null } catch { /* smoke 假路径等 */ }
|
|
384
456
|
|
|
385
457
|
let { baseArgs, resumeId } = stripResumeFlag(argv)
|
|
458
|
+
let activeSessionLogDir = null
|
|
386
459
|
let crashTimes = []
|
|
460
|
+
let memoryRestartTimes = []
|
|
387
461
|
let childAlive = false
|
|
388
462
|
// 崩溃前子进程通告的实时 permission mode(Shift+Tab 切换会更新通告)—
|
|
389
463
|
// 重启时经 --permission-mode 复原,argv 中的启动值被剥离让位。
|
|
@@ -406,7 +480,13 @@ export async function runSupervisor(distEntry, argv, options = {}) {
|
|
|
406
480
|
const p = JSON.parse(readFileSync(announceFile, 'utf8'))
|
|
407
481
|
if (p && p.v === 1 && typeof p.sessionId === 'string') parsed = p
|
|
408
482
|
} catch { return }
|
|
409
|
-
if (!parsed
|
|
483
|
+
if (!parsed) return
|
|
484
|
+
// logDir 由子进程计算,包含精确到秒的会话起点;旧版通告没有该字段时
|
|
485
|
+
// 才由 supervisor 以自身启动时间构造兼容落点。
|
|
486
|
+
activeSessionLogDir = typeof parsed.logDir === 'string' && parsed.logDir
|
|
487
|
+
? resolve(cwd, parsed.logDir)
|
|
488
|
+
: buildSessionLogDir(cwd, parsed.sessionId, supervisorStartedAt)
|
|
489
|
+
if (parsed.sessionId === announcedSid) return
|
|
410
490
|
announcedSid = parsed.sessionId
|
|
411
491
|
resumeId = parsed.sessionId
|
|
412
492
|
logFileOnly(isZh
|
|
@@ -429,7 +509,7 @@ export async function runSupervisor(distEntry, argv, options = {}) {
|
|
|
429
509
|
cleanupHeapSnapshots(heapSnapshotDirs(cwd))
|
|
430
510
|
let effectiveBase = permissionModeFromAnnounce ? stripPermissionModeFlag(baseArgs) : baseArgs
|
|
431
511
|
if (modelFromAnnounce !== null) effectiveBase = stripModelFlag(effectiveBase)
|
|
432
|
-
const args = [...
|
|
512
|
+
const args = [...launchPrefixArgs, ...effectiveBase]
|
|
433
513
|
if (permissionModeFromAnnounce) args.push('--permission-mode', permissionModeFromAnnounce)
|
|
434
514
|
if (modelFromAnnounce) args.push('--model', modelFromAnnounce)
|
|
435
515
|
if (resumeId) args.push('--resume', resumeId)
|
|
@@ -437,7 +517,7 @@ export async function runSupervisor(distEntry, argv, options = {}) {
|
|
|
437
517
|
...process.env,
|
|
438
518
|
STEM_SUPERVISED: '1',
|
|
439
519
|
STEM_SESSION_ANNOUNCE_FILE: announceFile,
|
|
440
|
-
})
|
|
520
|
+
}, { command: childCommand, cwd: childCwd })
|
|
441
521
|
childAlive = true
|
|
442
522
|
currentPid = child.pid ?? null
|
|
443
523
|
const lifecycleTimer = setInterval(checkAnnounceForLifecycle, 2000)
|
|
@@ -458,9 +538,50 @@ export async function runSupervisor(distEntry, argv, options = {}) {
|
|
|
458
538
|
: 'stem: exited with a configuration error (code 78); supervisor will not restart. Fix the configuration and run again.')
|
|
459
539
|
return EXIT_CODE_CONFIG
|
|
460
540
|
}
|
|
541
|
+
if (code === EXIT_CODE_MEMORY_RESTART) {
|
|
542
|
+
const t = now()
|
|
543
|
+
memoryRestartTimes = memoryRestartTimes.filter(x => t - x < CRASH_WINDOW_MS)
|
|
544
|
+
memoryRestartTimes.push(t)
|
|
545
|
+
resetTerminalFn()
|
|
546
|
+
let announce = null
|
|
547
|
+
try {
|
|
548
|
+
const parsed = JSON.parse(readFileSync(announceFile, 'utf8'))
|
|
549
|
+
if (parsed && parsed.v === 1 && typeof parsed.sessionId === 'string') announce = parsed
|
|
550
|
+
} catch { /* ignore */ }
|
|
551
|
+
if (!announce || typeof announce.queuePath !== 'string') {
|
|
552
|
+
logLine(isZh
|
|
553
|
+
? 'stem: 子进程请求内存恢复重启,但没有有效会话通告;为避免丢失上下文,已停止自动重启。'
|
|
554
|
+
: 'stem: child requested a memory recovery restart without a valid session announcement; automatic restart stopped to avoid losing context.')
|
|
555
|
+
return EXIT_CODE_MEMORY_RESTART
|
|
556
|
+
}
|
|
557
|
+
resumeId = announce.sessionId
|
|
558
|
+
if (typeof announce.permissionMode === 'string' && announce.permissionMode) {
|
|
559
|
+
permissionModeFromAnnounce = announce.permissionMode
|
|
560
|
+
}
|
|
561
|
+
if (typeof announce.model === 'string') modelFromAnnounce = announce.model
|
|
562
|
+
for (const p of Array.isArray(announce.jsonlPaths) ? announce.jsonlPaths : []) {
|
|
563
|
+
if (typeof p === 'string' && existsSync(p) && repairJsonlTail(p)) {
|
|
564
|
+
logLine(isZh ? `stem: 已修补会话记录 ${p}` : `stem: repaired session log ${p}`)
|
|
565
|
+
break
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
if (memoryRestartTimes.length >= CRASH_GIVE_UP_COUNT) {
|
|
569
|
+
logLine(isZh
|
|
570
|
+
? `stem: 60 秒内连续请求内存恢复重启 ${memoryRestartTimes.length} 次,停止自动重启。`
|
|
571
|
+
: `stem: memory recovery restart requested ${memoryRestartTimes.length} times within 60s; giving up on auto-restart.`)
|
|
572
|
+
return EXIT_CODE_MEMORY_RESTART
|
|
573
|
+
}
|
|
574
|
+
injectMemoryRestartPrompt(announce.queuePath, isZh)
|
|
575
|
+
onCrashCleanupPid(child.pid)
|
|
576
|
+
logLine(isZh
|
|
577
|
+
? `stem: 内存清理后仍处于硬水位,已保存会话 ${resumeId},正在受控重启并自动续接…`
|
|
578
|
+
: `stem: memory remained above the hard limit after cleanup; session ${resumeId} was saved and will resume after a controlled restart…`)
|
|
579
|
+
await sleep(250)
|
|
580
|
+
continue
|
|
581
|
+
}
|
|
461
582
|
|
|
462
583
|
// 崩溃路径。先读会话通告拿会话身份 — 所有崩溃落盘(supervisor.log /
|
|
463
|
-
// crash.json / 堆快照)
|
|
584
|
+
// crash.json / 堆快照)都按日期/时间会话目录归档,身份必须先到手;
|
|
464
585
|
// 子进程死在通告前 → 沿用现有 resumeId(可能为 null → 落 logs/ 顶层兜底)。
|
|
465
586
|
const t = now()
|
|
466
587
|
crashTimes = crashTimes.filter(x => t - x < CRASH_WINDOW_MS)
|
|
@@ -481,7 +602,9 @@ export async function runSupervisor(distEntry, argv, options = {}) {
|
|
|
481
602
|
if (typeof announce.model === 'string') modelFromAnnounce = announce.model
|
|
482
603
|
}
|
|
483
604
|
// 本次崩溃的会话归档目录(crash.json / 堆快照 / supervisor.log 共用)。
|
|
484
|
-
const sessionDir = resumeId
|
|
605
|
+
const sessionDir = resumeId
|
|
606
|
+
? (activeSessionLogDir ?? buildSessionLogDir(cwd, resumeId, new Date(t)))
|
|
607
|
+
: null
|
|
485
608
|
// 堆快照按会话归档:V8 落在 --diagnostic-dir(logs/)或 cwd(旧默认 /
|
|
486
609
|
// 建目录失败回落),搬进崩溃会话自己的目录,免得被后续清理当无主快照收走。
|
|
487
610
|
if (sessionDir) relocateHeapSnapshots([cwd, join(cwd, 'logs')], sessionDir)
|
|
@@ -514,6 +637,7 @@ export async function runSupervisor(distEntry, argv, options = {}) {
|
|
|
514
637
|
}
|
|
515
638
|
writeCrashMarker(sessionDir, announce.sessionId, { code, signal }, t, {
|
|
516
639
|
stderrTail: stderrTail8k,
|
|
640
|
+
command: childCommand,
|
|
517
641
|
args,
|
|
518
642
|
nodeVersion: process.version,
|
|
519
643
|
stemVersion,
|
|
@@ -540,8 +664,8 @@ export async function runSupervisor(distEntry, argv, options = {}) {
|
|
|
540
664
|
|
|
541
665
|
const delayMs = Math.min(1000 * 2 ** (crashTimes.length - 1), RESTART_BACKOFF_MAX_MS)
|
|
542
666
|
logLine(isZh
|
|
543
|
-
? `stem: 进程异常退出(code=${code ?? '-'}, signal=${signal ?? '-'}),${Math.round(delayMs / 1000)}s 后自动恢复会话${resumeId ? ` ${resumeId}` : ''}…(
|
|
544
|
-
: `stem: process crashed (code=${code ?? '-'}, signal=${signal ?? '-'}); resuming session${resumeId ? ` ${resumeId}` : ''} in ${Math.round(delayMs / 1000)}s… (set
|
|
667
|
+
? `stem: 进程异常退出(code=${code ?? '-'}, signal=${signal ?? '-'}),${Math.round(delayMs / 1000)}s 后自动恢复会话${resumeId ? ` ${resumeId}` : ''}…(STEM_SUPERVISOR_OFF=1 可关闭看护)`
|
|
668
|
+
: `stem: process crashed (code=${code ?? '-'}, signal=${signal ?? '-'}); resuming session${resumeId ? ` ${resumeId}` : ''} in ${Math.round(delayMs / 1000)}s… (set STEM_SUPERVISOR_OFF=1 to disable)`)
|
|
545
669
|
await sleep(delayMs)
|
|
546
670
|
|
|
547
671
|
// macOS 信号死亡 → ReportCrash 异步生成 .ips 报告(实测崩溃后 ~5s 才落盘,
|
package/bin/stem.mjs
CHANGED
|
@@ -6,14 +6,16 @@
|
|
|
6
6
|
// (OOM FATAL ERROR 在进程内不可拦截:同步分配风暴会锁死事件循环,任何 JS 层
|
|
7
7
|
// 守护都没有执行机会)时自动:修补会话 JSONL 尾部(崩溃可能截断半行,不修则
|
|
8
8
|
// --resume 直接拒载)→ 注入自包含续跑 prompt 到队列 sidecar → --resume 同一
|
|
9
|
-
//
|
|
10
|
-
//
|
|
9
|
+
// 会话重启。内存清理后 RSS 仍在硬档时以 exit 75 请求受控续接重启;正常退出
|
|
10
|
+
// (exit 0)与配置错误(exit 78)不重启。60 秒内连崩/连续内存重启 3 次放弃。
|
|
11
|
+
// STEM_SUPERVISOR_OFF=1 退回单次直跑;实现与语义锁定见
|
|
11
12
|
// bin/stem-supervisor-lib.mjs 与 scripts/smoke-supervisor.ts。
|
|
12
13
|
import { existsSync, mkdirSync, realpathSync } from 'node:fs'
|
|
13
14
|
import { dirname, join } from 'node:path'
|
|
14
15
|
import { devNull, totalmem } from 'node:os'
|
|
15
16
|
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
16
17
|
import { spawnSync } from 'node:child_process'
|
|
18
|
+
import { resolveBunCommand, shouldSupervise } from './stem-launcher-lib.mjs'
|
|
17
19
|
|
|
18
20
|
// realpath:npm link / 全局 bin 软链下定位真实包根,而非软链父目录。
|
|
19
21
|
const selfPath = realpathSync(fileURLToPath(import.meta.url))
|
|
@@ -57,11 +59,13 @@ function heapLimitMB() {
|
|
|
57
59
|
|
|
58
60
|
/** 子进程 node flags:heap 上限(未显式配置时)+ OOM 前堆快照(可关)。 */
|
|
59
61
|
function childNodeFlags() {
|
|
60
|
-
|
|
62
|
+
// Memory recovery performs an explicit collection after releasing transcript/render caches,
|
|
63
|
+
// then re-samples RSS before deciding whether a process replacement is necessary.
|
|
64
|
+
const flags = ['--expose-gc']
|
|
61
65
|
if (!heapConfigured()) flags.push(`--max-old-space-size=${heapLimitMB()}`)
|
|
62
66
|
// OOM 前自动堆快照:定位"未知同步分配风暴"元凶的唯一现实手段(手动
|
|
63
67
|
// /heapdump 在同步爆发下来不及)。代价是崩溃时写数 GB 快照;supervisor
|
|
64
|
-
// 崩溃后按会话归档进 logs
|
|
68
|
+
// 崩溃后按会话归档进 logs/<日期>/<时分秒-sessionId>/,每轮 spawn 前跨目录清理保最新
|
|
65
69
|
// 3 份。STEM_HEAP_SNAPSHOT=0 关闭。
|
|
66
70
|
if (
|
|
67
71
|
process.env.STEM_HEAP_SNAPSHOT !== '0' &&
|
|
@@ -69,7 +73,7 @@ function childNodeFlags() {
|
|
|
69
73
|
) {
|
|
70
74
|
flags.push('--heapsnapshot-near-heap-limit=1')
|
|
71
75
|
// 快照定向到 <cwd>/logs/(V8 默认写 cwd,散落且被误当项目文件);supervisor
|
|
72
|
-
//
|
|
76
|
+
// 崩溃后再按会话归档进日期/时间会话目录。注意 --diagnostic-dir 指向的目录
|
|
73
77
|
// 必须已存在 —— 实测(node 25)目录缺失时 V8 写快照直接原生断言崩溃,
|
|
74
78
|
// 所以先建目录,建失败就不加 flag 回落 cwd 默认。
|
|
75
79
|
if (!(process.env.NODE_OPTIONS ?? '').includes('--diagnostic-dir')) {
|
|
@@ -83,18 +87,9 @@ function childNodeFlags() {
|
|
|
83
87
|
return flags
|
|
84
88
|
}
|
|
85
89
|
|
|
86
|
-
/** 看护适用性:交互式 TUI 才看护;fast-echo / SDK 桥 / 已被看护的子进程不看。 */
|
|
87
|
-
function shouldSupervise(argv) {
|
|
88
|
-
if (process.env.STEM_NO_SUPERVISOR === '1') return false
|
|
89
|
-
if (process.env.STEM_SUPERVISED === '1') return false
|
|
90
|
-
if (!process.stdout.isTTY || !process.stdin.isTTY) return false
|
|
91
|
-
const nonInteractive = new Set(['--help', '-h', '--version', '-v', '--print'])
|
|
92
|
-
return !argv.some(a => nonInteractive.has(a) || a === '--sdk-url' || a.startsWith('--sdk-url='))
|
|
93
|
-
}
|
|
94
|
-
|
|
95
90
|
const distEntry = join(rootDir, 'dist', 'cli.mjs')
|
|
91
|
+
const argv = process.argv.slice(2)
|
|
96
92
|
if (process.env.STEM_DEV !== '1' && existsSync(distEntry)) {
|
|
97
|
-
const argv = process.argv.slice(2)
|
|
98
93
|
if (shouldSupervise(argv)) {
|
|
99
94
|
const { runSupervisor } = await import(pathToFileURL(join(rootDir, 'bin', 'stem-supervisor-lib.mjs')).href)
|
|
100
95
|
process.exit(await runSupervisor(distEntry, argv, { isZh, nodeFlags: childNodeFlags() }))
|
|
@@ -103,25 +98,37 @@ if (process.env.STEM_DEV !== '1' && existsSync(distEntry)) {
|
|
|
103
98
|
// 子任务场景会 OOM。V8 堆参数只能在进程启动时生效;用户已显式配置时同进程直跑。
|
|
104
99
|
await import(pathToFileURL(distEntry).href)
|
|
105
100
|
} else {
|
|
106
|
-
// 非看护场景(fast-echo / SDK 桥 /
|
|
101
|
+
// 非看护场景(fast-echo / SDK 桥 / STEM_SUPERVISOR_OFF)单次 re-exec,
|
|
107
102
|
// 把 --max-old-space-size 带给真正的 CLI 进程。
|
|
108
103
|
const result = spawnSync(
|
|
109
104
|
process.execPath,
|
|
110
|
-
[`--max-old-space-size=${heapLimitMB()}`, distEntry, ...argv],
|
|
105
|
+
['--expose-gc', `--max-old-space-size=${heapLimitMB()}`, distEntry, ...argv],
|
|
111
106
|
{ stdio: 'inherit' },
|
|
112
107
|
)
|
|
113
108
|
process.exit(result.status ?? 1)
|
|
114
109
|
}
|
|
115
110
|
} else {
|
|
116
|
-
// dev
|
|
111
|
+
// dev 模式:父启动器仍留在 Node 中看护,子进程才由 Bun 直跑源码。
|
|
112
|
+
// Bun 会自动加载 cwd 下的 .env,跳过时显式指向空设备。
|
|
117
113
|
const envFlag = skipDotenv
|
|
118
114
|
? [`--env-file=${devNull}`]
|
|
119
115
|
: existsSync(envFile)
|
|
120
116
|
? ['--env-file=.env']
|
|
121
117
|
: []
|
|
118
|
+
const devPrefixArgs = [...envFlag, './src/stem-entry/stem-entry-cli.tsx']
|
|
119
|
+
const bunCommand = resolveBunCommand()
|
|
120
|
+
if (shouldSupervise(argv)) {
|
|
121
|
+
const { runSupervisor } = await import(pathToFileURL(join(rootDir, 'bin', 'stem-supervisor-lib.mjs')).href)
|
|
122
|
+
process.exit(await runSupervisor(distEntry, argv, {
|
|
123
|
+
isZh,
|
|
124
|
+
childCommand: bunCommand,
|
|
125
|
+
childPrefixArgs: devPrefixArgs,
|
|
126
|
+
childCwd: rootDir,
|
|
127
|
+
}))
|
|
128
|
+
}
|
|
122
129
|
const result = spawnSync(
|
|
123
|
-
|
|
124
|
-
[...
|
|
130
|
+
bunCommand,
|
|
131
|
+
[...devPrefixArgs, ...argv],
|
|
125
132
|
{ cwd: rootDir, stdio: 'inherit' },
|
|
126
133
|
)
|
|
127
134
|
if (result.error && result.error.code === 'ENOENT') {
|