@mzzsfy/dsh-maintain 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -9
- package/package.json +1 -1
- package/src/client.js +126 -22
- package/src/core.mjs +58 -0
- package/src/index.js +369 -48
- package/src/runtime.mjs +69 -0
- package/src/upgrade.mjs +3 -2
- package/test/client-scope.test.mjs +54 -0
- package/test/core.test.mjs +135 -0
- package/test/parity.test.mjs +30 -3
- package/test/restart-ready.test.mjs +65 -9
- package/test/route.test.mjs +314 -23
- package/test/runtime.test.mjs +67 -0
- package/test/upgrade.test.mjs +172 -0
package/src/index.js
CHANGED
|
@@ -12,13 +12,20 @@ import {
|
|
|
12
12
|
TARGET_PACKAGE,
|
|
13
13
|
TAG_PLACEHOLDER,
|
|
14
14
|
buildUpgradeCommand,
|
|
15
|
+
classifyUpgradeFailure,
|
|
15
16
|
fetchDistTags,
|
|
16
17
|
isValidChannelName,
|
|
17
18
|
isValidRegistryBase,
|
|
19
|
+
isVersionPendingRestart,
|
|
20
|
+
judgeUpgradeFreshness,
|
|
18
21
|
judgeVersion,
|
|
19
22
|
resolveHostVersion,
|
|
23
|
+
UPGRADE_FAIL_FILE_LOCKED,
|
|
24
|
+
UPGRADE_FAIL_TRANSIENT_NETWORK,
|
|
25
|
+
VERDICT_UP_TO_DATE,
|
|
20
26
|
} from './core.mjs'
|
|
21
27
|
import { runUpgrade } from './upgrade.mjs'
|
|
28
|
+
import { detectRuntimeEnv, RUNTIME_KINDS } from './runtime.mjs'
|
|
22
29
|
|
|
23
30
|
export const name = 'dsh-maintain'
|
|
24
31
|
|
|
@@ -34,11 +41,177 @@ export const UPGRADE_TIMEOUT_MS = 10 * 60 * 1000
|
|
|
34
41
|
// 响应发出到执行退出的延迟:保证浏览器收到 200 并进入重启等待态,进程才离场;
|
|
35
42
|
// 导出仅供测试计算延迟窗口等待时长
|
|
36
43
|
export const RESTART_DELAY_MS = 2 * 1000
|
|
44
|
+
// 升级成功后自动重启的延迟:给 runCheck 与浮条终态一拍时间,观察器轮询可赶上;
|
|
45
|
+
// 导出仅供测试计算延迟窗口等待时长
|
|
46
|
+
export const AUTO_RESTART_DELAY_MS = 3 * 1000
|
|
37
47
|
// 轮询底层计时粒度;导出仅供 parity 测试与 client 提示文案对拍
|
|
38
48
|
export const TICK_MS = 60 * 1000
|
|
39
49
|
// 轮询间隔上界:超大值会让到期时间戳溢出为 Infinity,轮询静默失效
|
|
40
50
|
export const POLL_INTERVAL_MAX_SEC = 30 * 24 * 60 * 60
|
|
41
51
|
|
|
52
|
+
// 升级尝试次数上限(含首次):npm 文件锁重试收益递减,收敛防长期占锁;导出仅供测试与 parity 对拍
|
|
53
|
+
export const UPGRADE_MAX_ATTEMPTS = 3
|
|
54
|
+
// 可重试失败的退避序列,按重试序号取值,越界取末位;导出仅供测试与 parity 对拍
|
|
55
|
+
export const UPGRADE_RETRY_BACKOFF_MS = Object.freeze({
|
|
56
|
+
[UPGRADE_FAIL_FILE_LOCKED]: Object.freeze([5 * 1000, 15 * 1000]),
|
|
57
|
+
[UPGRADE_FAIL_TRANSIENT_NETWORK]: Object.freeze([3 * 1000, 9 * 1000]),
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
// 退避序列缺失返回 null,由调用方判不可重试:宁可不重试,不可零间隔轰击上游
|
|
61
|
+
function upgradeRetryBackoffMs(backoff, kind, retryIndex) {
|
|
62
|
+
const seq = backoff[kind]
|
|
63
|
+
if (!seq || seq.length === 0) return null
|
|
64
|
+
return seq[Math.min(retryIndex, seq.length - 1)]
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// 日志单行化:拼入 console.warn 的动态串不得携带换行,防日志结构被打散
|
|
68
|
+
const singleLine = (text) => String(text ?? '').replace(/[\r\n]+/g, ' ')
|
|
69
|
+
|
|
70
|
+
// 审计日志:升级/重启的触发、落定、拒绝各一行结构化输出,不引日志框架
|
|
71
|
+
const audit = (endpoint, outcome, extra) => {
|
|
72
|
+
console.warn('[dsh-maintain] audit endpoint=' + endpoint + ' outcome=' + outcome + (extra ? ' ' + extra : ''))
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// 活跃工作检测:agents/jobs/terminals 全软依赖方法面守卫,缺失或异常即降级放行(fail-open,
|
|
76
|
+
// 不因安全网缺失死锁重启)。terminals 主路径逐 agent 的 realm 作用域解析(PTY 注册表随
|
|
77
|
+
// agent 挂载且 isolate,共享根作用域因 realm 隔离恒缺,仅兜底);sessions 不作活跃指标;
|
|
78
|
+
// 禁止顶层 inject(阻塞装载)。导出仅供测试
|
|
79
|
+
// 降级留痕按服务键进程首见一次:status 每拍重检,不限频会刷屏
|
|
80
|
+
const degradedWarned = new Set()
|
|
81
|
+
|
|
82
|
+
export function collectActiveWork(ctx) {
|
|
83
|
+
const counts = { agents: 0, jobs: 0, terminals: 0 }
|
|
84
|
+
let detectionAvailable = true
|
|
85
|
+
const degrade = (key, detail) => {
|
|
86
|
+
detectionAvailable = false
|
|
87
|
+
if (degradedWarned.has(key)) return
|
|
88
|
+
degradedWarned.add(key)
|
|
89
|
+
console.warn('[dsh-maintain] 活跃工作检测降级,门控放行(' + key + '): ' + singleLine(detail))
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
let agentList = []
|
|
93
|
+
try {
|
|
94
|
+
const agents = ctx.get('agents')
|
|
95
|
+
if (!agents || typeof agents.list !== 'function') throw new Error('agents 服务方法面不可用')
|
|
96
|
+
const rows = agents.list()
|
|
97
|
+
agentList = Array.isArray(rows) ? rows : []
|
|
98
|
+
counts.agents = agentList.filter((agent) => agent && agent.status === 'running').length
|
|
99
|
+
} catch (error) {
|
|
100
|
+
degrade('agents', error && error.message ? error.message : String(error))
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
const jobs = ctx.get('jobs')
|
|
105
|
+
if (!jobs || typeof jobs.list !== 'function') throw new Error('jobs 服务方法面不可用')
|
|
106
|
+
const seen = new Set()
|
|
107
|
+
for (const caller of [undefined, ...agentList]) {
|
|
108
|
+
const rows = jobs.list(caller)
|
|
109
|
+
for (const job of Array.isArray(rows) ? rows : []) {
|
|
110
|
+
const key = String(job && job.id)
|
|
111
|
+
if (seen.has(key)) continue
|
|
112
|
+
seen.add(key)
|
|
113
|
+
// stopping 必须计活:kill 置 stopping 后未落定,进程仍在
|
|
114
|
+
if (job && (job.status === 'running' || job.status === 'stopping')) counts.jobs += 1
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
} catch (error) {
|
|
118
|
+
degrade('jobs', error && error.message ? error.message : String(error))
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
try {
|
|
122
|
+
const seen = new Set()
|
|
123
|
+
let sawTerminalsService = false
|
|
124
|
+
const scan = (terminals, owner) => {
|
|
125
|
+
if (!terminals || typeof terminals.list !== 'function') return
|
|
126
|
+
sawTerminalsService = true
|
|
127
|
+
const rows = terminals.list(owner)
|
|
128
|
+
for (const terminal of Array.isArray(rows) ? rows : []) {
|
|
129
|
+
const key = String(terminal && terminal.sessionId)
|
|
130
|
+
if (seen.has(key)) continue
|
|
131
|
+
seen.add(key)
|
|
132
|
+
if (terminal && terminal.status && terminal.status.kind === 'running') counts.terminals += 1
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
for (const agent of agentList) {
|
|
136
|
+
try {
|
|
137
|
+
const agentCtx = agent && agent.ctx
|
|
138
|
+
if (agentCtx && typeof agentCtx.get === 'function') scan(agentCtx.get('terminals'), agent)
|
|
139
|
+
} catch (error) {
|
|
140
|
+
degrade('terminals', 'realm 探测异常: ' + (error && error.message ? error.message : String(error)))
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
try {
|
|
144
|
+
scan(ctx.get('terminals'), undefined)
|
|
145
|
+
} catch (error) {
|
|
146
|
+
degrade('terminals', '根探测异常: ' + (error && error.message ? error.message : String(error)))
|
|
147
|
+
}
|
|
148
|
+
if (!sawTerminalsService) degrade('terminals', 'terminals 服务面不可用')
|
|
149
|
+
} catch (error) {
|
|
150
|
+
degrade('terminals', error && error.message ? error.message : String(error))
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return {
|
|
154
|
+
agents: counts.agents,
|
|
155
|
+
jobs: counts.jobs,
|
|
156
|
+
terminals: counts.terminals,
|
|
157
|
+
total: counts.agents + counts.jobs + counts.terminals,
|
|
158
|
+
detectionAvailable,
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const sleepMs = (ms) => new Promise((resolve) => { setTimeout(resolve, ms) })
|
|
163
|
+
|
|
164
|
+
// 升级尝试循环:单次执行语义与超时不变,失败经分类,可重试类按退避重试,
|
|
165
|
+
// 直至成功/不可重试/次数上限;尾流只在落定结果上,attempts 条目只留判定字段。
|
|
166
|
+
// runImpl/sleepImpl/backoff 注入仅供测试,默认真实执行器、定时睡眠与导出退避表
|
|
167
|
+
export async function runUpgradeWithRetry({ command, timeoutMs = UPGRADE_TIMEOUT_MS, maxAttempts = UPGRADE_MAX_ATTEMPTS, backoff = UPGRADE_RETRY_BACKOFF_MS, runImpl = runUpgrade, sleepImpl = sleepMs, onAttemptStart }) {
|
|
168
|
+
const attempts = []
|
|
169
|
+
for (let index = 0; index < maxAttempts; index += 1) {
|
|
170
|
+
if (index > 0 && typeof onAttemptStart === 'function') onAttemptStart()
|
|
171
|
+
const startedAt = Date.now()
|
|
172
|
+
let result = null
|
|
173
|
+
let thrownMessage = null
|
|
174
|
+
try {
|
|
175
|
+
result = await runImpl({ command, timeoutMs })
|
|
176
|
+
} catch (error) {
|
|
177
|
+
thrownMessage = error && error.message ? error.message : String(error)
|
|
178
|
+
}
|
|
179
|
+
if (thrownMessage !== null) {
|
|
180
|
+
attempts.push({ startedAt, finishedAt: Date.now(), ok: false, code: null, timedOut: false, kind: null })
|
|
181
|
+
return { attempts, ok: false, kind: null, stillRunning: false, stdoutTail: '', stderrTail: '', error: thrownMessage }
|
|
182
|
+
}
|
|
183
|
+
const failure = result.ok ? null : classifyUpgradeFailure(result)
|
|
184
|
+
attempts.push({
|
|
185
|
+
startedAt,
|
|
186
|
+
finishedAt: Date.now(),
|
|
187
|
+
ok: result.ok === true,
|
|
188
|
+
code: typeof result.code === 'number' ? result.code : null,
|
|
189
|
+
timedOut: result.timedOut === true,
|
|
190
|
+
kind: failure === null ? null : failure.kind,
|
|
191
|
+
})
|
|
192
|
+
const settle = { attempts, ok: result.ok === true, kind: failure === null ? null : failure.kind, stillRunning: result.stillRunning === true, stdoutTail: result.stdoutTail ?? '', stderrTail: result.stderrTail ?? '', error: null }
|
|
193
|
+
if (result.ok) return settle
|
|
194
|
+
if (failure.retryable !== true || index >= maxAttempts - 1) return settle
|
|
195
|
+
const backoffMs = upgradeRetryBackoffMs(backoff, failure.kind, index)
|
|
196
|
+
if (backoffMs === null) {
|
|
197
|
+
console.warn('[dsh-maintain] 可重试分类缺少退避序列,按不可重试落定: kind=' + singleLine(failure.kind))
|
|
198
|
+
return settle
|
|
199
|
+
}
|
|
200
|
+
await sleepImpl(backoffMs)
|
|
201
|
+
}
|
|
202
|
+
return { attempts, ok: false, kind: null, stillRunning: false, stdoutTail: '', stderrTail: '', error: null }
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// 升级后自动重启守卫:四条件缺一不可——成功、非 stale、非手动直跑、appExit 可用。
|
|
206
|
+
// 手动直跑(双 TTY)进程退出后无人拉起,只标 requiresManualRestart 交面板指引
|
|
207
|
+
export function judgeAutoRestart({ ok, stale, runtimeKind, hasExit }) {
|
|
208
|
+
if (ok !== true) return { schedule: false, requiresManualRestart: false }
|
|
209
|
+
if (stale === true) return { schedule: false, requiresManualRestart: false }
|
|
210
|
+
if (runtimeKind === RUNTIME_KINDS.MANUAL_START) return { schedule: false, requiresManualRestart: true }
|
|
211
|
+
if (hasExit !== true) return { schedule: false, requiresManualRestart: false }
|
|
212
|
+
return { schedule: true, requiresManualRestart: false }
|
|
213
|
+
}
|
|
214
|
+
|
|
42
215
|
// 浏览器半区调用的 API 路径清单:client.js 同名常量与之对拍(parity),防单侧改路径生产 404
|
|
43
216
|
export const API_PATHS = Object.freeze({
|
|
44
217
|
STATUS: '/api/maintain/status',
|
|
@@ -167,15 +340,6 @@ function readSettings(ctx) {
|
|
|
167
340
|
}
|
|
168
341
|
}
|
|
169
342
|
|
|
170
|
-
async function resolveCurrentHostVersion() {
|
|
171
|
-
return resolveHostVersion({
|
|
172
|
-
execPath: process.execPath,
|
|
173
|
-
platform: process.platform,
|
|
174
|
-
readFileImpl: readFile,
|
|
175
|
-
resolveImpl: undefined,
|
|
176
|
-
})
|
|
177
|
-
}
|
|
178
|
-
|
|
179
343
|
/** @param {import('@deepseek-ai/cordis').Context} ctx */
|
|
180
344
|
export function apply(ctx) {
|
|
181
345
|
// 启动器在挂载前提供 appExit(有界退出,5 秒兜底强制);缺失时重启能力关闭
|
|
@@ -189,12 +353,44 @@ export function apply(ctx) {
|
|
|
189
353
|
console.warn('[dsh-maintain] 检测到未过期的升级锁(可能存在残留升级子进程),升级端点保持拒绝直至锁过期: ' + UPGRADE_LOCK_PATH)
|
|
190
354
|
}
|
|
191
355
|
|
|
192
|
-
//
|
|
193
|
-
|
|
356
|
+
// 运行版本探测:服务面可注入受控实现(测试面),缺省走真实盘读;
|
|
357
|
+
// 约定恒 resolve 不 reject,真实实现全失败回 null,注入侧经 then 同步抛错同折进 rejection
|
|
358
|
+
function resolveCurrentHostVersion() {
|
|
359
|
+
const probe = ctx.get('hostVersionProbe')
|
|
360
|
+
if (typeof probe === 'function') return Promise.resolve().then(probe)
|
|
361
|
+
return resolveHostVersion({
|
|
362
|
+
execPath: process.execPath,
|
|
363
|
+
platform: process.platform,
|
|
364
|
+
readFileImpl: readFile,
|
|
365
|
+
resolveImpl: undefined,
|
|
366
|
+
})
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// 内存快照:仅存当前态,进程重启后从启动检查重新开始(设计约束:不持久化)。
|
|
370
|
+
// installedVersion 为磁盘实时版本(每次检查读取),与运行版本区分
|
|
371
|
+
const snapshot = { installedVersion: null, tags: null, checkedAt: null, error: null }
|
|
372
|
+
// 运行版本:apply 时读一次缓存,即宿主启动时的版本;verdict 以它判定。
|
|
373
|
+
// resolveCurrentHostVersion 约定恒 resolve 不 reject,此处 catch 锁定该约束,
|
|
374
|
+
// 防读盘异常以未处理拒绝形式逃逸
|
|
375
|
+
let runningVersion = null
|
|
376
|
+
const runningVersionReady = resolveCurrentHostVersion()
|
|
377
|
+
.then((version) => { runningVersion = version })
|
|
378
|
+
.catch(() => {})
|
|
379
|
+
|
|
380
|
+
// 运行环境:apply 时一次检测;探测异步,以 ready 链收口,异常回退 unknown(维持自动重启)
|
|
381
|
+
let runtimeEnv = { kind: RUNTIME_KINDS.UNKNOWN, declared: false }
|
|
382
|
+
const runtimeEnvReady = detectRuntimeEnv({
|
|
383
|
+
env: process.env,
|
|
384
|
+
platform: process.platform,
|
|
385
|
+
isTTY: { stdin: process.stdin.isTTY === true, stdout: process.stdout.isTTY === true },
|
|
386
|
+
})
|
|
387
|
+
.then((env) => { runtimeEnv = env })
|
|
388
|
+
.catch(() => {})
|
|
194
389
|
let upgrade = { running: false, last: null }
|
|
195
390
|
let checkInFlight = null
|
|
196
391
|
let nextDueAt = null
|
|
197
392
|
let restartScheduled = false
|
|
393
|
+
let autoRestartScheduled = false
|
|
198
394
|
|
|
199
395
|
function runCheck() {
|
|
200
396
|
if (checkInFlight) return checkInFlight
|
|
@@ -207,7 +403,7 @@ export function apply(ctx) {
|
|
|
207
403
|
// registry 不可达:保留上次 tags,仅记录错误
|
|
208
404
|
snapshot.error = error && error.message ? error.message : String(error)
|
|
209
405
|
}
|
|
210
|
-
snapshot.
|
|
406
|
+
snapshot.installedVersion = await resolveCurrentHostVersion()
|
|
211
407
|
snapshot.checkedAt = Date.now()
|
|
212
408
|
return snapshot
|
|
213
409
|
})()
|
|
@@ -245,9 +441,16 @@ export function apply(ctx) {
|
|
|
245
441
|
pollRunning = true
|
|
246
442
|
})
|
|
247
443
|
|
|
248
|
-
function
|
|
444
|
+
function judgeNow() {
|
|
445
|
+
// verdict 以运行版本判定:用户关心"跑的是不是最新";已装版本供升级复读校验
|
|
446
|
+
return judgeVersion({ currentVersion: runningVersion, tags: snapshot.tags, channel: readSettings(ctx).channel })
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
async function currentStatus() {
|
|
450
|
+
// 等运行版本首读与运行环境检测落定:apply 即发起,此处仅吸收启动窗口的微小延迟
|
|
451
|
+
await Promise.all([runningVersionReady, runtimeEnvReady])
|
|
249
452
|
const config = readSettings(ctx)
|
|
250
|
-
const judged =
|
|
453
|
+
const judged = judgeNow()
|
|
251
454
|
// running 即视为持锁:省一次盘读,且窗口期语义与 upgrade 路由的门闩一致
|
|
252
455
|
const lock = upgrade.running === true ? { startedAt: Date.now() } : readUpgradeLock()
|
|
253
456
|
return {
|
|
@@ -256,7 +459,9 @@ export function apply(ctx) {
|
|
|
256
459
|
bootAt: BOOT_AT,
|
|
257
460
|
pollRunning,
|
|
258
461
|
upgradeLockHeld: lock !== null && !upgradeLockStale(lock),
|
|
259
|
-
|
|
462
|
+
runningVersion,
|
|
463
|
+
installedVersion: snapshot.installedVersion,
|
|
464
|
+
restartPending: isVersionPendingRestart({ runningVersion, installedVersion: snapshot.installedVersion }),
|
|
260
465
|
channel: config.channel,
|
|
261
466
|
upgradeTemplate: config.upgradeCommandTemplate,
|
|
262
467
|
pollIntervalSec: config.pollIntervalSec,
|
|
@@ -268,6 +473,9 @@ export function apply(ctx) {
|
|
|
268
473
|
checkedAt: snapshot.checkedAt,
|
|
269
474
|
checkError: snapshot.error,
|
|
270
475
|
upgrade,
|
|
476
|
+
runtimeEnv,
|
|
477
|
+
autoRestartScheduled,
|
|
478
|
+
activeWork: collectActiveWork(ctx),
|
|
271
479
|
canRestart: typeof exit === 'function',
|
|
272
480
|
}
|
|
273
481
|
}
|
|
@@ -276,40 +484,114 @@ export function apply(ctx) {
|
|
|
276
484
|
const config = readSettings(ctx)
|
|
277
485
|
// 模板校验同步失败即同步 throw,由调用方 try/catch 转 400,不走异步通道
|
|
278
486
|
const command = buildUpgradeCommand({ template: config.upgradeCommandTemplate, tag: config.channel })
|
|
279
|
-
const last = {
|
|
487
|
+
const last = {
|
|
488
|
+
command,
|
|
489
|
+
startedAt: Date.now(),
|
|
490
|
+
ok: false,
|
|
491
|
+
finishedAt: null,
|
|
492
|
+
timedOut: false,
|
|
493
|
+
stillRunning: false,
|
|
494
|
+
code: null,
|
|
495
|
+
stdoutTail: '',
|
|
496
|
+
stderrTail: '',
|
|
497
|
+
error: null,
|
|
498
|
+
kind: null,
|
|
499
|
+
attempts: [],
|
|
500
|
+
previousVersion: null,
|
|
501
|
+
installedVersion: null,
|
|
502
|
+
stale: null,
|
|
503
|
+
reason: null,
|
|
504
|
+
}
|
|
280
505
|
// running 即串行化门闩:路由检查与本处置位之间无 await,单线程下无竞态窗口
|
|
281
506
|
upgrade = { running: true, last }
|
|
507
|
+
writeUpgradeLock(last.startedAt)
|
|
508
|
+
audit('upgrade', 'triggered', 'command=' + singleLine(command))
|
|
509
|
+
void performUpgrade(command, last, config.channel)
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
// 升级锁文件只在此写入:首次与每次重试覆写,startedAt 取当前尝试开始时刻,
|
|
513
|
+
// 过期阈值覆盖单次尝试的完整时长(超时 + 强杀宽限)
|
|
514
|
+
function writeUpgradeLock(startedAt) {
|
|
282
515
|
try {
|
|
283
|
-
writeFileSync(UPGRADE_LOCK_PATH, JSON.stringify({ startedAt
|
|
516
|
+
writeFileSync(UPGRADE_LOCK_PATH, JSON.stringify({ startedAt, pid: process.pid }), 'utf8')
|
|
284
517
|
} catch { /* 锁不可写仅损失跨进程防护,内存门闩仍生效 */ }
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
// 自动重启与手动重启共用 restartScheduled 互斥:调度窗口内到达的升级被 409,
|
|
521
|
+
// 手动重启端点幂等;autoRestartScheduled 单独给 status/client 分流终态文案
|
|
522
|
+
function scheduleAutoRestart() {
|
|
523
|
+
restartScheduled = true
|
|
524
|
+
autoRestartScheduled = true
|
|
525
|
+
audit('restart', 'scheduled', 'reason=upgrade-ok delayMs=' + AUTO_RESTART_DELAY_MS)
|
|
526
|
+
setTimeout(() => exit(0), AUTO_RESTART_DELAY_MS)
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
async function performUpgrade(command, last, channel) {
|
|
530
|
+
try {
|
|
531
|
+
// 触发前快照磁盘版本,成功落定后复读对比,版本未前进或未达目标即标 stale
|
|
532
|
+
last.previousVersion = await resolveCurrentHostVersion()
|
|
533
|
+
const settle = await runUpgradeWithRetry({
|
|
534
|
+
command,
|
|
535
|
+
onAttemptStart: () => writeUpgradeLock(Date.now()),
|
|
292
536
|
})
|
|
293
|
-
.
|
|
294
|
-
|
|
537
|
+
const final = settle.attempts[settle.attempts.length - 1] ?? null
|
|
538
|
+
Object.assign(last, {
|
|
539
|
+
attempts: settle.attempts,
|
|
540
|
+
ok: settle.ok,
|
|
541
|
+
kind: settle.kind,
|
|
542
|
+
error: settle.error,
|
|
543
|
+
finishedAt: Date.now(),
|
|
544
|
+
timedOut: final !== null ? final.timedOut : false,
|
|
545
|
+
stillRunning: settle.stillRunning,
|
|
546
|
+
code: final !== null ? final.code : null,
|
|
547
|
+
stdoutTail: settle.stdoutTail,
|
|
548
|
+
stderrTail: settle.stderrTail,
|
|
295
549
|
})
|
|
296
|
-
.
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
550
|
+
if (settle.ok) {
|
|
551
|
+
last.installedVersion = await resolveCurrentHostVersion()
|
|
552
|
+
const tags = snapshot.tags
|
|
553
|
+
const freshness = judgeUpgradeFreshness({
|
|
554
|
+
previousVersion: last.previousVersion,
|
|
555
|
+
installedVersion: last.installedVersion,
|
|
556
|
+
channelLatest: tags !== null && Object.prototype.hasOwnProperty.call(tags, channel) ? tags[channel] : null,
|
|
557
|
+
})
|
|
558
|
+
last.stale = freshness.stale
|
|
559
|
+
last.reason = freshness.reason
|
|
560
|
+
}
|
|
561
|
+
} catch (error) {
|
|
562
|
+
last.error = error && error.message ? error.message : String(error)
|
|
563
|
+
} finally {
|
|
564
|
+
if (last.finishedAt === null) last.finishedAt = Date.now()
|
|
565
|
+
upgrade = { running: false, last }
|
|
566
|
+
// stillRunning(强杀后进程树疑似仍在写全局目录)时保留锁文件,由过期机制收敛,
|
|
567
|
+
// 与"宿主死后幽灵锁"防线对称;正常落定即删
|
|
568
|
+
if (last.stillRunning === true) {
|
|
569
|
+
console.warn('[dsh-maintain] 升级超时强杀且进程疑似仍存活,升级锁保留至过期: ' + UPGRADE_LOCK_PATH)
|
|
570
|
+
} else {
|
|
571
|
+
try {
|
|
572
|
+
rmSync(UPGRADE_LOCK_PATH, { force: true })
|
|
573
|
+
} catch (lockError) {
|
|
574
|
+
console.warn('[dsh-maintain] 升级锁删除失败(将由过期机制收敛): ' + (lockError?.message ?? lockError))
|
|
307
575
|
}
|
|
308
|
-
|
|
309
|
-
|
|
576
|
+
}
|
|
577
|
+
console.warn('[dsh-maintain] 升级结束: ok=' + last.ok + ' kind=' + singleLine(last.kind)
|
|
578
|
+
+ (last.code !== null ? ' 退出码=' + last.code : '')
|
|
579
|
+
+ (last.error ? ' ' + singleLine(last.error) : '')
|
|
580
|
+
+ (last.stale === true ? ' stale=' + singleLine(last.reason) : ''))
|
|
581
|
+
audit('upgrade', last.ok === true ? 'ok' : 'failed',
|
|
582
|
+
'durationMs=' + (last.finishedAt - last.startedAt) + ' code=' + last.code + ' kind=' + singleLine(last.kind))
|
|
583
|
+
// 自动重启守卫:等环境检测落定后按四条件分流(手动直跑只标指引,不退出)
|
|
584
|
+
await runtimeEnvReady
|
|
585
|
+
const decision = judgeAutoRestart({ ok: last.ok === true, stale: last.stale === true, runtimeKind: runtimeEnv.kind, hasExit: typeof exit === 'function' })
|
|
586
|
+
if (decision.requiresManualRestart === true) last.requiresManualRestart = true
|
|
587
|
+
if (decision.schedule === true) {
|
|
588
|
+
// last 镜像调度标记:浮条终态文案按其分流(与 status.autoRestartScheduled 同值)
|
|
589
|
+
last.autoRestartScheduled = true
|
|
590
|
+
scheduleAutoRestart()
|
|
591
|
+
}
|
|
310
592
|
// 升级结束后自动重新检查版本并重排轮询(命令可能改了本地版本)
|
|
311
|
-
.then(
|
|
312
|
-
|
|
593
|
+
runCheck().then(scheduleNext, scheduleNext)
|
|
594
|
+
}
|
|
313
595
|
}
|
|
314
596
|
|
|
315
597
|
const WRITE = { crossOrigin: true }
|
|
@@ -318,7 +600,7 @@ export function apply(ctx) {
|
|
|
318
600
|
{
|
|
319
601
|
path: API_PATHS.STATUS,
|
|
320
602
|
handler: route('GET', {}, async (req, res) => {
|
|
321
|
-
sendJson(res, 200, currentStatus())
|
|
603
|
+
sendJson(res, 200, await currentStatus())
|
|
322
604
|
}),
|
|
323
605
|
},
|
|
324
606
|
{
|
|
@@ -326,7 +608,7 @@ export function apply(ctx) {
|
|
|
326
608
|
handler: route('POST', WRITE, async (req, res) => {
|
|
327
609
|
await drainInFlightThenCheck()
|
|
328
610
|
scheduleNext()
|
|
329
|
-
sendJson(res, 200, currentStatus())
|
|
611
|
+
sendJson(res, 200, await currentStatus())
|
|
330
612
|
}),
|
|
331
613
|
},
|
|
332
614
|
{
|
|
@@ -355,7 +637,7 @@ export function apply(ctx) {
|
|
|
355
637
|
await settings.update(NAMESPACE, { channel })
|
|
356
638
|
// 切通道无需重查:dist-tags 与 channel 无关,白名单已用现有 snapshot 校验
|
|
357
639
|
scheduleNext()
|
|
358
|
-
sendJson(res, 200, currentStatus())
|
|
640
|
+
sendJson(res, 200, await currentStatus())
|
|
359
641
|
}),
|
|
360
642
|
},
|
|
361
643
|
{
|
|
@@ -373,7 +655,7 @@ export function apply(ctx) {
|
|
|
373
655
|
return
|
|
374
656
|
}
|
|
375
657
|
await settings.update(NAMESPACE, { upgradeCommandTemplate: template })
|
|
376
|
-
sendJson(res, 200, currentStatus())
|
|
658
|
+
sendJson(res, 200, await currentStatus())
|
|
377
659
|
}),
|
|
378
660
|
},
|
|
379
661
|
{
|
|
@@ -398,7 +680,7 @@ export function apply(ctx) {
|
|
|
398
680
|
}
|
|
399
681
|
await settings.update(NAMESPACE, { pollIntervalSec: seconds })
|
|
400
682
|
scheduleNext()
|
|
401
|
-
sendJson(res, 200, currentStatus())
|
|
683
|
+
sendJson(res, 200, await currentStatus())
|
|
402
684
|
}),
|
|
403
685
|
},
|
|
404
686
|
{
|
|
@@ -418,7 +700,7 @@ export function apply(ctx) {
|
|
|
418
700
|
await settings.update(NAMESPACE, { registryBase: base })
|
|
419
701
|
await drainInFlightThenCheck()
|
|
420
702
|
scheduleNext()
|
|
421
|
-
sendJson(res, 200, currentStatus())
|
|
703
|
+
sendJson(res, 200, await currentStatus())
|
|
422
704
|
}),
|
|
423
705
|
},
|
|
424
706
|
{
|
|
@@ -427,15 +709,36 @@ export function apply(ctx) {
|
|
|
427
709
|
// 与重启调度双向互斥:restartScheduled 置位到 exit(0) 执行的窗口内触发升级,
|
|
428
710
|
// detached 孤儿会与新宿主交错
|
|
429
711
|
if (upgrade.running || restartScheduled) {
|
|
712
|
+
audit('upgrade', 'rejected', restartScheduled ? 'reason=restart-scheduled' : 'reason=running')
|
|
430
713
|
sendJson(res, 409, { error: restartScheduled ? '重启已调度,禁止触发升级' : '升级进行中' })
|
|
431
714
|
return
|
|
432
715
|
}
|
|
433
716
|
if (upgradeLockStale(readUpgradeLock()) === false) {
|
|
717
|
+
audit('upgrade', 'rejected', 'reason=lock')
|
|
434
718
|
sendJson(res, 409, { error: '存在未过期的升级锁(可能有残留升级子进程),等待锁过期或删除 ' + UPGRADE_LOCK_PATH })
|
|
435
719
|
return
|
|
436
720
|
}
|
|
721
|
+
// 防降级:运行版本已达通道最新时拒绝(重装旧版即降级,对齐 no-newer-version);
|
|
722
|
+
// 信息不足(verdict unknown)放行,宽松不误拒
|
|
723
|
+
const judged = judgeNow()
|
|
724
|
+
if (judged.verdict === VERDICT_UP_TO_DATE) {
|
|
725
|
+
audit('upgrade', 'rejected', 'reason=up-to-date')
|
|
726
|
+
sendJson(res, 409, { error: '当前已是通道最新版,无需升级;重装请走命令行' })
|
|
727
|
+
return
|
|
728
|
+
}
|
|
729
|
+
// 活跃工作门控:升级不可越(agent 会话/job 在跑时强行升级会被中断)
|
|
730
|
+
const activeWork = collectActiveWork(ctx)
|
|
731
|
+
if (activeWork.total > 0) {
|
|
732
|
+
audit('upgrade', 'rejected', 'reason=active-work total=' + activeWork.total)
|
|
733
|
+
sendJson(res, 409, {
|
|
734
|
+
error: '存在活跃工作(共 ' + activeWork.total + ' 项:agents ' + activeWork.agents + '/jobs ' + activeWork.jobs + '/terminals ' + activeWork.terminals + '),升级会中断它们,请等待完成',
|
|
735
|
+
items: activeWork,
|
|
736
|
+
detectionAvailable: activeWork.detectionAvailable,
|
|
737
|
+
})
|
|
738
|
+
return
|
|
739
|
+
}
|
|
437
740
|
triggerUpgrade()
|
|
438
|
-
sendJson(res, 200, currentStatus())
|
|
741
|
+
sendJson(res, 200, await currentStatus())
|
|
439
742
|
}),
|
|
440
743
|
},
|
|
441
744
|
{
|
|
@@ -444,10 +747,12 @@ export function apply(ctx) {
|
|
|
444
747
|
// 与升级门闩互斥:升级子进程经 detached+unref 存活于宿主死后,
|
|
445
748
|
// 重启后新宿主门闩归零会放行第二次升级,双 npm install 并发写全局目录
|
|
446
749
|
if (upgrade.running) {
|
|
750
|
+
audit('restart', 'rejected', 'reason=upgrade-running')
|
|
447
751
|
sendJson(res, 409, { error: '升级进行中,禁止重启;等待升级完成后重试' })
|
|
448
752
|
return
|
|
449
753
|
}
|
|
450
754
|
if (typeof exit !== 'function') {
|
|
755
|
+
audit('restart', 'rejected', 'reason=no-appExit')
|
|
451
756
|
sendJson(res, 500, { error: '启动器未提供 appExit,无法重启' })
|
|
452
757
|
return
|
|
453
758
|
}
|
|
@@ -455,7 +760,23 @@ export function apply(ctx) {
|
|
|
455
760
|
sendJson(res, 200, { ok: true, restarting: true })
|
|
456
761
|
return
|
|
457
762
|
}
|
|
763
|
+
// 活跃工作门控:重启可被 force 越过(升级后自动重启链路无法等待人工确认,
|
|
764
|
+
// 对齐 dsh-service:升级不可越/重启 force 可越)
|
|
765
|
+
const rawBody = await readBody(req)
|
|
766
|
+
// 'null' 字面量解析为 null:按空体处理,防 null.force 形态的内部错误外泄
|
|
767
|
+
const requestBody = rawBody ? (JSON.parse(rawBody) || {}) : {}
|
|
768
|
+
const activeWork = collectActiveWork(ctx)
|
|
769
|
+
if (activeWork.total > 0 && requestBody.force !== true) {
|
|
770
|
+
audit('restart', 'rejected', 'reason=active-work total=' + activeWork.total)
|
|
771
|
+
sendJson(res, 409, {
|
|
772
|
+
error: '存在活跃工作(共 ' + activeWork.total + ' 项:agents ' + activeWork.agents + '/jobs ' + activeWork.jobs + '/terminals ' + activeWork.terminals + '),重启会中断它们;确认无误请带 force 重试',
|
|
773
|
+
items: activeWork,
|
|
774
|
+
detectionAvailable: activeWork.detectionAvailable,
|
|
775
|
+
})
|
|
776
|
+
return
|
|
777
|
+
}
|
|
458
778
|
restartScheduled = true
|
|
779
|
+
audit('restart', 'triggered', requestBody.force === true ? 'forced=true' : 'forced=false')
|
|
459
780
|
sendJson(res, 200, { ok: true, restarting: true })
|
|
460
781
|
// 退出延迟与响应冲刷解耦:客户端在冲刷完成前断连会让 end 回调失效,绑定其上会使重启悬空;
|
|
461
782
|
// 同步发出响应后延迟退出,延迟本身保证回执先于进程离场送达
|
package/src/runtime.mjs
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// 运行环境检测:判定宿主进程是否处于托管启动形态,决定升级后可否自动重启。
|
|
2
|
+
// 纯函数,探测点经参数注入;顺序:env 显式覆盖 > 进程环境表 > POSIX 容器探测 > 双 TTY > unknown。
|
|
3
|
+
import { stat, readFile } from 'node:fs/promises'
|
|
4
|
+
|
|
5
|
+
export const RUNTIME_KINDS = Object.freeze({
|
|
6
|
+
DECLARED_MANAGED: 'declared-managed',
|
|
7
|
+
PM2: 'pm2',
|
|
8
|
+
SYSTEMD: 'systemd',
|
|
9
|
+
SUPERVISORD: 'supervisord',
|
|
10
|
+
KUBERNETES: 'kubernetes',
|
|
11
|
+
DOCKER: 'docker',
|
|
12
|
+
CONTAINER: 'container',
|
|
13
|
+
MANUAL_START: 'manual-start-likely',
|
|
14
|
+
UNKNOWN: 'unknown',
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
// 环境覆盖键:managed|manual,测试与用户显式纠偏用
|
|
18
|
+
const RUNTIME_ENV_OVERRIDE = 'DSH_MAINTAIN_RUNTIME_ENV'
|
|
19
|
+
const OVERRIDE_MANAGED = 'managed'
|
|
20
|
+
const OVERRIDE_MANUAL = 'manual'
|
|
21
|
+
|
|
22
|
+
// 各托管形态的特征环境变量表:命中任一即声明
|
|
23
|
+
const MANAGED_ENV_KEYS = Object.freeze([
|
|
24
|
+
[RUNTIME_KINDS.PM2, Object.freeze(['pm_id', 'PM2_HOME', 'pm_uptime'])],
|
|
25
|
+
[RUNTIME_KINDS.SYSTEMD, Object.freeze(['INVOCATION_ID', 'JOURNAL_STREAM', 'NOTIFY_SOCKET'])],
|
|
26
|
+
[RUNTIME_KINDS.SUPERVISORD, Object.freeze(['SUPERVISOR_ENABLED', 'SUPERVISOR_PROCESS_NAME'])],
|
|
27
|
+
[RUNTIME_KINDS.KUBERNETES, Object.freeze(['KUBERNETES_SERVICE_HOST'])],
|
|
28
|
+
])
|
|
29
|
+
|
|
30
|
+
const DOCKERENV_PATH = '/.dockerenv'
|
|
31
|
+
const CGROUP_PATH = '/proc/1/cgroup'
|
|
32
|
+
// cgroup 归类:/docker/ 与 dockerenv 同为 docker;kubepods 即 k8s;其余容器运行时归 container
|
|
33
|
+
const CGROUP_DOCKER_PATTERN = /\/docker\//
|
|
34
|
+
const CGROUP_K8S_PATTERN = /kubepods/
|
|
35
|
+
const CGROUP_CONTAINER_PATTERN = /containerd|lxc|podman/
|
|
36
|
+
|
|
37
|
+
const fallbackExists = async (path) => {
|
|
38
|
+
try {
|
|
39
|
+
await stat(path)
|
|
40
|
+
return true
|
|
41
|
+
} catch {
|
|
42
|
+
return false
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const fallbackRead = async (path) => readFile(path, 'utf8')
|
|
47
|
+
|
|
48
|
+
export async function detectRuntimeEnv({ env, platform, isTTY, existsImpl = fallbackExists, readFileImpl = fallbackRead }) {
|
|
49
|
+
const override = env[RUNTIME_ENV_OVERRIDE]
|
|
50
|
+
if (override === OVERRIDE_MANAGED) return { kind: RUNTIME_KINDS.DECLARED_MANAGED, declared: true }
|
|
51
|
+
if (override === OVERRIDE_MANUAL) return { kind: RUNTIME_KINDS.MANUAL_START, declared: false }
|
|
52
|
+
|
|
53
|
+
for (const [kind, keys] of MANAGED_ENV_KEYS) {
|
|
54
|
+
if (keys.some((key) => env[key])) return { kind, declared: true }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (platform !== 'win32') {
|
|
58
|
+
try {
|
|
59
|
+
if (await existsImpl(DOCKERENV_PATH)) return { kind: RUNTIME_KINDS.DOCKER, declared: true }
|
|
60
|
+
const cgroup = await readFileImpl(CGROUP_PATH)
|
|
61
|
+
if (CGROUP_DOCKER_PATTERN.test(cgroup)) return { kind: RUNTIME_KINDS.DOCKER, declared: true }
|
|
62
|
+
if (CGROUP_K8S_PATTERN.test(cgroup)) return { kind: RUNTIME_KINDS.KUBERNETES, declared: true }
|
|
63
|
+
if (CGROUP_CONTAINER_PATTERN.test(cgroup)) return { kind: RUNTIME_KINDS.CONTAINER, declared: true }
|
|
64
|
+
} catch { /* 探测失败按未声明继续 */ }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (isTTY.stdin === true && isTTY.stdout === true) return { kind: RUNTIME_KINDS.MANUAL_START, declared: false }
|
|
68
|
+
return { kind: RUNTIME_KINDS.UNKNOWN, declared: false }
|
|
69
|
+
}
|
package/src/upgrade.mjs
CHANGED
|
@@ -9,8 +9,9 @@ import { spawn } from 'node:child_process'
|
|
|
9
9
|
|
|
10
10
|
const OUTPUT_TAIL_BYTES = 2000
|
|
11
11
|
const SPAWN_ERROR_CODE = 127
|
|
12
|
-
// 树终止后的宽限期:子进程树拒死(权限/管道悬挂)
|
|
13
|
-
|
|
12
|
+
// 树终止后的宽限期:子进程树拒死(权限/管道悬挂)时强制收敛,防升级门闩永久卡死;
|
|
13
|
+
// 导出仅供 parity 测试作观察窗口上界的构成项对拍
|
|
14
|
+
export const KILL_GRACE_MS = 5 * 1000
|
|
14
15
|
|
|
15
16
|
function collectTail(stream) {
|
|
16
17
|
let text = ''
|