@ddtcorex/dsh-maestro-supervisor 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/boot-guard.d.ts +33 -0
- package/lib/boot-guard.js +68 -0
- package/lib/boot-lock.d.ts +56 -0
- package/lib/boot-lock.js +155 -0
- package/lib/cli.d.ts +22 -5
- package/lib/cli.js +120 -54
- package/lib/health-poller.d.ts +68 -1
- package/lib/health-poller.js +191 -133
- package/lib/plugin.js +15 -16
- package/lib/restart-exec.d.ts +35 -0
- package/lib/restart-exec.js +60 -0
- package/lib/restart-guards.d.ts +18 -0
- package/lib/restart-guards.js +33 -0
- package/lib/restart-tool.d.ts +2 -1
- package/lib/restart-tool.js +19 -1
- package/lib/restart-web.d.ts +26 -0
- package/lib/restart-web.js +58 -0
- package/lib/resume.d.ts +10 -0
- package/lib/resume.js +45 -28
- package/lib/snapshot.d.ts +52 -15
- package/lib/snapshot.js +168 -18
- package/lib/supervisor.d.ts +21 -4
- package/lib/supervisor.js +78 -36
- package/package.json +1 -1
- package/skills/dsh-safe-restart/SKILL.md +32 -2
- package/skills/dsh-safe-restart/scripts/restart-dsh-web.sh +162 -8
package/lib/health-poller.js
CHANGED
|
@@ -1,18 +1,12 @@
|
|
|
1
|
-
import { checkPlannedRestart } from './restart-guards.js';
|
|
1
|
+
import { checkPlannedRestart, BOOT_BOUNDARY_MARKER } from './restart-guards.js';
|
|
2
2
|
import { execSync as execSyncImpl } from 'node:child_process';
|
|
3
|
+
/**
|
|
4
|
+
* Wall-clock epoch ms of the current dsh-web unit start, or undefined when
|
|
5
|
+
* systemd does not manage the unit (portable host) or the lookup is disabled.
|
|
6
|
+
* `ActiveEnterTimestampMonotonic` is deliberately gone: a monotonic reading
|
|
7
|
+
* cannot be compared with a timestamp-less append-only log.
|
|
8
|
+
*/
|
|
3
9
|
export function getActiveEnterMs() {
|
|
4
|
-
if (process.env.VITEST)
|
|
5
|
-
return undefined;
|
|
6
|
-
try {
|
|
7
|
-
const out = execSyncImpl('systemctl --user show -p ActiveEnterTimestampMonotonic dsh-web.service 2>/dev/null', { encoding: 'utf8' });
|
|
8
|
-
const m = out.match(/ActiveEnterTimestampMonotonic=(\d+)/);
|
|
9
|
-
if (m)
|
|
10
|
-
return Number(m[1]);
|
|
11
|
-
}
|
|
12
|
-
catch { }
|
|
13
|
-
return undefined;
|
|
14
|
-
}
|
|
15
|
-
export function getActiveEnterWallMs() {
|
|
16
10
|
if (process.env.VITEST)
|
|
17
11
|
return undefined;
|
|
18
12
|
try {
|
|
@@ -30,25 +24,9 @@ export function getActiveEnterWallMs() {
|
|
|
30
24
|
catch { }
|
|
31
25
|
return undefined;
|
|
32
26
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const now = Date.now();
|
|
37
|
-
// 30s grace after ActiveEnterTimestamp (wall clock) — covers manual systemctl start without marker
|
|
38
|
-
const wall = wallMs ?? (() => {
|
|
39
|
-
try {
|
|
40
|
-
const fn = opts.getActiveEnterWallMs;
|
|
41
|
-
if (typeof fn === 'function')
|
|
42
|
-
return fn();
|
|
43
|
-
return getActiveEnterWallMs();
|
|
44
|
-
}
|
|
45
|
-
catch {
|
|
46
|
-
return undefined;
|
|
47
|
-
}
|
|
48
|
-
})();
|
|
49
|
-
if (typeof wall === 'number' && now - wall < 30000)
|
|
50
|
-
return true;
|
|
51
|
-
return false;
|
|
27
|
+
/** @deprecated kept for callers; identical to getActiveEnterMs(). */
|
|
28
|
+
export function getActiveEnterWallMs() {
|
|
29
|
+
return getActiveEnterMs();
|
|
52
30
|
}
|
|
53
31
|
// Specific parse/boot-failure markers only. Bare 'JSON'/'YAML' were removed
|
|
54
32
|
// (2026-08-31): they matched any line whose payload merely *contained* those
|
|
@@ -69,6 +47,92 @@ const ERROR_PATTERNS = [
|
|
|
69
47
|
'EADDRINUSE',
|
|
70
48
|
'address already in use',
|
|
71
49
|
];
|
|
50
|
+
/**
|
|
51
|
+
* Decide whether the current unit start can already be judged.
|
|
52
|
+
*
|
|
53
|
+
* `activeEnterAtMs` is WALL-CLOCK epoch ms from
|
|
54
|
+
* `systemctl --user show -p ActiveEnterTimestamp dsh-web.service`.
|
|
55
|
+
* Deliberately NOT `ActiveEnterTimestampMonotonic`: a monotonic reading can
|
|
56
|
+
* never be compared against an append-only log that carries no timestamps, and
|
|
57
|
+
* that mismatch is why the previous scan filter silently fell through and let
|
|
58
|
+
* the previous boot's crash text be read as this boot's (incident 2026-09-13).
|
|
59
|
+
*
|
|
60
|
+
* - 'unknown' — no boot anchor (systemd absent, lookup disabled, clock skew):
|
|
61
|
+
* behave exactly as before the fix; never suppress anything.
|
|
62
|
+
* - 'booting' — the unit started less than `bootGraceMs` ago and has not yet
|
|
63
|
+
* proven itself by actually serving a request: weak failures are
|
|
64
|
+
* suppressed and log lines are inconclusive.
|
|
65
|
+
* - 'settled' — the boot proved itself, or the grace window expired: judge
|
|
66
|
+
* normally.
|
|
67
|
+
*/
|
|
68
|
+
export function bootFreshness(opts) {
|
|
69
|
+
const { activeEnterAtMs, now, bootGraceMs, probeSucceeded } = opts;
|
|
70
|
+
if (activeEnterAtMs === undefined || !Number.isFinite(activeEnterAtMs))
|
|
71
|
+
return 'unknown';
|
|
72
|
+
if (now < activeEnterAtMs)
|
|
73
|
+
return 'unknown';
|
|
74
|
+
if (probeSucceeded)
|
|
75
|
+
return 'settled';
|
|
76
|
+
return now - activeEnterAtMs < bootGraceMs ? 'booting' : 'settled';
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Classify a fetch failure by what it says about the process.
|
|
80
|
+
*
|
|
81
|
+
* 'refused' — nothing is listening, so the process is gone: a strong down
|
|
82
|
+
* signal that must never be masked by a boot grace.
|
|
83
|
+
* 'timeout' — something may be alive but slow: weak, only meaningful once the
|
|
84
|
+
* boot grace expired.
|
|
85
|
+
* 'other' — anything we cannot attribute.
|
|
86
|
+
*
|
|
87
|
+
* Walks the `cause` chain because undici surfaces a refused connection as
|
|
88
|
+
* `TypeError: fetch failed` with the real `ECONNREFUSED` on `cause`.
|
|
89
|
+
*/
|
|
90
|
+
export function classifyFetchFailure(err) {
|
|
91
|
+
const parts = [];
|
|
92
|
+
let cur = err;
|
|
93
|
+
for (let depth = 0; cur != null && depth < 5; depth++) {
|
|
94
|
+
if (typeof cur === 'string') {
|
|
95
|
+
parts.push(cur);
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
const code = typeof cur.code === 'string' ? cur.code : '';
|
|
99
|
+
const name = typeof cur.name === 'string' ? cur.name : '';
|
|
100
|
+
const message = typeof cur.message === 'string' ? cur.message : '';
|
|
101
|
+
parts.push(`${code} ${name} ${message}`);
|
|
102
|
+
cur = cur.cause;
|
|
103
|
+
}
|
|
104
|
+
const text = parts.join(' ').toLowerCase();
|
|
105
|
+
if (/econnrefused|connection refused|ehostunreach|enetunreach/.test(text))
|
|
106
|
+
return 'refused';
|
|
107
|
+
if (/abort|timed? ?out|etimedout|und_err_connect_timeout/.test(text))
|
|
108
|
+
return 'timeout';
|
|
109
|
+
return 'other';
|
|
110
|
+
}
|
|
111
|
+
export const SUCCESS_MARKER = 'dsh web: http';
|
|
112
|
+
/** Index of the last boot-boundary line in the tail, or -1 when it predates the tail. */
|
|
113
|
+
export function lastBootBoundaryIndex(lines) {
|
|
114
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
115
|
+
if (lines[i].includes(BOOT_BOUNDARY_MARKER))
|
|
116
|
+
return i;
|
|
117
|
+
}
|
|
118
|
+
return -1;
|
|
119
|
+
}
|
|
120
|
+
/** Index of the last "dsh web: http" success marker (`lowerLines` must be lower-cased). */
|
|
121
|
+
export function lastSuccessMarkerIndex(lowerLines) {
|
|
122
|
+
for (let i = lowerLines.length - 1; i >= 0; i--) {
|
|
123
|
+
if (lowerLines[i].includes(SUCCESS_MARKER))
|
|
124
|
+
return i;
|
|
125
|
+
}
|
|
126
|
+
return -1;
|
|
127
|
+
}
|
|
128
|
+
/** Wall-clock ms parsed from a boot-boundary line, or undefined when unparseable. */
|
|
129
|
+
export function parseBootBoundaryMs(line) {
|
|
130
|
+
const m = /boot-boundary\s+(\S+)/.exec(line);
|
|
131
|
+
if (!m)
|
|
132
|
+
return undefined;
|
|
133
|
+
const ms = Date.parse(m[1]);
|
|
134
|
+
return Number.isNaN(ms) ? undefined : ms;
|
|
135
|
+
}
|
|
72
136
|
export async function pollHealth(opts = {}) {
|
|
73
137
|
// 5s was too tight for a busy plugin-tree boot: a lone AbortError from a slow
|
|
74
138
|
// (but otherwise fine) response was indistinguishable from a real crash, and
|
|
@@ -78,15 +142,19 @@ export async function pollHealth(opts = {}) {
|
|
|
78
142
|
// high load (loadavg 12) stalls; configurable via domains.supervisor.pollTimeoutMs.
|
|
79
143
|
// If opts.timeoutMs is not injected, read from supervisor config (maestro settings).
|
|
80
144
|
let effectiveTimeout = opts.timeoutMs;
|
|
81
|
-
|
|
145
|
+
let bootGraceMs = opts.bootGraceMs;
|
|
146
|
+
if (effectiveTimeout === undefined || bootGraceMs === undefined) {
|
|
82
147
|
try {
|
|
83
148
|
const { readSupervisorConfig } = await import('./config.js');
|
|
84
149
|
const cfg = await readSupervisorConfig();
|
|
85
|
-
if (typeof cfg.pollTimeoutMs === 'number' && cfg.pollTimeoutMs > 0)
|
|
150
|
+
if (effectiveTimeout === undefined && typeof cfg.pollTimeoutMs === 'number' && cfg.pollTimeoutMs > 0)
|
|
86
151
|
effectiveTimeout = cfg.pollTimeoutMs;
|
|
152
|
+
if (bootGraceMs === undefined && typeof cfg.bootGraceMs === 'number' && cfg.bootGraceMs > 0)
|
|
153
|
+
bootGraceMs = cfg.bootGraceMs;
|
|
87
154
|
}
|
|
88
155
|
catch { }
|
|
89
156
|
effectiveTimeout ??= 20000;
|
|
157
|
+
bootGraceMs ??= 180000;
|
|
90
158
|
}
|
|
91
159
|
const fetchFn = opts.fetch ?? defaultFetch(opts.url ?? 'http://127.0.0.1:3080/', effectiveTimeout);
|
|
92
160
|
const psAliveFn = opts.psAlive ?? defaultPsAlive;
|
|
@@ -111,64 +179,64 @@ export async function pollHealth(opts = {}) {
|
|
|
111
179
|
catch {
|
|
112
180
|
// ignore log read errors
|
|
113
181
|
}
|
|
114
|
-
// Suppression gate:
|
|
115
|
-
//
|
|
182
|
+
// Suppression gate: an in-flight planned restart, or a boot that has not yet
|
|
183
|
+
// proven itself, means transients are expected. A refused connection is
|
|
184
|
+
// never suppressed: nothing is listening, so the process is gone (D2).
|
|
116
185
|
const suppressed = checkPlannedRestart();
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
catch {
|
|
127
|
-
activeEnterMs = undefined;
|
|
186
|
+
let activeEnterAtMs = opts.activeEnterAtMs;
|
|
187
|
+
if (activeEnterAtMs === undefined) {
|
|
188
|
+
try {
|
|
189
|
+
const fn = opts.getActiveEnterMs ?? getActiveEnterMs;
|
|
190
|
+
activeEnterAtMs = fn();
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
activeEnterAtMs = undefined;
|
|
194
|
+
}
|
|
128
195
|
}
|
|
129
|
-
|
|
130
|
-
// Filter logTail to only consider lines after ActiveEnterTimestamp.
|
|
131
|
-
// Fallback: last 200 lines after last "dsh web: http" success marker (log is append-only).
|
|
196
|
+
const now = Date.now();
|
|
132
197
|
const lines = logContent.split('\n');
|
|
133
198
|
const lowerLines = lines.map(l => l.toLowerCase());
|
|
134
|
-
//
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
199
|
+
// Scope the scan to the current boot. The boundary line is only trusted for
|
|
200
|
+
// the unit start it was written for: a start more than a boot budget later
|
|
201
|
+
// (a systemd auto-restart, a manual start) is a different boot.
|
|
202
|
+
const boundaryIdx = lastBootBoundaryIndex(lines);
|
|
203
|
+
const boundaryMs = boundaryIdx === -1 ? undefined : parseBootBoundaryMs(lines[boundaryIdx]);
|
|
204
|
+
const scopedToThisBoot = boundaryMs !== undefined
|
|
205
|
+
&& activeEnterAtMs !== undefined
|
|
206
|
+
&& activeEnterAtMs >= boundaryMs
|
|
207
|
+
&& activeEnterAtMs - boundaryMs <= bootGraceMs;
|
|
208
|
+
const bootLines = scopedToThisBoot ? lines.slice(boundaryIdx + 1) : lines;
|
|
209
|
+
const bootLower = scopedToThisBoot ? lowerLines.slice(boundaryIdx + 1) : lowerLines;
|
|
210
|
+
// Proof of a finished boot is a serving probe, not a log line (see bootFreshness).
|
|
211
|
+
const probeSucceeded = httpCode === 200 || httpCode === 401;
|
|
212
|
+
const bootPhase = bootFreshness({ activeEnterAtMs, now, bootGraceMs, probeSucceeded });
|
|
213
|
+
const booting = bootPhase === 'booting';
|
|
214
|
+
const graceActive = suppressed || booting;
|
|
215
|
+
// Inside the current boot, only lines after its own success marker may be a
|
|
216
|
+
// post-start crash. With no marker and no scope proving these lines are this
|
|
217
|
+
// boot's, the scan is inconclusive (D3) instead of inheriting the previous
|
|
218
|
+
// boot's crash text — that inheritance is what produced the 2026-09-13
|
|
219
|
+
// rollback report ("rollback — degraded: This operation was aborted") from a
|
|
220
|
+
// healthy, still-booting instance reading the previous boot's EADDRINUSE.
|
|
221
|
+
const successIdx = lastSuccessMarkerIndex(bootLower);
|
|
142
222
|
let scanLines;
|
|
143
223
|
let scanLower;
|
|
144
|
-
if (
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
}
|
|
152
|
-
else {
|
|
153
|
-
scanLines = after;
|
|
154
|
-
scanLower = afterLower;
|
|
155
|
-
}
|
|
156
|
-
// If ActiveEnter is available, the same window applies — old EADDRINUSE before restart
|
|
157
|
-
// is before the success marker and thus excluded. No extra timestamp->line mapping needed.
|
|
224
|
+
if (successIdx !== -1) {
|
|
225
|
+
scanLines = bootLines.slice(successIdx + 1);
|
|
226
|
+
scanLower = bootLower.slice(successIdx + 1);
|
|
227
|
+
}
|
|
228
|
+
else if (booting && !scopedToThisBoot) {
|
|
229
|
+
scanLines = [];
|
|
230
|
+
scanLower = [];
|
|
158
231
|
}
|
|
159
232
|
else {
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
scanLines = lines;
|
|
167
|
-
scanLower = lowerLines;
|
|
168
|
-
}
|
|
233
|
+
scanLines = bootLines;
|
|
234
|
+
scanLower = bootLower;
|
|
235
|
+
}
|
|
236
|
+
if (scanLines.length > 200) {
|
|
237
|
+
scanLines = scanLines.slice(-200);
|
|
238
|
+
scanLower = scanLower.slice(-200);
|
|
169
239
|
}
|
|
170
|
-
// If ActiveEnter lookup succeeded/failed, we already applied the fallback window.
|
|
171
|
-
// When system has no success marker and no ActiveEnter, scanLines is still last 200.
|
|
172
240
|
let lastErrorIdx = -1;
|
|
173
241
|
let matchedLine = '';
|
|
174
242
|
for (let i = scanLines.length - 1; i >= 0; i--) {
|
|
@@ -183,12 +251,11 @@ export async function pollHealth(opts = {}) {
|
|
|
183
251
|
if (lastErrorIdx !== -1)
|
|
184
252
|
break;
|
|
185
253
|
}
|
|
254
|
+
let logError;
|
|
186
255
|
if (lastErrorIdx !== -1) {
|
|
187
|
-
// Within the windowed view, if a success appears after the error it would have been before the slice,
|
|
188
|
-
// but check anyway for safety (error before success within window)
|
|
189
256
|
let hasSuccessAfter = false;
|
|
190
257
|
for (let i = lastErrorIdx + 1; i < scanLines.length; i++) {
|
|
191
|
-
if (scanLower[i].includes(
|
|
258
|
+
if (scanLower[i].includes(SUCCESS_MARKER)) {
|
|
192
259
|
hasSuccessAfter = true;
|
|
193
260
|
break;
|
|
194
261
|
}
|
|
@@ -196,22 +263,33 @@ export async function pollHealth(opts = {}) {
|
|
|
196
263
|
if (!hasSuccessAfter)
|
|
197
264
|
logError = matchedLine;
|
|
198
265
|
}
|
|
199
|
-
else if (suppressed) {
|
|
200
|
-
// suppressed window with no windowed error — ensure stale errors before success are ignored
|
|
201
|
-
// (already handled by windowing)
|
|
202
|
-
}
|
|
203
|
-
// Legacy full-scan fallback for hasSuccessAfter across original lines when no windowed error
|
|
204
|
-
// but original had error before success — already suppressed by windowing, no need to re-check.
|
|
205
|
-
// Distinguish FULL (http !=200) vs DEGRADED (http 200 but log has plugin error)
|
|
206
|
-
// Suppression: during 30s grace (planned-restart OR recently started), transient fetch failures are not a crash
|
|
207
266
|
if (fetchError) {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
267
|
+
const kind = classifyFetchFailure(fetchError);
|
|
268
|
+
// D1: a timeout/abort while the current boot is unproven is a slow boot,
|
|
269
|
+
// not a crash — increment nothing. D2: a refused connection is different
|
|
270
|
+
// (nothing is listening, the process is gone) and is never masked.
|
|
271
|
+
if (booting && kind !== 'refused') {
|
|
272
|
+
return { up: true, httpCode, bootPhase, logTail: logContent.slice(-5000) };
|
|
273
|
+
}
|
|
274
|
+
// D5: inside the boot window a refused connection with a verifiably alive
|
|
275
|
+
// process is a boot symptom, not a crash — the listener may not be bound
|
|
276
|
+
// yet. Outside the window, or with a dead process, the verdict below is
|
|
277
|
+
// exactly the pre-D5 behaviour (degraded when the port still answers,
|
|
278
|
+
// down when it does not).
|
|
279
|
+
if (booting && kind === 'refused') {
|
|
280
|
+
let alive = false;
|
|
281
|
+
try {
|
|
282
|
+
alive = await psAliveFn();
|
|
283
|
+
}
|
|
284
|
+
catch {
|
|
285
|
+
alive = false;
|
|
286
|
+
}
|
|
287
|
+
if (alive) {
|
|
288
|
+
return { up: true, httpCode, bootPhase, logTail: logContent.slice(-5000) };
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
if (suppressed) {
|
|
292
|
+
return { up: true, httpCode, bootPhase, logTail: logContent.slice(-5000) };
|
|
215
293
|
}
|
|
216
294
|
// Corroborate with a cheap port-liveness check before declaring a crash.
|
|
217
295
|
// The HTTP fetch shares dsh-web's own event loop, so a busy-but-alive
|
|
@@ -232,6 +310,7 @@ export async function pollHealth(opts = {}) {
|
|
|
232
310
|
return {
|
|
233
311
|
up: true,
|
|
234
312
|
httpCode,
|
|
313
|
+
bootPhase,
|
|
235
314
|
error: logError ? `${fetchError} + ${logError}` : fetchError,
|
|
236
315
|
degraded: true,
|
|
237
316
|
logTail: logContent.slice(-5000),
|
|
@@ -240,52 +319,31 @@ export async function pollHealth(opts = {}) {
|
|
|
240
319
|
return {
|
|
241
320
|
up: false,
|
|
242
321
|
httpCode,
|
|
322
|
+
bootPhase,
|
|
243
323
|
error: logError ? `${fetchError} + ${logError}` : fetchError,
|
|
244
324
|
degraded: false,
|
|
245
325
|
logTail: logContent.slice(-5000),
|
|
246
326
|
};
|
|
247
327
|
}
|
|
248
|
-
// During grace, a windowed logError that is still present is considered
|
|
249
|
-
// as well — treat as up to avoid double restart.
|
|
328
|
+
// During grace, a windowed logError that is still present is considered
|
|
329
|
+
// stale/boot transient as well — treat as up to avoid double restart.
|
|
250
330
|
if (graceActive && logError) {
|
|
251
|
-
|
|
252
|
-
return {
|
|
253
|
-
up: true,
|
|
254
|
-
httpCode,
|
|
255
|
-
logTail: logContent.slice(-5000),
|
|
256
|
-
};
|
|
331
|
+
return { up: true, httpCode, bootPhase, logTail: logContent.slice(-5000) };
|
|
257
332
|
}
|
|
258
333
|
if (logError) {
|
|
259
|
-
// EADDRINUSE is fatal even with http 200 — old process still holds
|
|
260
|
-
// and new start failed; treat as FULL down so supervisor
|
|
334
|
+
// EADDRINUSE is fatal even with http 200 — an old process still holds
|
|
335
|
+
// the port and the new start failed; treat as FULL down so the supervisor
|
|
336
|
+
// kills + restarts.
|
|
261
337
|
const lowerErr = logError.toLowerCase();
|
|
262
338
|
const isFatalPortError = lowerErr.includes('eaddrinuse') || lowerErr.includes('address already in use');
|
|
263
339
|
if (isFatalPortError) {
|
|
264
|
-
return {
|
|
265
|
-
up: false,
|
|
266
|
-
httpCode,
|
|
267
|
-
error: logError,
|
|
268
|
-
degraded: false,
|
|
269
|
-
logTail: logContent.slice(-5000),
|
|
270
|
-
};
|
|
340
|
+
return { up: false, httpCode, bootPhase, error: logError, degraded: false, logTail: logContent.slice(-5000) };
|
|
271
341
|
}
|
|
272
342
|
// http 200 but log error → DEGRADED (isolatable), not FULL
|
|
273
343
|
if (httpCode === 200) {
|
|
274
|
-
return {
|
|
275
|
-
up: true,
|
|
276
|
-
httpCode,
|
|
277
|
-
error: logError,
|
|
278
|
-
degraded: true,
|
|
279
|
-
logTail: logContent.slice(-5000),
|
|
280
|
-
};
|
|
344
|
+
return { up: true, httpCode, bootPhase, error: logError, degraded: true, logTail: logContent.slice(-5000) };
|
|
281
345
|
}
|
|
282
|
-
return {
|
|
283
|
-
up: false,
|
|
284
|
-
httpCode,
|
|
285
|
-
error: logError,
|
|
286
|
-
degraded: false,
|
|
287
|
-
logTail: logContent.slice(-5000),
|
|
288
|
-
};
|
|
346
|
+
return { up: false, httpCode, bootPhase, error: logError, degraded: false, logTail: logContent.slice(-5000) };
|
|
289
347
|
}
|
|
290
348
|
// Also check psAlive as secondary signal — if fetch ok but ps dead, still down
|
|
291
349
|
try {
|
|
@@ -297,7 +355,7 @@ export async function pollHealth(opts = {}) {
|
|
|
297
355
|
catch {
|
|
298
356
|
// ignore
|
|
299
357
|
}
|
|
300
|
-
return { up: httpCode === 200 || httpCode === 401, httpCode, logTail: logContent.slice(-5000) };
|
|
358
|
+
return { up: httpCode === 200 || httpCode === 401, httpCode, bootPhase, logTail: logContent.slice(-5000) };
|
|
301
359
|
}
|
|
302
360
|
function defaultFetch(url, timeoutMs) {
|
|
303
361
|
return async () => {
|
package/lib/plugin.js
CHANGED
|
@@ -9,7 +9,7 @@ import * as fs from 'node:fs';
|
|
|
9
9
|
import * as path from 'node:path';
|
|
10
10
|
import * as os from 'node:os';
|
|
11
11
|
import { fileURLToPath } from 'node:url';
|
|
12
|
-
import { findInterrupted as defaultFindInterrupted, findDanglingOpenTurns as defaultFindDanglingOpenTurns } from './resume.js';
|
|
12
|
+
import { findInterrupted as defaultFindInterrupted, findDanglingOpenTurns as defaultFindDanglingOpenTurns, resolveSessionLogPath } from './resume.js';
|
|
13
13
|
import { readIntent, consumeIntent } from './intents.js';
|
|
14
14
|
import { appendResumeLog } from './resume-log.js';
|
|
15
15
|
import { makeSkillProvider } from './skill-provider.js';
|
|
@@ -294,12 +294,13 @@ async function readRouteFromHandle(persistence, sid) {
|
|
|
294
294
|
* first match so huge logs cost one streamed pass, not a full decode.
|
|
295
295
|
*/
|
|
296
296
|
async function readRouteFromRawLog(sessionsRoot, group, sessionId) {
|
|
297
|
-
const
|
|
298
|
-
|
|
297
|
+
const logPath = resolveSessionLogPath(path.join(sessionsRoot, group, sessionId));
|
|
298
|
+
if (logPath === undefined)
|
|
299
|
+
return undefined;
|
|
299
300
|
try {
|
|
300
|
-
if (
|
|
301
|
+
if (logPath.endsWith('.zstd')) {
|
|
301
302
|
const { execSync } = await import('node:child_process');
|
|
302
|
-
const out = execSync(`zstd -d -c ${JSON.stringify(
|
|
303
|
+
const out = execSync(`zstd -d -c ${JSON.stringify(logPath)} 2>/dev/null | grep -a -m1 '"type":"request/context"'`, { encoding: 'utf-8' });
|
|
303
304
|
const line = out.split('\n').find((l) => l.includes('request/context'));
|
|
304
305
|
if (line !== undefined) {
|
|
305
306
|
try {
|
|
@@ -309,18 +310,16 @@ async function readRouteFromRawLog(sessionsRoot, group, sessionId) {
|
|
|
309
310
|
}
|
|
310
311
|
return undefined;
|
|
311
312
|
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
return route;
|
|
321
|
-
}
|
|
322
|
-
catch { }
|
|
313
|
+
const content = fs.readFileSync(logPath, 'utf-8');
|
|
314
|
+
for (const line of content.split('\n')) {
|
|
315
|
+
if (!line.includes('"request/context"'))
|
|
316
|
+
continue;
|
|
317
|
+
try {
|
|
318
|
+
const route = routeFromEvents([JSON.parse(line)]);
|
|
319
|
+
if (route !== undefined)
|
|
320
|
+
return route;
|
|
323
321
|
}
|
|
322
|
+
catch { }
|
|
324
323
|
}
|
|
325
324
|
}
|
|
326
325
|
catch { }
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serialized systemd restart for `dsh-web.service`: stop, wait until the
|
|
3
|
+
* unit is inactive, then start — instead of one raw `systemctl restart`
|
|
4
|
+
* that boots the new process while the old one still holds :3082 and
|
|
5
|
+
* crash-loops on EADDRINUSE (2026-09-11 outage: 4 restarts in
|
|
6
|
+
* 15 min over a ~90s SIGTERM stop → 30 EADDRINUSE crashes).
|
|
7
|
+
*
|
|
8
|
+
* All side effects are injectable so tests never touch systemd.
|
|
9
|
+
*/
|
|
10
|
+
export interface SerializedRestartDeps {
|
|
11
|
+
/** Run a shell command; must throw on non-zero exit. */
|
|
12
|
+
exec?: (cmd: string) => void;
|
|
13
|
+
/** Whether the unit is currently active. */
|
|
14
|
+
isActive?: () => boolean;
|
|
15
|
+
sleep?: (ms: number) => Promise<void>;
|
|
16
|
+
/** Max time to wait for inactivity before starting anyway. */
|
|
17
|
+
stopTimeoutMs?: number;
|
|
18
|
+
/** Poll interval while waiting for inactivity. */
|
|
19
|
+
pollMs?: number;
|
|
20
|
+
/** Clock (injectable for tests). */
|
|
21
|
+
now?: () => number;
|
|
22
|
+
}
|
|
23
|
+
export declare function serializedSystemdRestart(deps?: SerializedRestartDeps): Promise<void>;
|
|
24
|
+
export declare const DSH_WEB_UNIT_NAME = "dsh-web.service";
|
|
25
|
+
export declare function dshWebUnitPath(): string;
|
|
26
|
+
/** Does systemd own `dsh-web.service` on this host? */
|
|
27
|
+
export declare function systemdUnitExists(unitPath?: string): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* The direct-node `nohup` fallback exists for portable hosts that have no
|
|
30
|
+
* systemd unit. When the unit exists, systemd owns the boot: spawning node
|
|
31
|
+
* behind its back is how a second instance appears — the two starts / one
|
|
32
|
+
* rollback report of 2026-09-13. Gate on "the unit does not exist", never on
|
|
33
|
+
* "this particular start failed".
|
|
34
|
+
*/
|
|
35
|
+
export declare function shouldUseNohupFallback(unitExists: boolean): boolean;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { execSync } from 'node:child_process';
|
|
2
|
+
import * as fs from 'node:fs';
|
|
3
|
+
import * as os from 'node:os';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
const STOP_TIMEOUT_MS = 120_000;
|
|
6
|
+
const POLL_MS = 2_000;
|
|
7
|
+
export async function serializedSystemdRestart(deps = {}) {
|
|
8
|
+
const exec = deps.exec ?? ((cmd) => {
|
|
9
|
+
execSync(cmd, { timeout: 15_000, stdio: 'pipe' });
|
|
10
|
+
});
|
|
11
|
+
const isActive = deps.isActive ?? (() => {
|
|
12
|
+
try {
|
|
13
|
+
execSync('systemctl --user is-active --quiet dsh-web.service', { timeout: 5_000, stdio: 'pipe' });
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
const sleep = deps.sleep ?? ((ms) => new Promise(r => setTimeout(r, ms)));
|
|
21
|
+
const stopTimeoutMs = deps.stopTimeoutMs ?? STOP_TIMEOUT_MS;
|
|
22
|
+
const pollMs = deps.pollMs ?? POLL_MS;
|
|
23
|
+
const now = deps.now ?? Date.now;
|
|
24
|
+
// Best-effort: the unit may already be down (crash) — the wait + start
|
|
25
|
+
// below still apply.
|
|
26
|
+
try {
|
|
27
|
+
exec('systemctl --user stop dsh-web.service');
|
|
28
|
+
}
|
|
29
|
+
catch { /* fall through to wait + start */ }
|
|
30
|
+
const deadline = now() + stopTimeoutMs;
|
|
31
|
+
while (isActive()) {
|
|
32
|
+
if (now() >= deadline)
|
|
33
|
+
break;
|
|
34
|
+
await sleep(pollMs);
|
|
35
|
+
}
|
|
36
|
+
exec('systemctl --user start dsh-web.service');
|
|
37
|
+
}
|
|
38
|
+
export const DSH_WEB_UNIT_NAME = 'dsh-web.service';
|
|
39
|
+
export function dshWebUnitPath() {
|
|
40
|
+
return process.env.DSH_WEB_UNIT_PATH ?? path.join(os.homedir(), '.config/systemd/user/dsh-web.service');
|
|
41
|
+
}
|
|
42
|
+
/** Does systemd own `dsh-web.service` on this host? */
|
|
43
|
+
export function systemdUnitExists(unitPath = dshWebUnitPath()) {
|
|
44
|
+
try {
|
|
45
|
+
return fs.existsSync(unitPath);
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The direct-node `nohup` fallback exists for portable hosts that have no
|
|
53
|
+
* systemd unit. When the unit exists, systemd owns the boot: spawning node
|
|
54
|
+
* behind its back is how a second instance appears — the two starts / one
|
|
55
|
+
* rollback report of 2026-09-13. Gate on "the unit does not exist", never on
|
|
56
|
+
* "this particular start failed".
|
|
57
|
+
*/
|
|
58
|
+
export function shouldUseNohupFallback(unitExists) {
|
|
59
|
+
return !unitExists;
|
|
60
|
+
}
|
package/lib/restart-guards.d.ts
CHANGED
|
@@ -1,4 +1,22 @@
|
|
|
1
1
|
export declare function buildKillStalePortsCommand(ports?: number[]): string;
|
|
2
|
+
/**
|
|
3
|
+
* Sentinel appended to dsh-web.log immediately before a supervised restart.
|
|
4
|
+
*
|
|
5
|
+
* The log is append-only and carries no timestamps, so this line is the only
|
|
6
|
+
* durable ordering between "the previous boot's crash" and "this boot's
|
|
7
|
+
* output". Without it the health scan inherited the previous, failed boot's
|
|
8
|
+
* `EADDRINUSE` stack and rolled back a healthy, still-booting instance
|
|
9
|
+
* (incident 2026-09-13).
|
|
10
|
+
*/
|
|
11
|
+
export declare const BOOT_BOUNDARY_MARKER = "[supervisor] boot-boundary";
|
|
12
|
+
/** The production dsh-web log (override with DSH_WEB_LOG for tests/tools). */
|
|
13
|
+
export declare function dshWebLogPath(): string;
|
|
14
|
+
/**
|
|
15
|
+
* Append the boot-boundary sentinel. Pass an explicit `logPath` in tests, or
|
|
16
|
+
* redirect with DSH_WEB_LOG. With no destination at all under VITEST this is a
|
|
17
|
+
* deliberate no-op so a unit test can never append to the operator's real log.
|
|
18
|
+
*/
|
|
19
|
+
export declare function markBootBoundary(logPath?: string): void;
|
|
2
20
|
export declare function isSelfCopyError(message: string): boolean;
|
|
3
21
|
export declare const PLANNED_RESTART_TTL_MS = 180000;
|
|
4
22
|
export declare function isPlannedRestartFresh(mtimeMs: number, nowMs: number, ttlMs?: number): boolean;
|
package/lib/restart-guards.js
CHANGED
|
@@ -11,6 +11,36 @@ export function buildKillStalePortsCommand(ports = [3080]) {
|
|
|
11
11
|
const filter = ports.map(p => `sport = :${p}`).join(' or ');
|
|
12
12
|
return `pids=$(ss -tlnp '( ${filter} )' 2>/dev/null | sed -n 's/.*pid=\\([0-9]*\\).*/\\1/p' | sort -u); if [ -n "$pids" ]; then echo "[supervisor] killing stale pids $pids"; kill $pids 2>/dev/null || true; sleep 2; fi`;
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* Sentinel appended to dsh-web.log immediately before a supervised restart.
|
|
16
|
+
*
|
|
17
|
+
* The log is append-only and carries no timestamps, so this line is the only
|
|
18
|
+
* durable ordering between "the previous boot's crash" and "this boot's
|
|
19
|
+
* output". Without it the health scan inherited the previous, failed boot's
|
|
20
|
+
* `EADDRINUSE` stack and rolled back a healthy, still-booting instance
|
|
21
|
+
* (incident 2026-09-13).
|
|
22
|
+
*/
|
|
23
|
+
export const BOOT_BOUNDARY_MARKER = '[supervisor] boot-boundary';
|
|
24
|
+
/** The production dsh-web log (override with DSH_WEB_LOG for tests/tools). */
|
|
25
|
+
export function dshWebLogPath() {
|
|
26
|
+
return process.env.DSH_WEB_LOG ?? path.join(os.homedir(), '.dsh/dsh-web.log');
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Append the boot-boundary sentinel. Pass an explicit `logPath` in tests, or
|
|
30
|
+
* redirect with DSH_WEB_LOG. With no destination at all under VITEST this is a
|
|
31
|
+
* deliberate no-op so a unit test can never append to the operator's real log.
|
|
32
|
+
*/
|
|
33
|
+
export function markBootBoundary(logPath) {
|
|
34
|
+
const explicit = logPath ?? process.env.DSH_WEB_LOG;
|
|
35
|
+
if (explicit === undefined && process.env.VITEST)
|
|
36
|
+
return;
|
|
37
|
+
const p = explicit ?? dshWebLogPath();
|
|
38
|
+
try {
|
|
39
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
40
|
+
fs.appendFileSync(p, `${BOOT_BOUNDARY_MARKER} ${new Date().toISOString()}\n`);
|
|
41
|
+
}
|
|
42
|
+
catch { }
|
|
43
|
+
}
|
|
14
44
|
// LKG rollback copies each entry with fs.cpSync — an entry that resolves back
|
|
15
45
|
// into itself (e.g. a symlink cycle reachable from ~/.dsh, observed in
|
|
16
46
|
// production pointing into ~/.npm/_npx/.../node_modules/unist-util-position)
|
|
@@ -43,6 +73,9 @@ export function writePlannedRestart(ttlMs = 30000) {
|
|
|
43
73
|
fs.chmodSync(p, 0o600);
|
|
44
74
|
}
|
|
45
75
|
catch { }
|
|
76
|
+
// The boundary belongs to the restart, not to the poller: every sanctioned
|
|
77
|
+
// restart path already writes this marker first.
|
|
78
|
+
markBootBoundary();
|
|
46
79
|
}
|
|
47
80
|
export function checkPlannedRestart(markerPath) {
|
|
48
81
|
// Legacy path explicit: check mtime of that file
|