@ddtcorex/dsh-maestro-supervisor 0.7.10 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,18 +1,12 @@
1
- import { checkPlannedRestart } from './restart-guards.js';
1
+ import { checkPlannedRestart, BOOT_BOUNDARY_MARKER } from './restart-guards.js';
2
2
  import { execSync as execSyncImpl } from 'node:child_process';
3
+ /**
4
+ * Wall-clock epoch ms of the current dsh-web unit start, or undefined when
5
+ * systemd does not manage the unit (portable host) or the lookup is disabled.
6
+ * `ActiveEnterTimestampMonotonic` is deliberately gone: a monotonic reading
7
+ * cannot be compared with a timestamp-less append-only log.
8
+ */
3
9
  export function getActiveEnterMs() {
4
- if (process.env.VITEST)
5
- return undefined;
6
- try {
7
- const out = execSyncImpl('systemctl --user show -p ActiveEnterTimestampMonotonic dsh-web.service 2>/dev/null', { encoding: 'utf8' });
8
- const m = out.match(/ActiveEnterTimestampMonotonic=(\d+)/);
9
- if (m)
10
- return Number(m[1]);
11
- }
12
- catch { }
13
- return undefined;
14
- }
15
- export function getActiveEnterWallMs() {
16
10
  if (process.env.VITEST)
17
11
  return undefined;
18
12
  try {
@@ -30,25 +24,9 @@ export function getActiveEnterWallMs() {
30
24
  catch { }
31
25
  return undefined;
32
26
  }
33
- function isRecentlyStarted(opts, wallMs) {
34
- if (process.env.VITEST)
35
- return false;
36
- const now = Date.now();
37
- // 30s grace after ActiveEnterTimestamp (wall clock) — covers manual systemctl start without marker
38
- const wall = wallMs ?? (() => {
39
- try {
40
- const fn = opts.getActiveEnterWallMs;
41
- if (typeof fn === 'function')
42
- return fn();
43
- return getActiveEnterWallMs();
44
- }
45
- catch {
46
- return undefined;
47
- }
48
- })();
49
- if (typeof wall === 'number' && now - wall < 30000)
50
- return true;
51
- return false;
27
+ /** @deprecated kept for callers; identical to getActiveEnterMs(). */
28
+ export function getActiveEnterWallMs() {
29
+ return getActiveEnterMs();
52
30
  }
53
31
  // Specific parse/boot-failure markers only. Bare 'JSON'/'YAML' were removed
54
32
  // (2026-08-31): they matched any line whose payload merely *contained* those
@@ -69,6 +47,92 @@ const ERROR_PATTERNS = [
69
47
  'EADDRINUSE',
70
48
  'address already in use',
71
49
  ];
50
+ /**
51
+ * Decide whether the current unit start can already be judged.
52
+ *
53
+ * `activeEnterAtMs` is WALL-CLOCK epoch ms from
54
+ * `systemctl --user show -p ActiveEnterTimestamp dsh-web.service`.
55
+ * Deliberately NOT `ActiveEnterTimestampMonotonic`: a monotonic reading can
56
+ * never be compared against an append-only log that carries no timestamps, and
57
+ * that mismatch is why the previous scan filter silently fell through and let
58
+ * the previous boot's crash text be read as this boot's (incident 2026-09-13).
59
+ *
60
+ * - 'unknown' — no boot anchor (systemd absent, lookup disabled, clock skew):
61
+ * behave exactly as before the fix; never suppress anything.
62
+ * - 'booting' — the unit started less than `bootGraceMs` ago and has not yet
63
+ * proven itself by actually serving a request: weak failures are
64
+ * suppressed and log lines are inconclusive.
65
+ * - 'settled' — the boot proved itself, or the grace window expired: judge
66
+ * normally.
67
+ */
68
+ export function bootFreshness(opts) {
69
+ const { activeEnterAtMs, now, bootGraceMs, probeSucceeded } = opts;
70
+ if (activeEnterAtMs === undefined || !Number.isFinite(activeEnterAtMs))
71
+ return 'unknown';
72
+ if (now < activeEnterAtMs)
73
+ return 'unknown';
74
+ if (probeSucceeded)
75
+ return 'settled';
76
+ return now - activeEnterAtMs < bootGraceMs ? 'booting' : 'settled';
77
+ }
78
+ /**
79
+ * Classify a fetch failure by what it says about the process.
80
+ *
81
+ * 'refused' — nothing is listening, so the process is gone: a strong down
82
+ * signal that must never be masked by a boot grace.
83
+ * 'timeout' — something may be alive but slow: weak, only meaningful once the
84
+ * boot grace expired.
85
+ * 'other' — anything we cannot attribute.
86
+ *
87
+ * Walks the `cause` chain because undici surfaces a refused connection as
88
+ * `TypeError: fetch failed` with the real `ECONNREFUSED` on `cause`.
89
+ */
90
+ export function classifyFetchFailure(err) {
91
+ const parts = [];
92
+ let cur = err;
93
+ for (let depth = 0; cur != null && depth < 5; depth++) {
94
+ if (typeof cur === 'string') {
95
+ parts.push(cur);
96
+ break;
97
+ }
98
+ const code = typeof cur.code === 'string' ? cur.code : '';
99
+ const name = typeof cur.name === 'string' ? cur.name : '';
100
+ const message = typeof cur.message === 'string' ? cur.message : '';
101
+ parts.push(`${code} ${name} ${message}`);
102
+ cur = cur.cause;
103
+ }
104
+ const text = parts.join(' ').toLowerCase();
105
+ if (/econnrefused|connection refused|ehostunreach|enetunreach/.test(text))
106
+ return 'refused';
107
+ if (/abort|timed? ?out|etimedout|und_err_connect_timeout/.test(text))
108
+ return 'timeout';
109
+ return 'other';
110
+ }
111
+ export const SUCCESS_MARKER = 'dsh web: http';
112
+ /** Index of the last boot-boundary line in the tail, or -1 when it predates the tail. */
113
+ export function lastBootBoundaryIndex(lines) {
114
+ for (let i = lines.length - 1; i >= 0; i--) {
115
+ if (lines[i].includes(BOOT_BOUNDARY_MARKER))
116
+ return i;
117
+ }
118
+ return -1;
119
+ }
120
+ /** Index of the last "dsh web: http" success marker (`lowerLines` must be lower-cased). */
121
+ export function lastSuccessMarkerIndex(lowerLines) {
122
+ for (let i = lowerLines.length - 1; i >= 0; i--) {
123
+ if (lowerLines[i].includes(SUCCESS_MARKER))
124
+ return i;
125
+ }
126
+ return -1;
127
+ }
128
+ /** Wall-clock ms parsed from a boot-boundary line, or undefined when unparseable. */
129
+ export function parseBootBoundaryMs(line) {
130
+ const m = /boot-boundary\s+(\S+)/.exec(line);
131
+ if (!m)
132
+ return undefined;
133
+ const ms = Date.parse(m[1]);
134
+ return Number.isNaN(ms) ? undefined : ms;
135
+ }
72
136
  export async function pollHealth(opts = {}) {
73
137
  // 5s was too tight for a busy plugin-tree boot: a lone AbortError from a slow
74
138
  // (but otherwise fine) response was indistinguishable from a real crash, and
@@ -78,15 +142,19 @@ export async function pollHealth(opts = {}) {
78
142
  // high load (loadavg 12) stalls; configurable via domains.supervisor.pollTimeoutMs.
79
143
  // If opts.timeoutMs is not injected, read from supervisor config (maestro settings).
80
144
  let effectiveTimeout = opts.timeoutMs;
81
- if (effectiveTimeout === undefined) {
145
+ let bootGraceMs = opts.bootGraceMs;
146
+ if (effectiveTimeout === undefined || bootGraceMs === undefined) {
82
147
  try {
83
148
  const { readSupervisorConfig } = await import('./config.js');
84
149
  const cfg = await readSupervisorConfig();
85
- if (typeof cfg.pollTimeoutMs === 'number' && cfg.pollTimeoutMs > 0)
150
+ if (effectiveTimeout === undefined && typeof cfg.pollTimeoutMs === 'number' && cfg.pollTimeoutMs > 0)
86
151
  effectiveTimeout = cfg.pollTimeoutMs;
152
+ if (bootGraceMs === undefined && typeof cfg.bootGraceMs === 'number' && cfg.bootGraceMs > 0)
153
+ bootGraceMs = cfg.bootGraceMs;
87
154
  }
88
155
  catch { }
89
156
  effectiveTimeout ??= 20000;
157
+ bootGraceMs ??= 180000;
90
158
  }
91
159
  const fetchFn = opts.fetch ?? defaultFetch(opts.url ?? 'http://127.0.0.1:3080/', effectiveTimeout);
92
160
  const psAliveFn = opts.psAlive ?? defaultPsAlive;
@@ -111,64 +179,64 @@ export async function pollHealth(opts = {}) {
111
179
  catch {
112
180
  // ignore log read errors
113
181
  }
114
- // Suppression gate: during 30s planned-restart window, boot transients are expected.
115
- // Fetch failures are treated as suppressed (up:true), and log scanning is windowed.
182
+ // Suppression gate: an in-flight planned restart, or a boot that has not yet
183
+ // proven itself, means transients are expected. A refused connection is
184
+ // never suppressed: nothing is listening, so the process is gone (D2).
116
185
  const suppressed = checkPlannedRestart();
117
- const recentlyStarted = isRecentlyStarted(opts);
118
- const graceActive = suppressed || recentlyStarted;
119
- // ActiveEnterTimestampMonotonic lookup — primary filter source; fallback to last success window
120
- let activeEnterMs;
121
- try {
122
- const fn = opts.getActiveEnterMs ?? getActiveEnterMs;
123
- activeEnterMs = fn();
124
- void activeEnterMs;
125
- }
126
- catch {
127
- activeEnterMs = undefined;
186
+ let activeEnterAtMs = opts.activeEnterAtMs;
187
+ if (activeEnterAtMs === undefined) {
188
+ try {
189
+ const fn = opts.getActiveEnterMs ?? getActiveEnterMs;
190
+ activeEnterAtMs = fn();
191
+ }
192
+ catch {
193
+ activeEnterAtMs = undefined;
194
+ }
128
195
  }
129
- let logError;
130
- // Filter logTail to only consider lines after ActiveEnterTimestamp.
131
- // Fallback: last 200 lines after last "dsh web: http" success marker (log is append-only).
196
+ const now = Date.now();
132
197
  const lines = logContent.split('\n');
133
198
  const lowerLines = lines.map(l => l.toLowerCase());
134
- // find last success marker
135
- let lastSuccessIdx = -1;
136
- for (let i = lowerLines.length - 1; i >= 0; i--) {
137
- if (lowerLines[i].includes('dsh web: http')) {
138
- lastSuccessIdx = i;
139
- break;
140
- }
141
- }
199
+ // Scope the scan to the current boot. The boundary line is only trusted for
200
+ // the unit start it was written for: a start more than a boot budget later
201
+ // (a systemd auto-restart, a manual start) is a different boot.
202
+ const boundaryIdx = lastBootBoundaryIndex(lines);
203
+ const boundaryMs = boundaryIdx === -1 ? undefined : parseBootBoundaryMs(lines[boundaryIdx]);
204
+ const scopedToThisBoot = boundaryMs !== undefined
205
+ && activeEnterAtMs !== undefined
206
+ && activeEnterAtMs >= boundaryMs
207
+ && activeEnterAtMs - boundaryMs <= bootGraceMs;
208
+ const bootLines = scopedToThisBoot ? lines.slice(boundaryIdx + 1) : lines;
209
+ const bootLower = scopedToThisBoot ? lowerLines.slice(boundaryIdx + 1) : lowerLines;
210
+ // Proof of a finished boot is a serving probe, not a log line (see bootFreshness).
211
+ const probeSucceeded = httpCode === 200 || httpCode === 401;
212
+ const bootPhase = bootFreshness({ activeEnterAtMs, now, bootGraceMs, probeSucceeded });
213
+ const booting = bootPhase === 'booting';
214
+ const graceActive = suppressed || booting;
215
+ // Inside the current boot, only lines after its own success marker may be a
216
+ // post-start crash. With no marker and no scope proving these lines are this
217
+ // boot's, the scan is inconclusive (D3) instead of inheriting the previous
218
+ // boot's crash text — that inheritance is what produced the 2026-09-13
219
+ // rollback report ("rollback — degraded: This operation was aborted") from a
220
+ // healthy, still-booting instance reading the previous boot's EADDRINUSE.
221
+ const successIdx = lastSuccessMarkerIndex(bootLower);
142
222
  let scanLines;
143
223
  let scanLower;
144
- if (lastSuccessIdx !== -1) {
145
- const after = lines.slice(lastSuccessIdx + 1);
146
- const afterLower = lowerLines.slice(lastSuccessIdx + 1);
147
- // keep only last 200 lines after success (fallback window)
148
- if (after.length > 200) {
149
- scanLines = after.slice(-200);
150
- scanLower = afterLower.slice(-200);
151
- }
152
- else {
153
- scanLines = after;
154
- scanLower = afterLower;
155
- }
156
- // If ActiveEnter is available, the same window applies — old EADDRINUSE before restart
157
- // is before the success marker and thus excluded. No extra timestamp->line mapping needed.
224
+ if (successIdx !== -1) {
225
+ scanLines = bootLines.slice(successIdx + 1);
226
+ scanLower = bootLower.slice(successIdx + 1);
227
+ }
228
+ else if (booting && !scopedToThisBoot) {
229
+ scanLines = [];
230
+ scanLower = [];
158
231
  }
159
232
  else {
160
- // no success marker: consider last 200 lines total (both primary and fallback)
161
- if (lines.length > 200) {
162
- scanLines = lines.slice(-200);
163
- scanLower = lowerLines.slice(-200);
164
- }
165
- else {
166
- scanLines = lines;
167
- scanLower = lowerLines;
168
- }
233
+ scanLines = bootLines;
234
+ scanLower = bootLower;
235
+ }
236
+ if (scanLines.length > 200) {
237
+ scanLines = scanLines.slice(-200);
238
+ scanLower = scanLower.slice(-200);
169
239
  }
170
- // If ActiveEnter lookup succeeded/failed, we already applied the fallback window.
171
- // When system has no success marker and no ActiveEnter, scanLines is still last 200.
172
240
  let lastErrorIdx = -1;
173
241
  let matchedLine = '';
174
242
  for (let i = scanLines.length - 1; i >= 0; i--) {
@@ -183,12 +251,11 @@ export async function pollHealth(opts = {}) {
183
251
  if (lastErrorIdx !== -1)
184
252
  break;
185
253
  }
254
+ let logError;
186
255
  if (lastErrorIdx !== -1) {
187
- // Within the windowed view, if a success appears after the error it would have been before the slice,
188
- // but check anyway for safety (error before success within window)
189
256
  let hasSuccessAfter = false;
190
257
  for (let i = lastErrorIdx + 1; i < scanLines.length; i++) {
191
- if (scanLower[i].includes('dsh web: http')) {
258
+ if (scanLower[i].includes(SUCCESS_MARKER)) {
192
259
  hasSuccessAfter = true;
193
260
  break;
194
261
  }
@@ -196,22 +263,33 @@ export async function pollHealth(opts = {}) {
196
263
  if (!hasSuccessAfter)
197
264
  logError = matchedLine;
198
265
  }
199
- else if (suppressed) {
200
- // suppressed window with no windowed error — ensure stale errors before success are ignored
201
- // (already handled by windowing)
202
- }
203
- // Legacy full-scan fallback for hasSuccessAfter across original lines when no windowed error
204
- // but original had error before success — already suppressed by windowing, no need to re-check.
205
- // Distinguish FULL (http !=200) vs DEGRADED (http 200 but log has plugin error)
206
- // Suppression: during 30s grace (planned-restart OR recently started), transient fetch failures are not a crash
207
266
  if (fetchError) {
208
- if (graceActive) {
209
- // treat fetch failed / http 404 as up:true suppressed (don't write error) also doubles effective downThreshold
210
- return {
211
- up: true,
212
- httpCode,
213
- logTail: logContent.slice(-5000),
214
- };
267
+ const kind = classifyFetchFailure(fetchError);
268
+ // D1: a timeout/abort while the current boot is unproven is a slow boot,
269
+ // not a crash — increment nothing. D2: a refused connection is different
270
+ // (nothing is listening, the process is gone) and is never masked.
271
+ if (booting && kind !== 'refused') {
272
+ return { up: true, httpCode, bootPhase, logTail: logContent.slice(-5000) };
273
+ }
274
+ // D5: inside the boot window a refused connection with a verifiably alive
275
+ // process is a boot symptom, not a crash — the listener may not be bound
276
+ // yet. Outside the window, or with a dead process, the verdict below is
277
+ // exactly the pre-D5 behaviour (degraded when the port still answers,
278
+ // down when it does not).
279
+ if (booting && kind === 'refused') {
280
+ let alive = false;
281
+ try {
282
+ alive = await psAliveFn();
283
+ }
284
+ catch {
285
+ alive = false;
286
+ }
287
+ if (alive) {
288
+ return { up: true, httpCode, bootPhase, logTail: logContent.slice(-5000) };
289
+ }
290
+ }
291
+ if (suppressed) {
292
+ return { up: true, httpCode, bootPhase, logTail: logContent.slice(-5000) };
215
293
  }
216
294
  // Corroborate with a cheap port-liveness check before declaring a crash.
217
295
  // The HTTP fetch shares dsh-web's own event loop, so a busy-but-alive
@@ -232,6 +310,7 @@ export async function pollHealth(opts = {}) {
232
310
  return {
233
311
  up: true,
234
312
  httpCode,
313
+ bootPhase,
235
314
  error: logError ? `${fetchError} + ${logError}` : fetchError,
236
315
  degraded: true,
237
316
  logTail: logContent.slice(-5000),
@@ -240,52 +319,31 @@ export async function pollHealth(opts = {}) {
240
319
  return {
241
320
  up: false,
242
321
  httpCode,
322
+ bootPhase,
243
323
  error: logError ? `${fetchError} + ${logError}` : fetchError,
244
324
  degraded: false,
245
325
  logTail: logContent.slice(-5000),
246
326
  };
247
327
  }
248
- // During grace, a windowed logError that is still present is considered stale/boot transient
249
- // as well — treat as up to avoid double restart. The fallback window already filters pre-restart errors.
328
+ // During grace, a windowed logError that is still present is considered
329
+ // stale/boot transient as well — treat as up to avoid double restart.
250
330
  if (graceActive && logError) {
251
- // Effective downThreshold doubling is handled by supervisor, but health also suppresses log tail
252
- return {
253
- up: true,
254
- httpCode,
255
- logTail: logContent.slice(-5000),
256
- };
331
+ return { up: true, httpCode, bootPhase, logTail: logContent.slice(-5000) };
257
332
  }
258
333
  if (logError) {
259
- // EADDRINUSE is fatal even with http 200 — old process still holds 3080
260
- // and new start failed; treat as FULL down so supervisor kills + restarts.
334
+ // EADDRINUSE is fatal even with http 200 — an old process still holds
335
+ // the port and the new start failed; treat as FULL down so the supervisor
336
+ // kills + restarts.
261
337
  const lowerErr = logError.toLowerCase();
262
338
  const isFatalPortError = lowerErr.includes('eaddrinuse') || lowerErr.includes('address already in use');
263
339
  if (isFatalPortError) {
264
- return {
265
- up: false,
266
- httpCode,
267
- error: logError,
268
- degraded: false,
269
- logTail: logContent.slice(-5000),
270
- };
340
+ return { up: false, httpCode, bootPhase, error: logError, degraded: false, logTail: logContent.slice(-5000) };
271
341
  }
272
342
  // http 200 but log error → DEGRADED (isolatable), not FULL
273
343
  if (httpCode === 200) {
274
- return {
275
- up: true,
276
- httpCode,
277
- error: logError,
278
- degraded: true,
279
- logTail: logContent.slice(-5000),
280
- };
344
+ return { up: true, httpCode, bootPhase, error: logError, degraded: true, logTail: logContent.slice(-5000) };
281
345
  }
282
- return {
283
- up: false,
284
- httpCode,
285
- error: logError,
286
- degraded: false,
287
- logTail: logContent.slice(-5000),
288
- };
346
+ return { up: false, httpCode, bootPhase, error: logError, degraded: false, logTail: logContent.slice(-5000) };
289
347
  }
290
348
  // Also check psAlive as secondary signal — if fetch ok but ps dead, still down
291
349
  try {
@@ -297,7 +355,7 @@ export async function pollHealth(opts = {}) {
297
355
  catch {
298
356
  // ignore
299
357
  }
300
- return { up: httpCode === 200 || httpCode === 401, httpCode, logTail: logContent.slice(-5000) };
358
+ return { up: httpCode === 200 || httpCode === 401, httpCode, bootPhase, logTail: logContent.slice(-5000) };
301
359
  }
302
360
  function defaultFetch(url, timeoutMs) {
303
361
  return async () => {
package/lib/intents.d.ts CHANGED
@@ -13,3 +13,20 @@ export declare function intentsDir(): string;
13
13
  export declare function intentPath(sessionId: string): string;
14
14
  export declare function readIntent(sessionId: string): RestartIntent | undefined;
15
15
  export declare function consumeIntent(sessionId: string): void;
16
+ /**
17
+ * Post-swap outcome written by the supervisor daemon after acting on a
18
+ * restart request (`<sessionId>.outcome.json`, mode 600, same dir). Read by
19
+ * `dsh_web_restart_status` so callers learn the new PID + HTTP status without
20
+ * hand-probing `ss`/`curl`.
21
+ */
22
+ export interface RestartOutcome {
23
+ state: 'ok' | 'failed';
24
+ oldPid?: number;
25
+ newPid?: number;
26
+ httpStatus?: number;
27
+ swappedAt: number;
28
+ error?: string;
29
+ }
30
+ export declare function outcomePath(sessionId: string): string;
31
+ export declare function writeRestartOutcome(sessionId: string, outcome: RestartOutcome): void;
32
+ export declare function readRestartOutcome(sessionId: string): RestartOutcome | undefined;
package/lib/intents.js CHANGED
@@ -1,4 +1,4 @@
1
- import { existsSync, readFileSync, unlinkSync } from 'node:fs';
1
+ import { existsSync, readFileSync, unlinkSync, mkdirSync, writeFileSync } from 'node:fs';
2
2
  import { join } from 'node:path';
3
3
  import { homedir } from 'node:os';
4
4
  export function intentsDir() {
@@ -25,3 +25,25 @@ export function consumeIntent(sessionId) {
25
25
  }
26
26
  catch { }
27
27
  }
28
+ export function outcomePath(sessionId) {
29
+ const safe = sessionId.replace(/[^A-Za-z0-9._-]/g, '_');
30
+ return join(intentsDir(), `${safe}.outcome.json`);
31
+ }
32
+ export function writeRestartOutcome(sessionId, outcome) {
33
+ try {
34
+ mkdirSync(intentsDir(), { recursive: true });
35
+ writeFileSync(outcomePath(sessionId), JSON.stringify(outcome), { encoding: 'utf8', mode: 0o600 });
36
+ }
37
+ catch { }
38
+ }
39
+ export function readRestartOutcome(sessionId) {
40
+ try {
41
+ const p = outcomePath(sessionId);
42
+ if (!existsSync(p))
43
+ return undefined;
44
+ return JSON.parse(readFileSync(p, 'utf8'));
45
+ }
46
+ catch {
47
+ return undefined;
48
+ }
49
+ }
package/lib/plugin.js CHANGED
@@ -9,7 +9,7 @@ import * as fs from 'node:fs';
9
9
  import * as path from 'node:path';
10
10
  import * as os from 'node:os';
11
11
  import { fileURLToPath } from 'node:url';
12
- import { findInterrupted as defaultFindInterrupted, findDanglingOpenTurns as defaultFindDanglingOpenTurns } from './resume.js';
12
+ import { findInterrupted as defaultFindInterrupted, findDanglingOpenTurns as defaultFindDanglingOpenTurns, resolveSessionLogPath } from './resume.js';
13
13
  import { readIntent, consumeIntent } from './intents.js';
14
14
  import { appendResumeLog } from './resume-log.js';
15
15
  import { makeSkillProvider } from './skill-provider.js';
@@ -294,12 +294,13 @@ async function readRouteFromHandle(persistence, sid) {
294
294
  * first match so huge logs cost one streamed pass, not a full decode.
295
295
  */
296
296
  async function readRouteFromRawLog(sessionsRoot, group, sessionId) {
297
- const zstdPath = path.join(sessionsRoot, group, sessionId, 'session.jsonl.zstd');
298
- const jsonlPath = path.join(sessionsRoot, group, sessionId, 'session.jsonl');
297
+ const logPath = resolveSessionLogPath(path.join(sessionsRoot, group, sessionId));
298
+ if (logPath === undefined)
299
+ return undefined;
299
300
  try {
300
- if (fs.existsSync(zstdPath)) {
301
+ if (logPath.endsWith('.zstd')) {
301
302
  const { execSync } = await import('node:child_process');
302
- const out = execSync(`zstd -d -c ${JSON.stringify(zstdPath)} 2>/dev/null | grep -a -m1 '"type":"request/context"'`, { encoding: 'utf-8' });
303
+ const out = execSync(`zstd -d -c ${JSON.stringify(logPath)} 2>/dev/null | grep -a -m1 '"type":"request/context"'`, { encoding: 'utf-8' });
303
304
  const line = out.split('\n').find((l) => l.includes('request/context'));
304
305
  if (line !== undefined) {
305
306
  try {
@@ -309,18 +310,16 @@ async function readRouteFromRawLog(sessionsRoot, group, sessionId) {
309
310
  }
310
311
  return undefined;
311
312
  }
312
- if (fs.existsSync(jsonlPath)) {
313
- const content = fs.readFileSync(jsonlPath, 'utf-8');
314
- for (const line of content.split('\n')) {
315
- if (!line.includes('"request/context"'))
316
- continue;
317
- try {
318
- const route = routeFromEvents([JSON.parse(line)]);
319
- if (route !== undefined)
320
- return route;
321
- }
322
- catch { }
313
+ const content = fs.readFileSync(logPath, 'utf-8');
314
+ for (const line of content.split('\n')) {
315
+ if (!line.includes('"request/context"'))
316
+ continue;
317
+ try {
318
+ const route = routeFromEvents([JSON.parse(line)]);
319
+ if (route !== undefined)
320
+ return route;
323
321
  }
322
+ catch { }
324
323
  }
325
324
  }
326
325
  catch { }
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Serialized systemd restart for `dsh-web.service`: stop, wait until the
3
+ * unit is inactive, then start — instead of one raw `systemctl restart`
4
+ * that boots the new process while the old one still holds :3082 and
5
+ * crash-loops on EADDRINUSE (2026-09-11 outage: 4 restarts in
6
+ * 15 min over a ~90s SIGTERM stop → 30 EADDRINUSE crashes).
7
+ *
8
+ * All side effects are injectable so tests never touch systemd.
9
+ */
10
+ export interface SerializedRestartDeps {
11
+ /** Run a shell command; must throw on non-zero exit. */
12
+ exec?: (cmd: string) => void;
13
+ /** Whether the unit is currently active. */
14
+ isActive?: () => boolean;
15
+ sleep?: (ms: number) => Promise<void>;
16
+ /** Max time to wait for inactivity before starting anyway. */
17
+ stopTimeoutMs?: number;
18
+ /** Poll interval while waiting for inactivity. */
19
+ pollMs?: number;
20
+ /** Clock (injectable for tests). */
21
+ now?: () => number;
22
+ }
23
+ export declare function serializedSystemdRestart(deps?: SerializedRestartDeps): Promise<void>;
24
+ export declare const DSH_WEB_UNIT_NAME = "dsh-web.service";
25
+ export declare function dshWebUnitPath(): string;
26
+ /** Does systemd own `dsh-web.service` on this host? */
27
+ export declare function systemdUnitExists(unitPath?: string): boolean;
28
+ /**
29
+ * The direct-node `nohup` fallback exists for portable hosts that have no
30
+ * systemd unit. When the unit exists, systemd owns the boot: spawning node
31
+ * behind its back is how a second instance appears — the two starts / one
32
+ * rollback report of 2026-09-13. Gate on "the unit does not exist", never on
33
+ * "this particular start failed".
34
+ */
35
+ export declare function shouldUseNohupFallback(unitExists: boolean): boolean;
@@ -0,0 +1,60 @@
1
+ import { execSync } from 'node:child_process';
2
+ import * as fs from 'node:fs';
3
+ import * as os from 'node:os';
4
+ import * as path from 'node:path';
5
+ const STOP_TIMEOUT_MS = 120_000;
6
+ const POLL_MS = 2_000;
7
+ export async function serializedSystemdRestart(deps = {}) {
8
+ const exec = deps.exec ?? ((cmd) => {
9
+ execSync(cmd, { timeout: 15_000, stdio: 'pipe' });
10
+ });
11
+ const isActive = deps.isActive ?? (() => {
12
+ try {
13
+ execSync('systemctl --user is-active --quiet dsh-web.service', { timeout: 5_000, stdio: 'pipe' });
14
+ return true;
15
+ }
16
+ catch {
17
+ return false;
18
+ }
19
+ });
20
+ const sleep = deps.sleep ?? ((ms) => new Promise(r => setTimeout(r, ms)));
21
+ const stopTimeoutMs = deps.stopTimeoutMs ?? STOP_TIMEOUT_MS;
22
+ const pollMs = deps.pollMs ?? POLL_MS;
23
+ const now = deps.now ?? Date.now;
24
+ // Best-effort: the unit may already be down (crash) — the wait + start
25
+ // below still apply.
26
+ try {
27
+ exec('systemctl --user stop dsh-web.service');
28
+ }
29
+ catch { /* fall through to wait + start */ }
30
+ const deadline = now() + stopTimeoutMs;
31
+ while (isActive()) {
32
+ if (now() >= deadline)
33
+ break;
34
+ await sleep(pollMs);
35
+ }
36
+ exec('systemctl --user start dsh-web.service');
37
+ }
38
+ export const DSH_WEB_UNIT_NAME = 'dsh-web.service';
39
+ export function dshWebUnitPath() {
40
+ return process.env.DSH_WEB_UNIT_PATH ?? path.join(os.homedir(), '.config/systemd/user/dsh-web.service');
41
+ }
42
+ /** Does systemd own `dsh-web.service` on this host? */
43
+ export function systemdUnitExists(unitPath = dshWebUnitPath()) {
44
+ try {
45
+ return fs.existsSync(unitPath);
46
+ }
47
+ catch {
48
+ return false;
49
+ }
50
+ }
51
+ /**
52
+ * The direct-node `nohup` fallback exists for portable hosts that have no
53
+ * systemd unit. When the unit exists, systemd owns the boot: spawning node
54
+ * behind its back is how a second instance appears — the two starts / one
55
+ * rollback report of 2026-09-13. Gate on "the unit does not exist", never on
56
+ * "this particular start failed".
57
+ */
58
+ export function shouldUseNohupFallback(unitExists) {
59
+ return !unitExists;
60
+ }
@@ -1,4 +1,22 @@
1
1
  export declare function buildKillStalePortsCommand(ports?: number[]): string;
2
+ /**
3
+ * Sentinel appended to dsh-web.log immediately before a supervised restart.
4
+ *
5
+ * The log is append-only and carries no timestamps, so this line is the only
6
+ * durable ordering between "the previous boot's crash" and "this boot's
7
+ * output". Without it the health scan inherited the previous, failed boot's
8
+ * `EADDRINUSE` stack and rolled back a healthy, still-booting instance
9
+ * (incident 2026-09-13).
10
+ */
11
+ export declare const BOOT_BOUNDARY_MARKER = "[supervisor] boot-boundary";
12
+ /** The production dsh-web log (override with DSH_WEB_LOG for tests/tools). */
13
+ export declare function dshWebLogPath(): string;
14
+ /**
15
+ * Append the boot-boundary sentinel. Pass an explicit `logPath` in tests, or
16
+ * redirect with DSH_WEB_LOG. With no destination at all under VITEST this is a
17
+ * deliberate no-op so a unit test can never append to the operator's real log.
18
+ */
19
+ export declare function markBootBoundary(logPath?: string): void;
2
20
  export declare function isSelfCopyError(message: string): boolean;
3
21
  export declare const PLANNED_RESTART_TTL_MS = 180000;
4
22
  export declare function isPlannedRestartFresh(mtimeMs: number, nowMs: number, ttlMs?: number): boolean;
@@ -11,9 +29,11 @@ export interface RestartRequest {
11
29
  ttl: number;
12
30
  callerSessionId?: string;
13
31
  reason?: string;
32
+ oldPid?: number;
14
33
  }
15
34
  export declare function writeRestartRequest(caller: {
16
35
  callerSessionId?: string;
17
36
  reason?: string;
37
+ oldPid?: number;
18
38
  }, ttlMs?: number): void;
19
39
  export declare function readRestartRequest(): RestartRequest | undefined;