@ddtcorex/dsh-maestro-supervisor 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/boot-guard.d.ts +33 -0
- package/lib/boot-guard.js +68 -0
- package/lib/boot-lock.d.ts +56 -0
- package/lib/boot-lock.js +155 -0
- package/lib/cli.d.ts +22 -5
- package/lib/cli.js +120 -54
- package/lib/health-poller.d.ts +68 -1
- package/lib/health-poller.js +191 -133
- package/lib/plugin.js +15 -16
- package/lib/restart-exec.d.ts +35 -0
- package/lib/restart-exec.js +60 -0
- package/lib/restart-guards.d.ts +18 -0
- package/lib/restart-guards.js +33 -0
- package/lib/restart-tool.d.ts +2 -1
- package/lib/restart-tool.js +19 -1
- package/lib/restart-web.d.ts +26 -0
- package/lib/restart-web.js +58 -0
- package/lib/resume.d.ts +10 -0
- package/lib/resume.js +45 -28
- package/lib/snapshot.d.ts +52 -15
- package/lib/snapshot.js +168 -18
- package/lib/supervisor.d.ts +21 -4
- package/lib/supervisor.js +78 -36
- package/package.json +1 -1
- package/skills/dsh-safe-restart/SKILL.md +32 -2
- package/skills/dsh-safe-restart/scripts/restart-dsh-web.sh +162 -8
package/lib/supervisor.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { classifyFetchFailure } from './health-poller.js';
|
|
1
2
|
import { runDebugAgent } from './debug-agent.js';
|
|
2
3
|
import { findInterrupted as defaultFindInterrupted, parseDuration } from './resume.js';
|
|
3
4
|
import * as fs from 'node:fs';
|
|
@@ -5,11 +6,10 @@ import * as path from 'node:path';
|
|
|
5
6
|
import * as os from 'node:os';
|
|
6
7
|
import { resolveHarnessRoot } from './paths.js';
|
|
7
8
|
import { readSupervisorConfig } from './config.js';
|
|
8
|
-
import { writePlannedRestart as defaultWritePlannedRestart, checkPlannedRestart as defaultCheckPlannedRestart, clearPlannedRestart
|
|
9
|
+
import { writePlannedRestart as defaultWritePlannedRestart, checkPlannedRestart as defaultCheckPlannedRestart, clearPlannedRestart } from './restart-guards.js';
|
|
9
10
|
import { writeRestartOutcome as defaultWriteOutcome } from './intents.js';
|
|
10
11
|
import { execFileSync } from 'node:child_process';
|
|
11
12
|
import { mintDshSessionCookie } from './dsh-session.js';
|
|
12
|
-
import { buildKillStalePortsCommand } from './restart-guards.js';
|
|
13
13
|
/** PID holding a 127.0.0.1 listener on the port, or undefined. Never throws. */
|
|
14
14
|
export function defaultListenerPid(port) {
|
|
15
15
|
try {
|
|
@@ -86,31 +86,22 @@ export class Supervisor {
|
|
|
86
86
|
return this.deps.clearPlannedRestart ?? clearPlannedRestart;
|
|
87
87
|
}
|
|
88
88
|
async restartWeb() {
|
|
89
|
-
this.
|
|
89
|
+
const grace = await this.getEffectiveBootGraceMs();
|
|
90
|
+
// Marker first, and its TTL is the boot budget — not 30 s. The observed
|
|
91
|
+
// boot took 1m52s: after +30 s every poll was judged as a crash, which is
|
|
92
|
+
// exactly how the 2026-09-13 rollback report was produced (D5).
|
|
93
|
+
this.getWritePlannedRestart()(grace);
|
|
90
94
|
if (this.deps.restartWeb) {
|
|
95
|
+
// Injected implementation (cli.ts daemon wiring, tests): it owns the
|
|
96
|
+
// boot lock, so the single-flight guarantee stays in one place.
|
|
91
97
|
await this.deps.restartWeb();
|
|
92
98
|
return;
|
|
93
99
|
}
|
|
94
|
-
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
}
|
|
99
|
-
catch { }
|
|
100
|
-
try {
|
|
101
|
-
execSync('systemctl --user is-active --quiet dsh-web.service && systemctl --user restart dsh-web.service || systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
|
|
102
|
-
return;
|
|
103
|
-
}
|
|
104
|
-
catch { }
|
|
105
|
-
try {
|
|
106
|
-
execSync('systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
|
|
107
|
-
return;
|
|
100
|
+
const { performSingleBootRestart } = await import('./restart-web.js');
|
|
101
|
+
const res = await performSingleBootRestart({ bootGraceMs: grace });
|
|
102
|
+
if (!res.restarted) {
|
|
103
|
+
await this.deps.notify(`restart skipped: ${res.reason ?? 'boot lock held'}`).catch(() => { });
|
|
108
104
|
}
|
|
109
|
-
catch { }
|
|
110
|
-
const { resolveDeepseekHarnessDir } = await import('./paths.js');
|
|
111
|
-
const harnessRoot = resolveDeepseekHarnessDir();
|
|
112
|
-
const logPath = path.join(os.homedir(), '.dsh/dsh-web.log');
|
|
113
|
-
execSync(`setsid nohup bash -c 'export NVM_DIR="$HOME/.nvm"; [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; cd ${JSON.stringify(harnessRoot)} && exec node --import tsx/esm apps/cli/src/bin.ts web --no-open >> ${JSON.stringify(logPath)} 2>&1' &`, { timeout: 5000 });
|
|
114
105
|
}
|
|
115
106
|
getRunDebugAgent() {
|
|
116
107
|
return this.deps.runDebugAgent ?? runDebugAgent;
|
|
@@ -257,6 +248,17 @@ export class Supervisor {
|
|
|
257
248
|
catch { }
|
|
258
249
|
return 20000;
|
|
259
250
|
}
|
|
251
|
+
async getEffectiveBootGraceMs() {
|
|
252
|
+
if (this.deps.bootGraceMs !== undefined)
|
|
253
|
+
return this.deps.bootGraceMs;
|
|
254
|
+
try {
|
|
255
|
+
const cfg = await readSupervisorConfig();
|
|
256
|
+
if (typeof cfg.bootGraceMs === 'number' && cfg.bootGraceMs > 0)
|
|
257
|
+
return cfg.bootGraceMs;
|
|
258
|
+
}
|
|
259
|
+
catch { }
|
|
260
|
+
return 180_000;
|
|
261
|
+
}
|
|
260
262
|
async findInterruptedRecent(withinMs) {
|
|
261
263
|
const ms = withinMs ?? this.getResumeWithinMs();
|
|
262
264
|
// Prefer injected mock for testability
|
|
@@ -327,6 +329,17 @@ export class Supervisor {
|
|
|
327
329
|
await this.deps.notify(`RESUME FAILED: ${ids.length} interrupted sessions (${ids.slice(0, 3).join(', ')}) — ${e?.message ?? String(e)}`).catch(() => { });
|
|
328
330
|
}
|
|
329
331
|
}
|
|
332
|
+
/**
|
|
333
|
+
* D4: a restore that could not put every entry back must be surfaced loudly.
|
|
334
|
+
* Without this, a half-restored tree reads exactly like a clean recovery in
|
|
335
|
+
* the log and in the operator's Telegram feed.
|
|
336
|
+
*/
|
|
337
|
+
async reportRollbackResult(summary, reportPath) {
|
|
338
|
+
if (!summary || !summary.skipped?.length)
|
|
339
|
+
return;
|
|
340
|
+
const first = summary.skipped.slice(0, 3).map(s => `${s.path} (${s.reason})`).join('; ');
|
|
341
|
+
await this.deps.notify(`ROLLBACK PARTIAL: ${summary.restored ?? 0} restored, ${summary.skipped.length} skipped — ${first} (report: ${reportPath})`).catch(() => { });
|
|
342
|
+
}
|
|
330
343
|
handleDebugResult(reportPath, res) {
|
|
331
344
|
if (res.fixed) {
|
|
332
345
|
void this.deps.notify(`FIXED: debug-agent fixed ${reportPath} — ${res.reason}`).catch(() => { });
|
|
@@ -367,9 +380,8 @@ export class Supervisor {
|
|
|
367
380
|
// own 30s TTL runs out.
|
|
368
381
|
this.lastRollback = this.deps.getTime ? this.deps.getTime() : Date.now();
|
|
369
382
|
await this.restartWeb();
|
|
370
|
-
//
|
|
371
|
-
//
|
|
372
|
-
this.getWritePlannedRestart()(PLANNED_RESTART_TTL_MS);
|
|
383
|
+
// restartWeb() already wrote the marker with the boot budget as its
|
|
384
|
+
// TTL; the marker is cleared on the first healthy poll instead.
|
|
373
385
|
await this.deps.notify(`restarted dsh-web after self-restart by session ${restartReq.callerSessionId}`);
|
|
374
386
|
}
|
|
375
387
|
catch (e) {
|
|
@@ -402,6 +414,14 @@ export class Supervisor {
|
|
|
402
414
|
// DEGRADED: http 200 but log has plugin error → report + notify, rollback after consecutive threshold
|
|
403
415
|
if (health.degraded) {
|
|
404
416
|
this.consecutiveDown = 0;
|
|
417
|
+
// D1: a degraded verdict while the current boot is unproven is a boot
|
|
418
|
+
// transient, not a plugin error — the incident's report line
|
|
419
|
+
// "rollback — degraded: This operation was aborted" was produced exactly
|
|
420
|
+
// here, five polls after a slow boot.
|
|
421
|
+
if (health.bootPhase === 'booting') {
|
|
422
|
+
this.consecutiveDegraded = 0;
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
405
425
|
// Check suppression first — don't count degraded during planned restart grace
|
|
406
426
|
let suppressedByMarkerDeg = false;
|
|
407
427
|
try {
|
|
@@ -484,7 +504,8 @@ export class Supervisor {
|
|
|
484
504
|
const degradedHealth = { up: false, httpCode: health.httpCode, error: `degraded → down: ${degradedError}`, logTail: logTail2, degraded: false };
|
|
485
505
|
const reportPath2 = await this.deps.writeReport({ ts: ts2, health: degradedHealth, action: `rollback — degraded: ${degradedError}`, logTail: logTail2, gitDiff: gitDiff2 }).catch(() => '');
|
|
486
506
|
try {
|
|
487
|
-
await this.deps.rollback();
|
|
507
|
+
const summary = await this.deps.rollback();
|
|
508
|
+
await this.reportRollbackResult(summary, reportPath2);
|
|
488
509
|
}
|
|
489
510
|
catch (e) {
|
|
490
511
|
await this.deps.notify(`rollback failed: ${e?.message ?? String(e)} (report: ${reportPath2})`).catch(() => { });
|
|
@@ -512,20 +533,33 @@ export class Supervisor {
|
|
|
512
533
|
if (health.up) {
|
|
513
534
|
this.consecutiveDown = 0;
|
|
514
535
|
this.consecutiveDegraded = 0;
|
|
515
|
-
//
|
|
516
|
-
//
|
|
517
|
-
//
|
|
518
|
-
|
|
536
|
+
// D5: a healthy poll is the authoritative "the restart succeeded"
|
|
537
|
+
// signal — clear the suppression marker here instead of waiting for its
|
|
538
|
+
// TTL, so a stale marker can never mute a later real crash.
|
|
539
|
+
let markerClearedThisTick = false;
|
|
540
|
+
try {
|
|
541
|
+
if (this.getCheckPlannedRestart()()) {
|
|
542
|
+
this.getClearPlannedRestart()();
|
|
543
|
+
markerClearedThisTick = true;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
catch { }
|
|
547
|
+
// Post-self-restart boot proved healthy: run the post-restart
|
|
548
|
+
// session-scan hook and re-arm the single-flight latch. A failed clear
|
|
549
|
+
// keeps the latch set so the same marker is never re-handled into a
|
|
550
|
+
// second restart.
|
|
519
551
|
if (this.awaitingHealthyBoot) {
|
|
520
552
|
this.awaitingHealthyBoot = false;
|
|
521
553
|
const req = this.pendingRestartRequest;
|
|
522
554
|
this.pendingRestartRequest = undefined;
|
|
523
|
-
let cleared =
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
555
|
+
let cleared = markerClearedThisTick;
|
|
556
|
+
if (!cleared) {
|
|
557
|
+
try {
|
|
558
|
+
this.getClearPlannedRestart()();
|
|
559
|
+
cleared = true;
|
|
560
|
+
}
|
|
561
|
+
catch { }
|
|
527
562
|
}
|
|
528
|
-
catch { }
|
|
529
563
|
if (cleared)
|
|
530
564
|
this.restartRequestHandled = false;
|
|
531
565
|
if (req) {
|
|
@@ -596,6 +630,13 @@ export class Supervisor {
|
|
|
596
630
|
// A lone timed-out poll (e.g. a slow plugin-tree boot) must not trigger
|
|
597
631
|
// rollback/restart: that restart produces its own transient errors on
|
|
598
632
|
// the next poll, which would otherwise re-trigger this same path forever.
|
|
633
|
+
// D1/D2: while the boot is unproven only a refused connection may advance
|
|
634
|
+
// the down counter — nothing is listening, so the process is gone. A
|
|
635
|
+
// timeout/abort (or anything unattributable) is a slow boot, not a crash.
|
|
636
|
+
if (health.bootPhase === 'booting' && classifyFetchFailure(health.error) !== 'refused') {
|
|
637
|
+
this.consecutiveDown = 0;
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
599
640
|
this.consecutiveDown++;
|
|
600
641
|
let downThreshold = await this.getEffectiveDownThreshold();
|
|
601
642
|
// When a planned restart marker is active, double the threshold (3→6 at
|
|
@@ -629,7 +670,8 @@ export class Supervisor {
|
|
|
629
670
|
const gitDiff = await this.collectGitDiff().catch(() => '');
|
|
630
671
|
const reportPath = await this.deps.writeReport({ ts, health, action: `rollback — ${health.error ?? 'down'}`, logTail, gitDiff }).catch(() => '');
|
|
631
672
|
try {
|
|
632
|
-
await this.deps.rollback();
|
|
673
|
+
const summary = await this.deps.rollback();
|
|
674
|
+
await this.reportRollbackResult(summary, reportPath);
|
|
633
675
|
}
|
|
634
676
|
catch (e) {
|
|
635
677
|
await this.deps.notify(`rollback failed: ${e?.message ?? String(e)} (report: ${reportPath})`).catch(() => { });
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: dsh-safe-restart
|
|
3
|
-
description: Use before updating any dsh-maestro-* client bundle or DSH Web asset,
|
|
3
|
+
description: Use before updating any dsh-maestro-* client bundle or DSH Web asset, before reloading the running dsh web host process, or when a rebuilt plugin's lib/ is newer than the running supervisor daemon; validate first and perform a user-approved host restart through the bundled guarded recipe. In-session agents use the `dsh_web_restart` tool; hand off to a human only if the tool is unavailable.
|
|
4
4
|
compatibility: dsh
|
|
5
5
|
---
|
|
6
6
|
|
|
@@ -37,7 +37,18 @@ validate first and get explicit user consent before a real swap.
|
|
|
37
37
|
or run a no-server composition check instead.
|
|
38
38
|
3. Verify HTTP 200 and the new marker on the candidate. Retain last-known-good
|
|
39
39
|
assets until the real swap has passed post-swap checks.
|
|
40
|
-
4.
|
|
40
|
+
4. Check the **second** long-lived process, not only `dsh web`: the standalone
|
|
41
|
+
`dsh-web-supervisor` daemon also caches this package's `lib/*.js` in RAM at
|
|
42
|
+
start, and it is the process that judges the new boot. A daemon older than
|
|
43
|
+
the newest build still applies the OLD rollback rules — on 2026-09-13 that
|
|
44
|
+
is exactly what rolled `dsh web` back in a loop every ~90s
|
|
45
|
+
(`rollback — degraded: This operation was aborted`) while `:3080` answered
|
|
46
|
+
in 1.4ms. Compare the daemon's start with the newest build:
|
|
47
|
+
`ps -o lstart= -p "$(pgrep -f 'lib/bin.js daemon')"` against
|
|
48
|
+
`stat -c '%y' <package>/lib/*.js`; a build newer than the daemon's start
|
|
49
|
+
means the daemon is **stale** and must be reloaded BEFORE the swap.
|
|
50
|
+
`restart-dsh-web.sh --check-supervisor` reports this and changes nothing.
|
|
51
|
+
5. Ask for explicit consent and timing. “restart đi” is consent; silence is
|
|
41
52
|
not.
|
|
42
53
|
|
|
43
54
|
## Run the bundled helper only after consent
|
|
@@ -62,6 +73,22 @@ the append-only log destination. The helper refuses a real swap without
|
|
|
62
73
|
and refuses to launch if ports are still occupied. `--dry-run` never runs
|
|
63
74
|
`kill` or `setsid`.
|
|
64
75
|
|
|
76
|
+
The helper reloads a stale supervisor daemon itself, before it stops `dsh web`,
|
|
77
|
+
and aborts the swap (exit 70) when that reload fails — keeping the old process
|
|
78
|
+
beats handing a fresh boot to old rollback rules. Both daemon modes are
|
|
79
|
+
standalone: no `--repo`, no listeners, no `--confirm`, and they never touch
|
|
80
|
+
`dsh web`.
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
bash <skill-resource-base>/scripts/restart-dsh-web.sh --check-supervisor # fresh|stale|absent
|
|
84
|
+
bash <skill-resource-base>/scripts/restart-dsh-web.sh --reload-supervisor # reload only when stale
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
`DSH_SUPERVISOR_RELOAD_WAIT` (seconds, default 15) bounds the wait for systemd
|
|
88
|
+
to bring the daemon back. Only systemd owns that relaunch
|
|
89
|
+
(`dsh-web-supervisor.service` is `Restart=always`), so a hand-started daemon is
|
|
90
|
+
reported and left running, never killed.
|
|
91
|
+
|
|
65
92
|
## In-session validation without restart
|
|
66
93
|
|
|
67
94
|
In-session agents have three tools (same row as `dsh_web_restart`) that never
|
|
@@ -114,6 +141,8 @@ Do not read the top of an old append-only log as liveness evidence. Instead:
|
|
|
114
141
|
- Restart for a client-only or static change.
|
|
115
142
|
- Hard-code a PID or a local developer's workspace path.
|
|
116
143
|
- Treat a 200 response alone as proof that a rebuilt plugin was loaded.
|
|
144
|
+
- Treat a rebuilt `lib/` as loaded while the supervisor daemon that judges the
|
|
145
|
+
boot still runs the previous one; the two processes load the code separately.
|
|
117
146
|
|
|
118
147
|
## Agent handoff (required when you are an agent)
|
|
119
148
|
|
|
@@ -122,3 +151,4 @@ If you are running inside `dsh web` (any `dsh-*` skill, `maestro-*` skill, or su
|
|
|
122
151
|
1. Run all preflight checks (`--dry-run`, `pnpm verify`, marker grep) and report the results.
|
|
123
152
|
2. Run the `dsh_web_restart` tool instead of executing the helper — the helper is for external agents and humans.
|
|
124
153
|
3. If the tool reports it is unavailable, print the exact detached `setsid nohup ... --confirm` command for the human to run; then wait for explicit human confirmation in a new terminal. The interrupted turn will rehydrate from the log when the browser reconnects — no data is lost.
|
|
154
|
+
4. A stale supervisor daemon blocks a correct restart just as much as a stale `lib/`, and in-session you cannot run the helper (`restart-dsh-web` in a command is refused by the self-kill guard, as is any `systemctl`). Compare the daemon start (`ps -o lstart= -p "$(pgrep -f 'lib/bin.js daemon')"` — quote the pid you actually resolved) with `stat -c '%y' <package>/lib/*.js`; when the build is newer, reload with a bare `kill -TERM <daemon pid>` after confirming `/proc/<pid>/cmdline` contains `bin.js daemon`. systemd restarts it in ~2s and `dsh web`, the browser and the session are untouched — never signal the `dsh web` pid for this. If that form is refused, or the daemon does not come back as a new pid, stop and hand the human the exact pid plus the reload command (same handoff as step 3); never fall back to `systemctl` or a broad `pkill`.
|
|
@@ -27,6 +27,8 @@ PLUGIN_DIR="${DSH_SUPERVISOR_PLUGIN_DIR:-$(cd "$(dirname "$SCRIPT_SRC")/../../..
|
|
|
27
27
|
confirmed=false
|
|
28
28
|
dry_run=false
|
|
29
29
|
auto_mode=false
|
|
30
|
+
check_supervisor=false
|
|
31
|
+
reload_supervisor=false
|
|
30
32
|
|
|
31
33
|
dry_boot_and_verify() {
|
|
32
34
|
local dsh_repo="$1"
|
|
@@ -57,16 +59,25 @@ dry_boot_and_verify() {
|
|
|
57
59
|
usage() {
|
|
58
60
|
cat <<'EOF'
|
|
59
61
|
Usage: restart-dsh-web.sh --repo <deepseek-harness> [--log <path>] [--confirm|--auto] [--dry-run]
|
|
62
|
+
restart-dsh-web.sh --check-supervisor
|
|
63
|
+
restart-dsh-web.sh --reload-supervisor
|
|
60
64
|
|
|
61
65
|
Safely hand over the DSH Web process that owns ports 3000 and 3080.
|
|
62
66
|
|
|
63
67
|
Options:
|
|
64
|
-
--repo <path>
|
|
65
|
-
--log <path>
|
|
66
|
-
--confirm
|
|
67
|
-
--auto
|
|
68
|
-
--dry-run
|
|
69
|
-
-
|
|
68
|
+
--repo <path> DeepSeek Harness checkout (or set DSH_REPO).
|
|
69
|
+
--log <path> Append-only launch log (or set DSH_RESTART_LOG).
|
|
70
|
+
--confirm Permit a real process handover (human-gated). Required unless --dry-run.
|
|
71
|
+
--auto Permit auto handover (supervisor, no consent prompt). Alias for --confirm with auto log prefix.
|
|
72
|
+
--dry-run Print the resolved process tree; never stop or launch anything.
|
|
73
|
+
--check-supervisor Report supervisor-daemon freshness (fresh|stale|absent); changes nothing.
|
|
74
|
+
--reload-supervisor Reload the daemon only when it predates the newest lib/*.js build.
|
|
75
|
+
-h, --help Show this help text.
|
|
76
|
+
|
|
77
|
+
The supervisor daemon caches this package's lib/*.js in RAM like `dsh web`
|
|
78
|
+
does, and a stale daemon rolls back a healthy boot with its old rules. Every
|
|
79
|
+
real restart therefore reloads it first when stale; DSH_SUPERVISOR_RELOAD_WAIT
|
|
80
|
+
(seconds, default 15) bounds the wait for systemd to bring it back.
|
|
70
81
|
EOF
|
|
71
82
|
}
|
|
72
83
|
|
|
@@ -75,6 +86,120 @@ fail() {
|
|
|
75
86
|
exit "${2:-1}"
|
|
76
87
|
}
|
|
77
88
|
|
|
89
|
+
# --- supervisor daemon freshness -------------------------------------------------
|
|
90
|
+
# `dsh web` is not the only process that caches this package's lib/*.js in RAM:
|
|
91
|
+
# the standalone dsh-web-supervisor daemon does too, and it is the process that
|
|
92
|
+
# decides whether a boot is healthy or must be rolled back. A daemon started
|
|
93
|
+
# BEFORE the newest build keeps running the old rollback rules — on 2026-09-13
|
|
94
|
+
# exactly that judged a slow boot as down and rolled `dsh web` back in a loop
|
|
95
|
+
# ("rollback - degraded: This operation was aborted") while the port answered in
|
|
96
|
+
# 1.4ms. So: never swap `dsh web` under a stale daemon.
|
|
97
|
+
#
|
|
98
|
+
# Only systemd owns a relaunch (dsh-web-supervisor.service has Restart=always);
|
|
99
|
+
# a daemon someone started by hand has no owner to bring it back, so this step
|
|
100
|
+
# reports and leaves it alone rather than killing it.
|
|
101
|
+
|
|
102
|
+
# MainPID of the supervisor unit, or empty unless that pid really is the daemon.
|
|
103
|
+
supervisor_main_pid() {
|
|
104
|
+
local pid cmdline
|
|
105
|
+
pid="$(systemctl --user show -p MainPID --value dsh-web-supervisor.service 2>/dev/null | tr -dc '0-9' || true)"
|
|
106
|
+
[[ -n "$pid" && "$pid" != 0 ]] || return 0
|
|
107
|
+
[[ -d "/proc/$pid" ]] || return 0
|
|
108
|
+
# Never signal a pid that is not this daemon, whatever systemd reports.
|
|
109
|
+
cmdline="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null || true)"
|
|
110
|
+
[[ "$cmdline" == *"bin.js daemon"* ]] || return 0
|
|
111
|
+
printf '%s' "$pid"
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
# Newest mtime among the package's built lib/*.js, or empty when unbuilt.
|
|
115
|
+
newest_lib_mtime() {
|
|
116
|
+
local newest="" file stamp
|
|
117
|
+
for file in "$PLUGIN_DIR"/lib/*.js; do
|
|
118
|
+
[[ -f "$file" ]] || continue
|
|
119
|
+
stamp="$(stat -c %Y "$file" 2>/dev/null || true)"
|
|
120
|
+
[[ -n "$stamp" ]] || continue
|
|
121
|
+
if [[ -z "$newest" || "$stamp" -gt "$newest" ]]; then newest="$stamp"; fi
|
|
122
|
+
done
|
|
123
|
+
printf '%s' "$newest"
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
# Classify the daemon: fresh | stale | absent | unknown.
|
|
127
|
+
supervisor_daemon_state() {
|
|
128
|
+
SUPERVISOR_PID="$(supervisor_main_pid)"
|
|
129
|
+
SUPERVISOR_LIB="$(newest_lib_mtime)"
|
|
130
|
+
if [[ -z "$SUPERVISOR_PID" ]]; then
|
|
131
|
+
SUPERVISOR_VERDICT=absent
|
|
132
|
+
return 0
|
|
133
|
+
fi
|
|
134
|
+
SUPERVISOR_START="$(stat -c %Y "/proc/$SUPERVISOR_PID" 2>/dev/null || true)"
|
|
135
|
+
if [[ -z "$SUPERVISOR_START" || -z "$SUPERVISOR_LIB" ]]; then
|
|
136
|
+
SUPERVISOR_VERDICT=unknown
|
|
137
|
+
return 0
|
|
138
|
+
fi
|
|
139
|
+
if (( SUPERVISOR_LIB > SUPERVISOR_START )); then
|
|
140
|
+
SUPERVISOR_VERDICT=stale
|
|
141
|
+
else
|
|
142
|
+
SUPERVISOR_VERDICT=fresh
|
|
143
|
+
fi
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
print_supervisor_state() {
|
|
147
|
+
case "$SUPERVISOR_VERDICT" in
|
|
148
|
+
fresh) printf '[restart] supervisor daemon: fresh pid=%s start=%s lib=%s\n' "$SUPERVISOR_PID" "$SUPERVISOR_START" "$SUPERVISOR_LIB" ;;
|
|
149
|
+
stale) printf '[restart] supervisor daemon: stale pid=%s start=%s lib=%s (built after this daemon started)\n' "$SUPERVISOR_PID" "$SUPERVISOR_START" "$SUPERVISOR_LIB" ;;
|
|
150
|
+
unknown) printf '[restart] supervisor daemon: unknown pid=%s (lib build time unavailable)\n' "$SUPERVISOR_PID" ;;
|
|
151
|
+
*) printf '[restart] supervisor daemon: absent\n' ;;
|
|
152
|
+
esac
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
# Reload the daemon when it predates the newest build; report the verdict either
|
|
156
|
+
# way. A failed reload is fatal: continuing would hand the swap to the very
|
|
157
|
+
# process that can roll it back in a loop.
|
|
158
|
+
ensure_supervisor_current() {
|
|
159
|
+
supervisor_daemon_state
|
|
160
|
+
if [[ "$SUPERVISOR_VERDICT" != stale ]]; then
|
|
161
|
+
print_supervisor_state
|
|
162
|
+
return 0
|
|
163
|
+
fi
|
|
164
|
+
local old_pid="$SUPERVISOR_PID"
|
|
165
|
+
local wait_s="${DSH_SUPERVISOR_RELOAD_WAIT:-15}"
|
|
166
|
+
local new_pid=""
|
|
167
|
+
print_supervisor_state
|
|
168
|
+
printf '[restart] reloading the supervisor daemon so it runs the built lib/*.js\n'
|
|
169
|
+
kill -TERM "$old_pid" 2>/dev/null || true
|
|
170
|
+
for _ in $(seq 1 "$wait_s"); do
|
|
171
|
+
sleep 1
|
|
172
|
+
new_pid="$(supervisor_main_pid)"
|
|
173
|
+
if [[ -n "$new_pid" && "$new_pid" != "$old_pid" ]]; then break; fi
|
|
174
|
+
new_pid=""
|
|
175
|
+
done
|
|
176
|
+
if [[ -z "$new_pid" ]]; then
|
|
177
|
+
printf '[restart] FAIL: supervisor daemon %s did not come back within %ss\n' "$old_pid" "$wait_s" >&2
|
|
178
|
+
return 1
|
|
179
|
+
fi
|
|
180
|
+
printf '[restart] supervisor daemon: reloaded %s -> %s\n' "$old_pid" "$new_pid"
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
# Single-flight + log scoping (D6, spec 2026-09-13-supervisor-safety-net-design):
|
|
184
|
+
# take the SAME boot.lock and append the SAME boot-boundary sentinel as the
|
|
185
|
+
# supervisor's own performSingleBootRestart, by calling the package's helper —
|
|
186
|
+
# never by re-implementing either marker here. The lock is owned by this shell's
|
|
187
|
+
# PID ($$), so it stays valid for as long as the script runs, and it is released
|
|
188
|
+
# on every exit path via the trap below.
|
|
189
|
+
BOOT_GUARD_HELPER="${PLUGIN_DIR}/lib/bin.js"
|
|
190
|
+
boot_guard_acquire() {
|
|
191
|
+
if [[ ! -f "$BOOT_GUARD_HELPER" ]]; then
|
|
192
|
+
fail "boot.lock helper is missing: $BOOT_GUARD_HELPER (run 'pnpm build' in the supervisor package)" 69
|
|
193
|
+
fi
|
|
194
|
+
if ! node "$BOOT_GUARD_HELPER" boot-guard acquire --pid "$$" >>"$log" 2>&1; then
|
|
195
|
+
fail 'another boot holds boot.lock (a dsh web boot or another restart is in flight)' 75
|
|
196
|
+
fi
|
|
197
|
+
}
|
|
198
|
+
boot_guard_release() {
|
|
199
|
+
[[ -f "$BOOT_GUARD_HELPER" ]] || return 0
|
|
200
|
+
node "$BOOT_GUARD_HELPER" boot-guard release --pid "$$" >>"$log" 2>&1 || true
|
|
201
|
+
}
|
|
202
|
+
|
|
78
203
|
while (($#)); do
|
|
79
204
|
case "$1" in
|
|
80
205
|
--repo)
|
|
@@ -100,6 +225,14 @@ while (($#)); do
|
|
|
100
225
|
dry_run=true
|
|
101
226
|
shift
|
|
102
227
|
;;
|
|
228
|
+
--check-supervisor)
|
|
229
|
+
check_supervisor=true
|
|
230
|
+
shift
|
|
231
|
+
;;
|
|
232
|
+
--reload-supervisor)
|
|
233
|
+
reload_supervisor=true
|
|
234
|
+
shift
|
|
235
|
+
;;
|
|
103
236
|
-h|--help)
|
|
104
237
|
usage
|
|
105
238
|
exit 0
|
|
@@ -110,6 +243,18 @@ while (($#)); do
|
|
|
110
243
|
esac
|
|
111
244
|
done
|
|
112
245
|
|
|
246
|
+
# Standalone supervisor-daemon modes: they never touch dsh web, so they need no
|
|
247
|
+
# --repo, no listeners and no consent.
|
|
248
|
+
if [[ "$check_supervisor" == true ]]; then
|
|
249
|
+
supervisor_daemon_state
|
|
250
|
+
print_supervisor_state
|
|
251
|
+
exit 0
|
|
252
|
+
fi
|
|
253
|
+
if [[ "$reload_supervisor" == true ]]; then
|
|
254
|
+
ensure_supervisor_current || fail 'supervisor daemon could not be reloaded — fix it before swapping dsh web' 70
|
|
255
|
+
exit 0
|
|
256
|
+
fi
|
|
257
|
+
|
|
113
258
|
[[ -n "$repo" ]] || fail 'provide --repo or DSH_REPO' 64
|
|
114
259
|
[[ -f "$repo/package.json" ]] || fail "repo has no package.json: $repo" 64
|
|
115
260
|
if [[ "$dry_run" != true && "$confirmed" != true ]]; then
|
|
@@ -222,10 +367,19 @@ printf '[restart] stopping process tree: %s\n' "$(tr '\n' ' ' <<<"$tree_pids")"
|
|
|
222
367
|
|
|
223
368
|
# Mark this as an intentional restart before the port goes down, so
|
|
224
369
|
# dsh-web-supervisor's health poll does not race us with its own rollback.
|
|
225
|
-
#
|
|
370
|
+
# The boot.lock below is the single-flight half of the same contract; both are
|
|
371
|
+
# released on every exit path (success or failure) via the trap.
|
|
372
|
+
mkdir -p "$(dirname "$log")"
|
|
373
|
+
boot_guard_acquire
|
|
226
374
|
mkdir -p "$(dirname "$marker")"
|
|
227
375
|
date -Iseconds > "$marker"
|
|
228
|
-
trap 'rm -f "$marker"' EXIT
|
|
376
|
+
trap 'boot_guard_release; rm -f "$marker"' EXIT
|
|
377
|
+
|
|
378
|
+
# Reload a stale supervisor daemon BEFORE the swap: it is the process that
|
|
379
|
+
# decides whether the boot below is healthy, and a daemon running pre-build
|
|
380
|
+
# rules can roll it back in a loop. Done while dsh web is still up, so the
|
|
381
|
+
# fresh daemon observes the planned-restart marker written just above.
|
|
382
|
+
ensure_supervisor_current || fail 'stale supervisor daemon could not be reloaded — refusing to swap dsh web under old rollback rules' 70
|
|
229
383
|
|
|
230
384
|
if [[ "$systemd_managed" == true ]]; then
|
|
231
385
|
# systemctl stop is a clean, intentional stop -- Restart=always does not
|