@ddtcorex/dsh-maestro-supervisor 0.7.10 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,33 @@
1
+ import { type BootLockDeps } from './boot-lock.js';
2
+ /**
3
+ * D6 — one restart implementation per concern.
4
+ *
5
+ * `performSingleBootRestart` (the daemon's path) takes `boot.lock` and appends
6
+ * the `[supervisor] boot-boundary` sentinel through `markBootBoundary()`. The
7
+ * manual skill script (`skills/dsh-safe-restart/scripts/restart-dsh-web.sh`)
8
+ * must do exactly the same, or a human restart is neither serialized with a
9
+ * supervised one (two boots racing for :3082) nor scoped in the append-only log
10
+ * (the poller reads the previous boot's crash as this boot's).
11
+ *
12
+ * This module is that shared entry point: the script shells out to
13
+ * `lib/bin.js boot-guard acquire|release --pid <shell pid>` instead of writing
14
+ * either marker itself.
15
+ */
16
+ /** Log the boundary sentinel belongs to (isolated home wins when set, for rehearsals). */
17
+ export declare function bootBoundaryLogPath(env?: NodeJS.ProcessEnv): string;
18
+ export interface BootGuard {
19
+ acquired: boolean;
20
+ release: () => void;
21
+ }
22
+ /**
23
+ * Acquire the shared boot lock for `deps.pid` and append the boundary sentinel.
24
+ * The sentinel is written first-thing after the lock, matching
25
+ * `performSingleBootRestart` (marker before the port goes down).
26
+ */
27
+ export declare function acquireBootGuard(deps?: BootLockDeps, logPath?: string): BootGuard;
28
+ /** Release the lock only if `pid` still owns it (see releaseBootLock). */
29
+ export declare function releaseBootGuard(pid: number, deps?: BootLockDeps): boolean;
30
+ export declare const BOOT_GUARD_BUSY_EXIT = 3;
31
+ export declare const BOOT_GUARD_USAGE_EXIT = 64;
32
+ /** CLI used by the skill script; returns the process exit code. */
33
+ export declare function runBootGuardCli(args: string[], deps?: BootLockDeps, logPath?: string): number;
@@ -0,0 +1,68 @@
1
+ import * as path from 'node:path';
2
+ import { acquireBootLock, releaseBootLock } from './boot-lock.js';
3
+ import { markBootBoundary, dshWebLogPath } from './restart-guards.js';
4
+ /**
5
+ * D6 — one restart implementation per concern.
6
+ *
7
+ * `performSingleBootRestart` (the daemon's path) takes `boot.lock` and appends
8
+ * the `[supervisor] boot-boundary` sentinel through `markBootBoundary()`. The
9
+ * manual skill script (`skills/dsh-safe-restart/scripts/restart-dsh-web.sh`)
10
+ * must do exactly the same, or a human restart is neither serialized with a
11
+ * supervised one (two boots racing for :3082) nor scoped in the append-only log
12
+ * (the poller reads the previous boot's crash as this boot's).
13
+ *
14
+ * This module is that shared entry point: the script shells out to
15
+ * `lib/bin.js boot-guard acquire|release --pid <shell pid>` instead of writing
16
+ * either marker itself.
17
+ */
18
+ /** Log the boundary sentinel belongs to (isolated home wins when set, for rehearsals). */
19
+ export function bootBoundaryLogPath(env = process.env) {
20
+ if (env.DSH_WEB_LOG)
21
+ return env.DSH_WEB_LOG;
22
+ if (env.DSH_HOME)
23
+ return path.join(env.DSH_HOME, 'dsh-web.log');
24
+ return dshWebLogPath();
25
+ }
26
+ /**
27
+ * Acquire the shared boot lock for `deps.pid` and append the boundary sentinel.
28
+ * The sentinel is written first-thing after the lock, matching
29
+ * `performSingleBootRestart` (marker before the port goes down).
30
+ */
31
+ export function acquireBootGuard(deps = {}, logPath) {
32
+ const lock = acquireBootLock(deps);
33
+ if (!lock)
34
+ return { acquired: false, release: () => { } };
35
+ markBootBoundary(logPath ?? bootBoundaryLogPath());
36
+ return { acquired: true, release: lock.release };
37
+ }
38
+ /** Release the lock only if `pid` still owns it (see releaseBootLock). */
39
+ export function releaseBootGuard(pid, deps = {}) {
40
+ return releaseBootLock(pid, deps);
41
+ }
42
+ export const BOOT_GUARD_BUSY_EXIT = 3;
43
+ export const BOOT_GUARD_USAGE_EXIT = 64;
44
+ /** CLI used by the skill script; returns the process exit code. */
45
+ export function runBootGuardCli(args, deps = {}, logPath) {
46
+ const action = args[0];
47
+ const pidIdx = args.indexOf('--pid');
48
+ const pid = pidIdx === -1 ? Number.NaN : Number(args[pidIdx + 1]);
49
+ if (!Number.isInteger(pid) || pid <= 0) {
50
+ console.error('boot-guard: --pid <owner pid> is required (pass the shell PID: $$)');
51
+ return BOOT_GUARD_USAGE_EXIT;
52
+ }
53
+ if (action === 'acquire') {
54
+ const guard = acquireBootGuard({ ...deps, pid }, logPath);
55
+ if (!guard.acquired) {
56
+ console.error('busy: another boot holds boot.lock');
57
+ return BOOT_GUARD_BUSY_EXIT;
58
+ }
59
+ console.log('acquired');
60
+ return 0;
61
+ }
62
+ if (action === 'release') {
63
+ releaseBootGuard(pid, deps);
64
+ return 0;
65
+ }
66
+ console.error(`boot-guard: unknown action: ${action ?? ''} (expected acquire|release)`);
67
+ return BOOT_GUARD_USAGE_EXIT;
68
+ }
@@ -0,0 +1,56 @@
1
+ /**
2
+ * One-boot-at-a-time guard (spec D4, incident 2026-09-13). Every path that
3
+ * starts dsh web takes this lock from the moment the restart is issued until
4
+ * the port answers, so a racing tick, the dsh_web_restart hand-off, or a
5
+ * rollback that fires while the first boot is still coming up skips instead of
6
+ * producing the second systemd start that made the incident's log tail toxic.
7
+ *
8
+ * The lock is an atomically created file (`O_EXCL`) carrying `{pid, ts}`. An
9
+ * existing lock is honored unless its owner is gone or it has outlived
10
+ * `staleMs` (the boot budget). Node exposes no flock(2) binding; O_EXCL plus
11
+ * PID liveness gives the same mutual exclusion deterministically, and every
12
+ * side effect is injectable so tests never touch the real ~/.dsh/.supervisor.
13
+ */
14
+ export declare const DEFAULT_BOOT_LOCK_STALE_MS = 180000;
15
+ export interface BootLockDeps {
16
+ lockPath?: string;
17
+ createExclusive?: (p: string, body: string) => boolean;
18
+ readLock?: (p: string) => string | undefined;
19
+ remove?: (p: string) => void;
20
+ pidAlive?: (pid: number) => boolean;
21
+ now?: () => number;
22
+ staleMs?: number;
23
+ sleep?: (ms: number) => Promise<void>;
24
+ portUp?: () => Promise<boolean>;
25
+ waitTimeoutMs?: number;
26
+ pollMs?: number;
27
+ /**
28
+ * Owner recorded in the lock file; defaults to this process. The skill script
29
+ * (D6) holds the lock with the shell's own PID, so liveness stays meaningful
30
+ * while the shell performs the restart.
31
+ */
32
+ pid?: number;
33
+ }
34
+ export interface BootLock {
35
+ path: string;
36
+ release: () => void;
37
+ }
38
+ export declare function bootLockPath(): string;
39
+ /** Take the boot lock, or return undefined when another live boot owns it. */
40
+ export declare function acquireBootLock(deps?: BootLockDeps): BootLock | undefined;
41
+ /**
42
+ * Release the lock only when `pid` still owns it. A boot that lost the race (or
43
+ * whose stale lock was taken over) must never delete the winner's lock file.
44
+ */
45
+ export declare function releaseBootLock(pid: number, deps?: BootLockDeps): boolean;
46
+ /** Poll the port until it answers or the boot budget expires. */
47
+ export declare function waitForPort(deps?: BootLockDeps): Promise<boolean>;
48
+ /**
49
+ * Run `fn` under the boot lock, then hold the lock until the port answers (or
50
+ * the boot budget expires). Returns `{ acquired: false }` — without running
51
+ * `fn` — when another boot already owns the lock.
52
+ */
53
+ export declare function withBootLock<T>(fn: () => Promise<T>, deps?: BootLockDeps): Promise<{
54
+ acquired: boolean;
55
+ value?: T;
56
+ }>;
@@ -0,0 +1,155 @@
1
+ import * as fs from 'node:fs';
2
+ import * as path from 'node:path';
3
+ import * as os from 'node:os';
4
+ /**
5
+ * One-boot-at-a-time guard (spec D4, incident 2026-09-13). Every path that
6
+ * starts dsh web takes this lock from the moment the restart is issued until
7
+ * the port answers, so a racing tick, the dsh_web_restart hand-off, or a
8
+ * rollback that fires while the first boot is still coming up skips instead of
9
+ * producing the second systemd start that made the incident's log tail toxic.
10
+ *
11
+ * The lock is an atomically created file (`O_EXCL`) carrying `{pid, ts}`. An
12
+ * existing lock is honored unless its owner is gone or it has outlived
13
+ * `staleMs` (the boot budget). Node exposes no flock(2) binding; O_EXCL plus
14
+ * PID liveness gives the same mutual exclusion deterministically, and every
15
+ * side effect is injectable so tests never touch the real ~/.dsh/.supervisor.
16
+ */
17
+ export const DEFAULT_BOOT_LOCK_STALE_MS = 180_000;
18
+ export function bootLockPath() {
19
+ return process.env.DSH_SUPERVISOR_BOOT_LOCK ?? path.join(os.homedir(), '.dsh/.supervisor/boot.lock');
20
+ }
21
+ function resolved(deps) {
22
+ return {
23
+ lockPath: deps.lockPath ?? bootLockPath(),
24
+ createExclusive: deps.createExclusive ?? ((p, body) => {
25
+ try {
26
+ fs.mkdirSync(path.dirname(p), { recursive: true });
27
+ fs.writeFileSync(p, body, { flag: 'wx', mode: 0o600 });
28
+ return true;
29
+ }
30
+ catch {
31
+ return false;
32
+ }
33
+ }),
34
+ readLock: deps.readLock ?? ((p) => {
35
+ try {
36
+ return fs.readFileSync(p, 'utf8');
37
+ }
38
+ catch {
39
+ return undefined;
40
+ }
41
+ }),
42
+ remove: deps.remove ?? ((p) => { try {
43
+ fs.unlinkSync(p);
44
+ }
45
+ catch { } }),
46
+ pidAlive: deps.pidAlive ?? ((pid) => {
47
+ if (!Number.isInteger(pid) || pid <= 0)
48
+ return false;
49
+ if (pid === process.pid)
50
+ return true;
51
+ try {
52
+ process.kill(pid, 0);
53
+ return true;
54
+ }
55
+ catch {
56
+ return false;
57
+ }
58
+ }),
59
+ now: deps.now ?? (() => Date.now()),
60
+ staleMs: deps.staleMs ?? DEFAULT_BOOT_LOCK_STALE_MS,
61
+ sleep: deps.sleep ?? ((ms) => new Promise(r => setTimeout(r, ms))),
62
+ portUp: deps.portUp ?? defaultPortUp,
63
+ waitTimeoutMs: deps.waitTimeoutMs ?? DEFAULT_BOOT_LOCK_STALE_MS,
64
+ pollMs: deps.pollMs ?? 3_000,
65
+ pid: deps.pid ?? process.pid,
66
+ };
67
+ }
68
+ function parseLock(raw) {
69
+ if (!raw)
70
+ return undefined;
71
+ try {
72
+ const j = JSON.parse(raw);
73
+ if (typeof j?.pid === 'number' && typeof j?.ts === 'number')
74
+ return { pid: j.pid, ts: j.ts };
75
+ }
76
+ catch { }
77
+ return undefined;
78
+ }
79
+ /** Take the boot lock, or return undefined when another live boot owns it. */
80
+ export function acquireBootLock(deps = {}) {
81
+ const d = resolved(deps);
82
+ const body = JSON.stringify({ pid: d.pid, ts: d.now() });
83
+ const take = () => d.createExclusive(d.lockPath, body) ? { path: d.lockPath, release: () => d.remove(d.lockPath) } : undefined;
84
+ const first = take();
85
+ if (first)
86
+ return first;
87
+ const existing = parseLock(d.readLock(d.lockPath));
88
+ const stale = existing === undefined || !d.pidAlive(existing.pid) || d.now() - existing.ts > d.staleMs;
89
+ if (!stale)
90
+ return undefined;
91
+ d.remove(d.lockPath);
92
+ return take();
93
+ }
94
+ /**
95
+ * Release the lock only when `pid` still owns it. A boot that lost the race (or
96
+ * whose stale lock was taken over) must never delete the winner's lock file.
97
+ */
98
+ export function releaseBootLock(pid, deps = {}) {
99
+ const d = resolved(deps);
100
+ const existing = parseLock(d.readLock(d.lockPath));
101
+ if (existing === undefined || existing.pid !== pid)
102
+ return false;
103
+ d.remove(d.lockPath);
104
+ return true;
105
+ }
106
+ async function defaultPortUp() {
107
+ try {
108
+ const ctrl = new AbortController();
109
+ const t = setTimeout(() => ctrl.abort(), 2000);
110
+ try {
111
+ // Any HTTP answer (200/303/401) means the boot is serving.
112
+ await fetch('http://127.0.0.1:3080/', { signal: ctrl.signal });
113
+ return true;
114
+ }
115
+ finally {
116
+ clearTimeout(t);
117
+ }
118
+ }
119
+ catch {
120
+ return false;
121
+ }
122
+ }
123
+ /** Poll the port until it answers or the boot budget expires. */
124
+ export async function waitForPort(deps = {}) {
125
+ const d = resolved(deps);
126
+ const deadline = d.now() + d.waitTimeoutMs;
127
+ for (;;) {
128
+ try {
129
+ if (await d.portUp())
130
+ return true;
131
+ }
132
+ catch { }
133
+ if (d.now() >= deadline)
134
+ return false;
135
+ await d.sleep(d.pollMs);
136
+ }
137
+ }
138
+ /**
139
+ * Run `fn` under the boot lock, then hold the lock until the port answers (or
140
+ * the boot budget expires). Returns `{ acquired: false }` — without running
141
+ * `fn` — when another boot already owns the lock.
142
+ */
143
+ export async function withBootLock(fn, deps = {}) {
144
+ const lock = acquireBootLock(deps);
145
+ if (!lock)
146
+ return { acquired: false };
147
+ try {
148
+ const value = await fn();
149
+ await waitForPort(deps);
150
+ return { acquired: true, value };
151
+ }
152
+ finally {
153
+ lock.release();
154
+ }
155
+ }
package/lib/cli.d.ts CHANGED
@@ -1,14 +1,31 @@
1
+ export interface RestoreResult {
2
+ /** Snapshot id that was restored. */
3
+ target: string;
4
+ /** Files (and symlinks) restored into the live DSH home. */
5
+ restored: number;
6
+ /** Entries that were not restored, each with a reason. Never silent. */
7
+ skipped: Array<{
8
+ path: string;
9
+ reason: string;
10
+ }>;
11
+ }
1
12
  /**
2
13
  * Copy a chosen LKG snapshot back into the DSH home. Recent snapshots are
3
14
  * tried newest-first, skipping any that still carry a failing plugin so a
4
- * broken bundle is not restored. `sessions/` is deliberately NOT restored
5
- * session logs are append-only truth and rolling them back to a snapshot
6
- * would drop every turn recorded after that snapshot.
7
- * @returns the rolled-back snapshot id.
15
+ * broken bundle is not restored. `sessions/` and every other runtime-data entry
16
+ * is deliberately NOT restored (D1) — session logs are append-only truth and
17
+ * rolling them back to a snapshot would drop every turn recorded after that
18
+ * snapshot.
19
+ *
20
+ * Never throws for a per-entry failure: the failures are collected into
21
+ * `skipped` and the caller reports them (D2/D4). The only throw left is "there
22
+ * is no snapshot at all", which the caller must know about.
23
+ *
24
+ * @returns the restore summary (snapshot id + what was/was not restored).
8
25
  */
9
26
  export declare function rollbackLKG(opts: {
10
27
  dshHome: string;
11
28
  lkgRoot: string;
12
29
  failingPlugin?: string;
13
- }): Promise<string>;
30
+ }): Promise<RestoreResult>;
14
31
  export declare function runCli(args: string[]): Promise<void>;
package/lib/cli.js CHANGED
@@ -1,18 +1,93 @@
1
1
  import { Supervisor } from './supervisor.js';
2
2
  import { pollHealth } from './health-poller.js';
3
- import { writeLKG, verifyLKG } from './snapshot.js';
3
+ import { writeLKG, verifyLKG, isLkgExcluded, failureReason } from './snapshot.js';
4
4
  import * as fs from 'node:fs';
5
5
  import * as path from 'node:path';
6
6
  import * as os from 'node:os';
7
- import { resolveHarnessRoot, resolveDeepseekHarnessDir } from './paths.js';
8
- import { buildKillStalePortsCommand, isSelfCopyError, checkPlannedRestart, writePlannedRestart, readRestartRequest, clearPlannedRestart } from './restart-guards.js';
7
+ import { resolveHarnessRoot } from './paths.js';
8
+ import { isSelfCopyError, checkPlannedRestart, readRestartRequest, clearPlannedRestart } from './restart-guards.js';
9
+ import { readSupervisorConfig } from './config.js';
10
+ /**
11
+ * Make a destination (recursively) writable before overwriting it (D3).
12
+ *
13
+ * `fs.cpSync` copies into an existing destination file with
14
+ * `O_WRONLY|O_CREAT|O_TRUNC`, so a mode-`0400` object fails with EACCES and
15
+ * aborts the whole restore — the exact 2026-09-13 failure
16
+ * (`EACCES, Permission denied '.../attachments/v1/objects/f8'`). Only entries
17
+ * that actually lack the needed bit are chmod-ed, so a healthy tree costs
18
+ * stats, not a chmod per file.
19
+ */
20
+ function makeWritable(target) {
21
+ let st;
22
+ try {
23
+ st = fs.lstatSync(target);
24
+ }
25
+ catch {
26
+ return;
27
+ }
28
+ if (st.isSymbolicLink())
29
+ return;
30
+ if (st.isDirectory()) {
31
+ if ((st.mode & 0o300) !== 0o300) {
32
+ try {
33
+ fs.chmodSync(target, st.mode | 0o300);
34
+ }
35
+ catch { }
36
+ }
37
+ let names = [];
38
+ try {
39
+ names = fs.readdirSync(target);
40
+ }
41
+ catch {
42
+ return;
43
+ }
44
+ for (const name of names)
45
+ makeWritable(path.join(target, name));
46
+ return;
47
+ }
48
+ if ((st.mode & 0o200) === 0) {
49
+ try {
50
+ fs.chmodSync(target, st.mode | 0o200);
51
+ }
52
+ catch { }
53
+ }
54
+ }
55
+ /** Files (and symlinks) a restore of this source entry would write. */
56
+ function countRestorable(srcPath) {
57
+ try {
58
+ const st = fs.lstatSync(srcPath);
59
+ if (st.isDirectory())
60
+ return walkFiles(srcPath).length;
61
+ return 1;
62
+ }
63
+ catch {
64
+ return 0;
65
+ }
66
+ }
67
+ function walkFiles(dir, base = dir) {
68
+ const out = [];
69
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
70
+ const full = path.join(dir, entry.name);
71
+ if (entry.isDirectory())
72
+ out.push(...walkFiles(full, base));
73
+ else if (entry.isFile())
74
+ out.push(path.relative(base, full));
75
+ }
76
+ return out;
77
+ }
9
78
  /**
10
79
  * Copy a chosen LKG snapshot back into the DSH home. Recent snapshots are
11
80
  * tried newest-first, skipping any that still carry a failing plugin so a
12
- * broken bundle is not restored. `sessions/` is deliberately NOT restored
13
- * session logs are append-only truth and rolling them back to a snapshot
14
- * would drop every turn recorded after that snapshot.
15
- * @returns the rolled-back snapshot id.
81
+ * broken bundle is not restored. `sessions/` and every other runtime-data entry
82
+ * is deliberately NOT restored (D1) — session logs are append-only truth and
83
+ * rolling them back to a snapshot would drop every turn recorded after that
84
+ * snapshot.
85
+ *
86
+ * Never throws for a per-entry failure: the failures are collected into
87
+ * `skipped` and the caller reports them (D2/D4). The only throw left is "there
88
+ * is no snapshot at all", which the caller must know about.
89
+ *
90
+ * @returns the restore summary (snapshot id + what was/was not restored).
16
91
  */
17
92
  export async function rollbackLKG(opts) {
18
93
  const { dshHome, lkgRoot, failingPlugin } = opts;
@@ -49,11 +124,16 @@ export async function rollbackLKG(opts) {
49
124
  }
50
125
  const target = chosen ?? entries[entries.length - 1];
51
126
  const src = path.join(lkgRoot, target);
127
+ const skipped = [];
128
+ let restored = 0;
52
129
  for (const entry of fs.readdirSync(src)) {
53
130
  if (entry === 'manifest.json')
54
131
  continue;
55
- if (entry === 'sessions') {
56
- console.log('[supervisor] rollback keeps live sessions (append-only truth) skipping sessions/');
132
+ if (isLkgExcluded(entry)) {
133
+ // Legacy snapshots (taken before D1) still carry runtime data. Restoring
134
+ // a stale sessions/ over live sessions can lose a log, and a 0400
135
+ // attachment blob is what aborted this very path — report, never restore.
136
+ skipped.push({ path: entry, reason: 'runtime data excluded from the LKG scope (not required to boot)' });
57
137
  continue;
58
138
  }
59
139
  const srcPath = path.join(src, entry);
@@ -65,15 +145,17 @@ export async function rollbackLKG(opts) {
65
145
  continue;
66
146
  }
67
147
  catch { }
148
+ makeWritable(destPath);
68
149
  fs.cpSync(srcPath, destPath, { recursive: true, force: true });
150
+ restored += countRestorable(srcPath);
69
151
  }
70
152
  catch (e) {
71
153
  if (isSelfCopyError(String(e?.message ?? '')))
72
154
  continue;
73
- throw e;
155
+ skipped.push({ path: entry, reason: failureReason(e) });
74
156
  }
75
157
  }
76
- return target;
158
+ return { target, restored, skipped };
77
159
  }
78
160
  export async function runCli(args) {
79
161
  const cmd = args[2] ?? '--help';
@@ -86,9 +168,14 @@ Commands:
86
168
  logs Tail supervisor reports
87
169
  rollback --to <ts> Rollback to LKG <ts>
88
170
  resume [--within <dur>] List interrupted sessions (filter by time, e.g. 5m, 30s, 1h)
171
+ boot-guard acquire|release --pid <pid> Take/release boot.lock + boot-boundary (used by the safe-restart script)
89
172
  `);
90
173
  return;
91
174
  }
175
+ if (cmd === 'boot-guard') {
176
+ const { runBootGuardCli } = await import('./boot-guard.js');
177
+ process.exit(runBootGuardCli(args.slice(3)));
178
+ }
92
179
  if (cmd === 'status') {
93
180
  const health = await pollHealth();
94
181
  console.log(`up: ${health.up}, httpCode: ${health.httpCode}, error: ${health.error ?? 'none'}`);
@@ -186,8 +273,16 @@ Commands:
186
273
  failingPlugin = m[0].replace(/^@ddtcorex\//, '');
187
274
  }
188
275
  catch { }
189
- const target = await rollbackLKG({ dshHome, lkgRoot, failingPlugin });
190
- console.log(`[supervisor] rolled back to ${target}${failingPlugin ? ` (avoiding ${failingPlugin})` : ''}`);
276
+ const res = await rollbackLKG({ dshHome, lkgRoot, failingPlugin });
277
+ console.log(`[supervisor] rolled back to ${res.target}${res.restored} file(s) restored${failingPlugin ? ` (avoiding ${failingPlugin})` : ''}`);
278
+ // D4: a partial restore is surfaced loudly, never silently.
279
+ if (res.skipped.length) {
280
+ console.log(`[supervisor] ROLLBACK PARTIAL: ${res.skipped.length} entr(ies) not restored`);
281
+ for (const s of res.skipped.slice(0, 10))
282
+ console.log(`[supervisor] skipped ${s.path}: ${s.reason}`);
283
+ if (res.skipped.length > 10)
284
+ console.log(`[supervisor] … ${res.skipped.length - 10} more`);
285
+ }
191
286
  // Reconcile node_modules from restored package.json (critical for link: deps)
192
287
  try {
193
288
  execSync('pnpm --dir ~/.dsh/profiles/web install --silent', { timeout: 30000, stdio: 'pipe' });
@@ -196,52 +291,23 @@ Commands:
196
291
  catch (e) {
197
292
  console.log(`[supervisor] pnpm install failed: ${e?.message ?? String(e)}`);
198
293
  }
294
+ return res;
199
295
  },
200
296
  restartWeb: async () => {
201
- const { execSync } = await import('node:child_process');
202
- // Single-owner: mark planned restart 30s before any systemctl/nohup
203
- // so pollHealth + tick suppress the transient down (no double restart).
297
+ // One implementation of the single-boot restart: marker (TTL = boot
298
+ // budget) boot.lock serialized systemd start → direct-node nohup
299
+ // only on hosts where the unit does not exist (D4/D5).
300
+ const { performSingleBootRestart, DEFAULT_BOOT_GRACE_MS } = await import('./restart-web.js');
301
+ let grace = DEFAULT_BOOT_GRACE_MS;
204
302
  try {
205
- writePlannedRestart(30000);
303
+ const cfg = await readSupervisorConfig();
304
+ if (typeof cfg.bootGraceMs === 'number' && cfg.bootGraceMs > 0)
305
+ grace = cfg.bootGraceMs;
206
306
  }
207
307
  catch { }
208
- // Kill stale MainThread holding 3080 before any restart attempt
209
- // (EADDRINUSE crash leaves old pid alive with http 200; new start would fail)
210
- // Scoped to :3080 only an unfiltered `ss -tlnp` matches every
211
- // listening process on the host, not just dsh web (regression: killed
212
- // unrelated services like redis/horizon on every restart).
213
- try {
214
- execSync(buildKillStalePortsCommand(), { timeout: 5000, stdio: 'pipe' });
215
- }
216
- catch { }
217
- // Prefer systemd — if dsh-web.service is installed, restart/start it
218
- try {
219
- execSync('systemctl --user is-active --quiet dsh-web.service && systemctl --user restart dsh-web.service || systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
220
- console.log('[supervisor] restarted dsh-web via systemd');
221
- return;
222
- }
223
- catch { }
224
- // Check if unit exists but not active — try start
225
- try {
226
- execSync('systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
227
- console.log('[supervisor] started dsh-web via systemd (fallback)');
228
- return;
229
- }
230
- catch { }
231
- // Last fallback: detached direct node (portable — sources nvm directly, falls back to system node)
232
- try {
233
- const harnessRoot = resolveDeepseekHarnessDir();
234
- const logPath = path.join(os.homedir(), '.dsh/dsh-web.log');
235
- try {
236
- writePlannedRestart(30000);
237
- }
238
- catch { }
239
- execSync(`setsid nohup bash -c 'export NVM_DIR="$HOME/.nvm"; [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; cd ${JSON.stringify(harnessRoot)} && exec node --import tsx/esm apps/cli/src/bin.ts web --no-open >> ${JSON.stringify(logPath)} 2>&1' &`, { timeout: 5000 });
240
- console.log('[supervisor] started dsh-web via nohup fallback (direct node, portable)');
241
- }
242
- catch (e) {
243
- throw new Error(`restartWeb failed: ${e?.message ?? String(e)}`);
244
- }
308
+ const res = await performSingleBootRestart({ bootGraceMs: grace });
309
+ if (!res.restarted)
310
+ console.log(`[supervisor] restart skipped: ${res.reason ?? 'boot lock held'}`);
245
311
  },
246
312
  notify: async (msg) => console.log(`[notify] ${msg}`),
247
313
  isPlannedRestartActive: () => checkPlannedRestart(),
@@ -4,8 +4,17 @@ export interface HealthState {
4
4
  error?: string;
5
5
  degraded?: boolean;
6
6
  logTail?: string;
7
+ /** Boot verdict for the unit start this poll observed (see bootFreshness). */
8
+ bootPhase?: BootFreshness;
7
9
  }
10
+ /**
11
+ * Wall-clock epoch ms of the current dsh-web unit start, or undefined when
12
+ * systemd does not manage the unit (portable host) or the lookup is disabled.
13
+ * `ActiveEnterTimestampMonotonic` is deliberately gone: a monotonic reading
14
+ * cannot be compared with a timestamp-less append-only log.
15
+ */
8
16
  export declare function getActiveEnterMs(): number | undefined;
17
+ /** @deprecated kept for callers; identical to getActiveEnterMs(). */
9
18
  export declare function getActiveEnterWallMs(): number | undefined;
10
19
  export interface PollHealthOpts {
11
20
  fetch?: () => Promise<{
@@ -16,8 +25,66 @@ export interface PollHealthOpts {
16
25
  logTail?: () => Promise<string>;
17
26
  url?: string;
18
27
  timeoutMs?: number;
19
- /** injectable for tests — overrides systemctl lookup */
28
+ /** injectable for tests — overrides the systemctl lookup */
20
29
  getActiveEnterMs?: () => number | undefined;
30
+ /** Wall-clock epoch ms of the current dsh-web unit start (see bootFreshness). */
31
+ activeEnterAtMs?: number;
32
+ /** Boot budget in ms; a younger boot without its own success marker is 'booting'. */
33
+ bootGraceMs?: number;
21
34
  }
35
+ export type BootFreshness = 'unknown' | 'booting' | 'settled';
36
+ /**
37
+ * Decide whether the current unit start can already be judged.
38
+ *
39
+ * `activeEnterAtMs` is WALL-CLOCK epoch ms from
40
+ * `systemctl --user show -p ActiveEnterTimestamp dsh-web.service`.
41
+ * Deliberately NOT `ActiveEnterTimestampMonotonic`: a monotonic reading can
42
+ * never be compared against an append-only log that carries no timestamps, and
43
+ * that mismatch is why the previous scan filter silently fell through and let
44
+ * the previous boot's crash text be read as this boot's (incident 2026-09-13).
45
+ *
46
+ * - 'unknown' — no boot anchor (systemd absent, lookup disabled, clock skew):
47
+ * behave exactly as before the fix; never suppress anything.
48
+ * - 'booting' — the unit started less than `bootGraceMs` ago and has not yet
49
+ * proven itself by actually serving a request: weak failures are
50
+ * suppressed and log lines are inconclusive.
51
+ * - 'settled' — the boot proved itself, or the grace window expired: judge
52
+ * normally.
53
+ */
54
+ export declare function bootFreshness(opts: {
55
+ activeEnterAtMs?: number;
56
+ now: number;
57
+ bootGraceMs: number;
58
+ /**
59
+ * Did THIS poll receive a serving response (200/401)?
60
+ *
61
+ * Deliberately not "the log shows `dsh web: http`": that line is printed the
62
+ * moment the raw webserver binds, seconds into a boot that can take ~90s to
63
+ * finish loading the plugin tree. Crediting it as proof ended the grace while
64
+ * the proxy in front was still coming up, so probes that hung during the rest
65
+ * of the boot were judged as crashes — the 2026-09-13 restart loop.
66
+ */
67
+ probeSucceeded: boolean;
68
+ }): BootFreshness;
69
+ /**
70
+ * Classify a fetch failure by what it says about the process.
71
+ *
72
+ * 'refused' — nothing is listening, so the process is gone: a strong down
73
+ * signal that must never be masked by a boot grace.
74
+ * 'timeout' — something may be alive but slow: weak, only meaningful once the
75
+ * boot grace expired.
76
+ * 'other' — anything we cannot attribute.
77
+ *
78
+ * Walks the `cause` chain because undici surfaces a refused connection as
79
+ * `TypeError: fetch failed` with the real `ECONNREFUSED` on `cause`.
80
+ */
81
+ export declare function classifyFetchFailure(err: unknown): 'refused' | 'timeout' | 'other';
82
+ export declare const SUCCESS_MARKER = "dsh web: http";
83
+ /** Index of the last boot-boundary line in the tail, or -1 when it predates the tail. */
84
+ export declare function lastBootBoundaryIndex(lines: string[]): number;
85
+ /** Index of the last "dsh web: http" success marker (`lowerLines` must be lower-cased). */
86
+ export declare function lastSuccessMarkerIndex(lowerLines: string[]): number;
87
+ /** Wall-clock ms parsed from a boot-boundary line, or undefined when unparseable. */
88
+ export declare function parseBootBoundaryMs(line: string): number | undefined;
22
89
  export declare function pollHealth(opts?: PollHealthOpts): Promise<HealthState>;
23
90
  export declare function collectLogTail(): Promise<string>;