@ddtcorex/dsh-maestro-supervisor 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/boot-guard.d.ts +33 -0
- package/lib/boot-guard.js +68 -0
- package/lib/boot-lock.d.ts +56 -0
- package/lib/boot-lock.js +155 -0
- package/lib/cli.d.ts +22 -5
- package/lib/cli.js +120 -54
- package/lib/health-poller.d.ts +68 -1
- package/lib/health-poller.js +191 -133
- package/lib/plugin.js +15 -16
- package/lib/restart-exec.d.ts +35 -0
- package/lib/restart-exec.js +60 -0
- package/lib/restart-guards.d.ts +18 -0
- package/lib/restart-guards.js +33 -0
- package/lib/restart-tool.d.ts +2 -1
- package/lib/restart-tool.js +19 -1
- package/lib/restart-web.d.ts +26 -0
- package/lib/restart-web.js +58 -0
- package/lib/resume.d.ts +10 -0
- package/lib/resume.js +45 -28
- package/lib/snapshot.d.ts +52 -15
- package/lib/snapshot.js +168 -18
- package/lib/supervisor.d.ts +21 -4
- package/lib/supervisor.js +78 -36
- package/package.json +1 -1
- package/skills/dsh-safe-restart/SKILL.md +32 -2
- package/skills/dsh-safe-restart/scripts/restart-dsh-web.sh +162 -8
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { type BootLockDeps } from './boot-lock.js';
|
|
2
|
+
/**
|
|
3
|
+
* D6 — one restart implementation per concern.
|
|
4
|
+
*
|
|
5
|
+
* `performSingleBootRestart` (the daemon's path) takes `boot.lock` and appends
|
|
6
|
+
* the `[supervisor] boot-boundary` sentinel through `markBootBoundary()`. The
|
|
7
|
+
* manual skill script (`skills/dsh-safe-restart/scripts/restart-dsh-web.sh`)
|
|
8
|
+
* must do exactly the same, or a human restart is neither serialized with a
|
|
9
|
+
* supervised one (two boots racing for :3082) nor scoped in the append-only log
|
|
10
|
+
* (the poller reads the previous boot's crash as this boot's).
|
|
11
|
+
*
|
|
12
|
+
* This module is that shared entry point: the script shells out to
|
|
13
|
+
* `lib/bin.js boot-guard acquire|release --pid <shell pid>` instead of writing
|
|
14
|
+
* either marker itself.
|
|
15
|
+
*/
|
|
16
|
+
/** Log the boundary sentinel belongs to (isolated home wins when set, for rehearsals). */
|
|
17
|
+
export declare function bootBoundaryLogPath(env?: NodeJS.ProcessEnv): string;
|
|
18
|
+
export interface BootGuard {
|
|
19
|
+
acquired: boolean;
|
|
20
|
+
release: () => void;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Acquire the shared boot lock for `deps.pid` and append the boundary sentinel.
|
|
24
|
+
* The sentinel is written first-thing after the lock, matching
|
|
25
|
+
* `performSingleBootRestart` (marker before the port goes down).
|
|
26
|
+
*/
|
|
27
|
+
export declare function acquireBootGuard(deps?: BootLockDeps, logPath?: string): BootGuard;
|
|
28
|
+
/** Release the lock only if `pid` still owns it (see releaseBootLock). */
|
|
29
|
+
export declare function releaseBootGuard(pid: number, deps?: BootLockDeps): boolean;
|
|
30
|
+
export declare const BOOT_GUARD_BUSY_EXIT = 3;
|
|
31
|
+
export declare const BOOT_GUARD_USAGE_EXIT = 64;
|
|
32
|
+
/** CLI used by the skill script; returns the process exit code. */
|
|
33
|
+
export declare function runBootGuardCli(args: string[], deps?: BootLockDeps, logPath?: string): number;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import * as path from 'node:path';
|
|
2
|
+
import { acquireBootLock, releaseBootLock } from './boot-lock.js';
|
|
3
|
+
import { markBootBoundary, dshWebLogPath } from './restart-guards.js';
|
|
4
|
+
/**
|
|
5
|
+
* D6 — one restart implementation per concern.
|
|
6
|
+
*
|
|
7
|
+
* `performSingleBootRestart` (the daemon's path) takes `boot.lock` and appends
|
|
8
|
+
* the `[supervisor] boot-boundary` sentinel through `markBootBoundary()`. The
|
|
9
|
+
* manual skill script (`skills/dsh-safe-restart/scripts/restart-dsh-web.sh`)
|
|
10
|
+
* must do exactly the same, or a human restart is neither serialized with a
|
|
11
|
+
* supervised one (two boots racing for :3082) nor scoped in the append-only log
|
|
12
|
+
* (the poller reads the previous boot's crash as this boot's).
|
|
13
|
+
*
|
|
14
|
+
* This module is that shared entry point: the script shells out to
|
|
15
|
+
* `lib/bin.js boot-guard acquire|release --pid <shell pid>` instead of writing
|
|
16
|
+
* either marker itself.
|
|
17
|
+
*/
|
|
18
|
+
/** Log the boundary sentinel belongs to (isolated home wins when set, for rehearsals). */
|
|
19
|
+
export function bootBoundaryLogPath(env = process.env) {
|
|
20
|
+
if (env.DSH_WEB_LOG)
|
|
21
|
+
return env.DSH_WEB_LOG;
|
|
22
|
+
if (env.DSH_HOME)
|
|
23
|
+
return path.join(env.DSH_HOME, 'dsh-web.log');
|
|
24
|
+
return dshWebLogPath();
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Acquire the shared boot lock for `deps.pid` and append the boundary sentinel.
|
|
28
|
+
* The sentinel is written first-thing after the lock, matching
|
|
29
|
+
* `performSingleBootRestart` (marker before the port goes down).
|
|
30
|
+
*/
|
|
31
|
+
export function acquireBootGuard(deps = {}, logPath) {
|
|
32
|
+
const lock = acquireBootLock(deps);
|
|
33
|
+
if (!lock)
|
|
34
|
+
return { acquired: false, release: () => { } };
|
|
35
|
+
markBootBoundary(logPath ?? bootBoundaryLogPath());
|
|
36
|
+
return { acquired: true, release: lock.release };
|
|
37
|
+
}
|
|
38
|
+
/** Release the lock only if `pid` still owns it (see releaseBootLock). */
|
|
39
|
+
export function releaseBootGuard(pid, deps = {}) {
|
|
40
|
+
return releaseBootLock(pid, deps);
|
|
41
|
+
}
|
|
42
|
+
export const BOOT_GUARD_BUSY_EXIT = 3;
|
|
43
|
+
export const BOOT_GUARD_USAGE_EXIT = 64;
|
|
44
|
+
/** CLI used by the skill script; returns the process exit code. */
|
|
45
|
+
export function runBootGuardCli(args, deps = {}, logPath) {
|
|
46
|
+
const action = args[0];
|
|
47
|
+
const pidIdx = args.indexOf('--pid');
|
|
48
|
+
const pid = pidIdx === -1 ? Number.NaN : Number(args[pidIdx + 1]);
|
|
49
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
50
|
+
console.error('boot-guard: --pid <owner pid> is required (pass the shell PID: $$)');
|
|
51
|
+
return BOOT_GUARD_USAGE_EXIT;
|
|
52
|
+
}
|
|
53
|
+
if (action === 'acquire') {
|
|
54
|
+
const guard = acquireBootGuard({ ...deps, pid }, logPath);
|
|
55
|
+
if (!guard.acquired) {
|
|
56
|
+
console.error('busy: another boot holds boot.lock');
|
|
57
|
+
return BOOT_GUARD_BUSY_EXIT;
|
|
58
|
+
}
|
|
59
|
+
console.log('acquired');
|
|
60
|
+
return 0;
|
|
61
|
+
}
|
|
62
|
+
if (action === 'release') {
|
|
63
|
+
releaseBootGuard(pid, deps);
|
|
64
|
+
return 0;
|
|
65
|
+
}
|
|
66
|
+
console.error(`boot-guard: unknown action: ${action ?? ''} (expected acquire|release)`);
|
|
67
|
+
return BOOT_GUARD_USAGE_EXIT;
|
|
68
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One-boot-at-a-time guard (spec D4, incident 2026-09-13). Every path that
|
|
3
|
+
* starts dsh web takes this lock from the moment the restart is issued until
|
|
4
|
+
* the port answers, so a racing tick, the dsh_web_restart hand-off, or a
|
|
5
|
+
* rollback that fires while the first boot is still coming up skips instead of
|
|
6
|
+
* producing the second systemd start that made the incident's log tail toxic.
|
|
7
|
+
*
|
|
8
|
+
* The lock is an atomically created file (`O_EXCL`) carrying `{pid, ts}`. An
|
|
9
|
+
* existing lock is honored unless its owner is gone or it has outlived
|
|
10
|
+
* `staleMs` (the boot budget). Node exposes no flock(2) binding; O_EXCL plus
|
|
11
|
+
* PID liveness gives the same mutual exclusion deterministically, and every
|
|
12
|
+
* side effect is injectable so tests never touch the real ~/.dsh/.supervisor.
|
|
13
|
+
*/
|
|
14
|
+
export declare const DEFAULT_BOOT_LOCK_STALE_MS = 180000;
|
|
15
|
+
export interface BootLockDeps {
|
|
16
|
+
lockPath?: string;
|
|
17
|
+
createExclusive?: (p: string, body: string) => boolean;
|
|
18
|
+
readLock?: (p: string) => string | undefined;
|
|
19
|
+
remove?: (p: string) => void;
|
|
20
|
+
pidAlive?: (pid: number) => boolean;
|
|
21
|
+
now?: () => number;
|
|
22
|
+
staleMs?: number;
|
|
23
|
+
sleep?: (ms: number) => Promise<void>;
|
|
24
|
+
portUp?: () => Promise<boolean>;
|
|
25
|
+
waitTimeoutMs?: number;
|
|
26
|
+
pollMs?: number;
|
|
27
|
+
/**
|
|
28
|
+
* Owner recorded in the lock file; defaults to this process. The skill script
|
|
29
|
+
* (D6) holds the lock with the shell's own PID, so liveness stays meaningful
|
|
30
|
+
* while the shell performs the restart.
|
|
31
|
+
*/
|
|
32
|
+
pid?: number;
|
|
33
|
+
}
|
|
34
|
+
export interface BootLock {
|
|
35
|
+
path: string;
|
|
36
|
+
release: () => void;
|
|
37
|
+
}
|
|
38
|
+
export declare function bootLockPath(): string;
|
|
39
|
+
/** Take the boot lock, or return undefined when another live boot owns it. */
|
|
40
|
+
export declare function acquireBootLock(deps?: BootLockDeps): BootLock | undefined;
|
|
41
|
+
/**
|
|
42
|
+
* Release the lock only when `pid` still owns it. A boot that lost the race (or
|
|
43
|
+
* whose stale lock was taken over) must never delete the winner's lock file.
|
|
44
|
+
*/
|
|
45
|
+
export declare function releaseBootLock(pid: number, deps?: BootLockDeps): boolean;
|
|
46
|
+
/** Poll the port until it answers or the boot budget expires. */
|
|
47
|
+
export declare function waitForPort(deps?: BootLockDeps): Promise<boolean>;
|
|
48
|
+
/**
|
|
49
|
+
* Run `fn` under the boot lock, then hold the lock until the port answers (or
|
|
50
|
+
* the boot budget expires). Returns `{ acquired: false }` — without running
|
|
51
|
+
* `fn` — when another boot already owns the lock.
|
|
52
|
+
*/
|
|
53
|
+
export declare function withBootLock<T>(fn: () => Promise<T>, deps?: BootLockDeps): Promise<{
|
|
54
|
+
acquired: boolean;
|
|
55
|
+
value?: T;
|
|
56
|
+
}>;
|
package/lib/boot-lock.js
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import * as os from 'node:os';
|
|
4
|
+
/**
|
|
5
|
+
* One-boot-at-a-time guard (spec D4, incident 2026-09-13). Every path that
|
|
6
|
+
* starts dsh web takes this lock from the moment the restart is issued until
|
|
7
|
+
* the port answers, so a racing tick, the dsh_web_restart hand-off, or a
|
|
8
|
+
* rollback that fires while the first boot is still coming up skips instead of
|
|
9
|
+
* producing the second systemd start that made the incident's log tail toxic.
|
|
10
|
+
*
|
|
11
|
+
* The lock is an atomically created file (`O_EXCL`) carrying `{pid, ts}`. An
|
|
12
|
+
* existing lock is honored unless its owner is gone or it has outlived
|
|
13
|
+
* `staleMs` (the boot budget). Node exposes no flock(2) binding; O_EXCL plus
|
|
14
|
+
* PID liveness gives the same mutual exclusion deterministically, and every
|
|
15
|
+
* side effect is injectable so tests never touch the real ~/.dsh/.supervisor.
|
|
16
|
+
*/
|
|
17
|
+
export const DEFAULT_BOOT_LOCK_STALE_MS = 180_000;
|
|
18
|
+
export function bootLockPath() {
|
|
19
|
+
return process.env.DSH_SUPERVISOR_BOOT_LOCK ?? path.join(os.homedir(), '.dsh/.supervisor/boot.lock');
|
|
20
|
+
}
|
|
21
|
+
function resolved(deps) {
|
|
22
|
+
return {
|
|
23
|
+
lockPath: deps.lockPath ?? bootLockPath(),
|
|
24
|
+
createExclusive: deps.createExclusive ?? ((p, body) => {
|
|
25
|
+
try {
|
|
26
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
27
|
+
fs.writeFileSync(p, body, { flag: 'wx', mode: 0o600 });
|
|
28
|
+
return true;
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
}),
|
|
34
|
+
readLock: deps.readLock ?? ((p) => {
|
|
35
|
+
try {
|
|
36
|
+
return fs.readFileSync(p, 'utf8');
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
}),
|
|
42
|
+
remove: deps.remove ?? ((p) => { try {
|
|
43
|
+
fs.unlinkSync(p);
|
|
44
|
+
}
|
|
45
|
+
catch { } }),
|
|
46
|
+
pidAlive: deps.pidAlive ?? ((pid) => {
|
|
47
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
48
|
+
return false;
|
|
49
|
+
if (pid === process.pid)
|
|
50
|
+
return true;
|
|
51
|
+
try {
|
|
52
|
+
process.kill(pid, 0);
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
}),
|
|
59
|
+
now: deps.now ?? (() => Date.now()),
|
|
60
|
+
staleMs: deps.staleMs ?? DEFAULT_BOOT_LOCK_STALE_MS,
|
|
61
|
+
sleep: deps.sleep ?? ((ms) => new Promise(r => setTimeout(r, ms))),
|
|
62
|
+
portUp: deps.portUp ?? defaultPortUp,
|
|
63
|
+
waitTimeoutMs: deps.waitTimeoutMs ?? DEFAULT_BOOT_LOCK_STALE_MS,
|
|
64
|
+
pollMs: deps.pollMs ?? 3_000,
|
|
65
|
+
pid: deps.pid ?? process.pid,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
function parseLock(raw) {
|
|
69
|
+
if (!raw)
|
|
70
|
+
return undefined;
|
|
71
|
+
try {
|
|
72
|
+
const j = JSON.parse(raw);
|
|
73
|
+
if (typeof j?.pid === 'number' && typeof j?.ts === 'number')
|
|
74
|
+
return { pid: j.pid, ts: j.ts };
|
|
75
|
+
}
|
|
76
|
+
catch { }
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
/** Take the boot lock, or return undefined when another live boot owns it. */
|
|
80
|
+
export function acquireBootLock(deps = {}) {
|
|
81
|
+
const d = resolved(deps);
|
|
82
|
+
const body = JSON.stringify({ pid: d.pid, ts: d.now() });
|
|
83
|
+
const take = () => d.createExclusive(d.lockPath, body) ? { path: d.lockPath, release: () => d.remove(d.lockPath) } : undefined;
|
|
84
|
+
const first = take();
|
|
85
|
+
if (first)
|
|
86
|
+
return first;
|
|
87
|
+
const existing = parseLock(d.readLock(d.lockPath));
|
|
88
|
+
const stale = existing === undefined || !d.pidAlive(existing.pid) || d.now() - existing.ts > d.staleMs;
|
|
89
|
+
if (!stale)
|
|
90
|
+
return undefined;
|
|
91
|
+
d.remove(d.lockPath);
|
|
92
|
+
return take();
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Release the lock only when `pid` still owns it. A boot that lost the race (or
|
|
96
|
+
* whose stale lock was taken over) must never delete the winner's lock file.
|
|
97
|
+
*/
|
|
98
|
+
export function releaseBootLock(pid, deps = {}) {
|
|
99
|
+
const d = resolved(deps);
|
|
100
|
+
const existing = parseLock(d.readLock(d.lockPath));
|
|
101
|
+
if (existing === undefined || existing.pid !== pid)
|
|
102
|
+
return false;
|
|
103
|
+
d.remove(d.lockPath);
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
async function defaultPortUp() {
|
|
107
|
+
try {
|
|
108
|
+
const ctrl = new AbortController();
|
|
109
|
+
const t = setTimeout(() => ctrl.abort(), 2000);
|
|
110
|
+
try {
|
|
111
|
+
// Any HTTP answer (200/303/401) means the boot is serving.
|
|
112
|
+
await fetch('http://127.0.0.1:3080/', { signal: ctrl.signal });
|
|
113
|
+
return true;
|
|
114
|
+
}
|
|
115
|
+
finally {
|
|
116
|
+
clearTimeout(t);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
/** Poll the port until it answers or the boot budget expires. */
|
|
124
|
+
export async function waitForPort(deps = {}) {
|
|
125
|
+
const d = resolved(deps);
|
|
126
|
+
const deadline = d.now() + d.waitTimeoutMs;
|
|
127
|
+
for (;;) {
|
|
128
|
+
try {
|
|
129
|
+
if (await d.portUp())
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
catch { }
|
|
133
|
+
if (d.now() >= deadline)
|
|
134
|
+
return false;
|
|
135
|
+
await d.sleep(d.pollMs);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Run `fn` under the boot lock, then hold the lock until the port answers (or
|
|
140
|
+
* the boot budget expires). Returns `{ acquired: false }` — without running
|
|
141
|
+
* `fn` — when another boot already owns the lock.
|
|
142
|
+
*/
|
|
143
|
+
export async function withBootLock(fn, deps = {}) {
|
|
144
|
+
const lock = acquireBootLock(deps);
|
|
145
|
+
if (!lock)
|
|
146
|
+
return { acquired: false };
|
|
147
|
+
try {
|
|
148
|
+
const value = await fn();
|
|
149
|
+
await waitForPort(deps);
|
|
150
|
+
return { acquired: true, value };
|
|
151
|
+
}
|
|
152
|
+
finally {
|
|
153
|
+
lock.release();
|
|
154
|
+
}
|
|
155
|
+
}
|
package/lib/cli.d.ts
CHANGED
|
@@ -1,14 +1,31 @@
|
|
|
1
|
+
export interface RestoreResult {
|
|
2
|
+
/** Snapshot id that was restored. */
|
|
3
|
+
target: string;
|
|
4
|
+
/** Files (and symlinks) restored into the live DSH home. */
|
|
5
|
+
restored: number;
|
|
6
|
+
/** Entries that were not restored, each with a reason. Never silent. */
|
|
7
|
+
skipped: Array<{
|
|
8
|
+
path: string;
|
|
9
|
+
reason: string;
|
|
10
|
+
}>;
|
|
11
|
+
}
|
|
1
12
|
/**
|
|
2
13
|
* Copy a chosen LKG snapshot back into the DSH home. Recent snapshots are
|
|
3
14
|
* tried newest-first, skipping any that still carry a failing plugin so a
|
|
4
|
-
* broken bundle is not restored. `sessions/`
|
|
5
|
-
* session logs are append-only truth and
|
|
6
|
-
* would drop every turn recorded after that
|
|
7
|
-
*
|
|
15
|
+
* broken bundle is not restored. `sessions/` and every other runtime-data entry
|
|
16
|
+
* is deliberately NOT restored (D1) — session logs are append-only truth and
|
|
17
|
+
* rolling them back to a snapshot would drop every turn recorded after that
|
|
18
|
+
* snapshot.
|
|
19
|
+
*
|
|
20
|
+
* Never throws for a per-entry failure: the failures are collected into
|
|
21
|
+
* `skipped` and the caller reports them (D2/D4). The only throw left is "there
|
|
22
|
+
* is no snapshot at all", which the caller must know about.
|
|
23
|
+
*
|
|
24
|
+
* @returns the restore summary (snapshot id + what was/was not restored).
|
|
8
25
|
*/
|
|
9
26
|
export declare function rollbackLKG(opts: {
|
|
10
27
|
dshHome: string;
|
|
11
28
|
lkgRoot: string;
|
|
12
29
|
failingPlugin?: string;
|
|
13
|
-
}): Promise<
|
|
30
|
+
}): Promise<RestoreResult>;
|
|
14
31
|
export declare function runCli(args: string[]): Promise<void>;
|
package/lib/cli.js
CHANGED
|
@@ -1,18 +1,93 @@
|
|
|
1
1
|
import { Supervisor } from './supervisor.js';
|
|
2
2
|
import { pollHealth } from './health-poller.js';
|
|
3
|
-
import { writeLKG, verifyLKG } from './snapshot.js';
|
|
3
|
+
import { writeLKG, verifyLKG, isLkgExcluded, failureReason } from './snapshot.js';
|
|
4
4
|
import * as fs from 'node:fs';
|
|
5
5
|
import * as path from 'node:path';
|
|
6
6
|
import * as os from 'node:os';
|
|
7
|
-
import { resolveHarnessRoot
|
|
8
|
-
import {
|
|
7
|
+
import { resolveHarnessRoot } from './paths.js';
|
|
8
|
+
import { isSelfCopyError, checkPlannedRestart, readRestartRequest, clearPlannedRestart } from './restart-guards.js';
|
|
9
|
+
import { readSupervisorConfig } from './config.js';
|
|
10
|
+
/**
|
|
11
|
+
* Make a destination (recursively) writable before overwriting it (D3).
|
|
12
|
+
*
|
|
13
|
+
* `fs.cpSync` copies into an existing destination file with
|
|
14
|
+
* `O_WRONLY|O_CREAT|O_TRUNC`, so a mode-`0400` object fails with EACCES and
|
|
15
|
+
* aborts the whole restore — the exact 2026-09-13 failure
|
|
16
|
+
* (`EACCES, Permission denied '.../attachments/v1/objects/f8'`). Only entries
|
|
17
|
+
* that actually lack the needed bit are chmod-ed, so a healthy tree costs
|
|
18
|
+
* stats, not a chmod per file.
|
|
19
|
+
*/
|
|
20
|
+
function makeWritable(target) {
|
|
21
|
+
let st;
|
|
22
|
+
try {
|
|
23
|
+
st = fs.lstatSync(target);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
if (st.isSymbolicLink())
|
|
29
|
+
return;
|
|
30
|
+
if (st.isDirectory()) {
|
|
31
|
+
if ((st.mode & 0o300) !== 0o300) {
|
|
32
|
+
try {
|
|
33
|
+
fs.chmodSync(target, st.mode | 0o300);
|
|
34
|
+
}
|
|
35
|
+
catch { }
|
|
36
|
+
}
|
|
37
|
+
let names = [];
|
|
38
|
+
try {
|
|
39
|
+
names = fs.readdirSync(target);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
for (const name of names)
|
|
45
|
+
makeWritable(path.join(target, name));
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
if ((st.mode & 0o200) === 0) {
|
|
49
|
+
try {
|
|
50
|
+
fs.chmodSync(target, st.mode | 0o200);
|
|
51
|
+
}
|
|
52
|
+
catch { }
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/** Files (and symlinks) a restore of this source entry would write. */
|
|
56
|
+
function countRestorable(srcPath) {
|
|
57
|
+
try {
|
|
58
|
+
const st = fs.lstatSync(srcPath);
|
|
59
|
+
if (st.isDirectory())
|
|
60
|
+
return walkFiles(srcPath).length;
|
|
61
|
+
return 1;
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
return 0;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
function walkFiles(dir, base = dir) {
|
|
68
|
+
const out = [];
|
|
69
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
70
|
+
const full = path.join(dir, entry.name);
|
|
71
|
+
if (entry.isDirectory())
|
|
72
|
+
out.push(...walkFiles(full, base));
|
|
73
|
+
else if (entry.isFile())
|
|
74
|
+
out.push(path.relative(base, full));
|
|
75
|
+
}
|
|
76
|
+
return out;
|
|
77
|
+
}
|
|
9
78
|
/**
|
|
10
79
|
* Copy a chosen LKG snapshot back into the DSH home. Recent snapshots are
|
|
11
80
|
* tried newest-first, skipping any that still carry a failing plugin so a
|
|
12
|
-
* broken bundle is not restored. `sessions/`
|
|
13
|
-
* session logs are append-only truth and
|
|
14
|
-
* would drop every turn recorded after that
|
|
15
|
-
*
|
|
81
|
+
* broken bundle is not restored. `sessions/` and every other runtime-data entry
|
|
82
|
+
* is deliberately NOT restored (D1) — session logs are append-only truth and
|
|
83
|
+
* rolling them back to a snapshot would drop every turn recorded after that
|
|
84
|
+
* snapshot.
|
|
85
|
+
*
|
|
86
|
+
* Never throws for a per-entry failure: the failures are collected into
|
|
87
|
+
* `skipped` and the caller reports them (D2/D4). The only throw left is "there
|
|
88
|
+
* is no snapshot at all", which the caller must know about.
|
|
89
|
+
*
|
|
90
|
+
* @returns the restore summary (snapshot id + what was/was not restored).
|
|
16
91
|
*/
|
|
17
92
|
export async function rollbackLKG(opts) {
|
|
18
93
|
const { dshHome, lkgRoot, failingPlugin } = opts;
|
|
@@ -49,11 +124,16 @@ export async function rollbackLKG(opts) {
|
|
|
49
124
|
}
|
|
50
125
|
const target = chosen ?? entries[entries.length - 1];
|
|
51
126
|
const src = path.join(lkgRoot, target);
|
|
127
|
+
const skipped = [];
|
|
128
|
+
let restored = 0;
|
|
52
129
|
for (const entry of fs.readdirSync(src)) {
|
|
53
130
|
if (entry === 'manifest.json')
|
|
54
131
|
continue;
|
|
55
|
-
if (entry
|
|
56
|
-
|
|
132
|
+
if (isLkgExcluded(entry)) {
|
|
133
|
+
// Legacy snapshots (taken before D1) still carry runtime data. Restoring
|
|
134
|
+
// a stale sessions/ over live sessions can lose a log, and a 0400
|
|
135
|
+
// attachment blob is what aborted this very path — report, never restore.
|
|
136
|
+
skipped.push({ path: entry, reason: 'runtime data excluded from the LKG scope (not required to boot)' });
|
|
57
137
|
continue;
|
|
58
138
|
}
|
|
59
139
|
const srcPath = path.join(src, entry);
|
|
@@ -65,15 +145,17 @@ export async function rollbackLKG(opts) {
|
|
|
65
145
|
continue;
|
|
66
146
|
}
|
|
67
147
|
catch { }
|
|
148
|
+
makeWritable(destPath);
|
|
68
149
|
fs.cpSync(srcPath, destPath, { recursive: true, force: true });
|
|
150
|
+
restored += countRestorable(srcPath);
|
|
69
151
|
}
|
|
70
152
|
catch (e) {
|
|
71
153
|
if (isSelfCopyError(String(e?.message ?? '')))
|
|
72
154
|
continue;
|
|
73
|
-
|
|
155
|
+
skipped.push({ path: entry, reason: failureReason(e) });
|
|
74
156
|
}
|
|
75
157
|
}
|
|
76
|
-
return target;
|
|
158
|
+
return { target, restored, skipped };
|
|
77
159
|
}
|
|
78
160
|
export async function runCli(args) {
|
|
79
161
|
const cmd = args[2] ?? '--help';
|
|
@@ -86,9 +168,14 @@ Commands:
|
|
|
86
168
|
logs Tail supervisor reports
|
|
87
169
|
rollback --to <ts> Rollback to LKG <ts>
|
|
88
170
|
resume [--within <dur>] List interrupted sessions (filter by time, e.g. 5m, 30s, 1h)
|
|
171
|
+
boot-guard acquire|release --pid <pid> Take/release boot.lock + boot-boundary (used by the safe-restart script)
|
|
89
172
|
`);
|
|
90
173
|
return;
|
|
91
174
|
}
|
|
175
|
+
if (cmd === 'boot-guard') {
|
|
176
|
+
const { runBootGuardCli } = await import('./boot-guard.js');
|
|
177
|
+
process.exit(runBootGuardCli(args.slice(3)));
|
|
178
|
+
}
|
|
92
179
|
if (cmd === 'status') {
|
|
93
180
|
const health = await pollHealth();
|
|
94
181
|
console.log(`up: ${health.up}, httpCode: ${health.httpCode}, error: ${health.error ?? 'none'}`);
|
|
@@ -186,8 +273,16 @@ Commands:
|
|
|
186
273
|
failingPlugin = m[0].replace(/^@ddtcorex\//, '');
|
|
187
274
|
}
|
|
188
275
|
catch { }
|
|
189
|
-
const
|
|
190
|
-
console.log(`[supervisor] rolled back to ${target}${failingPlugin ? ` (avoiding ${failingPlugin})` : ''}`);
|
|
276
|
+
const res = await rollbackLKG({ dshHome, lkgRoot, failingPlugin });
|
|
277
|
+
console.log(`[supervisor] rolled back to ${res.target} — ${res.restored} file(s) restored${failingPlugin ? ` (avoiding ${failingPlugin})` : ''}`);
|
|
278
|
+
// D4: a partial restore is surfaced loudly, never silently.
|
|
279
|
+
if (res.skipped.length) {
|
|
280
|
+
console.log(`[supervisor] ROLLBACK PARTIAL: ${res.skipped.length} entr(ies) not restored`);
|
|
281
|
+
for (const s of res.skipped.slice(0, 10))
|
|
282
|
+
console.log(`[supervisor] skipped ${s.path}: ${s.reason}`);
|
|
283
|
+
if (res.skipped.length > 10)
|
|
284
|
+
console.log(`[supervisor] … ${res.skipped.length - 10} more`);
|
|
285
|
+
}
|
|
191
286
|
// Reconcile node_modules from restored package.json (critical for link: deps)
|
|
192
287
|
try {
|
|
193
288
|
execSync('pnpm --dir ~/.dsh/profiles/web install --silent', { timeout: 30000, stdio: 'pipe' });
|
|
@@ -196,52 +291,23 @@ Commands:
|
|
|
196
291
|
catch (e) {
|
|
197
292
|
console.log(`[supervisor] pnpm install failed: ${e?.message ?? String(e)}`);
|
|
198
293
|
}
|
|
294
|
+
return res;
|
|
199
295
|
},
|
|
200
296
|
restartWeb: async () => {
|
|
201
|
-
|
|
202
|
-
//
|
|
203
|
-
//
|
|
297
|
+
// One implementation of the single-boot restart: marker (TTL = boot
|
|
298
|
+
// budget) → boot.lock → serialized systemd start → direct-node nohup
|
|
299
|
+
// only on hosts where the unit does not exist (D4/D5).
|
|
300
|
+
const { performSingleBootRestart, DEFAULT_BOOT_GRACE_MS } = await import('./restart-web.js');
|
|
301
|
+
let grace = DEFAULT_BOOT_GRACE_MS;
|
|
204
302
|
try {
|
|
205
|
-
|
|
303
|
+
const cfg = await readSupervisorConfig();
|
|
304
|
+
if (typeof cfg.bootGraceMs === 'number' && cfg.bootGraceMs > 0)
|
|
305
|
+
grace = cfg.bootGraceMs;
|
|
206
306
|
}
|
|
207
307
|
catch { }
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
// listening process on the host, not just dsh web (regression: killed
|
|
212
|
-
// unrelated services like redis/horizon on every restart).
|
|
213
|
-
try {
|
|
214
|
-
execSync(buildKillStalePortsCommand(), { timeout: 5000, stdio: 'pipe' });
|
|
215
|
-
}
|
|
216
|
-
catch { }
|
|
217
|
-
// Prefer systemd — if dsh-web.service is installed, restart/start it
|
|
218
|
-
try {
|
|
219
|
-
execSync('systemctl --user is-active --quiet dsh-web.service && systemctl --user restart dsh-web.service || systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
|
|
220
|
-
console.log('[supervisor] restarted dsh-web via systemd');
|
|
221
|
-
return;
|
|
222
|
-
}
|
|
223
|
-
catch { }
|
|
224
|
-
// Check if unit exists but not active — try start
|
|
225
|
-
try {
|
|
226
|
-
execSync('systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
|
|
227
|
-
console.log('[supervisor] started dsh-web via systemd (fallback)');
|
|
228
|
-
return;
|
|
229
|
-
}
|
|
230
|
-
catch { }
|
|
231
|
-
// Last fallback: detached direct node (portable — sources nvm directly, falls back to system node)
|
|
232
|
-
try {
|
|
233
|
-
const harnessRoot = resolveDeepseekHarnessDir();
|
|
234
|
-
const logPath = path.join(os.homedir(), '.dsh/dsh-web.log');
|
|
235
|
-
try {
|
|
236
|
-
writePlannedRestart(30000);
|
|
237
|
-
}
|
|
238
|
-
catch { }
|
|
239
|
-
execSync(`setsid nohup bash -c 'export NVM_DIR="$HOME/.nvm"; [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; cd ${JSON.stringify(harnessRoot)} && exec node --import tsx/esm apps/cli/src/bin.ts web --no-open >> ${JSON.stringify(logPath)} 2>&1' &`, { timeout: 5000 });
|
|
240
|
-
console.log('[supervisor] started dsh-web via nohup fallback (direct node, portable)');
|
|
241
|
-
}
|
|
242
|
-
catch (e) {
|
|
243
|
-
throw new Error(`restartWeb failed: ${e?.message ?? String(e)}`);
|
|
244
|
-
}
|
|
308
|
+
const res = await performSingleBootRestart({ bootGraceMs: grace });
|
|
309
|
+
if (!res.restarted)
|
|
310
|
+
console.log(`[supervisor] restart skipped: ${res.reason ?? 'boot lock held'}`);
|
|
245
311
|
},
|
|
246
312
|
notify: async (msg) => console.log(`[notify] ${msg}`),
|
|
247
313
|
isPlannedRestartActive: () => checkPlannedRestart(),
|
package/lib/health-poller.d.ts
CHANGED
|
@@ -4,8 +4,17 @@ export interface HealthState {
|
|
|
4
4
|
error?: string;
|
|
5
5
|
degraded?: boolean;
|
|
6
6
|
logTail?: string;
|
|
7
|
+
/** Boot verdict for the unit start this poll observed (see bootFreshness). */
|
|
8
|
+
bootPhase?: BootFreshness;
|
|
7
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* Wall-clock epoch ms of the current dsh-web unit start, or undefined when
|
|
12
|
+
* systemd does not manage the unit (portable host) or the lookup is disabled.
|
|
13
|
+
* `ActiveEnterTimestampMonotonic` is deliberately gone: a monotonic reading
|
|
14
|
+
* cannot be compared with a timestamp-less append-only log.
|
|
15
|
+
*/
|
|
8
16
|
export declare function getActiveEnterMs(): number | undefined;
|
|
17
|
+
/** @deprecated kept for callers; identical to getActiveEnterMs(). */
|
|
9
18
|
export declare function getActiveEnterWallMs(): number | undefined;
|
|
10
19
|
export interface PollHealthOpts {
|
|
11
20
|
fetch?: () => Promise<{
|
|
@@ -16,8 +25,66 @@ export interface PollHealthOpts {
|
|
|
16
25
|
logTail?: () => Promise<string>;
|
|
17
26
|
url?: string;
|
|
18
27
|
timeoutMs?: number;
|
|
19
|
-
/** injectable for tests — overrides systemctl lookup */
|
|
28
|
+
/** injectable for tests — overrides the systemctl lookup */
|
|
20
29
|
getActiveEnterMs?: () => number | undefined;
|
|
30
|
+
/** Wall-clock epoch ms of the current dsh-web unit start (see bootFreshness). */
|
|
31
|
+
activeEnterAtMs?: number;
|
|
32
|
+
/** Boot budget in ms; a younger boot without its own success marker is 'booting'. */
|
|
33
|
+
bootGraceMs?: number;
|
|
21
34
|
}
|
|
35
|
+
export type BootFreshness = 'unknown' | 'booting' | 'settled';
|
|
36
|
+
/**
|
|
37
|
+
* Decide whether the current unit start can already be judged.
|
|
38
|
+
*
|
|
39
|
+
* `activeEnterAtMs` is WALL-CLOCK epoch ms from
|
|
40
|
+
* `systemctl --user show -p ActiveEnterTimestamp dsh-web.service`.
|
|
41
|
+
* Deliberately NOT `ActiveEnterTimestampMonotonic`: a monotonic reading can
|
|
42
|
+
* never be compared against an append-only log that carries no timestamps, and
|
|
43
|
+
* that mismatch is why the previous scan filter silently fell through and let
|
|
44
|
+
* the previous boot's crash text be read as this boot's (incident 2026-09-13).
|
|
45
|
+
*
|
|
46
|
+
* - 'unknown' — no boot anchor (systemd absent, lookup disabled, clock skew):
|
|
47
|
+
* behave exactly as before the fix; never suppress anything.
|
|
48
|
+
* - 'booting' — the unit started less than `bootGraceMs` ago and has not yet
|
|
49
|
+
* proven itself by actually serving a request: weak failures are
|
|
50
|
+
* suppressed and log lines are inconclusive.
|
|
51
|
+
* - 'settled' — the boot proved itself, or the grace window expired: judge
|
|
52
|
+
* normally.
|
|
53
|
+
*/
|
|
54
|
+
export declare function bootFreshness(opts: {
|
|
55
|
+
activeEnterAtMs?: number;
|
|
56
|
+
now: number;
|
|
57
|
+
bootGraceMs: number;
|
|
58
|
+
/**
|
|
59
|
+
* Did THIS poll receive a serving response (200/401)?
|
|
60
|
+
*
|
|
61
|
+
* Deliberately not "the log shows `dsh web: http`": that line is printed the
|
|
62
|
+
* moment the raw webserver binds, seconds into a boot that can take ~90s to
|
|
63
|
+
* finish loading the plugin tree. Crediting it as proof ended the grace while
|
|
64
|
+
* the proxy in front was still coming up, so probes that hung during the rest
|
|
65
|
+
* of the boot were judged as crashes — the 2026-09-13 restart loop.
|
|
66
|
+
*/
|
|
67
|
+
probeSucceeded: boolean;
|
|
68
|
+
}): BootFreshness;
|
|
69
|
+
/**
|
|
70
|
+
* Classify a fetch failure by what it says about the process.
|
|
71
|
+
*
|
|
72
|
+
* 'refused' — nothing is listening, so the process is gone: a strong down
|
|
73
|
+
* signal that must never be masked by a boot grace.
|
|
74
|
+
* 'timeout' — something may be alive but slow: weak, only meaningful once the
|
|
75
|
+
* boot grace expired.
|
|
76
|
+
* 'other' — anything we cannot attribute.
|
|
77
|
+
*
|
|
78
|
+
* Walks the `cause` chain because undici surfaces a refused connection as
|
|
79
|
+
* `TypeError: fetch failed` with the real `ECONNREFUSED` on `cause`.
|
|
80
|
+
*/
|
|
81
|
+
export declare function classifyFetchFailure(err: unknown): 'refused' | 'timeout' | 'other';
|
|
82
|
+
export declare const SUCCESS_MARKER = "dsh web: http";
|
|
83
|
+
/** Index of the last boot-boundary line in the tail, or -1 when it predates the tail. */
|
|
84
|
+
export declare function lastBootBoundaryIndex(lines: string[]): number;
|
|
85
|
+
/** Index of the last "dsh web: http" success marker (`lowerLines` must be lower-cased). */
|
|
86
|
+
export declare function lastSuccessMarkerIndex(lowerLines: string[]): number;
|
|
87
|
+
/** Wall-clock ms parsed from a boot-boundary line, or undefined when unparseable. */
|
|
88
|
+
export declare function parseBootBoundaryMs(line: string): number | undefined;
|
|
22
89
|
export declare function pollHealth(opts?: PollHealthOpts): Promise<HealthState>;
|
|
23
90
|
export declare function collectLogTail(): Promise<string>;
|