@ddtcorex/dsh-maestro-supervisor 0.7.10 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/boot-guard.d.ts +33 -0
- package/lib/boot-guard.js +68 -0
- package/lib/boot-lock.d.ts +56 -0
- package/lib/boot-lock.js +155 -0
- package/lib/cli.d.ts +22 -5
- package/lib/cli.js +120 -54
- package/lib/health-poller.d.ts +68 -1
- package/lib/health-poller.js +191 -133
- package/lib/intents.d.ts +17 -0
- package/lib/intents.js +23 -1
- package/lib/plugin.js +15 -16
- package/lib/restart-exec.d.ts +35 -0
- package/lib/restart-exec.js +60 -0
- package/lib/restart-guards.d.ts +20 -0
- package/lib/restart-guards.js +34 -1
- package/lib/restart-tool.d.ts +30 -1
- package/lib/restart-tool.js +222 -8
- package/lib/restart-web.d.ts +26 -0
- package/lib/restart-web.js +58 -0
- package/lib/resume.d.ts +10 -0
- package/lib/resume.js +45 -28
- package/lib/snapshot.d.ts +52 -15
- package/lib/snapshot.js +168 -18
- package/lib/supervisor.d.ts +26 -4
- package/lib/supervisor.js +118 -36
- package/package.json +1 -1
- package/skills/dsh-safe-restart/SKILL.md +62 -2
- package/skills/dsh-safe-restart/scripts/restart-dsh-web.sh +162 -8
package/lib/snapshot.d.ts
CHANGED
|
@@ -1,22 +1,59 @@
|
|
|
1
|
-
interface
|
|
2
|
-
path: string;
|
|
3
|
-
sha256: string;
|
|
4
|
-
}
|
|
5
|
-
interface Manifest {
|
|
1
|
+
export interface SnapshotResult {
|
|
6
2
|
ts: string;
|
|
7
|
-
|
|
3
|
+
/** Files (and symlinks) copied into this snapshot. */
|
|
4
|
+
files: number;
|
|
5
|
+
/** Entries that could not be copied; the snapshot stays usable either way. */
|
|
6
|
+
skipped: Array<{
|
|
7
|
+
path: string;
|
|
8
|
+
reason: string;
|
|
9
|
+
}>;
|
|
8
10
|
}
|
|
9
|
-
export
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
export interface SnapshotDeps {
|
|
12
|
+
/**
|
|
13
|
+
* Copy one regular file, preserving `mode`. Injectable so a failing entry
|
|
14
|
+
* (the EACCES a read-only attachment object produces) can be table-tested
|
|
15
|
+
* without depending on the host's uid or filesystem quirks.
|
|
16
|
+
*/
|
|
17
|
+
copyFile?: (src: string, dest: string, mode: number) => void;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* DSH-home entries the last-known-good snapshot deliberately never copies.
|
|
21
|
+
*
|
|
22
|
+
* The LKG exists to recover a boot that fails while loading the plugin tree, so
|
|
23
|
+
* it holds boot **configuration**: `profiles/` (the plugin tree with its
|
|
24
|
+
* lockfile, `cordis.patch.yml` and sidecars), the per-plugin config directories
|
|
25
|
+
* and the settings documents. Everything below is runtime **data** — it is
|
|
26
|
+
* written continuously while `dsh web` runs, so restoring a snapshot of it can
|
|
27
|
+
* only lose newer state, and some of it is hostile to a bulk copy:
|
|
28
|
+
*
|
|
29
|
+
* - `sessions/` — append-only transcripts. Restoring a stale copy over live
|
|
30
|
+
* sessions drops every turn recorded after the snapshot, i.e. the recovery
|
|
31
|
+
* path would lose the log it is supposed to protect.
|
|
32
|
+
* - `attachments/` — content-addressed blobs stored mode `0400`. That read-only
|
|
33
|
+
* bit is exactly what made the 2026-09-13 rollback abort with
|
|
34
|
+
* `EACCES: Permission denied '.../attachments/v1/objects/f8'`.
|
|
35
|
+
* - `plugins-src/` — plugin source cache, re-fetched on demand (~400 MB host).
|
|
36
|
+
* - `.supervisor/` — the LKG root itself lives inside the DSH home, so copying
|
|
37
|
+
* it would recurse into every retained snapshot.
|
|
38
|
+
*
|
|
39
|
+
* This list is data, not a heuristic: the copy loop and the restore loop both
|
|
40
|
+
* consult `isLkgExcluded()`, and the table test pins the rule.
|
|
41
|
+
*/
|
|
42
|
+
export declare const LKG_EXCLUDED_ENTRIES: readonly string[];
|
|
43
|
+
/**
|
|
44
|
+
* True when a DSH-home-relative path must never enter (or leave) the LKG.
|
|
45
|
+
*
|
|
46
|
+
* The named entries match the first path segment; `*.log` matches by basename
|
|
47
|
+
* anywhere, because append-only logs are data at any depth and a restored stale
|
|
48
|
+
* `dsh-web.log` would poison the boot-boundary scan.
|
|
49
|
+
*/
|
|
50
|
+
export declare function isLkgExcluded(relPath: string): boolean;
|
|
51
|
+
/** Human-readable, bounded reason for a per-entry copy failure. */
|
|
52
|
+
export declare function failureReason(e: unknown): string;
|
|
53
|
+
export declare function writeLKG(dshHome: string, lkgRoot: string, deps?: SnapshotDeps): Promise<SnapshotResult>;
|
|
13
54
|
export declare function pruneByAge(root: string, maxAgeMs: number): Promise<void>;
|
|
14
55
|
export declare function pruneBySize(root: string, maxBytes: number): Promise<void>;
|
|
15
56
|
export declare function isDuplicateLKG(dshHome: string, lkgRoot: string): Promise<boolean>;
|
|
16
57
|
export declare function verifyLKG(lkgPath: string): Promise<boolean>;
|
|
17
58
|
export declare function rotateLKG(lkgRoot: string, keep?: number): Promise<void>;
|
|
18
|
-
export declare function writeFailed(dshHome: string, failedRoot: string): Promise<
|
|
19
|
-
ts: string;
|
|
20
|
-
manifest: Manifest;
|
|
21
|
-
}>;
|
|
22
|
-
export {};
|
|
59
|
+
export declare function writeFailed(dshHome: string, failedRoot: string): Promise<SnapshotResult>;
|
package/lib/snapshot.js
CHANGED
|
@@ -1,6 +1,51 @@
|
|
|
1
1
|
import * as fs from 'node:fs';
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import * as crypto from 'node:crypto';
|
|
4
|
+
/**
|
|
5
|
+
* DSH-home entries the last-known-good snapshot deliberately never copies.
|
|
6
|
+
*
|
|
7
|
+
* The LKG exists to recover a boot that fails while loading the plugin tree, so
|
|
8
|
+
* it holds boot **configuration**: `profiles/` (the plugin tree with its
|
|
9
|
+
* lockfile, `cordis.patch.yml` and sidecars), the per-plugin config directories
|
|
10
|
+
* and the settings documents. Everything below is runtime **data** — it is
|
|
11
|
+
* written continuously while `dsh web` runs, so restoring a snapshot of it can
|
|
12
|
+
* only lose newer state, and some of it is hostile to a bulk copy:
|
|
13
|
+
*
|
|
14
|
+
* - `sessions/` — append-only transcripts. Restoring a stale copy over live
|
|
15
|
+
* sessions drops every turn recorded after the snapshot, i.e. the recovery
|
|
16
|
+
* path would lose the log it is supposed to protect.
|
|
17
|
+
* - `attachments/` — content-addressed blobs stored mode `0400`. That read-only
|
|
18
|
+
* bit is exactly what made the 2026-09-13 rollback abort with
|
|
19
|
+
* `EACCES: Permission denied '.../attachments/v1/objects/f8'`.
|
|
20
|
+
* - `plugins-src/` — plugin source cache, re-fetched on demand (~400 MB host).
|
|
21
|
+
* - `.supervisor/` — the LKG root itself lives inside the DSH home, so copying
|
|
22
|
+
* it would recurse into every retained snapshot.
|
|
23
|
+
*
|
|
24
|
+
* This list is data, not a heuristic: the copy loop and the restore loop both
|
|
25
|
+
* consult `isLkgExcluded()`, and the table test pins the rule.
|
|
26
|
+
*/
|
|
27
|
+
export const LKG_EXCLUDED_ENTRIES = [
|
|
28
|
+
'sessions',
|
|
29
|
+
'attachments',
|
|
30
|
+
'plugins-src',
|
|
31
|
+
'.supervisor',
|
|
32
|
+
];
|
|
33
|
+
/**
|
|
34
|
+
* True when a DSH-home-relative path must never enter (or leave) the LKG.
|
|
35
|
+
*
|
|
36
|
+
* The named entries match the first path segment; `*.log` matches by basename
|
|
37
|
+
* anywhere, because append-only logs are data at any depth and a restored stale
|
|
38
|
+
* `dsh-web.log` would poison the boot-boundary scan.
|
|
39
|
+
*/
|
|
40
|
+
export function isLkgExcluded(relPath) {
|
|
41
|
+
const normalized = relPath.replace(/\\/g, '/').replace(/^\.\//, '').replace(/^\/+/, '');
|
|
42
|
+
const segments = normalized.split('/').filter(s => s.length > 0 && s !== '.');
|
|
43
|
+
if (!segments.length)
|
|
44
|
+
return false;
|
|
45
|
+
if (LKG_EXCLUDED_ENTRIES.includes(segments[0]))
|
|
46
|
+
return true;
|
|
47
|
+
return segments[segments.length - 1].endsWith('.log');
|
|
48
|
+
}
|
|
4
49
|
function sha256File(filePath) {
|
|
5
50
|
const data = fs.readFileSync(filePath);
|
|
6
51
|
return crypto.createHash('sha256').update(data).digest('hex');
|
|
@@ -16,7 +61,78 @@ function walkFiles(dir, base = dir) {
|
|
|
16
61
|
}
|
|
17
62
|
return out;
|
|
18
63
|
}
|
|
19
|
-
|
|
64
|
+
/** Human-readable, bounded reason for a per-entry copy failure. */
|
|
65
|
+
export function failureReason(e) {
|
|
66
|
+
const err = e;
|
|
67
|
+
const code = typeof err?.code === 'string' && err.code ? `${err.code}: ` : '';
|
|
68
|
+
const message = typeof err?.message === 'string' ? err.message : String(e);
|
|
69
|
+
return `${code}${message}`.slice(0, 300);
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Copy one snapshot entry, collecting — never throwing — per-entry failures
|
|
73
|
+
* (D2). `fs.cpSync` aborts the whole snapshot on the first unreadable object,
|
|
74
|
+
* which is precisely how one 0400 attachment file stopped the rollback that
|
|
75
|
+
* exists to rescue a broken boot.
|
|
76
|
+
*/
|
|
77
|
+
function copyEntry(state, src, dest, rel) {
|
|
78
|
+
if (isLkgExcluded(rel))
|
|
79
|
+
return;
|
|
80
|
+
let st;
|
|
81
|
+
try {
|
|
82
|
+
st = fs.lstatSync(src);
|
|
83
|
+
}
|
|
84
|
+
catch (e) {
|
|
85
|
+
state.skipped.push({ path: rel, reason: failureReason(e) });
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
if (st.isDirectory()) {
|
|
89
|
+
let names;
|
|
90
|
+
try {
|
|
91
|
+
// Directory modes are replicated only in their permission-to-traverse
|
|
92
|
+
// sense: the snapshot itself must stay readable/removable even when the
|
|
93
|
+
// source directory is not (a 0000 source dir must not produce a snapshot
|
|
94
|
+
// that nothing — including retention — can delete).
|
|
95
|
+
fs.mkdirSync(dest, { recursive: true, mode: (st.mode & 0o7777) | 0o700 });
|
|
96
|
+
names = fs.readdirSync(src);
|
|
97
|
+
}
|
|
98
|
+
catch (e) {
|
|
99
|
+
state.skipped.push({ path: rel, reason: failureReason(e) });
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
for (const name of names) {
|
|
103
|
+
copyEntry(state, path.join(src, name), path.join(dest, name), rel ? `${rel}/${name}` : name);
|
|
104
|
+
}
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
if (st.isSymbolicLink()) {
|
|
108
|
+
try {
|
|
109
|
+
const link = fs.readlinkSync(src);
|
|
110
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
111
|
+
try {
|
|
112
|
+
fs.unlinkSync(dest);
|
|
113
|
+
}
|
|
114
|
+
catch { }
|
|
115
|
+
fs.symlinkSync(link, dest);
|
|
116
|
+
state.copied++;
|
|
117
|
+
}
|
|
118
|
+
catch (e) {
|
|
119
|
+
state.skipped.push({ path: rel, reason: failureReason(e) });
|
|
120
|
+
}
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
if (st.isFile()) {
|
|
124
|
+
try {
|
|
125
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
126
|
+
state.copyFile(src, dest, st.mode & 0o7777);
|
|
127
|
+
state.copied++;
|
|
128
|
+
}
|
|
129
|
+
catch (e) {
|
|
130
|
+
state.skipped.push({ path: rel, reason: failureReason(e) });
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// Sockets/FIFOs/devices are not boot configuration: ignored on purpose.
|
|
134
|
+
}
|
|
135
|
+
export async function writeLKG(dshHome, lkgRoot, deps = {}) {
|
|
20
136
|
// Dedupe: skip snapshot if current state identical to latest LKG (prevents 5-min unconditional growth)
|
|
21
137
|
try {
|
|
22
138
|
if (await isDuplicateLKG(dshHome, lkgRoot)) {
|
|
@@ -31,36 +147,68 @@ export async function writeLKG(dshHome, lkgRoot) {
|
|
|
31
147
|
const latest = entries[entries.length - 1];
|
|
32
148
|
const manifestPath = path.join(lkgRoot, latest, 'manifest.json');
|
|
33
149
|
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
|
|
34
|
-
return { ts: latest, manifest };
|
|
150
|
+
return { ts: latest, files: manifest.files?.length ?? 0, skipped: [] };
|
|
35
151
|
}
|
|
36
152
|
}
|
|
37
153
|
catch { }
|
|
38
154
|
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
39
155
|
const dest = path.join(lkgRoot, ts);
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
156
|
+
try {
|
|
157
|
+
fs.mkdirSync(dest, { recursive: true });
|
|
158
|
+
}
|
|
159
|
+
catch (e) {
|
|
160
|
+
// No snapshot is better than an exception thrown into the rollback path.
|
|
161
|
+
return { ts, files: 0, skipped: [{ path: lkgRoot, reason: failureReason(e) }] };
|
|
162
|
+
}
|
|
163
|
+
const state = {
|
|
164
|
+
dshHome,
|
|
165
|
+
copyFile: deps.copyFile ?? ((src, dst, mode) => {
|
|
166
|
+
fs.copyFileSync(src, dst);
|
|
167
|
+
try {
|
|
168
|
+
fs.chmodSync(dst, mode);
|
|
169
|
+
}
|
|
170
|
+
catch { }
|
|
171
|
+
}),
|
|
172
|
+
copied: 0,
|
|
173
|
+
skipped: [],
|
|
174
|
+
};
|
|
175
|
+
// Copy only the boot configuration (D1), one entry at a time so a single
|
|
176
|
+
// unreadable file is recorded and skipped instead of aborting the snapshot (D2).
|
|
177
|
+
let topLevel = [];
|
|
178
|
+
try {
|
|
179
|
+
if (fs.existsSync(dshHome))
|
|
180
|
+
topLevel = fs.readdirSync(dshHome);
|
|
181
|
+
}
|
|
182
|
+
catch (e) {
|
|
183
|
+
state.skipped.push({ path: '.', reason: failureReason(e) });
|
|
184
|
+
}
|
|
185
|
+
for (const entry of topLevel) {
|
|
186
|
+
copyEntry(state, path.join(dshHome, entry), path.join(dest, entry), entry);
|
|
187
|
+
}
|
|
188
|
+
let fileList = [];
|
|
189
|
+
try {
|
|
190
|
+
fileList = fs.existsSync(dest) ? walkFiles(dest) : [];
|
|
191
|
+
}
|
|
192
|
+
catch (e) {
|
|
193
|
+
state.skipped.push({ path: 'manifest', reason: failureReason(e) });
|
|
194
|
+
}
|
|
52
195
|
const manifest = {
|
|
53
196
|
ts,
|
|
54
|
-
files:
|
|
197
|
+
files: fileList
|
|
55
198
|
.filter(f => f !== 'manifest.json')
|
|
56
199
|
.map(f => ({ path: f, sha256: sha256File(path.join(dest, f)) })),
|
|
57
200
|
};
|
|
58
|
-
|
|
201
|
+
try {
|
|
202
|
+
fs.writeFileSync(path.join(dest, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
|
203
|
+
}
|
|
204
|
+
catch (e) {
|
|
205
|
+
state.skipped.push({ path: 'manifest.json', reason: failureReason(e) });
|
|
206
|
+
}
|
|
59
207
|
// Retention: keep only 3 most recent, plus age (7d) and size (5GB) caps — prevents unbounded 40GB+ growth
|
|
60
208
|
await rotateLKG(lkgRoot, 3).catch(() => { });
|
|
61
209
|
await pruneByAge(lkgRoot, 7 * 24 * 60 * 60 * 1000).catch(() => { });
|
|
62
210
|
await pruneBySize(lkgRoot, 5 * 1024 * 1024 * 1024).catch(() => { });
|
|
63
|
-
return { ts,
|
|
211
|
+
return { ts, files: state.copied, skipped: state.skipped };
|
|
64
212
|
}
|
|
65
213
|
export async function pruneByAge(root, maxAgeMs) {
|
|
66
214
|
if (!fs.existsSync(root))
|
|
@@ -150,7 +298,9 @@ export async function isDuplicateLKG(dshHome, lkgRoot) {
|
|
|
150
298
|
let newestFileMtime = 0;
|
|
151
299
|
if (fs.existsSync(dshHome)) {
|
|
152
300
|
for (const entry of fs.readdirSync(dshHome)) {
|
|
153
|
-
|
|
301
|
+
// Same scope as the copy loop: runtime data changes constantly and
|
|
302
|
+
// must not defeat the dedupe for the configuration being snapshotted.
|
|
303
|
+
if (isLkgExcluded(entry))
|
|
154
304
|
continue;
|
|
155
305
|
try {
|
|
156
306
|
const s = fs.statSync(path.join(dshHome, entry));
|
package/lib/supervisor.d.ts
CHANGED
|
@@ -1,15 +1,25 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type HealthState } from './health-poller.js';
|
|
2
2
|
import type { RestartRequest } from './restart-guards.js';
|
|
3
|
+
import { type RestartOutcome } from './intents.js';
|
|
3
4
|
import { type MintCookieOpts } from './dsh-session.js';
|
|
5
|
+
/** Partial-restore report a rollback may hand back (cli.ts wires RestoreResult). */
|
|
6
|
+
export interface RollbackSummary {
|
|
7
|
+
target?: string;
|
|
8
|
+
restored?: number;
|
|
9
|
+
skipped?: Array<{
|
|
10
|
+
path: string;
|
|
11
|
+
reason: string;
|
|
12
|
+
}>;
|
|
13
|
+
}
|
|
4
14
|
export interface SupervisorDeps {
|
|
5
15
|
pollHealth: () => Promise<HealthState>;
|
|
6
16
|
writeLKG: () => Promise<{
|
|
7
17
|
ts: string;
|
|
8
|
-
manifest
|
|
18
|
+
manifest?: any;
|
|
9
19
|
}>;
|
|
10
20
|
writeFailed: () => Promise<{
|
|
11
21
|
ts: string;
|
|
12
|
-
manifest
|
|
22
|
+
manifest?: any;
|
|
13
23
|
}>;
|
|
14
24
|
writeReport: (opts: {
|
|
15
25
|
ts: string;
|
|
@@ -18,12 +28,13 @@ export interface SupervisorDeps {
|
|
|
18
28
|
logTail?: string;
|
|
19
29
|
gitDiff?: string;
|
|
20
30
|
}) => Promise<string>;
|
|
21
|
-
rollback: (ts?: string) => Promise<void>;
|
|
31
|
+
rollback: (ts?: string) => Promise<RollbackSummary | void>;
|
|
22
32
|
restartWeb?: () => Promise<void>;
|
|
23
33
|
notify: (msg: string) => Promise<void>;
|
|
24
34
|
intervalMs?: number;
|
|
25
35
|
debounceMs?: number;
|
|
26
36
|
downThreshold?: number;
|
|
37
|
+
bootGraceMs?: number;
|
|
27
38
|
getTime?: () => number;
|
|
28
39
|
isPlannedRestartActive?: () => boolean | Promise<boolean>;
|
|
29
40
|
writePlannedRestart?: (ttlMs?: number) => void;
|
|
@@ -45,7 +56,11 @@ export interface SupervisorDeps {
|
|
|
45
56
|
}>;
|
|
46
57
|
readRestartRequest?: () => RestartRequest | undefined;
|
|
47
58
|
onRestartRequestHandled?: (req: RestartRequest) => void;
|
|
59
|
+
writeOutcome?: (sessionId: string, outcome: RestartOutcome) => void;
|
|
60
|
+
listenerPid?: (port: number) => number | undefined;
|
|
48
61
|
}
|
|
62
|
+
/** PID holding a 127.0.0.1 listener on the port, or undefined. Never throws. */
|
|
63
|
+
export declare function defaultListenerPid(port: number): number | undefined;
|
|
49
64
|
export declare function resumeViaRpc(ids: string[], fetchFn?: (url: string, init: RequestInit) => Promise<Response>, extraHeaders?: Record<string, string>): Promise<{
|
|
50
65
|
resumed: string[];
|
|
51
66
|
}>;
|
|
@@ -86,9 +101,16 @@ export declare class Supervisor {
|
|
|
86
101
|
private getEffectiveDownThreshold;
|
|
87
102
|
private getEffectiveDegradedThreshold;
|
|
88
103
|
private getEffectivePollTimeoutMs;
|
|
104
|
+
private getEffectiveBootGraceMs;
|
|
89
105
|
private findInterruptedRecent;
|
|
90
106
|
private collectGitDiff;
|
|
91
107
|
private attemptAutoResume;
|
|
108
|
+
/**
|
|
109
|
+
* D4: a restore that could not put every entry back must be surfaced loudly.
|
|
110
|
+
* Without this, a half-restored tree reads exactly like a clean recovery in
|
|
111
|
+
* the log and in the operator's Telegram feed.
|
|
112
|
+
*/
|
|
113
|
+
private reportRollbackResult;
|
|
92
114
|
private handleDebugResult;
|
|
93
115
|
tick(): Promise<void>;
|
|
94
116
|
start(): Promise<void>;
|
package/lib/supervisor.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { classifyFetchFailure } from './health-poller.js';
|
|
1
2
|
import { runDebugAgent } from './debug-agent.js';
|
|
2
3
|
import { findInterrupted as defaultFindInterrupted, parseDuration } from './resume.js';
|
|
3
4
|
import * as fs from 'node:fs';
|
|
@@ -5,9 +6,23 @@ import * as path from 'node:path';
|
|
|
5
6
|
import * as os from 'node:os';
|
|
6
7
|
import { resolveHarnessRoot } from './paths.js';
|
|
7
8
|
import { readSupervisorConfig } from './config.js';
|
|
8
|
-
import { writePlannedRestart as defaultWritePlannedRestart, checkPlannedRestart as defaultCheckPlannedRestart, clearPlannedRestart
|
|
9
|
+
import { writePlannedRestart as defaultWritePlannedRestart, checkPlannedRestart as defaultCheckPlannedRestart, clearPlannedRestart } from './restart-guards.js';
|
|
10
|
+
import { writeRestartOutcome as defaultWriteOutcome } from './intents.js';
|
|
11
|
+
import { execFileSync } from 'node:child_process';
|
|
9
12
|
import { mintDshSessionCookie } from './dsh-session.js';
|
|
10
|
-
|
|
13
|
+
/** PID holding a 127.0.0.1 listener on the port, or undefined. Never throws. */
|
|
14
|
+
export function defaultListenerPid(port) {
|
|
15
|
+
try {
|
|
16
|
+
const out = execFileSync('ss', ['-tlnp'], { encoding: 'utf8' });
|
|
17
|
+
for (const line of out.split('\n')) {
|
|
18
|
+
const m = new RegExp(`127\\.0\\.0\\.1:${port}\\s[^]*?pid=(\\d+)`).exec(line);
|
|
19
|
+
if (m)
|
|
20
|
+
return Number(m[1]);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
catch { }
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
11
26
|
export async function resumeViaRpc(ids, fetchFn = globalThis.fetch, extraHeaders = {}) {
|
|
12
27
|
const rpcId = crypto.randomUUID();
|
|
13
28
|
const response = await fetchFn('http://127.0.0.1:3080/dsh-maestro-supervisor-resume/resume', {
|
|
@@ -71,31 +86,22 @@ export class Supervisor {
|
|
|
71
86
|
return this.deps.clearPlannedRestart ?? clearPlannedRestart;
|
|
72
87
|
}
|
|
73
88
|
async restartWeb() {
|
|
74
|
-
this.
|
|
89
|
+
const grace = await this.getEffectiveBootGraceMs();
|
|
90
|
+
// Marker first, and its TTL is the boot budget — not 30 s. The observed
|
|
91
|
+
// boot took 1m52s: after +30 s every poll was judged as a crash, which is
|
|
92
|
+
// exactly how the 2026-09-13 rollback report was produced (D5).
|
|
93
|
+
this.getWritePlannedRestart()(grace);
|
|
75
94
|
if (this.deps.restartWeb) {
|
|
95
|
+
// Injected implementation (cli.ts daemon wiring, tests): it owns the
|
|
96
|
+
// boot lock, so the single-flight guarantee stays in one place.
|
|
76
97
|
await this.deps.restartWeb();
|
|
77
98
|
return;
|
|
78
99
|
}
|
|
79
|
-
|
|
80
|
-
const
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
}
|
|
84
|
-
catch { }
|
|
85
|
-
try {
|
|
86
|
-
execSync('systemctl --user is-active --quiet dsh-web.service && systemctl --user restart dsh-web.service || systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
|
|
87
|
-
return;
|
|
88
|
-
}
|
|
89
|
-
catch { }
|
|
90
|
-
try {
|
|
91
|
-
execSync('systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
|
|
92
|
-
return;
|
|
100
|
+
const { performSingleBootRestart } = await import('./restart-web.js');
|
|
101
|
+
const res = await performSingleBootRestart({ bootGraceMs: grace });
|
|
102
|
+
if (!res.restarted) {
|
|
103
|
+
await this.deps.notify(`restart skipped: ${res.reason ?? 'boot lock held'}`).catch(() => { });
|
|
93
104
|
}
|
|
94
|
-
catch { }
|
|
95
|
-
const { resolveDeepseekHarnessDir } = await import('./paths.js');
|
|
96
|
-
const harnessRoot = resolveDeepseekHarnessDir();
|
|
97
|
-
const logPath = path.join(os.homedir(), '.dsh/dsh-web.log');
|
|
98
|
-
execSync(`setsid nohup bash -c 'export NVM_DIR="$HOME/.nvm"; [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; cd ${JSON.stringify(harnessRoot)} && exec node --import tsx/esm apps/cli/src/bin.ts web --no-open >> ${JSON.stringify(logPath)} 2>&1' &`, { timeout: 5000 });
|
|
99
105
|
}
|
|
100
106
|
getRunDebugAgent() {
|
|
101
107
|
return this.deps.runDebugAgent ?? runDebugAgent;
|
|
@@ -242,6 +248,17 @@ export class Supervisor {
|
|
|
242
248
|
catch { }
|
|
243
249
|
return 20000;
|
|
244
250
|
}
|
|
251
|
+
async getEffectiveBootGraceMs() {
|
|
252
|
+
if (this.deps.bootGraceMs !== undefined)
|
|
253
|
+
return this.deps.bootGraceMs;
|
|
254
|
+
try {
|
|
255
|
+
const cfg = await readSupervisorConfig();
|
|
256
|
+
if (typeof cfg.bootGraceMs === 'number' && cfg.bootGraceMs > 0)
|
|
257
|
+
return cfg.bootGraceMs;
|
|
258
|
+
}
|
|
259
|
+
catch { }
|
|
260
|
+
return 180_000;
|
|
261
|
+
}
|
|
245
262
|
async findInterruptedRecent(withinMs) {
|
|
246
263
|
const ms = withinMs ?? this.getResumeWithinMs();
|
|
247
264
|
// Prefer injected mock for testability
|
|
@@ -312,6 +329,17 @@ export class Supervisor {
|
|
|
312
329
|
await this.deps.notify(`RESUME FAILED: ${ids.length} interrupted sessions (${ids.slice(0, 3).join(', ')}) — ${e?.message ?? String(e)}`).catch(() => { });
|
|
313
330
|
}
|
|
314
331
|
}
|
|
332
|
+
/**
|
|
333
|
+
* D4: a restore that could not put every entry back must be surfaced loudly.
|
|
334
|
+
* Without this, a half-restored tree reads exactly like a clean recovery in
|
|
335
|
+
* the log and in the operator's Telegram feed.
|
|
336
|
+
*/
|
|
337
|
+
async reportRollbackResult(summary, reportPath) {
|
|
338
|
+
if (!summary || !summary.skipped?.length)
|
|
339
|
+
return;
|
|
340
|
+
const first = summary.skipped.slice(0, 3).map(s => `${s.path} (${s.reason})`).join('; ');
|
|
341
|
+
await this.deps.notify(`ROLLBACK PARTIAL: ${summary.restored ?? 0} restored, ${summary.skipped.length} skipped — ${first} (report: ${reportPath})`).catch(() => { });
|
|
342
|
+
}
|
|
315
343
|
handleDebugResult(reportPath, res) {
|
|
316
344
|
if (res.fixed) {
|
|
317
345
|
void this.deps.notify(`FIXED: debug-agent fixed ${reportPath} — ${res.reason}`).catch(() => { });
|
|
@@ -352,13 +380,24 @@ export class Supervisor {
|
|
|
352
380
|
// own 30s TTL runs out.
|
|
353
381
|
this.lastRollback = this.deps.getTime ? this.deps.getTime() : Date.now();
|
|
354
382
|
await this.restartWeb();
|
|
355
|
-
//
|
|
356
|
-
//
|
|
357
|
-
this.getWritePlannedRestart()(PLANNED_RESTART_TTL_MS);
|
|
383
|
+
// restartWeb() already wrote the marker with the boot budget as its
|
|
384
|
+
// TTL; the marker is cleared on the first healthy poll instead.
|
|
358
385
|
await this.deps.notify(`restarted dsh-web after self-restart by session ${restartReq.callerSessionId}`);
|
|
359
386
|
}
|
|
360
387
|
catch (e) {
|
|
361
388
|
await this.deps.notify(`self-restart dsh-web failed: ${e?.message ?? String(e)}`).catch(() => { });
|
|
389
|
+
if (restartReq.callerSessionId) {
|
|
390
|
+
try {
|
|
391
|
+
;
|
|
392
|
+
(this.deps.writeOutcome ?? defaultWriteOutcome)(restartReq.callerSessionId, {
|
|
393
|
+
state: 'failed',
|
|
394
|
+
oldPid: restartReq.oldPid,
|
|
395
|
+
swappedAt: this.deps.getTime ? this.deps.getTime() : Date.now(),
|
|
396
|
+
error: e?.message ?? String(e),
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
catch { }
|
|
400
|
+
}
|
|
362
401
|
}
|
|
363
402
|
finally {
|
|
364
403
|
// Hold the marker and latch until health.up: clearing here would
|
|
@@ -375,6 +414,14 @@ export class Supervisor {
|
|
|
375
414
|
// DEGRADED: http 200 but log has plugin error → report + notify, rollback after consecutive threshold
|
|
376
415
|
if (health.degraded) {
|
|
377
416
|
this.consecutiveDown = 0;
|
|
417
|
+
// D1: a degraded verdict while the current boot is unproven is a boot
|
|
418
|
+
// transient, not a plugin error — the incident's report line
|
|
419
|
+
// "rollback — degraded: This operation was aborted" was produced exactly
|
|
420
|
+
// here, five polls after a slow boot.
|
|
421
|
+
if (health.bootPhase === 'booting') {
|
|
422
|
+
this.consecutiveDegraded = 0;
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
378
425
|
// Check suppression first — don't count degraded during planned restart grace
|
|
379
426
|
let suppressedByMarkerDeg = false;
|
|
380
427
|
try {
|
|
@@ -457,7 +504,8 @@ export class Supervisor {
|
|
|
457
504
|
const degradedHealth = { up: false, httpCode: health.httpCode, error: `degraded → down: ${degradedError}`, logTail: logTail2, degraded: false };
|
|
458
505
|
const reportPath2 = await this.deps.writeReport({ ts: ts2, health: degradedHealth, action: `rollback — degraded: ${degradedError}`, logTail: logTail2, gitDiff: gitDiff2 }).catch(() => '');
|
|
459
506
|
try {
|
|
460
|
-
await this.deps.rollback();
|
|
507
|
+
const summary = await this.deps.rollback();
|
|
508
|
+
await this.reportRollbackResult(summary, reportPath2);
|
|
461
509
|
}
|
|
462
510
|
catch (e) {
|
|
463
511
|
await this.deps.notify(`rollback failed: ${e?.message ?? String(e)} (report: ${reportPath2})`).catch(() => { });
|
|
@@ -485,20 +533,33 @@ export class Supervisor {
|
|
|
485
533
|
if (health.up) {
|
|
486
534
|
this.consecutiveDown = 0;
|
|
487
535
|
this.consecutiveDegraded = 0;
|
|
488
|
-
//
|
|
489
|
-
//
|
|
490
|
-
//
|
|
491
|
-
|
|
536
|
+
// D5: a healthy poll is the authoritative "the restart succeeded"
|
|
537
|
+
// signal — clear the suppression marker here instead of waiting for its
|
|
538
|
+
// TTL, so a stale marker can never mute a later real crash.
|
|
539
|
+
let markerClearedThisTick = false;
|
|
540
|
+
try {
|
|
541
|
+
if (this.getCheckPlannedRestart()()) {
|
|
542
|
+
this.getClearPlannedRestart()();
|
|
543
|
+
markerClearedThisTick = true;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
catch { }
|
|
547
|
+
// Post-self-restart boot proved healthy: run the post-restart
|
|
548
|
+
// session-scan hook and re-arm the single-flight latch. A failed clear
|
|
549
|
+
// keeps the latch set so the same marker is never re-handled into a
|
|
550
|
+
// second restart.
|
|
492
551
|
if (this.awaitingHealthyBoot) {
|
|
493
552
|
this.awaitingHealthyBoot = false;
|
|
494
553
|
const req = this.pendingRestartRequest;
|
|
495
554
|
this.pendingRestartRequest = undefined;
|
|
496
|
-
let cleared =
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
555
|
+
let cleared = markerClearedThisTick;
|
|
556
|
+
if (!cleared) {
|
|
557
|
+
try {
|
|
558
|
+
this.getClearPlannedRestart()();
|
|
559
|
+
cleared = true;
|
|
560
|
+
}
|
|
561
|
+
catch { }
|
|
500
562
|
}
|
|
501
|
-
catch { }
|
|
502
563
|
if (cleared)
|
|
503
564
|
this.restartRequestHandled = false;
|
|
504
565
|
if (req) {
|
|
@@ -506,6 +567,19 @@ export class Supervisor {
|
|
|
506
567
|
this.deps.onRestartRequestHandled?.(req);
|
|
507
568
|
}
|
|
508
569
|
catch { }
|
|
570
|
+
if (req.callerSessionId) {
|
|
571
|
+
try {
|
|
572
|
+
const listen = this.deps.listenerPid ?? defaultListenerPid;
|
|
573
|
+
(this.deps.writeOutcome ?? defaultWriteOutcome)(req.callerSessionId, {
|
|
574
|
+
state: 'ok',
|
|
575
|
+
oldPid: req.oldPid,
|
|
576
|
+
newPid: listen(3082),
|
|
577
|
+
httpStatus: health?.httpCode ?? 200,
|
|
578
|
+
swappedAt: this.deps.getTime ? this.deps.getTime() : Date.now(),
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
catch { }
|
|
582
|
+
}
|
|
509
583
|
}
|
|
510
584
|
}
|
|
511
585
|
// Throttle LKG writes to at most once per 5 minutes
|
|
@@ -556,6 +630,13 @@ export class Supervisor {
|
|
|
556
630
|
// A lone timed-out poll (e.g. a slow plugin-tree boot) must not trigger
|
|
557
631
|
// rollback/restart: that restart produces its own transient errors on
|
|
558
632
|
// the next poll, which would otherwise re-trigger this same path forever.
|
|
633
|
+
// D1/D2: while the boot is unproven only a refused connection may advance
|
|
634
|
+
// the down counter — nothing is listening, so the process is gone. A
|
|
635
|
+
// timeout/abort (or anything unattributable) is a slow boot, not a crash.
|
|
636
|
+
if (health.bootPhase === 'booting' && classifyFetchFailure(health.error) !== 'refused') {
|
|
637
|
+
this.consecutiveDown = 0;
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
559
640
|
this.consecutiveDown++;
|
|
560
641
|
let downThreshold = await this.getEffectiveDownThreshold();
|
|
561
642
|
// When a planned restart marker is active, double the threshold (3→6 at
|
|
@@ -589,7 +670,8 @@ export class Supervisor {
|
|
|
589
670
|
const gitDiff = await this.collectGitDiff().catch(() => '');
|
|
590
671
|
const reportPath = await this.deps.writeReport({ ts, health, action: `rollback — ${health.error ?? 'down'}`, logTail, gitDiff }).catch(() => '');
|
|
591
672
|
try {
|
|
592
|
-
await this.deps.rollback();
|
|
673
|
+
const summary = await this.deps.rollback();
|
|
674
|
+
await this.reportRollbackResult(summary, reportPath);
|
|
593
675
|
}
|
|
594
676
|
catch (e) {
|
|
595
677
|
await this.deps.notify(`rollback failed: ${e?.message ?? String(e)} (report: ${reportPath})`).catch(() => { });
|
package/package.json
CHANGED