@ddtcorex/dsh-maestro-supervisor 0.5.4 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +168 -16
- package/cordis.patch.yml +11 -0
- package/lib/bin.d.ts +2 -0
- package/lib/bin.js +3 -0
- package/lib/cli.js +60 -1
- package/lib/client.js +150 -0
- package/lib/debug-agent.js +9 -5
- package/lib/health-poller.js +44 -7
- package/lib/index.d.ts +7 -2
- package/lib/index.js +7 -3
- package/lib/paths.d.ts +6 -0
- package/lib/paths.js +44 -0
- package/lib/plugin.d.ts +42 -0
- package/lib/plugin.js +290 -0
- package/lib/resume.d.ts +20 -1
- package/lib/resume.js +181 -9
- package/lib/supervisor.d.ts +12 -0
- package/lib/supervisor.js +205 -16
- package/lib/types/client/auto-reload.d.ts +8 -0
- package/lib/types/client/auto-reload.d.ts.map +1 -0
- package/lib/types/client/index.d.ts +2 -0
- package/lib/types/client/index.d.ts.map +1 -0
- package/package.json +31 -5
package/lib/supervisor.js
CHANGED
|
@@ -1,5 +1,29 @@
|
|
|
1
1
|
import { runDebugAgent } from './debug-agent.js';
|
|
2
|
-
import { findInterrupted as defaultFindInterrupted } from './resume.js';
|
|
2
|
+
import { findInterrupted as defaultFindInterrupted, parseDuration } from './resume.js';
|
|
3
|
+
import * as fs from 'node:fs';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
import * as os from 'node:os';
|
|
6
|
+
import { resolveHarnessRoot } from './paths.js';
|
|
7
|
+
export async function resumeViaRpc(ids, fetchFn = globalThis.fetch) {
|
|
8
|
+
const rpcId = crypto.randomUUID();
|
|
9
|
+
const response = await fetchFn('http://127.0.0.1:3080/dsh-maestro-supervisor-resume/resume', {
|
|
10
|
+
method: 'POST',
|
|
11
|
+
headers: { 'content-type': 'application/json' },
|
|
12
|
+
body: JSON.stringify({ type: 'client-request', rpcId, method: 'resume', payload: { ids } }),
|
|
13
|
+
});
|
|
14
|
+
if (!response.ok)
|
|
15
|
+
throw new Error(`resume RPC returned HTTP ${response.status}`);
|
|
16
|
+
const envelope = await response.json();
|
|
17
|
+
const resumed = envelope?.type === 'server-response'
|
|
18
|
+
&& envelope?.rpcId === rpcId
|
|
19
|
+
&& envelope?.result?.ok === true
|
|
20
|
+
&& Array.isArray(envelope?.result?.value?.resumed)
|
|
21
|
+
? envelope.result.value.resumed.filter((id) => typeof id === 'string')
|
|
22
|
+
: undefined;
|
|
23
|
+
if (resumed === undefined)
|
|
24
|
+
throw new Error('resume RPC returned an invalid result');
|
|
25
|
+
return { resumed };
|
|
26
|
+
}
|
|
3
27
|
export class Supervisor {
|
|
4
28
|
deps;
|
|
5
29
|
lastRollback = 0;
|
|
@@ -16,10 +40,141 @@ export class Supervisor {
|
|
|
16
40
|
getFindInterrupted() {
|
|
17
41
|
return this.deps.findInterrupted ?? defaultFindInterrupted;
|
|
18
42
|
}
|
|
43
|
+
getResumeSessions() {
|
|
44
|
+
return this.deps.resumeSessions ?? resumeViaRpc;
|
|
45
|
+
}
|
|
46
|
+
getAutoResumeEnabled() {
|
|
47
|
+
// Priority: env > supervisor config.json > maestro settings.json > default true (enabled)
|
|
48
|
+
// Configures whether interrupted sessions are auto-resumed after restart (vs only notify).
|
|
49
|
+
// For Settings UI: boolean toggle — true = auto-resume within window, false = notify only.
|
|
50
|
+
const env = process.env.DSH_SUPERVISOR_AUTO_RESUME;
|
|
51
|
+
if (env !== undefined) {
|
|
52
|
+
const v = env.trim().toLowerCase();
|
|
53
|
+
if (['1', 'true', 'yes', 'on', 'enabled'].includes(v))
|
|
54
|
+
return true;
|
|
55
|
+
if (['0', 'false', 'no', 'off', 'disabled'].includes(v))
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
try {
|
|
59
|
+
const cfgPath = path.join(os.homedir(), '.dsh/.supervisor/config.json');
|
|
60
|
+
if (fs.existsSync(cfgPath)) {
|
|
61
|
+
const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf-8'));
|
|
62
|
+
const raw = cfg.autoResumeEnabled ?? cfg.autoResume;
|
|
63
|
+
if (typeof raw === 'boolean')
|
|
64
|
+
return raw;
|
|
65
|
+
if (typeof raw === 'string') {
|
|
66
|
+
const v = raw.trim().toLowerCase();
|
|
67
|
+
if (['1', 'true', 'yes', 'on'].includes(v))
|
|
68
|
+
return true;
|
|
69
|
+
if (['0', 'false', 'no', 'off'].includes(v))
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const maestroPath = path.join(os.homedir(), '.dsh/maestro/settings.json');
|
|
74
|
+
if (fs.existsSync(maestroPath)) {
|
|
75
|
+
const j = JSON.parse(fs.readFileSync(maestroPath, 'utf-8'));
|
|
76
|
+
const raw = j?.domains?.supervisor?.autoResumeEnabled ?? j?.supervisor?.autoResumeEnabled ?? j?.domains?.supervisor?.autoResume ?? j?.supervisor?.autoResume;
|
|
77
|
+
if (typeof raw === 'boolean')
|
|
78
|
+
return raw;
|
|
79
|
+
if (typeof raw === 'string') {
|
|
80
|
+
const v = raw.trim().toLowerCase();
|
|
81
|
+
if (['1', 'true', 'yes', 'on'].includes(v))
|
|
82
|
+
return true;
|
|
83
|
+
if (['0', 'false', 'no', 'off'].includes(v))
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
catch { }
|
|
89
|
+
return true; // default enabled
|
|
90
|
+
}
|
|
91
|
+
getResumeWithinMs() {
|
|
92
|
+
// Priority: env > supervisor config.json > maestro settings.json > default 5 (minutes)
|
|
93
|
+
// Note: config value is in MINUTES (number 5 = 5 minutes). String "5m"/"30s"/"1h" also supported via parseDuration.
|
|
94
|
+
const env = process.env.DSH_SUPERVISOR_RESUME_WITHIN;
|
|
95
|
+
if (env) {
|
|
96
|
+
// Bare number in env like "5" → treat as minutes for ergonomics
|
|
97
|
+
if (/^\d+$/.test(env.trim())) {
|
|
98
|
+
const n = parseInt(env.trim(), 10);
|
|
99
|
+
if (!isNaN(n))
|
|
100
|
+
return n * 60 * 1000;
|
|
101
|
+
}
|
|
102
|
+
const v = parseDuration(env);
|
|
103
|
+
if (v !== undefined)
|
|
104
|
+
return v;
|
|
105
|
+
const n = parseInt(env, 10);
|
|
106
|
+
if (!isNaN(n))
|
|
107
|
+
return n;
|
|
108
|
+
}
|
|
109
|
+
try {
|
|
110
|
+
const cfgPath = path.join(os.homedir(), '.dsh/.supervisor/config.json');
|
|
111
|
+
if (fs.existsSync(cfgPath)) {
|
|
112
|
+
const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf-8'));
|
|
113
|
+
const raw = cfg.autoResumeWithin ?? cfg.resumeWithin;
|
|
114
|
+
if (typeof raw === 'string') {
|
|
115
|
+
if (/^\d+$/.test(raw.trim()))
|
|
116
|
+
return parseInt(raw.trim(), 10) * 60 * 1000; // bare string digits → minutes
|
|
117
|
+
const v = parseDuration(raw);
|
|
118
|
+
if (v !== undefined)
|
|
119
|
+
return v;
|
|
120
|
+
}
|
|
121
|
+
else if (typeof raw === 'number')
|
|
122
|
+
return raw * 60 * 1000; // number is MINUTES
|
|
123
|
+
}
|
|
124
|
+
const maestroPath = path.join(os.homedir(), '.dsh/maestro/settings.json');
|
|
125
|
+
if (fs.existsSync(maestroPath)) {
|
|
126
|
+
const j = JSON.parse(fs.readFileSync(maestroPath, 'utf-8'));
|
|
127
|
+
const raw = j?.domains?.supervisor?.autoResumeWithin ?? j?.supervisor?.autoResumeWithin;
|
|
128
|
+
if (typeof raw === 'string') {
|
|
129
|
+
if (/^\d+$/.test(raw.trim()))
|
|
130
|
+
return parseInt(raw.trim(), 10) * 60 * 1000;
|
|
131
|
+
const v = parseDuration(raw);
|
|
132
|
+
if (v !== undefined)
|
|
133
|
+
return v;
|
|
134
|
+
}
|
|
135
|
+
else if (typeof raw === 'number')
|
|
136
|
+
return raw * 60 * 1000;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
catch { }
|
|
140
|
+
return 5 * 60 * 1000; // default 5 minutes
|
|
141
|
+
}
|
|
142
|
+
async findInterruptedRecent(withinMs) {
|
|
143
|
+
const ms = withinMs ?? this.getResumeWithinMs();
|
|
144
|
+
// Prefer injected mock for testability
|
|
145
|
+
if (this.deps.findInterrupted) {
|
|
146
|
+
try {
|
|
147
|
+
const res = await this.deps.findInterrupted();
|
|
148
|
+
// If mock doesn't filter by time, we still return as-is (test expects all)
|
|
149
|
+
// For real filtering when mock is not time-aware, try to filter via resume module if possible
|
|
150
|
+
if (ms !== undefined && res.interrupted.length) {
|
|
151
|
+
try {
|
|
152
|
+
const { findInterrupted } = await import('./resume.js');
|
|
153
|
+
// Re-query with time filter for real filesystem; if mock was used for test, keep mock result
|
|
154
|
+
if (process.env.VITEST)
|
|
155
|
+
return res;
|
|
156
|
+
return findInterrupted(undefined, { withinMs: ms });
|
|
157
|
+
}
|
|
158
|
+
catch { }
|
|
159
|
+
}
|
|
160
|
+
return res;
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
// fallback to real
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
try {
|
|
167
|
+
const { findInterrupted } = await import('./resume.js');
|
|
168
|
+
return findInterrupted(undefined, { withinMs: ms });
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
return this.getFindInterrupted()();
|
|
172
|
+
}
|
|
173
|
+
}
|
|
19
174
|
async collectGitDiff() {
|
|
20
175
|
try {
|
|
21
176
|
const { execSync } = await import('node:child_process');
|
|
22
|
-
const ws =
|
|
177
|
+
const ws = resolveHarnessRoot();
|
|
23
178
|
try {
|
|
24
179
|
const out = execSync(`git -C ${JSON.stringify(ws)} status --porcelain 2>/dev/null | head -n 50`, { encoding: 'utf-8', timeout: 2000 });
|
|
25
180
|
if (out.trim()) {
|
|
@@ -35,13 +190,32 @@ export class Supervisor {
|
|
|
35
190
|
return '';
|
|
36
191
|
}
|
|
37
192
|
}
|
|
193
|
+
async attemptAutoResume(ids) {
|
|
194
|
+
if (!ids.length)
|
|
195
|
+
return;
|
|
196
|
+
if (!this.getAutoResumeEnabled()) {
|
|
197
|
+
await this.deps.notify(`RESUME: ${ids.length} interrupted sessions (${ids.slice(0, 3).join(', ')}) — auto-resume disabled`).catch(() => { });
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
try {
|
|
201
|
+
const { resumed } = await this.getResumeSessions()(ids);
|
|
202
|
+
if (!resumed.length) {
|
|
203
|
+
await this.deps.notify(`RESUME SKIPPED: no interrupted sessions could be re-attached (${ids.slice(0, 3).join(', ')})`).catch(() => { });
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
await this.deps.notify(`RESUME: ${resumed.length} interrupted sessions — continue triggered (${resumed.slice(0, 3).join(', ')})`).catch(() => { });
|
|
207
|
+
}
|
|
208
|
+
catch (e) {
|
|
209
|
+
await this.deps.notify(`RESUME FAILED: ${ids.length} interrupted sessions (${ids.slice(0, 3).join(', ')}) — ${e?.message ?? String(e)}`).catch(() => { });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
38
212
|
handleDebugResult(reportPath, res) {
|
|
39
213
|
if (res.fixed) {
|
|
40
214
|
void this.deps.notify(`FIXED: debug-agent fixed ${reportPath} — ${res.reason}`).catch(() => { });
|
|
41
|
-
// After fix, try to resume interrupted sessions
|
|
42
|
-
void this.
|
|
215
|
+
// After fix, try to resume interrupted sessions (only recent, default 5 from config, in minutes)
|
|
216
|
+
void this.findInterruptedRecent().then(r => {
|
|
43
217
|
if (r.interrupted.length)
|
|
44
|
-
void this.
|
|
218
|
+
void this.attemptAutoResume(r.interrupted).catch(() => { });
|
|
45
219
|
}).catch(() => { });
|
|
46
220
|
}
|
|
47
221
|
else if (res.reason.includes('max attempts')) {
|
|
@@ -71,23 +245,22 @@ export class Supervisor {
|
|
|
71
245
|
await this.deps.notify(`DEGRADED: ${health.error ?? 'plugin'} (report: ${reportPath})`).catch(() => { });
|
|
72
246
|
// Phase 3: debug + resume — use injected fn if provided (even in VITEST), otherwise fire-and-forget real impl (skip in VITEST)
|
|
73
247
|
const runner = this.getRunDebugAgent();
|
|
74
|
-
const finder = this.getFindInterrupted();
|
|
75
248
|
const isInjected = !!this.deps.runDebugAgent;
|
|
76
249
|
if (isInjected) {
|
|
77
250
|
void runner({ reportPath, health }).then(res => this.handleDebugResult(reportPath, res)).catch(() => { });
|
|
78
251
|
setTimeout(() => {
|
|
79
|
-
|
|
252
|
+
this.findInterruptedRecent().then(r => {
|
|
80
253
|
if (r.interrupted.length)
|
|
81
|
-
this.
|
|
254
|
+
void this.attemptAutoResume(r.interrupted).catch(() => { });
|
|
82
255
|
}).catch(() => { });
|
|
83
256
|
}, 0);
|
|
84
257
|
}
|
|
85
258
|
else if (!process.env.VITEST) {
|
|
86
259
|
void runner({ reportPath, health }).then(res => this.handleDebugResult(reportPath, res)).catch(() => { });
|
|
87
260
|
setTimeout(() => {
|
|
88
|
-
|
|
261
|
+
this.findInterruptedRecent().then(r => {
|
|
89
262
|
if (r.interrupted.length)
|
|
90
|
-
this.
|
|
263
|
+
void this.attemptAutoResume(r.interrupted).catch(() => { });
|
|
91
264
|
}).catch(() => { });
|
|
92
265
|
}, 0);
|
|
93
266
|
}
|
|
@@ -124,26 +297,40 @@ export class Supervisor {
|
|
|
124
297
|
const logTail = health.logTail ?? '';
|
|
125
298
|
const gitDiff = await this.collectGitDiff().catch(() => '');
|
|
126
299
|
const reportPath = await this.deps.writeReport({ ts, health, action: `rollback — ${health.error ?? 'down'}`, logTail, gitDiff }).catch(() => '');
|
|
127
|
-
|
|
300
|
+
try {
|
|
301
|
+
await this.deps.rollback();
|
|
302
|
+
}
|
|
303
|
+
catch (e) {
|
|
304
|
+
await this.deps.notify(`rollback failed: ${e?.message ?? String(e)} (report: ${reportPath})`).catch(() => { });
|
|
305
|
+
}
|
|
306
|
+
// Always attempt to (re)start dsh web — survives reboot even when rollback is a no-op
|
|
307
|
+
if (this.deps.restartWeb) {
|
|
308
|
+
try {
|
|
309
|
+
await this.deps.restartWeb();
|
|
310
|
+
await this.deps.notify(`restarted dsh-web after rollback (report: ${reportPath})`).catch(() => { });
|
|
311
|
+
}
|
|
312
|
+
catch (e) {
|
|
313
|
+
await this.deps.notify(`restart dsh-web failed: ${e?.message ?? String(e)} (report: ${reportPath})`).catch(() => { });
|
|
314
|
+
}
|
|
315
|
+
}
|
|
128
316
|
await this.deps.notify(`CRASH detected → rollback (report: ${reportPath}, error: ${health.error ?? 'down'})`).catch(() => { });
|
|
129
317
|
const runner = this.getRunDebugAgent();
|
|
130
|
-
const finder = this.getFindInterrupted();
|
|
131
318
|
const isInjected = !!this.deps.runDebugAgent;
|
|
132
319
|
if (isInjected) {
|
|
133
320
|
void runner({ reportPath, health }).then(res => this.handleDebugResult(reportPath, res)).catch(() => { });
|
|
134
321
|
setTimeout(() => {
|
|
135
|
-
|
|
322
|
+
this.findInterruptedRecent().then(r => {
|
|
136
323
|
if (r.interrupted.length)
|
|
137
|
-
this.
|
|
324
|
+
void this.attemptAutoResume(r.interrupted).catch(() => { });
|
|
138
325
|
}).catch(() => { });
|
|
139
326
|
}, 0);
|
|
140
327
|
}
|
|
141
328
|
else if (!process.env.VITEST) {
|
|
142
329
|
void runner({ reportPath, health }).then(res => this.handleDebugResult(reportPath, res)).catch(() => { });
|
|
143
330
|
setTimeout(() => {
|
|
144
|
-
|
|
331
|
+
this.findInterruptedRecent().then(r => {
|
|
145
332
|
if (r.interrupted.length)
|
|
146
|
-
this.
|
|
333
|
+
void this.attemptAutoResume(r.interrupted).catch(() => { });
|
|
147
334
|
}).catch(() => { });
|
|
148
335
|
}, 0);
|
|
149
336
|
}
|
|
@@ -157,6 +344,8 @@ export class Supervisor {
|
|
|
157
344
|
return;
|
|
158
345
|
const intervalMs = this.deps.intervalMs ?? 3000;
|
|
159
346
|
this.timer = setInterval(() => { this.tick().catch(() => { }); }, intervalMs);
|
|
347
|
+
// Immediate tick so a reboot is recovered in ~0-3s, not 3s
|
|
348
|
+
this.tick().catch(() => { });
|
|
160
349
|
}
|
|
161
350
|
stop() {
|
|
162
351
|
if (this.timer) {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-maestro-supervisor — client auto-reload for DSH Web after restart.
|
|
3
|
+
* Hybrid: polls `HEAD /` when the server is down (offline/WebSocket close)
|
|
4
|
+
* and reloads as soon as it is back. The host also pushes a reload via
|
|
5
|
+
* `POST /dsh-maestro-supervisor-reload` (loopback) when it recovers.
|
|
6
|
+
*/
|
|
7
|
+
export declare function apply(ctx: any): void;
|
|
8
|
+
//# sourceMappingURL=auto-reload.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auto-reload.d.ts","sourceRoot":"","sources":["../../../src/client/auto-reload.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,wBAAgB,KAAK,CAAC,GAAG,EAAE,GAAG,GAAG,IAAI,CA4FpC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/client/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,14 +1,40 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ddtcorex/dsh-maestro-supervisor",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "Supervisor daemon for DSH Web resilience — auto-detect crashes, rollback to LKG, report",
|
|
5
5
|
"type": "module",
|
|
6
|
+
"main": "./lib/index.js",
|
|
7
|
+
"types": "./lib/types/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./lib/types/index.d.ts",
|
|
11
|
+
"default": "./lib/index.js"
|
|
12
|
+
},
|
|
13
|
+
"./client": {
|
|
14
|
+
"types": "./lib/types/client/auto-reload.d.ts",
|
|
15
|
+
"default": "./lib/client.js"
|
|
16
|
+
},
|
|
17
|
+
"./cordis.patch.yml": "./cordis.patch.yml",
|
|
18
|
+
"./package.json": "./package.json"
|
|
19
|
+
},
|
|
6
20
|
"bin": {
|
|
7
|
-
"dsh-web-supervisor": "./lib/
|
|
21
|
+
"dsh-web-supervisor": "./lib/bin.js"
|
|
22
|
+
},
|
|
23
|
+
"dsh": {
|
|
24
|
+
"bundle": {
|
|
25
|
+
"patch": "./cordis.patch.yml"
|
|
26
|
+
},
|
|
27
|
+
"client": {
|
|
28
|
+
"platform": "web",
|
|
29
|
+
"inject": [
|
|
30
|
+
"@deepseek-ai/dsh-client-runtime"
|
|
31
|
+
]
|
|
32
|
+
}
|
|
8
33
|
},
|
|
9
34
|
"files": [
|
|
10
35
|
"lib",
|
|
11
|
-
"README.md"
|
|
36
|
+
"README.md",
|
|
37
|
+
"cordis.patch.yml"
|
|
12
38
|
],
|
|
13
39
|
"devDependencies": {
|
|
14
40
|
"@types/node": "^26.3.0",
|
|
@@ -16,8 +42,8 @@
|
|
|
16
42
|
"vitest": "^3.2.4"
|
|
17
43
|
},
|
|
18
44
|
"scripts": {
|
|
19
|
-
"build": "tsc -p tsconfig.json",
|
|
20
|
-
"verify": "tsc --noEmit",
|
|
45
|
+
"build": "tsc -p tsconfig.json && tsc -p tsconfig.client.json && node scripts/build-client.mjs",
|
|
46
|
+
"verify": "tsc --noEmit && tsc -p tsconfig.client.json --noEmit",
|
|
21
47
|
"test": "vitest run",
|
|
22
48
|
"test:watch": "vitest"
|
|
23
49
|
}
|