@ddtcorex/dsh-maestro-supervisor 0.6.8 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli.d.ts +13 -0
- package/lib/cli.js +87 -55
- package/lib/plugin.d.ts +1 -1
- package/lib/plugin.js +73 -1
- package/lib/restart-guards.d.ts +11 -0
- package/lib/restart-guards.js +23 -0
- package/lib/restart-tool.d.ts +70 -0
- package/lib/restart-tool.js +241 -0
- package/lib/scan.d.ts +20 -0
- package/lib/scan.js +71 -0
- package/lib/self-kill-guard.d.ts +26 -0
- package/lib/self-kill-guard.js +63 -0
- package/lib/skill-provider.d.ts +6 -0
- package/lib/skill-provider.js +70 -0
- package/lib/supervisor.d.ts +9 -0
- package/lib/supervisor.js +73 -1
- package/package.json +2 -1
- package/skills/dsh-safe-restart/SKILL.md +90 -0
- package/skills/dsh-safe-restart/scripts/restart-dsh-web.sh +279 -0
package/lib/cli.d.ts
CHANGED
|
@@ -1 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copy a chosen LKG snapshot back into the DSH home. Recent snapshots are
|
|
3
|
+
* tried newest-first, skipping any that still carry a failing plugin so a
|
|
4
|
+
* broken bundle is not restored. `sessions/` is deliberately NOT restored —
|
|
5
|
+
* session logs are append-only truth and rolling them back to a snapshot
|
|
6
|
+
* would drop every turn recorded after that snapshot.
|
|
7
|
+
* @returns the rolled-back snapshot id.
|
|
8
|
+
*/
|
|
9
|
+
export declare function rollbackLKG(opts: {
|
|
10
|
+
dshHome: string;
|
|
11
|
+
lkgRoot: string;
|
|
12
|
+
failingPlugin?: string;
|
|
13
|
+
}): Promise<string>;
|
|
1
14
|
export declare function runCli(args: string[]): Promise<void>;
|
package/lib/cli.js
CHANGED
|
@@ -5,7 +5,76 @@ import * as fs from 'node:fs';
|
|
|
5
5
|
import * as path from 'node:path';
|
|
6
6
|
import * as os from 'node:os';
|
|
7
7
|
import { resolveHarnessRoot, resolveDeepseekHarnessDir } from './paths.js';
|
|
8
|
-
import { buildKillStalePortsCommand, isSelfCopyError, checkPlannedRestart, writePlannedRestart } from './restart-guards.js';
|
|
8
|
+
import { buildKillStalePortsCommand, isSelfCopyError, checkPlannedRestart, writePlannedRestart, readRestartRequest, clearPlannedRestart } from './restart-guards.js';
|
|
9
|
+
/**
|
|
10
|
+
* Copy a chosen LKG snapshot back into the DSH home. Recent snapshots are
|
|
11
|
+
* tried newest-first, skipping any that still carry a failing plugin so a
|
|
12
|
+
* broken bundle is not restored. `sessions/` is deliberately NOT restored —
|
|
13
|
+
* session logs are append-only truth and rolling them back to a snapshot
|
|
14
|
+
* would drop every turn recorded after that snapshot.
|
|
15
|
+
* @returns the rolled-back snapshot id.
|
|
16
|
+
*/
|
|
17
|
+
export async function rollbackLKG(opts) {
|
|
18
|
+
const { dshHome, lkgRoot, failingPlugin } = opts;
|
|
19
|
+
const entries = fs.existsSync(lkgRoot) ? fs.readdirSync(lkgRoot).sort() : [];
|
|
20
|
+
if (!entries.length)
|
|
21
|
+
throw new Error('no LKG to rollback to');
|
|
22
|
+
const candidates = [...entries].reverse().slice(0, 3);
|
|
23
|
+
let chosen;
|
|
24
|
+
for (const cand of candidates) {
|
|
25
|
+
if (!failingPlugin) {
|
|
26
|
+
chosen = cand;
|
|
27
|
+
break;
|
|
28
|
+
}
|
|
29
|
+
try {
|
|
30
|
+
const pkgPath = path.join(lkgRoot, cand, 'profiles/web/package.json');
|
|
31
|
+
if (!fs.existsSync(pkgPath)) {
|
|
32
|
+
chosen = cand;
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
|
|
36
|
+
const bundles = pkg?.dsh?.profile?.bundles ?? [];
|
|
37
|
+
const deps = pkg?.dependencies ?? {};
|
|
38
|
+
const hasFailing = bundles.some((b) => b.includes(failingPlugin)) || Object.keys(deps).some(k => k.includes(failingPlugin));
|
|
39
|
+
if (!hasFailing) {
|
|
40
|
+
chosen = cand;
|
|
41
|
+
break;
|
|
42
|
+
}
|
|
43
|
+
console.log(`[supervisor] skipping LKG ${cand} still contains failing plugin ${failingPlugin}`);
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
chosen = cand;
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const target = chosen ?? entries[entries.length - 1];
|
|
51
|
+
const src = path.join(lkgRoot, target);
|
|
52
|
+
for (const entry of fs.readdirSync(src)) {
|
|
53
|
+
if (entry === 'manifest.json')
|
|
54
|
+
continue;
|
|
55
|
+
if (entry === 'sessions') {
|
|
56
|
+
console.log('[supervisor] rollback keeps live sessions (append-only truth) — skipping sessions/');
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const srcPath = path.join(src, entry);
|
|
60
|
+
const destPath = path.join(dshHome, entry);
|
|
61
|
+
try {
|
|
62
|
+
// Skip if src and dest are the same file (e.g. symlink to same target like ~/.dsh/AGENTS.md)
|
|
63
|
+
try {
|
|
64
|
+
if (fs.existsSync(srcPath) && fs.existsSync(destPath) && fs.realpathSync(srcPath) === fs.realpathSync(destPath))
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
catch { }
|
|
68
|
+
fs.cpSync(srcPath, destPath, { recursive: true, force: true });
|
|
69
|
+
}
|
|
70
|
+
catch (e) {
|
|
71
|
+
if (isSelfCopyError(String(e?.message ?? '')))
|
|
72
|
+
continue;
|
|
73
|
+
throw e;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return target;
|
|
77
|
+
}
|
|
9
78
|
export async function runCli(args) {
|
|
10
79
|
const cmd = args[2] ?? '--help';
|
|
11
80
|
if (cmd === '--help' || cmd === '-h' || cmd === 'help') {
|
|
@@ -108,10 +177,6 @@ Commands:
|
|
|
108
177
|
},
|
|
109
178
|
rollback: async () => {
|
|
110
179
|
const { execSync } = await import('node:child_process');
|
|
111
|
-
const entries = fs.existsSync(lkgRoot) ? fs.readdirSync(lkgRoot).sort() : [];
|
|
112
|
-
if (!entries.length)
|
|
113
|
-
throw new Error('no LKG to rollback to');
|
|
114
|
-
// Try newest to oldest (up to 3) to find a clean LKG for plugin failures
|
|
115
180
|
// Extract failing plugin from current log tail if possible
|
|
116
181
|
let failingPlugin;
|
|
117
182
|
try {
|
|
@@ -121,56 +186,7 @@ Commands:
|
|
|
121
186
|
failingPlugin = m[0].replace(/^@ddtcorex\//, '');
|
|
122
187
|
}
|
|
123
188
|
catch { }
|
|
124
|
-
const
|
|
125
|
-
let chosen;
|
|
126
|
-
for (const cand of candidates) {
|
|
127
|
-
if (!failingPlugin) {
|
|
128
|
-
chosen = cand;
|
|
129
|
-
break;
|
|
130
|
-
}
|
|
131
|
-
try {
|
|
132
|
-
const pkgPath = path.join(lkgRoot, cand, 'profiles/web/package.json');
|
|
133
|
-
if (!fs.existsSync(pkgPath)) {
|
|
134
|
-
chosen = cand;
|
|
135
|
-
break;
|
|
136
|
-
}
|
|
137
|
-
const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
|
|
138
|
-
const bundles = pkg?.dsh?.profile?.bundles ?? [];
|
|
139
|
-
const deps = pkg?.dependencies ?? {};
|
|
140
|
-
const hasFailing = bundles.some((b) => b.includes(failingPlugin)) || Object.keys(deps).some(k => k.includes(failingPlugin));
|
|
141
|
-
if (!hasFailing) {
|
|
142
|
-
chosen = cand;
|
|
143
|
-
break;
|
|
144
|
-
}
|
|
145
|
-
console.log(`[supervisor] skipping LKG ${cand} still contains failing plugin ${failingPlugin}`);
|
|
146
|
-
}
|
|
147
|
-
catch {
|
|
148
|
-
chosen = cand;
|
|
149
|
-
break;
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
const target = chosen ?? entries[entries.length - 1];
|
|
153
|
-
const src = path.join(lkgRoot, target);
|
|
154
|
-
for (const entry of fs.readdirSync(src)) {
|
|
155
|
-
if (entry === 'manifest.json')
|
|
156
|
-
continue;
|
|
157
|
-
const srcPath = path.join(src, entry);
|
|
158
|
-
const destPath = path.join(dshHome, entry);
|
|
159
|
-
try {
|
|
160
|
-
// Skip if src and dest are the same file (e.g. symlink to same target like ~/.dsh/AGENTS.md)
|
|
161
|
-
try {
|
|
162
|
-
if (fs.existsSync(srcPath) && fs.existsSync(destPath) && fs.realpathSync(srcPath) === fs.realpathSync(destPath))
|
|
163
|
-
continue;
|
|
164
|
-
}
|
|
165
|
-
catch { }
|
|
166
|
-
fs.cpSync(srcPath, destPath, { recursive: true, force: true });
|
|
167
|
-
}
|
|
168
|
-
catch (e) {
|
|
169
|
-
if (isSelfCopyError(String(e?.message ?? '')))
|
|
170
|
-
continue;
|
|
171
|
-
throw e;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
189
|
+
const target = await rollbackLKG({ dshHome, lkgRoot, failingPlugin });
|
|
174
190
|
console.log(`[supervisor] rolled back to ${target}${failingPlugin ? ` (avoiding ${failingPlugin})` : ''}`);
|
|
175
191
|
// Reconcile node_modules from restored package.json (critical for link: deps)
|
|
176
192
|
try {
|
|
@@ -229,6 +245,22 @@ Commands:
|
|
|
229
245
|
},
|
|
230
246
|
notify: async (msg) => console.log(`[notify] ${msg}`),
|
|
231
247
|
isPlannedRestartActive: () => checkPlannedRestart(),
|
|
248
|
+
// dsh_web_restart marker ownership: the daemon acts on the marker. After
|
|
249
|
+
// the restart, scan recent session logs for torn tails (an in-flight
|
|
250
|
+
// session truncated by the restart) and report; the marker is cleared in
|
|
251
|
+
// the supervisor's own finally block and again here as a safety net.
|
|
252
|
+
readRestartRequest: () => readRestartRequest(),
|
|
253
|
+
onRestartRequestHandled: async () => {
|
|
254
|
+
const { scanSessions } = await import('./scan.js');
|
|
255
|
+
const res = await scanSessions(path.join(os.homedir(), '.dsh'), { withinMs: 10 * 60 * 1000 }).catch(() => ({ scanned: 0, torn: [] }));
|
|
256
|
+
if (res.torn.length) {
|
|
257
|
+
console.log(`[supervisor] post-self-restart scan: ${res.torn.length} torn session log(s)`);
|
|
258
|
+
}
|
|
259
|
+
else {
|
|
260
|
+
console.log('[supervisor] post-self-restart scan: clean');
|
|
261
|
+
}
|
|
262
|
+
void clearPlannedRestart();
|
|
263
|
+
},
|
|
232
264
|
});
|
|
233
265
|
await supervisor.start();
|
|
234
266
|
// keep process alive
|
package/lib/plugin.d.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* and web restart; this plugin handles the in-process resume.
|
|
7
7
|
*/
|
|
8
8
|
import { findInterrupted as defaultFindInterrupted, findDanglingOpenTurns as defaultFindDanglingOpenTurns } from './resume.js';
|
|
9
|
-
export declare const inject: readonly ["sessions", "agents", "connection"];
|
|
9
|
+
export declare const inject: readonly ["sessions", "agents", "connection", "skills"];
|
|
10
10
|
export interface SupervisorPluginConfig {
|
|
11
11
|
autoResumeWithin?: number | string;
|
|
12
12
|
autoResumeEnabled?: boolean;
|
package/lib/plugin.js
CHANGED
|
@@ -8,8 +8,13 @@
|
|
|
8
8
|
import * as fs from 'node:fs';
|
|
9
9
|
import * as path from 'node:path';
|
|
10
10
|
import * as os from 'node:os';
|
|
11
|
+
import { fileURLToPath } from 'node:url';
|
|
11
12
|
import { findInterrupted as defaultFindInterrupted, findDanglingOpenTurns as defaultFindDanglingOpenTurns } from './resume.js';
|
|
12
|
-
|
|
13
|
+
import { makeSkillProvider } from './skill-provider.js';
|
|
14
|
+
import { registerRestartTool } from './restart-tool.js';
|
|
15
|
+
import { makePreExecuteGuard } from './self-kill-guard.js';
|
|
16
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
17
|
+
export const inject = ['sessions', 'agents', 'connection', 'skills'];
|
|
13
18
|
function parseDuration(s) {
|
|
14
19
|
if (!s)
|
|
15
20
|
return undefined;
|
|
@@ -280,6 +285,25 @@ export function createResumeRpcHandler(ctx, opts = {}) {
|
|
|
280
285
|
return { ok: true, value: { resumed: await resume(ctx, ids) } };
|
|
281
286
|
};
|
|
282
287
|
}
|
|
288
|
+
/** Resolve the package-root skills/ dir regardless of module layout. The built
|
|
289
|
+
* host lib is flat (lib/plugin.js → ../skills), but under vitest the same
|
|
290
|
+
* module loads from src/host/ (→ ../../skills). Walking to the nearest
|
|
291
|
+
* package.json yields the same package-root skills/ in both layouts. */
|
|
292
|
+
function resolveSkillsDir(fromDir) {
|
|
293
|
+
let dir = fromDir;
|
|
294
|
+
for (let i = 0; i < 6; i++) {
|
|
295
|
+
try {
|
|
296
|
+
if (fs.existsSync(path.join(dir, 'package.json')))
|
|
297
|
+
return path.join(dir, 'skills');
|
|
298
|
+
}
|
|
299
|
+
catch { }
|
|
300
|
+
const parent = path.dirname(dir);
|
|
301
|
+
if (parent === dir)
|
|
302
|
+
break;
|
|
303
|
+
dir = parent;
|
|
304
|
+
}
|
|
305
|
+
return path.join(fromDir, '..', 'skills');
|
|
306
|
+
}
|
|
283
307
|
function ensureSystemdKeepalive(ctx) {
|
|
284
308
|
// Best-effort: ensure dsh-web-keepalive.service exists and is enabled, and linger is on.
|
|
285
309
|
// This is the user-level auto-fix for the 11:42:58 crash where manager session 97
|
|
@@ -365,6 +389,36 @@ export function apply(ctx, config = {}) {
|
|
|
365
389
|
ensureSystemdKeepalive(ctx);
|
|
366
390
|
}
|
|
367
391
|
catch { }
|
|
392
|
+
try {
|
|
393
|
+
const skills = ctx.get?.('skills');
|
|
394
|
+
if (skills?.registerProvider) {
|
|
395
|
+
ctx.effect(() => {
|
|
396
|
+
let unregister;
|
|
397
|
+
try {
|
|
398
|
+
// Package-root skills/ is resolved at runtime by walking to the
|
|
399
|
+
// nearest package.json (robust to lib/ vs src/host/ layouts).
|
|
400
|
+
unregister = skills.registerProvider(() => makeSkillProvider(resolveSkillsDir(__dirname)));
|
|
401
|
+
}
|
|
402
|
+
catch (e) {
|
|
403
|
+
ctx.logger?.warn?.(`[supervisor] skill provider failed: ${e?.message ?? String(e)}`);
|
|
404
|
+
}
|
|
405
|
+
return () => { try {
|
|
406
|
+
unregister?.();
|
|
407
|
+
}
|
|
408
|
+
catch { } };
|
|
409
|
+
}, 'supervisor:skill');
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
catch { }
|
|
413
|
+
try {
|
|
414
|
+
ctx.effect(() => registerRestartTool(ctx), 'supervisor:restart-tool');
|
|
415
|
+
}
|
|
416
|
+
catch (e) {
|
|
417
|
+
try {
|
|
418
|
+
ctx.logger?.warn?.(`[supervisor] restart tool effect failed: ${e?.message ?? String(e)}`);
|
|
419
|
+
}
|
|
420
|
+
catch { }
|
|
421
|
+
}
|
|
368
422
|
ctx.effect(() => {
|
|
369
423
|
let disposed = false;
|
|
370
424
|
let timer = null;
|
|
@@ -400,6 +454,24 @@ export function apply(ctx, config = {}) {
|
|
|
400
454
|
}
|
|
401
455
|
};
|
|
402
456
|
}, 'supervisor:auto-resume');
|
|
457
|
+
try {
|
|
458
|
+
// Deny bash/shell self-kill commands in-tree; the safe restart path is
|
|
459
|
+
// dsh_web_restart (supervisor daemon owns the actual restart).
|
|
460
|
+
const guard = makePreExecuteGuard();
|
|
461
|
+
ctx.effect(() => {
|
|
462
|
+
const un = ctx.on?.('tools/pre-execute', guard) ?? null;
|
|
463
|
+
return () => { try {
|
|
464
|
+
un?.();
|
|
465
|
+
}
|
|
466
|
+
catch { } };
|
|
467
|
+
}, 'supervisor:self-kill-guard');
|
|
468
|
+
}
|
|
469
|
+
catch (e) {
|
|
470
|
+
try {
|
|
471
|
+
ctx.logger?.warn?.(`[supervisor] self-kill guard effect failed: ${e?.message ?? String(e)}`);
|
|
472
|
+
}
|
|
473
|
+
catch { }
|
|
474
|
+
}
|
|
403
475
|
}
|
|
404
476
|
catch (e) {
|
|
405
477
|
try {
|
package/lib/restart-guards.d.ts
CHANGED
|
@@ -6,3 +6,14 @@ export declare function plannedRestartPath(): string;
|
|
|
6
6
|
export declare function writePlannedRestart(ttlMs?: number): void;
|
|
7
7
|
export declare function checkPlannedRestart(markerPath?: string): boolean;
|
|
8
8
|
export declare function clearPlannedRestart(): void;
|
|
9
|
+
export interface RestartRequest {
|
|
10
|
+
ts: number;
|
|
11
|
+
ttl: number;
|
|
12
|
+
callerSessionId?: string;
|
|
13
|
+
reason?: string;
|
|
14
|
+
}
|
|
15
|
+
export declare function writeRestartRequest(caller: {
|
|
16
|
+
callerSessionId?: string;
|
|
17
|
+
reason?: string;
|
|
18
|
+
}, ttlMs?: number): void;
|
|
19
|
+
export declare function readRestartRequest(): RestartRequest | undefined;
|
package/lib/restart-guards.js
CHANGED
|
@@ -92,3 +92,26 @@ export function clearPlannedRestart() {
|
|
|
92
92
|
}
|
|
93
93
|
catch { }
|
|
94
94
|
}
|
|
95
|
+
export function writeRestartRequest(caller, ttlMs = 180_000) {
|
|
96
|
+
const p = plannedRestartPath();
|
|
97
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
98
|
+
const body = { ts: Date.now(), ttl: ttlMs, ...caller };
|
|
99
|
+
fs.writeFileSync(p, JSON.stringify(body), { mode: 0o600 });
|
|
100
|
+
try {
|
|
101
|
+
fs.chmodSync(p, 0o600);
|
|
102
|
+
}
|
|
103
|
+
catch { }
|
|
104
|
+
}
|
|
105
|
+
export function readRestartRequest() {
|
|
106
|
+
try {
|
|
107
|
+
const raw = fs.readFileSync(plannedRestartPath(), 'utf8');
|
|
108
|
+
const j = JSON.parse(raw);
|
|
109
|
+
if (typeof j.ts === 'number' && typeof j.ttl === 'number') {
|
|
110
|
+
if (Date.now() - j.ts >= j.ttl)
|
|
111
|
+
return undefined;
|
|
112
|
+
return { ts: j.ts, ttl: j.ttl, callerSessionId: j.callerSessionId, reason: j.reason };
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
catch { }
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh_web_restart tool — the safe restart path for the model running inside
|
|
3
|
+
* dsh web. Scheduling a restart through this tool instead of a raw kill keeps
|
|
4
|
+
* the restart inside the supervisor's ownership loop:
|
|
5
|
+
*
|
|
6
|
+
* 1. dry-boot gate — if the plugin tree changed since the latest LKG, boot a
|
|
7
|
+
* copy of the live profile on an ephemeral DSH_HOME first and only
|
|
8
|
+
* schedule when that boot serves HTTP.
|
|
9
|
+
* 2. intent sidecar — record the caller session + reason under
|
|
10
|
+
* ~/.dsh/.supervisor/intents/ for attribution.
|
|
11
|
+
* 3. hand-off — write the restart-request marker (planned-restart.json with
|
|
12
|
+
* callerSessionId) that the supervisor daemon owns and acts on
|
|
13
|
+
* (out-of-band). This tool NEVER restarts the host in-tree.
|
|
14
|
+
*/
|
|
15
|
+
import { writeRestartRequest } from './restart-guards.js';
|
|
16
|
+
/**
|
|
17
|
+
* Boot a copy of the live web profile on an isolated DSH_HOME and verify the
|
|
18
|
+
* plugin tree loads and serves. Returns ok + a one-line detail for the tool
|
|
19
|
+
* message. The spawned tree is killed (best-effort) and the temp home removed.
|
|
20
|
+
* Unit tests mock this (never spawn a real node boot in tests).
|
|
21
|
+
*
|
|
22
|
+
* NOTE (bin.ts finding): the `web` alias already implies `--profile web`, and
|
|
23
|
+
* the web app's own commander program (no allowUnknownOption) rejects a stray
|
|
24
|
+
* `--profile` in its inner args — so the spawn passes only `web --no-open
|
|
25
|
+
* --port <port>`.
|
|
26
|
+
*/
|
|
27
|
+
export declare function dryBootVerify(harnessRoot: string, opts?: {
|
|
28
|
+
timeoutMs?: number;
|
|
29
|
+
}): Promise<{
|
|
30
|
+
ok: boolean;
|
|
31
|
+
detail: string;
|
|
32
|
+
}>;
|
|
33
|
+
/** Minimal file metadata the drift check reads; injectable for deterministic tests. */
|
|
34
|
+
export interface FileStat {
|
|
35
|
+
mtimeMs: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Whether the live plugin tree differs from the latest LKG snapshot. Two
|
|
39
|
+
* signals are combined:
|
|
40
|
+
*
|
|
41
|
+
* 1. manifest drift — the live profile's web `package.json` text vs baseline;
|
|
42
|
+
* 2. plugin-lib drift — any `@ddtcorex` plugin `lib/` file newer than the
|
|
43
|
+
* snapshot moment. Link-installed plugins resolve to the same workspace
|
|
44
|
+
* files in both live and LKG, so the stored copies cannot be compared
|
|
45
|
+
* byte-wise; the snapshot itself is the meaningful baseline and a rebuilt
|
|
46
|
+
* `lib/` bumps a file past it even when the manifest text is unchanged.
|
|
47
|
+
* writeLKG writes `manifest.json` LAST, so its FILE mtime is the
|
|
48
|
+
* authoritative snapshot moment; the snapshot dir mtime is only a
|
|
49
|
+
* fallback for legacy snapshots without a manifest.
|
|
50
|
+
*
|
|
51
|
+
* `statFile` (default `statSync`) reads the metadata so tests can inject a
|
|
52
|
+
* controlled reader instead of relying on filesystem utimes (which CI runners
|
|
53
|
+
* do not reliably reflect). No LKG baseline, a missing file on either side, or
|
|
54
|
+
* any stat/read error means "changed" — the caller falls back to the dry-boot
|
|
55
|
+
* gate.
|
|
56
|
+
*/
|
|
57
|
+
export declare function isPluginTreeChanged(harnessRoot: string, lkgDir?: string, opts?: {
|
|
58
|
+
statFile?: (p: string) => FileStat;
|
|
59
|
+
}): boolean;
|
|
60
|
+
/**
|
|
61
|
+
* Register the dsh_web_restart tool. Registration is fail-safe (warns, never
|
|
62
|
+
* throws) and the returned function disposes the registration. `deps` are
|
|
63
|
+
* injectable for tests.
|
|
64
|
+
*/
|
|
65
|
+
export declare function registerRestartTool(ctx: any, deps?: {
|
|
66
|
+
sessionIdOf?: (exec: any) => string | undefined;
|
|
67
|
+
dryBoot?: typeof dryBootVerify;
|
|
68
|
+
writeRestartRequest?: typeof writeRestartRequest;
|
|
69
|
+
harnessRoot?: string;
|
|
70
|
+
}): () => void;
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh_web_restart tool — the safe restart path for the model running inside
|
|
3
|
+
* dsh web. Scheduling a restart through this tool instead of a raw kill keeps
|
|
4
|
+
* the restart inside the supervisor's ownership loop:
|
|
5
|
+
*
|
|
6
|
+
* 1. dry-boot gate — if the plugin tree changed since the latest LKG, boot a
|
|
7
|
+
* copy of the live profile on an ephemeral DSH_HOME first and only
|
|
8
|
+
* schedule when that boot serves HTTP.
|
|
9
|
+
* 2. intent sidecar — record the caller session + reason under
|
|
10
|
+
* ~/.dsh/.supervisor/intents/ for attribution.
|
|
11
|
+
* 3. hand-off — write the restart-request marker (planned-restart.json with
|
|
12
|
+
* callerSessionId) that the supervisor daemon owns and acts on
|
|
13
|
+
* (out-of-band). This tool NEVER restarts the host in-tree.
|
|
14
|
+
*/
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
import { mkdtempSync, rmSync, cpSync, existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
|
17
|
+
import { tmpdir, homedir } from 'node:os';
|
|
18
|
+
import { spawn } from 'node:child_process';
|
|
19
|
+
import { createRequire } from 'node:module';
|
|
20
|
+
import { writeRestartRequest } from './restart-guards.js';
|
|
21
|
+
/**
|
|
22
|
+
* Boot a copy of the live web profile on an isolated DSH_HOME and verify the
|
|
23
|
+
* plugin tree loads and serves. Returns ok + a one-line detail for the tool
|
|
24
|
+
* message. The spawned tree is killed (best-effort) and the temp home removed.
|
|
25
|
+
* Unit tests mock this (never spawn a real node boot in tests).
|
|
26
|
+
*
|
|
27
|
+
* NOTE (bin.ts finding): the `web` alias already implies `--profile web`, and
|
|
28
|
+
* the web app's own commander program (no allowUnknownOption) rejects a stray
|
|
29
|
+
* `--profile` in its inner args — so the spawn passes only `web --no-open
|
|
30
|
+
* --port <port>`.
|
|
31
|
+
*/
|
|
32
|
+
export async function dryBootVerify(harnessRoot, opts = {}) {
|
|
33
|
+
const timeoutMs = opts.timeoutMs ?? 60_000;
|
|
34
|
+
const home = homedir();
|
|
35
|
+
const liveProfile = join(home, '.dsh', 'profiles', 'web');
|
|
36
|
+
if (!existsSync(liveProfile))
|
|
37
|
+
return { ok: true, detail: 'skipped (no live web profile)' };
|
|
38
|
+
const tmpHome = mkdtempSync(join(tmpdir(), 'dsh-dryboot-'));
|
|
39
|
+
const logs = [];
|
|
40
|
+
let child = null;
|
|
41
|
+
try {
|
|
42
|
+
cpSync(liveProfile, join(tmpHome, 'profiles', 'web'), { recursive: true, preserveTimestamps: true });
|
|
43
|
+
const port = String(9000 + Math.floor(Math.random() * 1000));
|
|
44
|
+
const url = `http://127.0.0.1:${port}/`;
|
|
45
|
+
child = spawn('node', ['--import', 'tsx/esm', 'apps/cli/src/bin.ts', 'web', '--no-open', '--port', port], {
|
|
46
|
+
cwd: harnessRoot, env: { ...process.env, DSH_HOME: tmpHome }, stdio: ['ignore', 'pipe', 'pipe'],
|
|
47
|
+
});
|
|
48
|
+
child.stdout?.on('data', (d) => logs.push(d.toString()));
|
|
49
|
+
child.stderr?.on('data', (d) => logs.push(d.toString()));
|
|
50
|
+
const deadline = Date.now() + timeoutMs;
|
|
51
|
+
let code = -1;
|
|
52
|
+
while (Date.now() < deadline) {
|
|
53
|
+
if (child.exitCode !== null) {
|
|
54
|
+
code = child.exitCode;
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
try {
|
|
58
|
+
const r = await fetch(url);
|
|
59
|
+
if (r.status === 200 || r.status === 401) {
|
|
60
|
+
code = 0;
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
catch { }
|
|
65
|
+
await new Promise(r => setTimeout(r, 500));
|
|
66
|
+
}
|
|
67
|
+
const tail = logs.join('').slice(-3000);
|
|
68
|
+
const loadErr = /ERR_MODULE_NOT_FOUND|assertChannel|must declare output|failed to apply loader entry/.exec(tail);
|
|
69
|
+
return { ok: code === 0 && !loadErr, detail: loadErr ? loadErr[0] : (code === 0 ? 'dry-boot ok' : `dry-boot failed (exit ${code})`) };
|
|
70
|
+
}
|
|
71
|
+
catch (e) {
|
|
72
|
+
return { ok: false, detail: `dry-boot error: ${e?.message ?? String(e)}` };
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
// Kill on every exit path — a successful boot included. The dry boot only
|
|
76
|
+
// exists to verify the tree; leaving the child up would orphan a dsh web
|
|
77
|
+
// on the ephemeral port whose temp DSH_HOME is removed below, and a stale
|
|
78
|
+
// orphan could later answer a port collision with a false-positive 200.
|
|
79
|
+
try {
|
|
80
|
+
child?.kill('SIGKILL');
|
|
81
|
+
}
|
|
82
|
+
catch { }
|
|
83
|
+
try {
|
|
84
|
+
rmSync(tmpHome, { recursive: true, force: true });
|
|
85
|
+
}
|
|
86
|
+
catch { }
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Whether the live plugin tree differs from the latest LKG snapshot. Two
|
|
91
|
+
* signals are combined:
|
|
92
|
+
*
|
|
93
|
+
* 1. manifest drift — the live profile's web `package.json` text vs baseline;
|
|
94
|
+
* 2. plugin-lib drift — any `@ddtcorex` plugin `lib/` file newer than the
|
|
95
|
+
* snapshot moment. Link-installed plugins resolve to the same workspace
|
|
96
|
+
* files in both live and LKG, so the stored copies cannot be compared
|
|
97
|
+
* byte-wise; the snapshot itself is the meaningful baseline and a rebuilt
|
|
98
|
+
* `lib/` bumps a file past it even when the manifest text is unchanged.
|
|
99
|
+
* writeLKG writes `manifest.json` LAST, so its FILE mtime is the
|
|
100
|
+
* authoritative snapshot moment; the snapshot dir mtime is only a
|
|
101
|
+
* fallback for legacy snapshots without a manifest.
|
|
102
|
+
*
|
|
103
|
+
* `statFile` (default `statSync`) reads the metadata so tests can inject a
|
|
104
|
+
* controlled reader instead of relying on filesystem utimes (which CI runners
|
|
105
|
+
* do not reliably reflect). No LKG baseline, a missing file on either side, or
|
|
106
|
+
* any stat/read error means "changed" — the caller falls back to the dry-boot
|
|
107
|
+
* gate.
|
|
108
|
+
*/
|
|
109
|
+
export function isPluginTreeChanged(harnessRoot, lkgDir = join(homedir(), '.dsh/.supervisor/lkg'), opts = {}) {
|
|
110
|
+
void harnessRoot;
|
|
111
|
+
const statFile = opts.statFile ?? ((p) => statSync(p));
|
|
112
|
+
try {
|
|
113
|
+
const entries = existsSync(lkgDir) ? readdirSync(lkgDir).sort() : [];
|
|
114
|
+
const latest = entries[entries.length - 1];
|
|
115
|
+
if (!latest)
|
|
116
|
+
return true; // no baseline → assume changed
|
|
117
|
+
const lkgHome = join(lkgDir, latest, 'profiles', 'web');
|
|
118
|
+
const live = join(homedir(), '.dsh', 'profiles', 'web');
|
|
119
|
+
const lkgManifest = join(lkgHome, 'package.json');
|
|
120
|
+
const liveManifest = join(live, 'package.json');
|
|
121
|
+
if (!existsSync(lkgManifest) || !existsSync(liveManifest))
|
|
122
|
+
return true;
|
|
123
|
+
if (readFileSync(liveManifest, 'utf8') !== readFileSync(lkgManifest, 'utf8'))
|
|
124
|
+
return true;
|
|
125
|
+
const snapshotManifest = join(lkgDir, latest, 'manifest.json');
|
|
126
|
+
const baseline = existsSync(snapshotManifest)
|
|
127
|
+
? statFile(snapshotManifest).mtimeMs
|
|
128
|
+
: statFile(join(lkgDir, latest)).mtimeMs;
|
|
129
|
+
const livePlugins = join(live, 'node_modules', '@ddtcorex');
|
|
130
|
+
if (existsSync(livePlugins)) {
|
|
131
|
+
for (const name of readdirSync(livePlugins)) {
|
|
132
|
+
const libDir = join(livePlugins, name, 'lib');
|
|
133
|
+
if (!existsSync(libDir))
|
|
134
|
+
continue;
|
|
135
|
+
if (newestFileMtime(libDir, statFile) > baseline)
|
|
136
|
+
return true;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return false;
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/** Newest mtime under a directory; recursion threads the injected stat reader. */
|
|
146
|
+
function newestFileMtime(dir, statFile) {
|
|
147
|
+
let newest = 0;
|
|
148
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
149
|
+
const p = join(dir, entry.name);
|
|
150
|
+
if (entry.isDirectory())
|
|
151
|
+
newest = Math.max(newest, newestFileMtime(p, statFile));
|
|
152
|
+
else
|
|
153
|
+
newest = Math.max(newest, statFile(p).mtimeMs);
|
|
154
|
+
}
|
|
155
|
+
return newest;
|
|
156
|
+
}
|
|
157
|
+
function currentSessionId(exec, fallback) {
|
|
158
|
+
// dsh-tools dispatch hands a ToolRunContext — the exec itself has no
|
|
159
|
+
// sessionId/session/caller fields; session identity lives on the agent
|
|
160
|
+
// (exec.agent.id is the branded SessionId, exec.agent.session.id also
|
|
161
|
+
// exists). Probe those first so production dispatches are identifiable.
|
|
162
|
+
return exec?.agent?.id ?? exec?.agent?.session?.id ?? exec?.sessionId ?? exec?.session?.id ?? exec?.caller?.sessionId ?? fallback?.(exec);
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Register the dsh_web_restart tool. Registration is fail-safe (warns, never
|
|
166
|
+
* throws) and the returned function disposes the registration. `deps` are
|
|
167
|
+
* injectable for tests.
|
|
168
|
+
*/
|
|
169
|
+
export function registerRestartTool(ctx, deps = {}) {
|
|
170
|
+
const doDryBoot = deps.dryBoot ?? dryBootVerify;
|
|
171
|
+
const doWrite = deps.writeRestartRequest ?? writeRestartRequest;
|
|
172
|
+
const doSessionId = deps.sessionIdOf ?? currentSessionId;
|
|
173
|
+
let dispose;
|
|
174
|
+
try {
|
|
175
|
+
dispose = ctx.tools.register({
|
|
176
|
+
name: 'dsh_web_restart',
|
|
177
|
+
description: 'Schedule a safe restart of the dsh web host. Verifies the plugin tree first (dry-boot), records an intent for the calling session, and hands the restart to the supervisor daemon (out-of-band). Never restarts in-tree itself.',
|
|
178
|
+
parameters: {
|
|
179
|
+
type: 'object',
|
|
180
|
+
properties: {
|
|
181
|
+
reason: { type: 'string', description: 'Why the restart is happening (recorded in the intent)' },
|
|
182
|
+
pluginChanged: { type: 'boolean', description: 'Override for the auto-detected plugin-tree change check' },
|
|
183
|
+
},
|
|
184
|
+
additionalProperties: false,
|
|
185
|
+
},
|
|
186
|
+
output: {
|
|
187
|
+
schema: { type: 'object', additionalProperties: true, properties: { ok: { type: 'boolean' }, detail: { type: 'string' } } },
|
|
188
|
+
render: (_args, value) => [{ type: 'text', text: value.detail }],
|
|
189
|
+
},
|
|
190
|
+
execute: async (args, exec) => {
|
|
191
|
+
const harnessRoot = deps.harnessRoot ?? (await import('./paths.js')).resolveDeepseekHarnessDir();
|
|
192
|
+
const lkgDir = join(homedir(), '.dsh/.supervisor/lkg');
|
|
193
|
+
const changed = args.pluginChanged === true || (args.pluginChanged !== false && isPluginTreeChanged(harnessRoot, lkgDir));
|
|
194
|
+
if (changed) {
|
|
195
|
+
const gate = await doDryBoot(harnessRoot);
|
|
196
|
+
if (!gate.ok)
|
|
197
|
+
return { ok: false, detail: `dry-boot failed — restart refused. ${gate.detail}` };
|
|
198
|
+
}
|
|
199
|
+
const callerSessionId = doSessionId(exec);
|
|
200
|
+
if (!callerSessionId) {
|
|
201
|
+
// The daemon's grace branch keys on callerSessionId — without it the
|
|
202
|
+
// marker would be written but no restart would ever be supervised.
|
|
203
|
+
return { ok: false, detail: 'cannot identify the calling session — restart not scheduled' };
|
|
204
|
+
}
|
|
205
|
+
doWrite({ callerSessionId, reason: typeof args.reason === 'string' ? args.reason : undefined }, 180_000);
|
|
206
|
+
writeIntentSidecar(callerSessionId, args.reason);
|
|
207
|
+
return { ok: true, detail: `restart scheduled (≈30s) — caller ${callerSessionId}` };
|
|
208
|
+
},
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
catch (e) {
|
|
212
|
+
try {
|
|
213
|
+
ctx.logger?.warn?.(`[supervisor] dsh_web_restart tool failed: ${e?.message ?? String(e)}`);
|
|
214
|
+
}
|
|
215
|
+
catch { }
|
|
216
|
+
}
|
|
217
|
+
return () => { try {
|
|
218
|
+
if (typeof dispose === 'function')
|
|
219
|
+
dispose();
|
|
220
|
+
}
|
|
221
|
+
catch { } };
|
|
222
|
+
}
|
|
223
|
+
function writeIntentSidecar(sessionId, reason) {
|
|
224
|
+
try {
|
|
225
|
+
if (!sessionId)
|
|
226
|
+
return;
|
|
227
|
+
const dir = join(homedir(), '.dsh/.supervisor/intents');
|
|
228
|
+
const require = createRequire(import.meta.url);
|
|
229
|
+
const { mkdirSync, writeFileSync, chmodSync } = require('node:fs');
|
|
230
|
+
mkdirSync(dir, { recursive: true });
|
|
231
|
+
// Flatten slash-namespaced ids ('proj/abc' → 'proj_abc') so the sidecar is
|
|
232
|
+
// a single file under intents/ and never needs a nested intents/proj/ dir.
|
|
233
|
+
const safeId = sessionId.replace(/[^A-Za-z0-9._-]/g, '_');
|
|
234
|
+
writeFileSync(join(dir, `${safeId}.json`), JSON.stringify({ ts: Date.now(), sessionId, reason: reason ?? '' }), 'utf8');
|
|
235
|
+
try {
|
|
236
|
+
chmodSync(join(dir, `${safeId}.json`), 0o600);
|
|
237
|
+
}
|
|
238
|
+
catch { }
|
|
239
|
+
}
|
|
240
|
+
catch { }
|
|
241
|
+
}
|
package/lib/scan.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Post-restart session scan: walk recent session logs under
|
|
3
|
+
* <dshHome>/sessions/<project>/<session>/ and flag torn tails (a zstd frame
|
|
4
|
+
* that fails to decode, or a plain-text log that is unreadable). Runs after
|
|
5
|
+
* an intentional dsh-web restart so the supervisor can report whether any
|
|
6
|
+
* in-flight session log was left truncated by the restart.
|
|
7
|
+
*/
|
|
8
|
+
interface ScanOptions {
|
|
9
|
+
withinMs?: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Scan session logs whose mtime falls within the window. A file whose decode
|
|
13
|
+
* fails (torn zstd frame or unreadable plain text) is reported as torn. A
|
|
14
|
+
* missing sessions root yields an empty scan, never an error.
|
|
15
|
+
*/
|
|
16
|
+
export declare function scanSessions(dshHome: string, opts?: ScanOptions): Promise<{
|
|
17
|
+
scanned: number;
|
|
18
|
+
torn: string[];
|
|
19
|
+
}>;
|
|
20
|
+
export {};
|