@ddtcorex/dsh-maestro-supervisor 0.6.7 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli.d.ts +13 -0
- package/lib/cli.js +87 -55
- package/lib/debug-agent.js +52 -16
- package/lib/health-poller.js +4 -2
- package/lib/plugin.d.ts +1 -1
- package/lib/plugin.js +73 -1
- package/lib/restart-guards.d.ts +11 -0
- package/lib/restart-guards.js +23 -0
- package/lib/restart-tool.d.ts +70 -0
- package/lib/restart-tool.js +241 -0
- package/lib/scan.d.ts +20 -0
- package/lib/scan.js +71 -0
- package/lib/self-kill-guard.d.ts +26 -0
- package/lib/self-kill-guard.js +63 -0
- package/lib/skill-provider.d.ts +6 -0
- package/lib/skill-provider.js +70 -0
- package/lib/supervisor.d.ts +9 -0
- package/lib/supervisor.js +73 -1
- package/package.json +2 -1
- package/skills/dsh-safe-restart/SKILL.md +90 -0
- package/skills/dsh-safe-restart/scripts/restart-dsh-web.sh +279 -0
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh_web_restart tool — the safe restart path for the model running inside
|
|
3
|
+
* dsh web. Scheduling a restart through this tool instead of a raw kill keeps
|
|
4
|
+
* the restart inside the supervisor's ownership loop:
|
|
5
|
+
*
|
|
6
|
+
* 1. dry-boot gate — if the plugin tree changed since the latest LKG, boot a
|
|
7
|
+
* copy of the live profile on an ephemeral DSH_HOME first and only
|
|
8
|
+
* schedule when that boot serves HTTP.
|
|
9
|
+
* 2. intent sidecar — record the caller session + reason under
|
|
10
|
+
* ~/.dsh/.supervisor/intents/ for attribution.
|
|
11
|
+
* 3. hand-off — write the restart-request marker (planned-restart.json with
|
|
12
|
+
* callerSessionId) that the supervisor daemon owns and acts on
|
|
13
|
+
* (out-of-band). This tool NEVER restarts the host in-tree.
|
|
14
|
+
*/
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
import { mkdtempSync, rmSync, cpSync, existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
|
17
|
+
import { tmpdir, homedir } from 'node:os';
|
|
18
|
+
import { spawn } from 'node:child_process';
|
|
19
|
+
import { createRequire } from 'node:module';
|
|
20
|
+
import { writeRestartRequest } from './restart-guards.js';
|
|
21
|
+
/**
|
|
22
|
+
* Boot a copy of the live web profile on an isolated DSH_HOME and verify the
|
|
23
|
+
* plugin tree loads and serves. Returns ok + a one-line detail for the tool
|
|
24
|
+
* message. The spawned tree is killed (best-effort) and the temp home removed.
|
|
25
|
+
* Unit tests mock this (never spawn a real node boot in tests).
|
|
26
|
+
*
|
|
27
|
+
* NOTE (bin.ts finding): the `web` alias already implies `--profile web`, and
|
|
28
|
+
* the web app's own commander program (no allowUnknownOption) rejects a stray
|
|
29
|
+
* `--profile` in its inner args — so the spawn passes only `web --no-open
|
|
30
|
+
* --port <port>`.
|
|
31
|
+
*/
|
|
32
|
+
export async function dryBootVerify(harnessRoot, opts = {}) {
|
|
33
|
+
const timeoutMs = opts.timeoutMs ?? 60_000;
|
|
34
|
+
const home = homedir();
|
|
35
|
+
const liveProfile = join(home, '.dsh', 'profiles', 'web');
|
|
36
|
+
if (!existsSync(liveProfile))
|
|
37
|
+
return { ok: true, detail: 'skipped (no live web profile)' };
|
|
38
|
+
const tmpHome = mkdtempSync(join(tmpdir(), 'dsh-dryboot-'));
|
|
39
|
+
const logs = [];
|
|
40
|
+
let child = null;
|
|
41
|
+
try {
|
|
42
|
+
cpSync(liveProfile, join(tmpHome, 'profiles', 'web'), { recursive: true, preserveTimestamps: true });
|
|
43
|
+
const port = String(9000 + Math.floor(Math.random() * 1000));
|
|
44
|
+
const url = `http://127.0.0.1:${port}/`;
|
|
45
|
+
child = spawn('node', ['--import', 'tsx/esm', 'apps/cli/src/bin.ts', 'web', '--no-open', '--port', port], {
|
|
46
|
+
cwd: harnessRoot, env: { ...process.env, DSH_HOME: tmpHome }, stdio: ['ignore', 'pipe', 'pipe'],
|
|
47
|
+
});
|
|
48
|
+
child.stdout?.on('data', (d) => logs.push(d.toString()));
|
|
49
|
+
child.stderr?.on('data', (d) => logs.push(d.toString()));
|
|
50
|
+
const deadline = Date.now() + timeoutMs;
|
|
51
|
+
let code = -1;
|
|
52
|
+
while (Date.now() < deadline) {
|
|
53
|
+
if (child.exitCode !== null) {
|
|
54
|
+
code = child.exitCode;
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
try {
|
|
58
|
+
const r = await fetch(url);
|
|
59
|
+
if (r.status === 200 || r.status === 401) {
|
|
60
|
+
code = 0;
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
catch { }
|
|
65
|
+
await new Promise(r => setTimeout(r, 500));
|
|
66
|
+
}
|
|
67
|
+
const tail = logs.join('').slice(-3000);
|
|
68
|
+
const loadErr = /ERR_MODULE_NOT_FOUND|assertChannel|must declare output|failed to apply loader entry/.exec(tail);
|
|
69
|
+
return { ok: code === 0 && !loadErr, detail: loadErr ? loadErr[0] : (code === 0 ? 'dry-boot ok' : `dry-boot failed (exit ${code})`) };
|
|
70
|
+
}
|
|
71
|
+
catch (e) {
|
|
72
|
+
return { ok: false, detail: `dry-boot error: ${e?.message ?? String(e)}` };
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
// Kill on every exit path — a successful boot included. The dry boot only
|
|
76
|
+
// exists to verify the tree; leaving the child up would orphan a dsh web
|
|
77
|
+
// on the ephemeral port whose temp DSH_HOME is removed below, and a stale
|
|
78
|
+
// orphan could later answer a port collision with a false-positive 200.
|
|
79
|
+
try {
|
|
80
|
+
child?.kill('SIGKILL');
|
|
81
|
+
}
|
|
82
|
+
catch { }
|
|
83
|
+
try {
|
|
84
|
+
rmSync(tmpHome, { recursive: true, force: true });
|
|
85
|
+
}
|
|
86
|
+
catch { }
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Whether the live plugin tree differs from the latest LKG snapshot. Two
|
|
91
|
+
* signals are combined:
|
|
92
|
+
*
|
|
93
|
+
* 1. manifest drift — the live profile's web `package.json` text vs baseline;
|
|
94
|
+
* 2. plugin-lib drift — any `@ddtcorex` plugin `lib/` file newer than the
|
|
95
|
+
* snapshot moment. Link-installed plugins resolve to the same workspace
|
|
96
|
+
* files in both live and LKG, so the stored copies cannot be compared
|
|
97
|
+
* byte-wise; the snapshot itself is the meaningful baseline and a rebuilt
|
|
98
|
+
* `lib/` bumps a file past it even when the manifest text is unchanged.
|
|
99
|
+
* writeLKG writes `manifest.json` LAST, so its FILE mtime is the
|
|
100
|
+
* authoritative snapshot moment; the snapshot dir mtime is only a
|
|
101
|
+
* fallback for legacy snapshots without a manifest.
|
|
102
|
+
*
|
|
103
|
+
* `statFile` (default `statSync`) reads the metadata so tests can inject a
|
|
104
|
+
* controlled reader instead of relying on filesystem utimes (which CI runners
|
|
105
|
+
* do not reliably reflect). No LKG baseline, a missing file on either side, or
|
|
106
|
+
* any stat/read error means "changed" — the caller falls back to the dry-boot
|
|
107
|
+
* gate.
|
|
108
|
+
*/
|
|
109
|
+
export function isPluginTreeChanged(harnessRoot, lkgDir = join(homedir(), '.dsh/.supervisor/lkg'), opts = {}) {
|
|
110
|
+
void harnessRoot;
|
|
111
|
+
const statFile = opts.statFile ?? ((p) => statSync(p));
|
|
112
|
+
try {
|
|
113
|
+
const entries = existsSync(lkgDir) ? readdirSync(lkgDir).sort() : [];
|
|
114
|
+
const latest = entries[entries.length - 1];
|
|
115
|
+
if (!latest)
|
|
116
|
+
return true; // no baseline → assume changed
|
|
117
|
+
const lkgHome = join(lkgDir, latest, 'profiles', 'web');
|
|
118
|
+
const live = join(homedir(), '.dsh', 'profiles', 'web');
|
|
119
|
+
const lkgManifest = join(lkgHome, 'package.json');
|
|
120
|
+
const liveManifest = join(live, 'package.json');
|
|
121
|
+
if (!existsSync(lkgManifest) || !existsSync(liveManifest))
|
|
122
|
+
return true;
|
|
123
|
+
if (readFileSync(liveManifest, 'utf8') !== readFileSync(lkgManifest, 'utf8'))
|
|
124
|
+
return true;
|
|
125
|
+
const snapshotManifest = join(lkgDir, latest, 'manifest.json');
|
|
126
|
+
const baseline = existsSync(snapshotManifest)
|
|
127
|
+
? statFile(snapshotManifest).mtimeMs
|
|
128
|
+
: statFile(join(lkgDir, latest)).mtimeMs;
|
|
129
|
+
const livePlugins = join(live, 'node_modules', '@ddtcorex');
|
|
130
|
+
if (existsSync(livePlugins)) {
|
|
131
|
+
for (const name of readdirSync(livePlugins)) {
|
|
132
|
+
const libDir = join(livePlugins, name, 'lib');
|
|
133
|
+
if (!existsSync(libDir))
|
|
134
|
+
continue;
|
|
135
|
+
if (newestFileMtime(libDir, statFile) > baseline)
|
|
136
|
+
return true;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return false;
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/** Newest mtime under a directory; recursion threads the injected stat reader. */
|
|
146
|
+
function newestFileMtime(dir, statFile) {
|
|
147
|
+
let newest = 0;
|
|
148
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
149
|
+
const p = join(dir, entry.name);
|
|
150
|
+
if (entry.isDirectory())
|
|
151
|
+
newest = Math.max(newest, newestFileMtime(p, statFile));
|
|
152
|
+
else
|
|
153
|
+
newest = Math.max(newest, statFile(p).mtimeMs);
|
|
154
|
+
}
|
|
155
|
+
return newest;
|
|
156
|
+
}
|
|
157
|
+
function currentSessionId(exec, fallback) {
|
|
158
|
+
// dsh-tools dispatch hands a ToolRunContext — the exec itself has no
|
|
159
|
+
// sessionId/session/caller fields; session identity lives on the agent
|
|
160
|
+
// (exec.agent.id is the branded SessionId, exec.agent.session.id also
|
|
161
|
+
// exists). Probe those first so production dispatches are identifiable.
|
|
162
|
+
return exec?.agent?.id ?? exec?.agent?.session?.id ?? exec?.sessionId ?? exec?.session?.id ?? exec?.caller?.sessionId ?? fallback?.(exec);
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Register the dsh_web_restart tool. Registration is fail-safe (warns, never
|
|
166
|
+
* throws) and the returned function disposes the registration. `deps` are
|
|
167
|
+
* injectable for tests.
|
|
168
|
+
*/
|
|
169
|
+
export function registerRestartTool(ctx, deps = {}) {
|
|
170
|
+
const doDryBoot = deps.dryBoot ?? dryBootVerify;
|
|
171
|
+
const doWrite = deps.writeRestartRequest ?? writeRestartRequest;
|
|
172
|
+
const doSessionId = deps.sessionIdOf ?? currentSessionId;
|
|
173
|
+
let dispose;
|
|
174
|
+
try {
|
|
175
|
+
dispose = ctx.tools.register({
|
|
176
|
+
name: 'dsh_web_restart',
|
|
177
|
+
description: 'Schedule a safe restart of the dsh web host. Verifies the plugin tree first (dry-boot), records an intent for the calling session, and hands the restart to the supervisor daemon (out-of-band). Never restarts in-tree itself.',
|
|
178
|
+
parameters: {
|
|
179
|
+
type: 'object',
|
|
180
|
+
properties: {
|
|
181
|
+
reason: { type: 'string', description: 'Why the restart is happening (recorded in the intent)' },
|
|
182
|
+
pluginChanged: { type: 'boolean', description: 'Override for the auto-detected plugin-tree change check' },
|
|
183
|
+
},
|
|
184
|
+
additionalProperties: false,
|
|
185
|
+
},
|
|
186
|
+
output: {
|
|
187
|
+
schema: { type: 'object', additionalProperties: true, properties: { ok: { type: 'boolean' }, detail: { type: 'string' } } },
|
|
188
|
+
render: (_args, value) => [{ type: 'text', text: value.detail }],
|
|
189
|
+
},
|
|
190
|
+
execute: async (args, exec) => {
|
|
191
|
+
const harnessRoot = deps.harnessRoot ?? (await import('./paths.js')).resolveDeepseekHarnessDir();
|
|
192
|
+
const lkgDir = join(homedir(), '.dsh/.supervisor/lkg');
|
|
193
|
+
const changed = args.pluginChanged === true || (args.pluginChanged !== false && isPluginTreeChanged(harnessRoot, lkgDir));
|
|
194
|
+
if (changed) {
|
|
195
|
+
const gate = await doDryBoot(harnessRoot);
|
|
196
|
+
if (!gate.ok)
|
|
197
|
+
return { ok: false, detail: `dry-boot failed — restart refused. ${gate.detail}` };
|
|
198
|
+
}
|
|
199
|
+
const callerSessionId = doSessionId(exec);
|
|
200
|
+
if (!callerSessionId) {
|
|
201
|
+
// The daemon's grace branch keys on callerSessionId — without it the
|
|
202
|
+
// marker would be written but no restart would ever be supervised.
|
|
203
|
+
return { ok: false, detail: 'cannot identify the calling session — restart not scheduled' };
|
|
204
|
+
}
|
|
205
|
+
doWrite({ callerSessionId, reason: typeof args.reason === 'string' ? args.reason : undefined }, 180_000);
|
|
206
|
+
writeIntentSidecar(callerSessionId, args.reason);
|
|
207
|
+
return { ok: true, detail: `restart scheduled (≈30s) — caller ${callerSessionId}` };
|
|
208
|
+
},
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
catch (e) {
|
|
212
|
+
try {
|
|
213
|
+
ctx.logger?.warn?.(`[supervisor] dsh_web_restart tool failed: ${e?.message ?? String(e)}`);
|
|
214
|
+
}
|
|
215
|
+
catch { }
|
|
216
|
+
}
|
|
217
|
+
return () => { try {
|
|
218
|
+
if (typeof dispose === 'function')
|
|
219
|
+
dispose();
|
|
220
|
+
}
|
|
221
|
+
catch { } };
|
|
222
|
+
}
|
|
223
|
+
function writeIntentSidecar(sessionId, reason) {
|
|
224
|
+
try {
|
|
225
|
+
if (!sessionId)
|
|
226
|
+
return;
|
|
227
|
+
const dir = join(homedir(), '.dsh/.supervisor/intents');
|
|
228
|
+
const require = createRequire(import.meta.url);
|
|
229
|
+
const { mkdirSync, writeFileSync, chmodSync } = require('node:fs');
|
|
230
|
+
mkdirSync(dir, { recursive: true });
|
|
231
|
+
// Flatten slash-namespaced ids ('proj/abc' → 'proj_abc') so the sidecar is
|
|
232
|
+
// a single file under intents/ and never needs a nested intents/proj/ dir.
|
|
233
|
+
const safeId = sessionId.replace(/[^A-Za-z0-9._-]/g, '_');
|
|
234
|
+
writeFileSync(join(dir, `${safeId}.json`), JSON.stringify({ ts: Date.now(), sessionId, reason: reason ?? '' }), 'utf8');
|
|
235
|
+
try {
|
|
236
|
+
chmodSync(join(dir, `${safeId}.json`), 0o600);
|
|
237
|
+
}
|
|
238
|
+
catch { }
|
|
239
|
+
}
|
|
240
|
+
catch { }
|
|
241
|
+
}
|
package/lib/scan.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Post-restart session scan: walk recent session logs under
|
|
3
|
+
* <dshHome>/sessions/<project>/<session>/ and flag torn tails (a zstd frame
|
|
4
|
+
* that fails to decode, or a plain-text log that is unreadable). Runs after
|
|
5
|
+
* an intentional dsh-web restart so the supervisor can report whether any
|
|
6
|
+
* in-flight session log was left truncated by the restart.
|
|
7
|
+
*/
|
|
8
|
+
interface ScanOptions {
|
|
9
|
+
withinMs?: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Scan session logs whose mtime falls within the window. A file whose decode
|
|
13
|
+
* fails (torn zstd frame or unreadable plain text) is reported as torn. A
|
|
14
|
+
* missing sessions root yields an empty scan, never an error.
|
|
15
|
+
*/
|
|
16
|
+
export declare function scanSessions(dshHome: string, opts?: ScanOptions): Promise<{
|
|
17
|
+
scanned: number;
|
|
18
|
+
torn: string[];
|
|
19
|
+
}>;
|
|
20
|
+
export {};
|
package/lib/scan.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Post-restart session scan: walk recent session logs under
|
|
3
|
+
* <dshHome>/sessions/<project>/<session>/ and flag torn tails (a zstd frame
|
|
4
|
+
* that fails to decode, or a plain-text log that is unreadable). Runs after
|
|
5
|
+
* an intentional dsh-web restart so the supervisor can report whether any
|
|
6
|
+
* in-flight session log was left truncated by the restart.
|
|
7
|
+
*/
|
|
8
|
+
import { readdirSync, statSync } from 'node:fs';
|
|
9
|
+
import { join, extname, basename } from 'node:path';
|
|
10
|
+
import { execFileSync } from 'node:child_process';
|
|
11
|
+
async function decodeOk(file) {
|
|
12
|
+
try {
|
|
13
|
+
if (extname(file) === '.zstd') {
|
|
14
|
+
execFileSync('zstd', ['-d', '-c', file], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 30_000 });
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
const { readFileSync } = await import('node:fs');
|
|
18
|
+
readFileSync(file, 'utf8');
|
|
19
|
+
}
|
|
20
|
+
return true;
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Scan session logs whose mtime falls within the window. A file whose decode
|
|
28
|
+
* fails (torn zstd frame or unreadable plain text) is reported as torn. A
|
|
29
|
+
* missing sessions root yields an empty scan, never an error.
|
|
30
|
+
*/
|
|
31
|
+
export async function scanSessions(dshHome, opts = {}) {
|
|
32
|
+
const sessionsRoot = join(dshHome, 'sessions');
|
|
33
|
+
const now = Date.now();
|
|
34
|
+
const withinMs = opts.withinMs ?? 10 * 60 * 1000;
|
|
35
|
+
const files = [];
|
|
36
|
+
try {
|
|
37
|
+
for (const proj of readdirSync(sessionsRoot)) {
|
|
38
|
+
const projDir = join(sessionsRoot, proj);
|
|
39
|
+
if (!statSync(projDir).isDirectory())
|
|
40
|
+
continue;
|
|
41
|
+
for (const sess of readdirSync(projDir)) {
|
|
42
|
+
const sessDir = join(projDir, sess);
|
|
43
|
+
if (!statSync(sessDir).isDirectory())
|
|
44
|
+
continue;
|
|
45
|
+
for (const f of readdirSync(sessDir)) {
|
|
46
|
+
const name = basename(f);
|
|
47
|
+
if (!name.endsWith('.zstd') && !name.endsWith('.jsonl'))
|
|
48
|
+
continue;
|
|
49
|
+
const fp = join(sessDir, f);
|
|
50
|
+
let mtime = 0;
|
|
51
|
+
try {
|
|
52
|
+
mtime = statSync(fp).mtimeMs;
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
if (now - mtime > withinMs)
|
|
58
|
+
continue;
|
|
59
|
+
files.push(fp);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
catch { /* sessions root absent */ }
|
|
65
|
+
const torn = [];
|
|
66
|
+
for (const f of files) {
|
|
67
|
+
if (!(await decodeOk(f)))
|
|
68
|
+
torn.push(f);
|
|
69
|
+
}
|
|
70
|
+
return { scanned: files.length, torn };
|
|
71
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Self-kill guard for `tools/pre-execute`: deny bash/shell commands that would
|
|
3
|
+
* kill or restart the very dsh web process the agent is running inside. The
|
|
4
|
+
* model must route restarts through the supervisor's dsh_web_restart tool, not
|
|
5
|
+
* by killing the host.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Whether a shell command is a self-kill. `livePids` are the pids currently
|
|
9
|
+
* holding listening sockets; a `kill <pid>` whose pid is one of ours is a
|
|
10
|
+
* self-kill regardless of anything else in the command. The kill parser accepts
|
|
11
|
+
* flag forms (`kill -9 <pid>`, `kill -TERM <pid>`). A command that is
|
|
12
|
+
* essentially JUST a `kill <unrelated-pid>` is allowed (idempotent cleanups
|
|
13
|
+
* are common), as are kill attempts whose output reports "not found"/"done" —
|
|
14
|
+
* but any compound that chains a restart/kill after it (or before the end)
|
|
15
|
+
* stays denied.
|
|
16
|
+
*/
|
|
17
|
+
export declare function isSelfKillCommand(cmd: string, livePids: number[]): boolean;
|
|
18
|
+
/**
|
|
19
|
+
* Build a `tools/pre-execute` waterfall listener: deny matching
|
|
20
|
+
* bash/shell/exec commands, otherwise delegate to `next()`. Live pids default
|
|
21
|
+
* to the pids holding listening sockets (`ss -tlnp`) so `kill <pid>` of a
|
|
22
|
+
* live host process is caught even when the command names no tool.
|
|
23
|
+
*/
|
|
24
|
+
export declare function makePreExecuteGuard(opts?: {
|
|
25
|
+
livePids?: () => number[];
|
|
26
|
+
}): (exec: any, next: () => Promise<any>) => Promise<any>;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Self-kill guard for `tools/pre-execute`: deny bash/shell commands that would
|
|
3
|
+
* kill or restart the very dsh web process the agent is running inside. The
|
|
4
|
+
* model must route restarts through the supervisor's dsh_web_restart tool, not
|
|
5
|
+
* by killing the host.
|
|
6
|
+
*/
|
|
7
|
+
import { createRequire } from 'node:module';
|
|
8
|
+
const require = createRequire(import.meta.url);
|
|
9
|
+
// Patterns that restart/stop/start or kill dsh web (systemctl --user units,
|
|
10
|
+
// pkill/killall over the dsh tree, killing holders of :3080, and the
|
|
11
|
+
// dsh-safe-web-update helper itself). The bare `kill\s+` alternative is
|
|
12
|
+
// narrowed below so a kill of an unrelated pid is not denied as a self-kill.
|
|
13
|
+
const SELF_KILL_RE = /(systemctl\s+--?user\s+.*(restart|stop|start).*dsh-web|pkill\s+.*dsh|killall\s+.*dsh|ss\s+.*3080.*kill|restart-dsh-web|kill\s+)/i;
|
|
14
|
+
/**
|
|
15
|
+
* Whether a shell command is a self-kill. `livePids` are the pids currently
|
|
16
|
+
* holding listening sockets; a `kill <pid>` whose pid is one of ours is a
|
|
17
|
+
* self-kill regardless of anything else in the command. The kill parser accepts
|
|
18
|
+
* flag forms (`kill -9 <pid>`, `kill -TERM <pid>`). A command that is
|
|
19
|
+
* essentially JUST a `kill <unrelated-pid>` is allowed (idempotent cleanups
|
|
20
|
+
* are common), as are kill attempts whose output reports "not found"/"done" —
|
|
21
|
+
* but any compound that chains a restart/kill after it (or before the end)
|
|
22
|
+
* stays denied.
|
|
23
|
+
*/
|
|
24
|
+
export function isSelfKillCommand(cmd, livePids) {
|
|
25
|
+
if (/kill\s+(?:-\S+\s+)?(\d+)/i.test(cmd)) {
|
|
26
|
+
const pid = Number(cmd.match(/kill\s+(?:-\S+\s+)?(\d+)/i)?.[1]);
|
|
27
|
+
if (livePids.includes(pid))
|
|
28
|
+
return true;
|
|
29
|
+
}
|
|
30
|
+
return SELF_KILL_RE.test(cmd)
|
|
31
|
+
// Exclude only a command that is JUST `kill [flags] <unrelated pid>` —
|
|
32
|
+
// anchored end-to-end so `kill 1234 && systemctl restart dsh-web` cannot
|
|
33
|
+
// whitelist the compound through its prefix.
|
|
34
|
+
&& !/^kill\s+(?:-\S+\s+)?\d+\s*$/i.test(cmd.trim())
|
|
35
|
+
&& !/kill\s+(?:-\S+\s+)?(\d+)\s+.*(not found|done)/i.test(cmd);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Build a `tools/pre-execute` waterfall listener: deny matching
|
|
39
|
+
* bash/shell/exec commands, otherwise delegate to `next()`. Live pids default
|
|
40
|
+
* to the pids holding listening sockets (`ss -tlnp`) so `kill <pid>` of a
|
|
41
|
+
* live host process is caught even when the command names no tool.
|
|
42
|
+
*/
|
|
43
|
+
export function makePreExecuteGuard(opts = {}) {
|
|
44
|
+
const livePids = opts.livePids ?? (() => {
|
|
45
|
+
try {
|
|
46
|
+
const { execSync } = require('node:child_process');
|
|
47
|
+
const out = execSync(`ss -tlnp 2>/dev/null | grep -oP 'pid=\\K[0-9]+' | sort -u`, { encoding: 'utf8' });
|
|
48
|
+
return out.trim().split('\n').filter(Boolean).map(Number);
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
return [];
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
return async (exec, next) => {
|
|
55
|
+
// dsh-tools hands the frozen ToolExecution (name + arguments); the guard
|
|
56
|
+
// also accepts the `args` shape for tests/embedded hosts.
|
|
57
|
+
const cmd = String(exec?.args?.command ?? exec?.args?.input ?? exec?.arguments?.command ?? exec?.arguments?.input ?? '');
|
|
58
|
+
if ((exec?.name === 'bash' || exec?.name === 'shell' || exec?.name === 'exec') && isSelfKillCommand(cmd, livePids())) {
|
|
59
|
+
return { kind: 'deny', reason: 'DENIED — this command restarts the dsh web process you are running inside. Use the dsh_web_restart tool (supervisor) for a safe restart.' };
|
|
60
|
+
}
|
|
61
|
+
return next();
|
|
62
|
+
};
|
|
63
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { SkillCandidate, SkillDefinition, SkillLookupOptions } from '@deepseek-ai/dsh-skill';
|
|
2
|
+
export declare function makeSkillProvider(skillsDir: string): {
|
|
3
|
+
name: string;
|
|
4
|
+
list(_options: SkillLookupOptions): Promise<SkillCandidate[]>;
|
|
5
|
+
get(candidate: SkillCandidate, _options: SkillLookupOptions): Promise<SkillDefinition | undefined>;
|
|
6
|
+
};
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { readFile, stat } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
const SKILL_NAME = 'dsh-safe-restart';
|
|
4
|
+
/** Minimal frontmatter reader for our own SKILL.md — enough to serve the provider contract. */
|
|
5
|
+
function parseFrontmatter(raw) {
|
|
6
|
+
const m = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
|
|
7
|
+
if (!m)
|
|
8
|
+
return { name: SKILL_NAME, description: '', body: raw };
|
|
9
|
+
const fm = m[1].split('\n').reduce((acc, line) => {
|
|
10
|
+
const kv = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
|
|
11
|
+
if (kv)
|
|
12
|
+
acc[kv[1]] = kv[2].replace(/^["']|["']$/g, '');
|
|
13
|
+
return acc;
|
|
14
|
+
}, {});
|
|
15
|
+
return { name: fm.name || SKILL_NAME, description: fm.description || '', body: m[2] };
|
|
16
|
+
}
|
|
17
|
+
export function makeSkillProvider(skillsDir) {
|
|
18
|
+
return {
|
|
19
|
+
// The dsh-skill service attributes candidates through the provider
|
|
20
|
+
// object's own `name` (maestro-skills returns `{ name, list, get }`); a
|
|
21
|
+
// missing name surfaces at runtime as `skill provider "undefined" returned
|
|
22
|
+
// skill ... for provider "maestro-supervisor"` and fails every turn.
|
|
23
|
+
name: 'maestro-supervisor',
|
|
24
|
+
async list(_options) {
|
|
25
|
+
const entry = join(skillsDir, SKILL_NAME);
|
|
26
|
+
const st = await stat(entry).catch(() => null);
|
|
27
|
+
if (!st?.isDirectory())
|
|
28
|
+
return [];
|
|
29
|
+
const skillFilePath = join(entry, 'SKILL.md');
|
|
30
|
+
const fileSt = await stat(skillFilePath).catch(() => null);
|
|
31
|
+
if (!fileSt?.isFile())
|
|
32
|
+
return [];
|
|
33
|
+
const raw = await readFile(skillFilePath, 'utf-8').catch(() => null);
|
|
34
|
+
if (raw === null)
|
|
35
|
+
return [];
|
|
36
|
+
const { name, description } = parseFrontmatter(raw);
|
|
37
|
+
return [{
|
|
38
|
+
name,
|
|
39
|
+
description,
|
|
40
|
+
invocation: { modelInvocable: true, userInvocable: true },
|
|
41
|
+
source: 'custom',
|
|
42
|
+
provider: 'maestro-supervisor',
|
|
43
|
+
rank: 360,
|
|
44
|
+
locator: skillFilePath,
|
|
45
|
+
path: skillFilePath,
|
|
46
|
+
resourceBase: { kind: 'directory', path: entry },
|
|
47
|
+
metadata: { name, description },
|
|
48
|
+
}];
|
|
49
|
+
},
|
|
50
|
+
async get(candidate, _options) {
|
|
51
|
+
try {
|
|
52
|
+
const raw = await readFile(candidate.path, 'utf-8');
|
|
53
|
+
const { name, description, body } = parseFrontmatter(raw);
|
|
54
|
+
return {
|
|
55
|
+
name, description,
|
|
56
|
+
invocation: candidate.invocation,
|
|
57
|
+
source: candidate.source,
|
|
58
|
+
provider: candidate.provider,
|
|
59
|
+
resourceBase: candidate.resourceBase,
|
|
60
|
+
path: candidate.path,
|
|
61
|
+
content: body,
|
|
62
|
+
metadata: candidate.metadata,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return undefined;
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
}
|
package/lib/supervisor.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { HealthState } from './health-poller.js';
|
|
2
|
+
import type { RestartRequest } from './restart-guards.js';
|
|
2
3
|
export interface SupervisorDeps {
|
|
3
4
|
pollHealth: () => Promise<HealthState>;
|
|
4
5
|
writeLKG: () => Promise<{
|
|
@@ -26,6 +27,7 @@ export interface SupervisorDeps {
|
|
|
26
27
|
isPlannedRestartActive?: () => boolean | Promise<boolean>;
|
|
27
28
|
writePlannedRestart?: (ttlMs?: number) => void;
|
|
28
29
|
checkPlannedRestart?: () => boolean;
|
|
30
|
+
clearPlannedRestart?: () => void;
|
|
29
31
|
runDebugAgent?: (opts: {
|
|
30
32
|
reportPath: string;
|
|
31
33
|
health: HealthState;
|
|
@@ -40,6 +42,8 @@ export interface SupervisorDeps {
|
|
|
40
42
|
resumeSessions?: (ids: string[]) => Promise<{
|
|
41
43
|
resumed: string[];
|
|
42
44
|
}>;
|
|
45
|
+
readRestartRequest?: () => RestartRequest | undefined;
|
|
46
|
+
onRestartRequestHandled?: (req: RestartRequest) => void;
|
|
43
47
|
}
|
|
44
48
|
export declare function resumeViaRpc(ids: string[], fetchFn?: (url: string, init: RequestInit) => Promise<Response>): Promise<{
|
|
45
49
|
resumed: string[];
|
|
@@ -53,9 +57,14 @@ export declare class Supervisor {
|
|
|
53
57
|
private consecutiveDown;
|
|
54
58
|
private consecutiveDegraded;
|
|
55
59
|
private timer;
|
|
60
|
+
private restartRequestHandled;
|
|
61
|
+
private restartRequestTimer;
|
|
62
|
+
private awaitingHealthyBoot;
|
|
63
|
+
private pendingRestartRequest;
|
|
56
64
|
constructor(deps: SupervisorDeps);
|
|
57
65
|
private getWritePlannedRestart;
|
|
58
66
|
private getCheckPlannedRestart;
|
|
67
|
+
private getClearPlannedRestart;
|
|
59
68
|
restartWeb(): Promise<void>;
|
|
60
69
|
private getRunDebugAgent;
|
|
61
70
|
private getFindInterrupted;
|
package/lib/supervisor.js
CHANGED
|
@@ -5,7 +5,7 @@ import * as path from 'node:path';
|
|
|
5
5
|
import * as os from 'node:os';
|
|
6
6
|
import { resolveHarnessRoot } from './paths.js';
|
|
7
7
|
import { readSupervisorConfig } from './config.js';
|
|
8
|
-
import { writePlannedRestart as defaultWritePlannedRestart, checkPlannedRestart as defaultCheckPlannedRestart } from './restart-guards.js';
|
|
8
|
+
import { writePlannedRestart as defaultWritePlannedRestart, checkPlannedRestart as defaultCheckPlannedRestart, clearPlannedRestart, PLANNED_RESTART_TTL_MS } from './restart-guards.js';
|
|
9
9
|
import { buildKillStalePortsCommand } from './restart-guards.js';
|
|
10
10
|
export async function resumeViaRpc(ids, fetchFn = globalThis.fetch) {
|
|
11
11
|
const rpcId = crypto.randomUUID();
|
|
@@ -36,6 +36,10 @@ export class Supervisor {
|
|
|
36
36
|
consecutiveDown = 0;
|
|
37
37
|
consecutiveDegraded = 0;
|
|
38
38
|
timer = null;
|
|
39
|
+
restartRequestHandled = false;
|
|
40
|
+
restartRequestTimer = null;
|
|
41
|
+
awaitingHealthyBoot = false;
|
|
42
|
+
pendingRestartRequest;
|
|
39
43
|
constructor(deps) {
|
|
40
44
|
this.deps = deps;
|
|
41
45
|
}
|
|
@@ -45,6 +49,9 @@ export class Supervisor {
|
|
|
45
49
|
getCheckPlannedRestart() {
|
|
46
50
|
return this.deps.checkPlannedRestart ?? defaultCheckPlannedRestart;
|
|
47
51
|
}
|
|
52
|
+
getClearPlannedRestart() {
|
|
53
|
+
return this.deps.clearPlannedRestart ?? clearPlannedRestart;
|
|
54
|
+
}
|
|
48
55
|
async restartWeb() {
|
|
49
56
|
this.getWritePlannedRestart()(30000);
|
|
50
57
|
if (this.deps.restartWeb) {
|
|
@@ -291,6 +298,44 @@ export class Supervisor {
|
|
|
291
298
|
}
|
|
292
299
|
async tick() {
|
|
293
300
|
const health = await this.deps.pollHealth();
|
|
301
|
+
// dsh_web_restart hand-off: the tool wrote a caller marker and handed the
|
|
302
|
+
// restart to the daemon (out-of-band). Honor one request per marker — one
|
|
303
|
+
// grace timer → restartWeb once → notify. The suppression marker and the
|
|
304
|
+
// single-flight latch are held until a post-restart health.up tick clears
|
|
305
|
+
// them, so the in-flight down of our own restart can never be mistaken for
|
|
306
|
+
// a crash and raced with a rollback + second restart by the crash path.
|
|
307
|
+
const restartReq = this.deps.readRestartRequest ? this.deps.readRestartRequest() : undefined;
|
|
308
|
+
if (restartReq?.callerSessionId && !this.restartRequestHandled) {
|
|
309
|
+
this.restartRequestHandled = true;
|
|
310
|
+
this.restartRequestTimer = setTimeout(() => {
|
|
311
|
+
void (async () => {
|
|
312
|
+
try {
|
|
313
|
+
// Debounce crash handling from the moment the restart is issued —
|
|
314
|
+
// the crash path sets lastRollback the same way. A slow boot must
|
|
315
|
+
// not be classified as a crash even after the suppression marker's
|
|
316
|
+
// own 30s TTL runs out.
|
|
317
|
+
this.lastRollback = this.deps.getTime ? this.deps.getTime() : Date.now();
|
|
318
|
+
await this.restartWeb();
|
|
319
|
+
// Supervisor.restartWeb() wrote a 30s marker; extend it past a slow
|
|
320
|
+
// boot. It is cleared only once the boot proves healthy below.
|
|
321
|
+
this.getWritePlannedRestart()(PLANNED_RESTART_TTL_MS);
|
|
322
|
+
await this.deps.notify(`restarted dsh-web after self-restart by session ${restartReq.callerSessionId}`);
|
|
323
|
+
}
|
|
324
|
+
catch (e) {
|
|
325
|
+
await this.deps.notify(`self-restart dsh-web failed: ${e?.message ?? String(e)}`).catch(() => { });
|
|
326
|
+
}
|
|
327
|
+
finally {
|
|
328
|
+
// Hold the marker and latch until health.up: clearing here would
|
|
329
|
+
// drop crash suppression mid-restart, and re-arming here would let
|
|
330
|
+
// a still-present marker re-fire into a second restart.
|
|
331
|
+
this.awaitingHealthyBoot = true;
|
|
332
|
+
this.pendingRestartRequest = restartReq;
|
|
333
|
+
this.restartRequestTimer = null;
|
|
334
|
+
}
|
|
335
|
+
})();
|
|
336
|
+
}, 5000);
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
294
339
|
// DEGRADED: http 200 but log has plugin error → report + notify, rollback after consecutive threshold
|
|
295
340
|
if (health.degraded) {
|
|
296
341
|
this.consecutiveDown = 0;
|
|
@@ -404,6 +449,29 @@ export class Supervisor {
|
|
|
404
449
|
if (health.up) {
|
|
405
450
|
this.consecutiveDown = 0;
|
|
406
451
|
this.consecutiveDegraded = 0;
|
|
452
|
+
// Post-self-restart boot proved healthy: clear the suppression marker,
|
|
453
|
+
// run the post-restart session-scan hook and re-arm the single-flight
|
|
454
|
+
// latch. A failed clear keeps the latch set so the same marker is never
|
|
455
|
+
// re-handled into a second restart.
|
|
456
|
+
if (this.awaitingHealthyBoot) {
|
|
457
|
+
this.awaitingHealthyBoot = false;
|
|
458
|
+
const req = this.pendingRestartRequest;
|
|
459
|
+
this.pendingRestartRequest = undefined;
|
|
460
|
+
let cleared = false;
|
|
461
|
+
try {
|
|
462
|
+
this.getClearPlannedRestart()();
|
|
463
|
+
cleared = true;
|
|
464
|
+
}
|
|
465
|
+
catch { }
|
|
466
|
+
if (cleared)
|
|
467
|
+
this.restartRequestHandled = false;
|
|
468
|
+
if (req) {
|
|
469
|
+
try {
|
|
470
|
+
this.deps.onRestartRequestHandled?.(req);
|
|
471
|
+
}
|
|
472
|
+
catch { }
|
|
473
|
+
}
|
|
474
|
+
}
|
|
407
475
|
// Throttle LKG writes to at most once per 5 minutes
|
|
408
476
|
const now = this.deps.getTime ? this.deps.getTime() : Date.now();
|
|
409
477
|
if (now - this.lastLKGWrite > 5 * 60 * 1000) {
|
|
@@ -543,5 +611,9 @@ export class Supervisor {
|
|
|
543
611
|
clearInterval(this.timer);
|
|
544
612
|
this.timer = null;
|
|
545
613
|
}
|
|
614
|
+
if (this.restartRequestTimer) {
|
|
615
|
+
clearTimeout(this.restartRequestTimer);
|
|
616
|
+
this.restartRequestTimer = null;
|
|
617
|
+
}
|
|
546
618
|
}
|
|
547
619
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ddtcorex/dsh-maestro-supervisor",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Supervisor daemon for DSH Web resilience — auto-detect crashes, rollback to LKG, report",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./lib/index.js",
|
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
},
|
|
34
34
|
"files": [
|
|
35
35
|
"lib",
|
|
36
|
+
"skills",
|
|
36
37
|
"README.md",
|
|
37
38
|
"cordis.patch.yml"
|
|
38
39
|
],
|