@ddtcorex/dsh-maestro-supervisor 0.7.2 → 0.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/debug-agent.d.ts +0 -2
- package/lib/debug-agent.js +4 -340
- package/lib/plugin.d.ts +93 -0
- package/lib/plugin.js +233 -2
- package/lib/resume-tools.d.ts +101 -0
- package/lib/resume-tools.js +248 -0
- package/lib/session-health.d.ts +83 -0
- package/lib/session-health.js +268 -0
- package/package.json +3 -2
- package/skills/dsh-safe-restart/scripts/restart-dsh-web.sh +34 -7
package/lib/plugin.js
CHANGED
|
@@ -14,6 +14,9 @@ import { readIntent, consumeIntent } from './intents.js';
|
|
|
14
14
|
import { makeSkillProvider } from './skill-provider.js';
|
|
15
15
|
import { registerRestartTool } from './restart-tool.js';
|
|
16
16
|
import { makePreExecuteGuard } from './self-kill-guard.js';
|
|
17
|
+
import { runSessionHealthCheck } from './session-health.js';
|
|
18
|
+
import { warnCoreToolLoss, recordResumedSession, recordResumeProbe, registerResumeToolHealthService, } from './resume-tools.js';
|
|
19
|
+
export * from './resume-tools.js';
|
|
17
20
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
18
21
|
export const inject = ['sessions', 'agents', 'connection', 'tools', 'skills'];
|
|
19
22
|
function parseDuration(s) {
|
|
@@ -79,6 +82,43 @@ function getAutoResumeEnabled(config) {
|
|
|
79
82
|
catch { }
|
|
80
83
|
return true;
|
|
81
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* C2 — resolve the mitigation policy with exactly the same precedence chain as
|
|
87
|
+
* getAutoResumeEnabled: (1) the Cordis-supplied plugin config (cordis.patch.yml
|
|
88
|
+
* `config:` block / whatever apply() receives — highest precedence), (2) env
|
|
89
|
+
* DSH_SUPERVISOR_RESUME_CORE_TOOL_POLICY, (3) ~/.dsh/.supervisor/config.json,
|
|
90
|
+
* (4) ~/.dsh/maestro/settings.json (domains.supervisor.resumeCoreToolPolicy),
|
|
91
|
+
* (5) 'warn'. Any other value falls through to the default 'warn'.
|
|
92
|
+
*/
|
|
93
|
+
function getResumeCoreToolPolicy(config) {
|
|
94
|
+
if (config?.resumeCoreToolPolicy === 'warn' || config?.resumeCoreToolPolicy === 'park') {
|
|
95
|
+
return config.resumeCoreToolPolicy;
|
|
96
|
+
}
|
|
97
|
+
const env = process.env.DSH_SUPERVISOR_RESUME_CORE_TOOL_POLICY;
|
|
98
|
+
if (env) {
|
|
99
|
+
const v = env.trim().toLowerCase();
|
|
100
|
+
if (v === 'warn' || v === 'park')
|
|
101
|
+
return v;
|
|
102
|
+
}
|
|
103
|
+
try {
|
|
104
|
+
const cfgPath = path.join(os.homedir(), '.dsh/.supervisor/config.json');
|
|
105
|
+
if (fs.existsSync(cfgPath)) {
|
|
106
|
+
const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf-8'));
|
|
107
|
+
const raw = cfg.resumeCoreToolPolicy;
|
|
108
|
+
if (raw === 'warn' || raw === 'park')
|
|
109
|
+
return raw;
|
|
110
|
+
}
|
|
111
|
+
const maestroPath = path.join(os.homedir(), '.dsh/maestro/settings.json');
|
|
112
|
+
if (fs.existsSync(maestroPath)) {
|
|
113
|
+
const j = JSON.parse(fs.readFileSync(maestroPath, 'utf-8'));
|
|
114
|
+
const raw = j?.domains?.supervisor?.resumeCoreToolPolicy ?? j?.supervisor?.resumeCoreToolPolicy;
|
|
115
|
+
if (raw === 'warn' || raw === 'park')
|
|
116
|
+
return raw;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
catch { }
|
|
120
|
+
return 'warn';
|
|
121
|
+
}
|
|
82
122
|
function getResumeWithinMs(config) {
|
|
83
123
|
// Same precedence rule as getAutoResumeEnabled: explicit config wins first.
|
|
84
124
|
if (config?.autoResumeWithin !== undefined) {
|
|
@@ -134,6 +174,41 @@ function getResumeWithinMs(config) {
|
|
|
134
174
|
catch { }
|
|
135
175
|
return 5 * 60 * 1000;
|
|
136
176
|
}
|
|
177
|
+
/**
|
|
178
|
+
* Core tools that must be visible on a resumed session. Part C targets the
|
|
179
|
+
* post-restart bash loss (`Error: unknown tool "bash"`); extend this list to
|
|
180
|
+
* widen the probe (e.g. 'cordis_inspect_query').
|
|
181
|
+
*/
|
|
182
|
+
export const CRITICAL_TOOLS = ['bash'];
|
|
183
|
+
/** Default: the resumed agent's tool scope is its top-level session id. */
|
|
184
|
+
export const defaultResolveToolScope = (_ctx, sessionId) => sessionId;
|
|
185
|
+
/**
|
|
186
|
+
* Snapshot one session's visible tool view for the journal: which CRITICAL_TOOLS
|
|
187
|
+
* are missing from the SCOPED registry (not the global view) and how many tools
|
|
188
|
+
* are visible. When the tools service is absent or lacks `get`, the probe is
|
|
189
|
+
* skipped and reports no missing tools. The log line is the Part D trigger —
|
|
190
|
+
* `bash=false` at resume marks the loss the moment it happens.
|
|
191
|
+
* @param tools - the harness ToolRegistry service, or undefined when unavailable.
|
|
192
|
+
* @param scope - the session's tool scope (defaults to the top-level session id).
|
|
193
|
+
* @param logger - optional ctx logger; the probe writes its line when present.
|
|
194
|
+
*/
|
|
195
|
+
export function probeToolView(tools, scope, logger) {
|
|
196
|
+
const probe = { missing: [], visible: 0 };
|
|
197
|
+
try {
|
|
198
|
+
const get = tools?.get;
|
|
199
|
+
if (typeof get !== 'function')
|
|
200
|
+
return probe;
|
|
201
|
+
probe.missing = [...CRITICAL_TOOLS].filter((name) => get(name, scope) === undefined);
|
|
202
|
+
const schemas = tools?.schemas?.(scope);
|
|
203
|
+
probe.visible = Array.isArray(schemas) ? schemas.length : 0;
|
|
204
|
+
}
|
|
205
|
+
catch { }
|
|
206
|
+
try {
|
|
207
|
+
logger?.info?.(`[supervisor] resumed ${scope}: bash=${!probe.missing.includes('bash')} visibleTools=${probe.visible} missing=${probe.missing.join(',') || 'none'}`);
|
|
208
|
+
}
|
|
209
|
+
catch { }
|
|
210
|
+
return probe;
|
|
211
|
+
}
|
|
137
212
|
export async function runAutoResume(ctx, opts = {}) {
|
|
138
213
|
try {
|
|
139
214
|
const doFind = opts.findInterrupted ?? defaultFindInterrupted;
|
|
@@ -163,7 +238,7 @@ export async function runAutoResume(ctx, opts = {}) {
|
|
|
163
238
|
return;
|
|
164
239
|
}
|
|
165
240
|
ctx.logger?.info?.(`[supervisor] auto-resume: ${merged.length}/${scanned} interrupted within ${withinMs}ms: ${merged.slice(0, 3).join(', ')}`);
|
|
166
|
-
await doResume(ctx, merged);
|
|
241
|
+
await doResume(ctx, merged, { ...(opts?.config !== undefined ? { config: opts.config } : {}) });
|
|
167
242
|
}
|
|
168
243
|
catch (e) {
|
|
169
244
|
try {
|
|
@@ -175,6 +250,9 @@ export async function runAutoResume(ctx, opts = {}) {
|
|
|
175
250
|
export async function resumeInterrupted(ctx, ids, deps = {}) {
|
|
176
251
|
const doReadIntent = deps.readIntent ?? readIntent;
|
|
177
252
|
const doConsumeIntent = deps.consumeIntent ?? consumeIntent;
|
|
253
|
+
const doProbe = deps.probeToolView ?? probeToolView;
|
|
254
|
+
const doResolveToolScope = deps.resolveToolScope ?? defaultResolveToolScope;
|
|
255
|
+
const coreToolPolicy = getResumeCoreToolPolicy(deps.config);
|
|
178
256
|
const resumed = [];
|
|
179
257
|
for (const id of ids) {
|
|
180
258
|
try {
|
|
@@ -276,6 +354,28 @@ export async function resumeInterrupted(ctx, ids, deps = {}) {
|
|
|
276
354
|
catch { }
|
|
277
355
|
resumed.push(id);
|
|
278
356
|
ctx.logger?.info?.(`[supervisor] auto-resume: sent recovery continue for ${id}`);
|
|
357
|
+
// C1 observability probe — snapshot the resumed session's SCOPED tool
|
|
358
|
+
// view at the success point so a post-resume bash loss surfaces in the
|
|
359
|
+
// journal the moment it happens (Part D reads this line). Defensive:
|
|
360
|
+
// absent ctx.tools just skips the probe, never fails the resume.
|
|
361
|
+
// C2 mitigation — when the probe reports a CORE tool lost from the
|
|
362
|
+
// resumed session's SCOPED view, notify the operator + inject the tool-
|
|
363
|
+
// inventory System message (park policy additionally parks the id for a
|
|
364
|
+
// manual reopen). Both record the session as "currently resumed" and
|
|
365
|
+
// the probe as the freshest observation for maestro_resume_tool_health.
|
|
366
|
+
try {
|
|
367
|
+
const probeTools = ctx.get?.('tools') ?? ctx.tools;
|
|
368
|
+
const probe = doProbe(probeTools, doResolveToolScope(ctx, sessionId), ctx.logger);
|
|
369
|
+
recordResumedSession(sessionId);
|
|
370
|
+
if (probe.missing.length) {
|
|
371
|
+
await warnCoreToolLoss(ctx, sessionId, doResolveToolScope(ctx, sessionId), probe, coreToolPolicy, {
|
|
372
|
+
notify: deps.notify,
|
|
373
|
+
injectSessionMessage: deps.injectSessionMessage,
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
recordResumeProbe(probe);
|
|
377
|
+
}
|
|
378
|
+
catch { }
|
|
279
379
|
}
|
|
280
380
|
catch (e) {
|
|
281
381
|
ctx.logger?.warn?.(`[supervisor] auto-resume failed ${id}: ${e?.message ?? String(e)}`);
|
|
@@ -301,9 +401,122 @@ export function createResumeRpcHandler(ctx, opts = {}) {
|
|
|
301
401
|
if (!ids.length) {
|
|
302
402
|
return { ok: false, error: { code: 'bad-request', message: 'resume requires at least one session id' } };
|
|
303
403
|
}
|
|
304
|
-
|
|
404
|
+
// Forward the resume-relevant surface (config so resumeCoreToolPolicy
|
|
405
|
+
// resolves, plus the C2 injectable seams). Optional keys keep the deps
|
|
406
|
+
// object minimal — absent opts only yields `{}`.
|
|
407
|
+
const resumeDeps = {
|
|
408
|
+
...(opts.config !== undefined ? { config: opts.config } : {}),
|
|
409
|
+
...(opts.notify !== undefined ? { notify: opts.notify } : {}),
|
|
410
|
+
...(opts.injectSessionMessage !== undefined ? { injectSessionMessage: opts.injectSessionMessage } : {}),
|
|
411
|
+
};
|
|
412
|
+
return { ok: true, value: { resumed: await resume(ctx, ids, resumeDeps) } };
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* Session-log root resolution shared with the safe-restart pre-flight script
|
|
417
|
+
* (skills/dsh-safe-restart/scripts/restart-dsh-web.sh): SESSIONS_ROOT wins,
|
|
418
|
+
* else DSH_HOME/sessions, else ~/.dsh/sessions. Kept in lockstep with the
|
|
419
|
+
* shell derivation so the RPC/tool and the pre-flight always scan the same
|
|
420
|
+
* store.
|
|
421
|
+
*/
|
|
422
|
+
function defaultSessionsRoot() {
|
|
423
|
+
if (typeof process.env.SESSIONS_ROOT === 'string' && process.env.SESSIONS_ROOT)
|
|
424
|
+
return process.env.SESSIONS_ROOT;
|
|
425
|
+
return path.join(process.env.DSH_HOME || path.join(os.homedir(), '.dsh'), 'sessions');
|
|
426
|
+
}
|
|
427
|
+
/** Extract a non-empty payload.root, else the resolved default session root. */
|
|
428
|
+
function resolveHealthRoot(payload, config) {
|
|
429
|
+
const root = payload?.root;
|
|
430
|
+
if (typeof root === 'string' && root)
|
|
431
|
+
return root;
|
|
432
|
+
return config?.sessionLogRoot ?? defaultSessionsRoot();
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Loopback RPC handler for /dsh-maestro-supervisor-session-health. Runs the
|
|
436
|
+
* A1 session-log health check over the resolved root with repair on and
|
|
437
|
+
* quarantine off (mirrors the safe-restart pre-flight — single-frame logs get
|
|
438
|
+
* re-encoded, corrupt logs stay in place and are only counted). Same
|
|
439
|
+
* `{ ok, value | error }` envelope shape as the resume handler.
|
|
440
|
+
*/
|
|
441
|
+
export function createSessionHealthRpcHandler(ctx, deps = {}) {
|
|
442
|
+
const run = deps.run ?? runSessionHealthCheck;
|
|
443
|
+
const config = deps.config ?? ctx.config;
|
|
444
|
+
return async (_endpoint, payload, _signal) => {
|
|
445
|
+
try {
|
|
446
|
+
return { ok: true, value: await run(resolveHealthRoot(payload, config), { repair: true, quarantine: false }) };
|
|
447
|
+
}
|
|
448
|
+
catch (e) {
|
|
449
|
+
return { ok: false, error: { code: 'session-health-failed', message: e?.message ?? String(e) } };
|
|
450
|
+
}
|
|
305
451
|
};
|
|
306
452
|
}
|
|
453
|
+
/** dsh.tools definition for the maestro_session_health host tool. */
|
|
454
|
+
function makeSessionHealthToolDef(config) {
|
|
455
|
+
return {
|
|
456
|
+
name: 'maestro_session_health',
|
|
457
|
+
description: 'Scan session logs under the operator DSH home sessions dir and report unhealthy shapes (single-frame whole logs, corrupt first frames). ' +
|
|
458
|
+
'Single-frame logs are re-encoded into the canonical multi-frame form (with a backup); corrupt logs are only counted unless quarantine is enabled. ' +
|
|
459
|
+
'Safe to run before or after a dsh web restart.',
|
|
460
|
+
parameters: {
|
|
461
|
+
type: 'object',
|
|
462
|
+
properties: {
|
|
463
|
+
root: { type: 'string', description: 'Session-log root to scan; defaults to the configured/operator DSH home sessions dir' },
|
|
464
|
+
repair: { type: 'boolean', description: 'Re-encode single-frame whole logs into canonical multi-frame form (default true)' },
|
|
465
|
+
quarantine: { type: 'boolean', description: 'Move corrupt-first-frame logs aside instead of leaving them in place (default false)' },
|
|
466
|
+
},
|
|
467
|
+
additionalProperties: false,
|
|
468
|
+
},
|
|
469
|
+
output: {
|
|
470
|
+
schema: { type: 'object', additionalProperties: true, properties: { ok: { type: 'boolean' }, counts: { type: 'object' } } },
|
|
471
|
+
render: (_args, value) => [{ type: 'text', text: `fixed=${value?.counts?.fixed ?? 0} quarantined=${value?.counts?.quarantined ?? 0} remaining=${value?.counts?.remaining ?? 0}` }],
|
|
472
|
+
},
|
|
473
|
+
execute: async (args) => {
|
|
474
|
+
const counts = await runSessionHealthCheck(resolveHealthRoot(args, config), {
|
|
475
|
+
repair: args?.repair !== false,
|
|
476
|
+
quarantine: args?.quarantine === true,
|
|
477
|
+
});
|
|
478
|
+
return { ok: true, counts };
|
|
479
|
+
},
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Register the session-health RPC handle (loopback authority) and the
|
|
484
|
+
* maestro_session_health host tool. Fail-safe like the other registrations:
|
|
485
|
+
* any registration error is logged, never thrown, and the returned disposer
|
|
486
|
+
* unregisters everything that did succeed.
|
|
487
|
+
*/
|
|
488
|
+
export function registerSessionHealthService(ctx, config = {}) {
|
|
489
|
+
const disposers = [];
|
|
490
|
+
try {
|
|
491
|
+
const conn = ctx.connection ?? ctx.get?.('connection');
|
|
492
|
+
if (conn?.rpc?.handle) {
|
|
493
|
+
disposers.push(conn.rpc.handle('/dsh-maestro-supervisor-session-health', createSessionHealthRpcHandler(ctx, { config }), { authority: 'loopback' }));
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
catch (e) {
|
|
497
|
+
try {
|
|
498
|
+
ctx.logger?.warn?.(`[supervisor] session-health RPC registration failed: ${e?.message ?? String(e)}`);
|
|
499
|
+
}
|
|
500
|
+
catch { }
|
|
501
|
+
}
|
|
502
|
+
try {
|
|
503
|
+
if (typeof ctx.tools?.register === 'function') {
|
|
504
|
+
disposers.push(ctx.tools.register(makeSessionHealthToolDef(config)));
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
catch (e) {
|
|
508
|
+
try {
|
|
509
|
+
ctx.logger?.warn?.(`[supervisor] session-health tool registration failed: ${e?.message ?? String(e)}`);
|
|
510
|
+
}
|
|
511
|
+
catch { }
|
|
512
|
+
}
|
|
513
|
+
return () => { for (const d of disposers) {
|
|
514
|
+
try {
|
|
515
|
+
d();
|
|
516
|
+
}
|
|
517
|
+
catch { }
|
|
518
|
+
} };
|
|
519
|
+
}
|
|
307
520
|
/** Resolve the package-root skills/ dir regardless of module layout. The built
|
|
308
521
|
* host lib is flat (lib/plugin.js → ../skills), but under vitest the same
|
|
309
522
|
* module loads from src/host/ (→ ../../skills). Walking to the nearest
|
|
@@ -438,6 +651,24 @@ export function apply(ctx, config = {}) {
|
|
|
438
651
|
}
|
|
439
652
|
catch { }
|
|
440
653
|
}
|
|
654
|
+
try {
|
|
655
|
+
ctx.effect(() => registerSessionHealthService(ctx, config), 'supervisor:session-health');
|
|
656
|
+
}
|
|
657
|
+
catch (e) {
|
|
658
|
+
try {
|
|
659
|
+
ctx.logger?.warn?.(`[supervisor] session-health effect failed: ${e?.message ?? String(e)}`);
|
|
660
|
+
}
|
|
661
|
+
catch { }
|
|
662
|
+
}
|
|
663
|
+
try {
|
|
664
|
+
ctx.effect(() => registerResumeToolHealthService(ctx), 'supervisor:resume-tool-health');
|
|
665
|
+
}
|
|
666
|
+
catch (e) {
|
|
667
|
+
try {
|
|
668
|
+
ctx.logger?.warn?.(`[supervisor] resume-tool-health effect failed: ${e?.message ?? String(e)}`);
|
|
669
|
+
}
|
|
670
|
+
catch { }
|
|
671
|
+
}
|
|
441
672
|
ctx.effect(() => {
|
|
442
673
|
let disposed = false;
|
|
443
674
|
let timer = null;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* C2 — deterministic plugin-layer mitigation for resumed sessions that lose
|
|
3
|
+
* core tools after a dsh web restart (`Error: unknown tool "bash"`).
|
|
4
|
+
*
|
|
5
|
+
* The supervisor cannot edit the harness tool layer (hard workspace rule) and
|
|
6
|
+
* the exact layer-rebuild root cause is only observable on the live system
|
|
7
|
+
* (Part D). So when the post-resume probe (C1) reports a core tool missing
|
|
8
|
+
* from the resumed session's SCOPED view, this module runs a host-side
|
|
9
|
+
* mitigation:
|
|
10
|
+
*
|
|
11
|
+
* 1. a short notifier line so the operator sees the loss immediately;
|
|
12
|
+
* 2. a "System:"-prefixed inventory user-message injected into the session
|
|
13
|
+
* (same createUserMessage/followup pattern as the resume-intent message),
|
|
14
|
+
* so the model stops calling the lost tool and uses what remains;
|
|
15
|
+
* 3. optionally (resumeCoreToolPolicy: 'park') records the session id in a
|
|
16
|
+
* module-level parked set exposed through the `maestro_resume_tool_health`
|
|
17
|
+
* host tool + loopback RPC, marking it for manual reopen.
|
|
18
|
+
*
|
|
19
|
+
* The module-level state (`parkedCoreToolLossIds`, `lastResumeProbe`,
|
|
20
|
+
* `resumedSessions`) is per-`dsh web`-process: a host restart clears it, which
|
|
21
|
+
* is exactly right — these are per-boot facts.
|
|
22
|
+
*/
|
|
23
|
+
import { type ToolViewProbe, type ToolViewProbeFn, type ToolScopeResolver, type ToolsLike, type ResumeCoreToolPolicy } from './plugin.js';
|
|
24
|
+
/** Reset the per-process mitigation state — tests + fresh boot reuse. */
|
|
25
|
+
export declare function resetResumeToolHealthState(): void;
|
|
26
|
+
/** Record a session the auto-resume confirmed as resumed (on-demand probe target). */
|
|
27
|
+
export declare function recordResumedSession(sessionId: string): void;
|
|
28
|
+
/** Record the last REAL post-resume tool-view observation ({ missing, visible }). */
|
|
29
|
+
export declare function recordResumeProbe(probe: ToolViewProbe): void;
|
|
30
|
+
/**
|
|
31
|
+
* Operator-facing notify line. The park variant appends the manual-reopen
|
|
32
|
+
* marker so the operator knows the session was NOT auto-continued for tools.
|
|
33
|
+
*/
|
|
34
|
+
export declare function buildCoreToolLossNotifyLine(sessionId: string, missing: string[], policy: ResumeCoreToolPolicy): string;
|
|
35
|
+
/**
|
|
36
|
+
* The SYSTEM message injected into the resumed session. Tells the model the
|
|
37
|
+
* CURRENT inventory (scoped `schemas()` — the tools the session can actually
|
|
38
|
+
* call) so it stops issuing the lost tool instead of looping on unknown-tool
|
|
39
|
+
* errors. Names joined with ', '.
|
|
40
|
+
*/
|
|
41
|
+
export declare function buildToolInventoryMessage(missing: string[], available: string[]): string;
|
|
42
|
+
export interface WarnCoreToolLossDeps {
|
|
43
|
+
/** Loose notifier — default wires to the package notifier (swallows errors). */
|
|
44
|
+
notify?: (line: string) => Promise<void>;
|
|
45
|
+
/** Push a message into the resumed session — default follows the createUserMessage / followup pattern. */
|
|
46
|
+
injectSessionMessage?: (sessionId: string, content: string) => unknown;
|
|
47
|
+
/** Tool registry whose SCOPED schemas() yields the available-tool inventory. Defaults to ctx.tools. */
|
|
48
|
+
tools?: ToolsLike;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* C2 mitigation entry: called by the resume flow when the post-resume probe
|
|
52
|
+
* found a core tool missing. Notifies, injects the inventory message, and —
|
|
53
|
+
* under 'park' policy — records the session id. Only fires on an actual
|
|
54
|
+
* CRITICAL_TOOLS loss; a non-core missing name (e.g. cordis_inspect_query)
|
|
55
|
+
* is a no-op. All seams are injectable for tests; every step is defensive.
|
|
56
|
+
*/
|
|
57
|
+
export declare function warnCoreToolLoss(ctx: any, sessionId: string, scope: string, probe: ToolViewProbe, policy: ResumeCoreToolPolicy, opts?: WarnCoreToolLossDeps): Promise<void>;
|
|
58
|
+
export interface ResumeToolHealthSnapshot {
|
|
59
|
+
lastResumeProbe: ToolViewProbe | null;
|
|
60
|
+
parked: string[];
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Build the RPC value. When the tool registry is reachable AND at least one
|
|
64
|
+
* current resumed session still lives, re-probes each session's SCOPED view
|
|
65
|
+
* on demand and records the aggregate as the freshest `lastResumeProbe`;
|
|
66
|
+
* otherwise returns the stored last observation unchanged ('tool registry
|
|
67
|
+
* unreachable -> lastResumeProbe' contract). Session ids whose agent is gone
|
|
68
|
+
* are skipped (they are no longer "current resumed sessions").
|
|
69
|
+
*/
|
|
70
|
+
export declare function snapshotResumeToolHealth(ctx: any, deps?: {
|
|
71
|
+
probeToolView?: ToolViewProbeFn;
|
|
72
|
+
resolveToolScope?: ToolScopeResolver;
|
|
73
|
+
}): ResumeToolHealthSnapshot;
|
|
74
|
+
/**
|
|
75
|
+
* Loopback RPC handler for /dsh-maestro-supervisor-resume-tool-health.
|
|
76
|
+
* Same `{ ok, value | error }` envelope shape as the other supervisor RPCs.
|
|
77
|
+
*/
|
|
78
|
+
export declare function createResumeToolHealthRpcHandler(ctx: any, deps?: {
|
|
79
|
+
probeToolView?: ToolViewProbeFn;
|
|
80
|
+
resolveToolScope?: ToolScopeResolver;
|
|
81
|
+
}): (_endpoint: string, _payload: unknown, _signal: AbortSignal) => Promise<{
|
|
82
|
+
ok: boolean;
|
|
83
|
+
value: ResumeToolHealthSnapshot;
|
|
84
|
+
error?: undefined;
|
|
85
|
+
} | {
|
|
86
|
+
ok: boolean;
|
|
87
|
+
error: {
|
|
88
|
+
code: string;
|
|
89
|
+
message: any;
|
|
90
|
+
};
|
|
91
|
+
value?: undefined;
|
|
92
|
+
}>;
|
|
93
|
+
/** dsh.tools definition for the maestro_resume_tool_health host tool. */
|
|
94
|
+
export declare function makeResumeToolHealthToolDef(ctx: any): any;
|
|
95
|
+
/**
|
|
96
|
+
* Register the resume-tool-health RPC handle (loopback authority) and the
|
|
97
|
+
* maestro_resume_tool_health host tool. Fail-safe like the other
|
|
98
|
+
* registrations: any registration error is logged, never thrown, and the
|
|
99
|
+
* returned disposer unregisters everything that did succeed.
|
|
100
|
+
*/
|
|
101
|
+
export declare function registerResumeToolHealthService(ctx: any): () => void;
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* C2 — deterministic plugin-layer mitigation for resumed sessions that lose
|
|
3
|
+
* core tools after a dsh web restart (`Error: unknown tool "bash"`).
|
|
4
|
+
*
|
|
5
|
+
* The supervisor cannot edit the harness tool layer (hard workspace rule) and
|
|
6
|
+
* the exact layer-rebuild root cause is only observable on the live system
|
|
7
|
+
* (Part D). So when the post-resume probe (C1) reports a core tool missing
|
|
8
|
+
* from the resumed session's SCOPED view, this module runs a host-side
|
|
9
|
+
* mitigation:
|
|
10
|
+
*
|
|
11
|
+
* 1. a short notifier line so the operator sees the loss immediately;
|
|
12
|
+
* 2. a "System:"-prefixed inventory user-message injected into the session
|
|
13
|
+
* (same createUserMessage/followup pattern as the resume-intent message),
|
|
14
|
+
* so the model stops calling the lost tool and uses what remains;
|
|
15
|
+
* 3. optionally (resumeCoreToolPolicy: 'park') records the session id in a
|
|
16
|
+
* module-level parked set exposed through the `maestro_resume_tool_health`
|
|
17
|
+
* host tool + loopback RPC, marking it for manual reopen.
|
|
18
|
+
*
|
|
19
|
+
* The module-level state (`parkedCoreToolLossIds`, `lastResumeProbe`,
|
|
20
|
+
* `resumedSessions`) is per-`dsh web`-process: a host restart clears it, which
|
|
21
|
+
* is exactly right — these are per-boot facts.
|
|
22
|
+
*/
|
|
23
|
+
import { notify } from './notifier.js';
|
|
24
|
+
import { probeToolView, defaultResolveToolScope, CRITICAL_TOOLS, } from './plugin.js';
|
|
25
|
+
// --- module-level mitigation state ---------------------------------------------
|
|
26
|
+
let lastResumeProbe = null;
|
|
27
|
+
const parkedCoreToolLossIds = new Set();
|
|
28
|
+
const resumedSessions = new Set();
|
|
29
|
+
/** Reset the per-process mitigation state — tests + fresh boot reuse. */
|
|
30
|
+
export function resetResumeToolHealthState() {
|
|
31
|
+
lastResumeProbe = null;
|
|
32
|
+
parkedCoreToolLossIds.clear();
|
|
33
|
+
resumedSessions.clear();
|
|
34
|
+
}
|
|
35
|
+
/** Record a session the auto-resume confirmed as resumed (on-demand probe target). */
|
|
36
|
+
export function recordResumedSession(sessionId) {
|
|
37
|
+
resumedSessions.add(sessionId);
|
|
38
|
+
}
|
|
39
|
+
/** Record the last REAL post-resume tool-view observation ({ missing, visible }). */
|
|
40
|
+
export function recordResumeProbe(probe) {
|
|
41
|
+
lastResumeProbe = probe;
|
|
42
|
+
}
|
|
43
|
+
// --- message builders -----------------------------------------------------------
|
|
44
|
+
/**
|
|
45
|
+
* Operator-facing notify line. The park variant appends the manual-reopen
|
|
46
|
+
* marker so the operator knows the session was NOT auto-continued for tools.
|
|
47
|
+
*/
|
|
48
|
+
export function buildCoreToolLossNotifyLine(sessionId, missing, policy) {
|
|
49
|
+
const base = `[supervisor] resumed ${sessionId}: core tool lost [${missing.join(',')}] — tool view incomplete; reopen session if it persists`;
|
|
50
|
+
return policy === 'park' ? `${base} (manual reopen required)` : base;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The SYSTEM message injected into the resumed session. Tells the model the
|
|
54
|
+
* CURRENT inventory (scoped `schemas()` — the tools the session can actually
|
|
55
|
+
* call) so it stops issuing the lost tool instead of looping on unknown-tool
|
|
56
|
+
* errors. Names joined with ', '.
|
|
57
|
+
*/
|
|
58
|
+
export function buildToolInventoryMessage(missing, available) {
|
|
59
|
+
const names = missing.join(', ');
|
|
60
|
+
const verb = missing.length === 1 ? 'tool is' : 'tools are';
|
|
61
|
+
return `System: the ${names} ${verb} unavailable in this session's restored tool view. Available tools: ${available.join(', ')}. Do not call ${names}; use the available tools.`;
|
|
62
|
+
}
|
|
63
|
+
/** Scoped schemas() = exactly what the session can still call; never the global view. */
|
|
64
|
+
function resolveAvailableToolNames(tools, scope) {
|
|
65
|
+
try {
|
|
66
|
+
const schemas = typeof tools?.schemas === 'function' ? tools.schemas : undefined;
|
|
67
|
+
if (!schemas)
|
|
68
|
+
return [];
|
|
69
|
+
const list = schemas(scope);
|
|
70
|
+
if (!Array.isArray(list))
|
|
71
|
+
return [];
|
|
72
|
+
return list.map((t) => (t && typeof t.name === 'string' ? t.name : '')).filter((n) => n.length > 0);
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Default session-message injection: mirrors the resume-intent push in
|
|
80
|
+
* plugin.ts `resumeInterrupted` — `createUserMessage` (dynamic import so the
|
|
81
|
+
* plugin loads without the LLM package ever installed) + `agent.followup`.
|
|
82
|
+
* Never throws; a missing agent just no-ops.
|
|
83
|
+
*/
|
|
84
|
+
function makeDefaultSessionMessageInjector(ctx) {
|
|
85
|
+
return async (sessionId, content) => {
|
|
86
|
+
try {
|
|
87
|
+
const agents = ctx.get?.('agents') ?? ctx.agents;
|
|
88
|
+
const agent = typeof agents?.get === 'function' ? agents.get(sessionId) : undefined;
|
|
89
|
+
if (typeof agent?.followup !== 'function')
|
|
90
|
+
return;
|
|
91
|
+
const { createUserMessage } = await import('@deepseek-ai/dsh-llm').catch(() => ({
|
|
92
|
+
createUserMessage: (input) => ({ ...input, role: 'user', id: crypto.randomUUID() }),
|
|
93
|
+
}));
|
|
94
|
+
agent.followup(createUserMessage({
|
|
95
|
+
content: [{ type: 'text', text: content }],
|
|
96
|
+
source: { kind: 'user' },
|
|
97
|
+
}));
|
|
98
|
+
}
|
|
99
|
+
catch { }
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* C2 mitigation entry: called by the resume flow when the post-resume probe
|
|
104
|
+
* found a core tool missing. Notifies, injects the inventory message, and —
|
|
105
|
+
* under 'park' policy — records the session id. Only fires on an actual
|
|
106
|
+
* CRITICAL_TOOLS loss; a non-core missing name (e.g. cordis_inspect_query)
|
|
107
|
+
* is a no-op. All seams are injectable for tests; every step is defensive.
|
|
108
|
+
*/
|
|
109
|
+
export async function warnCoreToolLoss(ctx, sessionId, scope, probe, policy, opts = {}) {
|
|
110
|
+
const lostCore = probe.missing.filter((n) => CRITICAL_TOOLS.includes(n));
|
|
111
|
+
if (!lostCore.length)
|
|
112
|
+
return;
|
|
113
|
+
const doNotify = opts.notify ?? ((line) => notify(line));
|
|
114
|
+
try {
|
|
115
|
+
await doNotify(buildCoreToolLossNotifyLine(sessionId, lostCore, policy));
|
|
116
|
+
}
|
|
117
|
+
catch { }
|
|
118
|
+
try {
|
|
119
|
+
const tools = opts.tools ?? (ctx.get?.('tools') ?? ctx.tools);
|
|
120
|
+
const doInject = opts.injectSessionMessage ?? makeDefaultSessionMessageInjector(ctx);
|
|
121
|
+
await doInject(sessionId, buildToolInventoryMessage(lostCore, resolveAvailableToolNames(tools, scope)));
|
|
122
|
+
}
|
|
123
|
+
catch { }
|
|
124
|
+
if (policy === 'park')
|
|
125
|
+
parkedCoreToolLossIds.add(sessionId);
|
|
126
|
+
}
|
|
127
|
+
function mergeResumeProbes(base, next) {
|
|
128
|
+
if (!base)
|
|
129
|
+
return next;
|
|
130
|
+
return {
|
|
131
|
+
missing: Array.from(new Set([...base.missing, ...next.missing])),
|
|
132
|
+
visible: base.visible + next.visible,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Build the RPC value. When the tool registry is reachable AND at least one
|
|
137
|
+
* current resumed session still lives, re-probes each session's SCOPED view
|
|
138
|
+
* on demand and records the aggregate as the freshest `lastResumeProbe`;
|
|
139
|
+
* otherwise returns the stored last observation unchanged ('tool registry
|
|
140
|
+
* unreachable -> lastResumeProbe' contract). Session ids whose agent is gone
|
|
141
|
+
* are skipped (they are no longer "current resumed sessions").
|
|
142
|
+
*/
|
|
143
|
+
export function snapshotResumeToolHealth(ctx, deps = {}) {
|
|
144
|
+
const doProbe = deps.probeToolView ?? probeToolView;
|
|
145
|
+
const doResolve = deps.resolveToolScope ?? defaultResolveToolScope;
|
|
146
|
+
const tools = ctx.get?.('tools') ?? ctx.tools;
|
|
147
|
+
let reachable = false;
|
|
148
|
+
let aggregated = null;
|
|
149
|
+
for (const id of resumedSessions) {
|
|
150
|
+
try {
|
|
151
|
+
const agents = ctx.get?.('agents') ?? ctx.agents;
|
|
152
|
+
if (typeof agents?.get === 'function' && agents.get(id) === undefined)
|
|
153
|
+
continue;
|
|
154
|
+
aggregated = mergeResumeProbes(aggregated, doProbe(tools, doResolve(ctx, id)));
|
|
155
|
+
reachable = true;
|
|
156
|
+
}
|
|
157
|
+
catch { }
|
|
158
|
+
}
|
|
159
|
+
if (reachable && aggregated)
|
|
160
|
+
lastResumeProbe = aggregated;
|
|
161
|
+
return { lastResumeProbe, parked: [...parkedCoreToolLossIds] };
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Loopback RPC handler for /dsh-maestro-supervisor-resume-tool-health.
|
|
165
|
+
* Same `{ ok, value | error }` envelope shape as the other supervisor RPCs.
|
|
166
|
+
*/
|
|
167
|
+
export function createResumeToolHealthRpcHandler(ctx, deps = {}) {
|
|
168
|
+
return async (_endpoint, _payload, _signal) => {
|
|
169
|
+
try {
|
|
170
|
+
return { ok: true, value: snapshotResumeToolHealth(ctx, deps) };
|
|
171
|
+
}
|
|
172
|
+
catch (e) {
|
|
173
|
+
return { ok: false, error: { code: 'resume-tool-health-failed', message: e?.message ?? String(e) } };
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
/** dsh.tools definition for the maestro_resume_tool_health host tool. */
|
|
178
|
+
export function makeResumeToolHealthToolDef(ctx) {
|
|
179
|
+
return {
|
|
180
|
+
name: 'maestro_resume_tool_health',
|
|
181
|
+
description: 'Report the tool-view health of sessions auto-resumed after a dsh web restart. ' +
|
|
182
|
+
'Returns the last post-resume probe (missing core tools, visible count) plus the sessions ' +
|
|
183
|
+
'parked for manual reopen under resumeCoreToolPolicy: park. The probe is refreshed on demand ' +
|
|
184
|
+
'against the currently resumed sessions when the tool registry is reachable.',
|
|
185
|
+
parameters: { type: 'object', properties: {}, additionalProperties: false },
|
|
186
|
+
output: {
|
|
187
|
+
schema: {
|
|
188
|
+
type: 'object',
|
|
189
|
+
additionalProperties: true,
|
|
190
|
+
properties: {
|
|
191
|
+
lastResumeProbe: {
|
|
192
|
+
type: 'object',
|
|
193
|
+
properties: {
|
|
194
|
+
missing: { type: 'array', items: { type: 'string' } },
|
|
195
|
+
visible: { type: 'number' },
|
|
196
|
+
},
|
|
197
|
+
},
|
|
198
|
+
parked: { type: 'array', items: { type: 'string' } },
|
|
199
|
+
},
|
|
200
|
+
},
|
|
201
|
+
render: (_args, value) => {
|
|
202
|
+
const probe = value?.lastResumeProbe;
|
|
203
|
+
const missing = Array.isArray(probe?.missing) && probe.missing.length ? probe.missing.join(',') : 'none';
|
|
204
|
+
const visible = typeof probe?.visible === 'number' ? probe.visible : 'n/a';
|
|
205
|
+
return [{ type: 'text', text: `missing=${missing} visible=${visible} parked=${Array.isArray(value?.parked) ? value.parked.length : 0}` }];
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
execute: async () => snapshotResumeToolHealth(ctx),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Register the resume-tool-health RPC handle (loopback authority) and the
|
|
213
|
+
* maestro_resume_tool_health host tool. Fail-safe like the other
|
|
214
|
+
* registrations: any registration error is logged, never thrown, and the
|
|
215
|
+
* returned disposer unregisters everything that did succeed.
|
|
216
|
+
*/
|
|
217
|
+
export function registerResumeToolHealthService(ctx) {
|
|
218
|
+
const disposers = [];
|
|
219
|
+
try {
|
|
220
|
+
const conn = ctx.connection ?? ctx.get?.('connection');
|
|
221
|
+
if (conn?.rpc?.handle) {
|
|
222
|
+
disposers.push(conn.rpc.handle('/dsh-maestro-supervisor-resume-tool-health', createResumeToolHealthRpcHandler(ctx), { authority: 'loopback' }));
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
catch (e) {
|
|
226
|
+
try {
|
|
227
|
+
ctx.logger?.warn?.(`[supervisor] resume-tool-health RPC registration failed: ${e?.message ?? String(e)}`);
|
|
228
|
+
}
|
|
229
|
+
catch { }
|
|
230
|
+
}
|
|
231
|
+
try {
|
|
232
|
+
if (typeof ctx.tools?.register === 'function') {
|
|
233
|
+
disposers.push(ctx.tools.register(makeResumeToolHealthToolDef(ctx)));
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
catch (e) {
|
|
237
|
+
try {
|
|
238
|
+
ctx.logger?.warn?.(`[supervisor] resume-tool-health tool registration failed: ${e?.message ?? String(e)}`);
|
|
239
|
+
}
|
|
240
|
+
catch { }
|
|
241
|
+
}
|
|
242
|
+
return () => { for (const d of disposers) {
|
|
243
|
+
try {
|
|
244
|
+
d();
|
|
245
|
+
}
|
|
246
|
+
catch { }
|
|
247
|
+
} };
|
|
248
|
+
}
|