@ddtcorex/dsh-maestro-supervisor 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/debug-agent.d.ts +14 -0
- package/lib/debug-agent.js +23 -0
- package/lib/health-poller.js +31 -7
- package/lib/resume.d.ts +5 -0
- package/lib/resume.js +41 -0
- package/lib/supervisor.d.ts +1 -0
- package/lib/supervisor.js +36 -0
- package/package.json +1 -1
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface DebugAgentOpts {
|
|
2
|
+
reportPath: string;
|
|
3
|
+
health: {
|
|
4
|
+
error?: string;
|
|
5
|
+
httpCode?: number;
|
|
6
|
+
};
|
|
7
|
+
attempts?: number;
|
|
8
|
+
cooldownMs?: number;
|
|
9
|
+
}
|
|
10
|
+
export declare function runDebugAgent(opts: DebugAgentOpts): Promise<{
|
|
11
|
+
fixed: boolean;
|
|
12
|
+
reason: string;
|
|
13
|
+
}>;
|
|
14
|
+
export declare function _resetDebugAgentForTest(): void;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
let lastRun = 0;
|
|
2
|
+
let attempts = 0;
|
|
3
|
+
const MAX_ATTEMPTS = 3;
|
|
4
|
+
export async function runDebugAgent(opts) {
|
|
5
|
+
const now = Date.now();
|
|
6
|
+
const cooldownMs = opts.cooldownMs ?? 10 * 60 * 1000;
|
|
7
|
+
if (now - lastRun < cooldownMs) {
|
|
8
|
+
return { fixed: false, reason: 'cooldown' };
|
|
9
|
+
}
|
|
10
|
+
if (attempts >= MAX_ATTEMPTS) {
|
|
11
|
+
return { fixed: false, reason: 'max attempts' };
|
|
12
|
+
}
|
|
13
|
+
lastRun = now;
|
|
14
|
+
attempts++;
|
|
15
|
+
// Stub: Phase 3 full LLM wiring will use systematic-debugging preset
|
|
16
|
+
// For now just log and return not fixed — supervisor will still resume sessions
|
|
17
|
+
// Real implementation: spawn subagent via @ddtcorex/dsh-maestro-notifier or openai
|
|
18
|
+
return { fixed: false, reason: `stub — would debug ${opts.reportPath} (attempt ${attempts})` };
|
|
19
|
+
}
|
|
20
|
+
export function _resetDebugAgentForTest() {
|
|
21
|
+
lastRun = 0;
|
|
22
|
+
attempts = 0;
|
|
23
|
+
}
|
package/lib/health-poller.js
CHANGED
|
@@ -46,13 +46,30 @@ export async function pollHealth(opts = {}) {
|
|
|
46
46
|
break;
|
|
47
47
|
}
|
|
48
48
|
}
|
|
49
|
-
//
|
|
50
|
-
if (fetchError
|
|
49
|
+
// Distinguish FULL (http !=200) vs DEGRADED (http 200 but log has plugin error)
|
|
50
|
+
if (fetchError) {
|
|
51
51
|
return {
|
|
52
52
|
up: false,
|
|
53
53
|
httpCode,
|
|
54
|
-
error: logError
|
|
55
|
-
degraded:
|
|
54
|
+
error: logError ? `${fetchError} + ${logError}` : fetchError,
|
|
55
|
+
degraded: false,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
if (logError) {
|
|
59
|
+
// http 200 but log error → DEGRADED (isolatable), not FULL
|
|
60
|
+
if (httpCode === 200) {
|
|
61
|
+
return {
|
|
62
|
+
up: true,
|
|
63
|
+
httpCode,
|
|
64
|
+
error: logError,
|
|
65
|
+
degraded: true,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
up: false,
|
|
70
|
+
httpCode,
|
|
71
|
+
error: logError,
|
|
72
|
+
degraded: false,
|
|
56
73
|
};
|
|
57
74
|
}
|
|
58
75
|
// Also check psAlive as secondary signal — if fetch ok but ps dead, still down
|
|
@@ -95,9 +112,16 @@ async function defaultLogTail() {
|
|
|
95
112
|
try {
|
|
96
113
|
const { readFileSync } = await import('node:fs');
|
|
97
114
|
const { homedir } = await import('node:os');
|
|
98
|
-
const
|
|
99
|
-
const
|
|
100
|
-
|
|
115
|
+
const candidates = [`${homedir()}/.dsh/dsh-web.log`, `${homedir()}/.dsh.log`];
|
|
116
|
+
for (const logPath of candidates) {
|
|
117
|
+
try {
|
|
118
|
+
const content = readFileSync(logPath, 'utf-8');
|
|
119
|
+
if (content)
|
|
120
|
+
return content.slice(-5000);
|
|
121
|
+
}
|
|
122
|
+
catch { }
|
|
123
|
+
}
|
|
124
|
+
return '';
|
|
101
125
|
}
|
|
102
126
|
catch {
|
|
103
127
|
return '';
|
package/lib/resume.d.ts
ADDED
package/lib/resume.js
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import * as os from 'node:os';
|
|
4
|
+
export async function findInterrupted(dshHome) {
|
|
5
|
+
const home = dshHome ?? path.join(os.homedir(), '.dsh');
|
|
6
|
+
const sessionsRoot = path.join(home, 'sessions');
|
|
7
|
+
let scanned = 0;
|
|
8
|
+
const interrupted = [];
|
|
9
|
+
try {
|
|
10
|
+
const groups = fs.readdirSync(sessionsRoot, { withFileTypes: true });
|
|
11
|
+
for (const g of groups) {
|
|
12
|
+
if (!g.isDirectory())
|
|
13
|
+
continue;
|
|
14
|
+
const groupPath = path.join(sessionsRoot, g.name);
|
|
15
|
+
const sessions = fs.readdirSync(groupPath, { withFileTypes: true });
|
|
16
|
+
for (const s of sessions) {
|
|
17
|
+
if (!s.isDirectory())
|
|
18
|
+
continue;
|
|
19
|
+
scanned++;
|
|
20
|
+
const zstdPath = path.join(groupPath, s.name, 'session.jsonl.zstd');
|
|
21
|
+
const jsonlPath = path.join(groupPath, s.name, 'session.jsonl');
|
|
22
|
+
try {
|
|
23
|
+
let content = '';
|
|
24
|
+
if (fs.existsSync(zstdPath)) {
|
|
25
|
+
const { execSync } = await import('node:child_process');
|
|
26
|
+
content = execSync(`zstd -d -c ${JSON.stringify(zstdPath)} 2>/dev/null | tail -5`, { encoding: 'utf-8' });
|
|
27
|
+
}
|
|
28
|
+
else if (fs.existsSync(jsonlPath)) {
|
|
29
|
+
content = fs.readFileSync(jsonlPath, 'utf-8').slice(-5000);
|
|
30
|
+
}
|
|
31
|
+
if (content.toLowerCase().includes('interrupted')) {
|
|
32
|
+
interrupted.push(`${g.name}/${s.name}`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
catch { }
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
catch { }
|
|
40
|
+
return { scanned, interrupted };
|
|
41
|
+
}
|
package/lib/supervisor.d.ts
CHANGED
package/lib/supervisor.js
CHANGED
|
@@ -1,14 +1,41 @@
|
|
|
1
|
+
import { runDebugAgent } from './debug-agent.js';
|
|
2
|
+
import { findInterrupted } from './resume.js';
|
|
1
3
|
export class Supervisor {
|
|
2
4
|
deps;
|
|
3
5
|
lastRollback = 0;
|
|
4
6
|
rollingBack = false;
|
|
5
7
|
lastLKGWrite = 0;
|
|
8
|
+
lastDegradedNotify = 0;
|
|
6
9
|
timer = null;
|
|
7
10
|
constructor(deps) {
|
|
8
11
|
this.deps = deps;
|
|
9
12
|
}
|
|
10
13
|
async tick() {
|
|
11
14
|
const health = await this.deps.pollHealth();
|
|
15
|
+
// DEGRADED: http 200 but log has plugin error → report, notify, no rollback
|
|
16
|
+
if (health.degraded) {
|
|
17
|
+
const now = this.deps.getTime ? this.deps.getTime() : Date.now();
|
|
18
|
+
if (now - this.lastDegradedNotify < 60000)
|
|
19
|
+
return;
|
|
20
|
+
this.lastDegradedNotify = now;
|
|
21
|
+
try {
|
|
22
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
23
|
+
const reportPath = await this.deps.writeReport({ ts, health, action: `degraded — ${health.error ?? 'plugin'}` }).catch(() => '');
|
|
24
|
+
await this.deps.notify(`DEGRADED: ${health.error ?? 'plugin'} (report: ${reportPath})`).catch(() => { });
|
|
25
|
+
// Phase 3: fire-and-forget debug + resume (never block tick, skip in test)
|
|
26
|
+
if (!process.env.VITEST) {
|
|
27
|
+
void runDebugAgent({ reportPath, health }).catch(() => { });
|
|
28
|
+
setTimeout(() => {
|
|
29
|
+
findInterrupted().then(r => {
|
|
30
|
+
if (r.interrupted.length)
|
|
31
|
+
this.deps.notify(`RESUME: ${r.interrupted.length} interrupted sessions (${r.interrupted.slice(0, 3).join(', ')})`).catch(() => { });
|
|
32
|
+
}).catch(() => { });
|
|
33
|
+
}, 0);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
catch { }
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
12
39
|
if (health.up) {
|
|
13
40
|
// Throttle LKG writes to at most once per 5 minutes
|
|
14
41
|
const now = this.deps.getTime ? this.deps.getTime() : Date.now();
|
|
@@ -38,6 +65,15 @@ export class Supervisor {
|
|
|
38
65
|
const reportPath = await this.deps.writeReport({ ts, health, action: `rollback — ${health.error ?? 'down'}` }).catch(() => '');
|
|
39
66
|
await this.deps.rollback();
|
|
40
67
|
await this.deps.notify(`CRASH detected → rollback (report: ${reportPath}, error: ${health.error ?? 'down'})`).catch(() => { });
|
|
68
|
+
if (!process.env.VITEST) {
|
|
69
|
+
void runDebugAgent({ reportPath, health }).catch(() => { });
|
|
70
|
+
setTimeout(() => {
|
|
71
|
+
findInterrupted().then(r => {
|
|
72
|
+
if (r.interrupted.length)
|
|
73
|
+
this.deps.notify(`RESUME: ${r.interrupted.length} interrupted sessions`).catch(() => { });
|
|
74
|
+
}).catch(() => { });
|
|
75
|
+
}, 0);
|
|
76
|
+
}
|
|
41
77
|
}
|
|
42
78
|
finally {
|
|
43
79
|
this.rollingBack = false;
|