@ddtcorex/dsh-maestro-supervisor 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli.js +34 -3
- package/lib/debug-agent.js +151 -32
- package/lib/health-poller.d.ts +2 -0
- package/lib/health-poller.js +36 -5
- package/lib/snapshot.d.ts +3 -0
- package/lib/snapshot.js +128 -2
- package/lib/supervisor.d.ts +3 -0
- package/lib/supervisor.js +25 -2
- package/package.json +1 -1
package/lib/cli.js
CHANGED
|
@@ -44,9 +44,40 @@ Commands:
|
|
|
44
44
|
pollHealth: () => pollHealth(),
|
|
45
45
|
writeLKG: () => writeLKG(dshHome, lkgRoot),
|
|
46
46
|
writeFailed: () => writeLKG(dshHome, failedRoot),
|
|
47
|
-
writeReport: async ({ ts, health, action }) => {
|
|
48
|
-
const { writeReport } = await import('./report.js');
|
|
49
|
-
|
|
47
|
+
writeReport: async ({ ts, health, action, logTail, gitDiff }) => {
|
|
48
|
+
const { writeReport, collectGitDiff } = await import('./report.js');
|
|
49
|
+
const { collectLogTail } = await import('./health-poller.js');
|
|
50
|
+
// Prefer supervisor-provided tail/diff (from health.logTail); fallback to live collect
|
|
51
|
+
let tail = logTail ?? health.logTail ?? '';
|
|
52
|
+
if (!tail) {
|
|
53
|
+
try {
|
|
54
|
+
tail = await collectLogTail();
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
tail = '';
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
let diff = gitDiff ?? '';
|
|
61
|
+
if (!diff) {
|
|
62
|
+
try {
|
|
63
|
+
const harnessRoot = process.env.MAESTRO_HARNESS_ROOT ?? path.join(os.homedir(), 'Work/htdocs/maestro-harness');
|
|
64
|
+
diff = await collectGitDiff(harnessRoot).catch(() => '');
|
|
65
|
+
if (!diff) {
|
|
66
|
+
// fallback: try git diff in cwd
|
|
67
|
+
const { execSync } = await import('node:child_process');
|
|
68
|
+
try {
|
|
69
|
+
diff = execSync('git diff 2>/dev/null | head -n 200', { encoding: 'utf-8', timeout: 2000 });
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
diff = '';
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
diff = '';
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return writeReport({ reportsRoot, ts, health, gitDiff: diff, logTail: tail, action });
|
|
50
81
|
},
|
|
51
82
|
rollback: async () => {
|
|
52
83
|
// Find latest LKG and restore
|
package/lib/debug-agent.js
CHANGED
|
@@ -143,8 +143,22 @@ function buildPrompt(ctx) {
|
|
|
143
143
|
}
|
|
144
144
|
function tryApplyLLMFix(response, writeFile, exec) {
|
|
145
145
|
try {
|
|
146
|
-
//
|
|
147
|
-
|
|
146
|
+
// First try to extract inner JSON if response is OpenAI wrapper (choices) — unwrap message.content
|
|
147
|
+
let candidate = response;
|
|
148
|
+
try {
|
|
149
|
+
const outer = JSON.parse(response);
|
|
150
|
+
if (outer.choices?.[0]?.message?.content)
|
|
151
|
+
candidate = outer.choices[0].message.content;
|
|
152
|
+
else if (outer.output) {
|
|
153
|
+
// openai-responses wrapper: find message with output_text
|
|
154
|
+
const msg = outer.output.find((o) => o.type === 'message' && o.content?.[0]?.text);
|
|
155
|
+
if (msg)
|
|
156
|
+
candidate = msg.content[0].text;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
catch { }
|
|
160
|
+
// Also handle case where response is already the inner JSON string or contains it
|
|
161
|
+
const jsonMatch = candidate.match(/\{[\s\S]*\}/);
|
|
148
162
|
if (!jsonMatch)
|
|
149
163
|
return false;
|
|
150
164
|
const obj = JSON.parse(jsonMatch[0]);
|
|
@@ -163,7 +177,19 @@ function tryApplyLLMFix(response, writeFile, exec) {
|
|
|
163
177
|
}
|
|
164
178
|
function extractSummary(response) {
|
|
165
179
|
try {
|
|
166
|
-
|
|
180
|
+
let candidate = response;
|
|
181
|
+
try {
|
|
182
|
+
const outer = JSON.parse(response);
|
|
183
|
+
if (outer.choices?.[0]?.message?.content)
|
|
184
|
+
candidate = outer.choices[0].message.content;
|
|
185
|
+
else if (outer.output) {
|
|
186
|
+
const msg = outer.output.find((o) => o.type === 'message' && o.content?.[0]?.text);
|
|
187
|
+
if (msg)
|
|
188
|
+
candidate = msg.content[0].text;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
catch { }
|
|
192
|
+
const m = candidate.match(/\{[\s\S]*\}/);
|
|
167
193
|
if (m) {
|
|
168
194
|
const obj = JSON.parse(m[0]);
|
|
169
195
|
if (obj.analysis)
|
|
@@ -206,51 +232,144 @@ async function defaultFetchLLM(prompt) {
|
|
|
206
232
|
const cfg = await resolveLLMConfig();
|
|
207
233
|
if (!cfg.key)
|
|
208
234
|
throw new Error('no api key — set DEEPSEEK_API_KEY / OMNI_ROUTE_API_KEY / OPENCODE_GO_API_KEY / OPENAI_API_KEY or ~/.dsh/.credentials.yaml or AI_API_KEY');
|
|
209
|
-
|
|
235
|
+
// Build URLs from cfg.url (may be base or full endpoint). Support any provider/model — try completions then responses.
|
|
236
|
+
const base = cfg.url.replace(/\/v1\/(chat\/completions|responses).*/, '/v1').replace(/\/$/, '');
|
|
237
|
+
const completionsUrl = cfg.url.includes('/chat/completions') ? cfg.url : (cfg.url.includes('/responses') ? cfg.url.replace('/responses', '/chat/completions') : `${base}/chat/completions`);
|
|
238
|
+
const responsesUrl = cfg.url.includes('/responses') ? cfg.url : `${base}/responses`;
|
|
239
|
+
// Try openai-completions first (works for deepseek-v4, openai, openrouter, omni-route completions)
|
|
240
|
+
try {
|
|
241
|
+
const res = await fetch(completionsUrl, {
|
|
242
|
+
method: 'POST',
|
|
243
|
+
headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${cfg.key}` },
|
|
244
|
+
body: JSON.stringify({
|
|
245
|
+
model: cfg.model,
|
|
246
|
+
temperature: 0.2,
|
|
247
|
+
messages: [
|
|
248
|
+
{ role: 'system', content: 'You are a systematic-debugging agent for DSH Web resilience. Follow the 4 phases: root cause, pattern analysis, hypothesis, implementation. Always propose minimal single-file fix.' },
|
|
249
|
+
{ role: 'user', content: prompt },
|
|
250
|
+
],
|
|
251
|
+
}),
|
|
252
|
+
});
|
|
253
|
+
if (res.ok) {
|
|
254
|
+
const j = await res.json();
|
|
255
|
+
const content = j.choices?.[0]?.message?.content;
|
|
256
|
+
if (content)
|
|
257
|
+
return content;
|
|
258
|
+
// Some providers return different shape but still ok — return stringified
|
|
259
|
+
return JSON.stringify(j);
|
|
260
|
+
}
|
|
261
|
+
// If completions fails, fall through to responses for providers that use openai-responses (e.g. opencode-go muse-spark)
|
|
262
|
+
const txt = await res.text().catch(() => '');
|
|
263
|
+
// Only fall through for 4xx/5xx that likely indicate wrong API — otherwise throw
|
|
264
|
+
if (res.status >= 400 && res.status < 600) {
|
|
265
|
+
// try responses as fallback for any provider/model
|
|
266
|
+
}
|
|
267
|
+
else {
|
|
268
|
+
throw new Error(`LLM ${res.status} ${txt}`);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
catch (e) {
|
|
272
|
+
// Network error — if completionsUrl was tried and failed, try responses as generic fallback
|
|
273
|
+
if (!e.message?.includes('LLM ')) {
|
|
274
|
+
// will try responses below
|
|
275
|
+
}
|
|
276
|
+
else {
|
|
277
|
+
throw e;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
// Fallback: try openai-responses (used by opencode-go muse-spark, minimax-m3, qwen3.7, etc.) — works for any provider that supports it
|
|
281
|
+
const res2 = await fetch(responsesUrl, {
|
|
210
282
|
method: 'POST',
|
|
211
|
-
headers: {
|
|
212
|
-
'Content-Type': 'application/json',
|
|
213
|
-
'Authorization': `Bearer ${cfg.key}`,
|
|
214
|
-
},
|
|
283
|
+
headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${cfg.key}` },
|
|
215
284
|
body: JSON.stringify({
|
|
216
285
|
model: cfg.model,
|
|
217
|
-
|
|
218
|
-
messages: [
|
|
286
|
+
input: [
|
|
219
287
|
{ role: 'system', content: 'You are a systematic-debugging agent for DSH Web resilience. Follow the 4 phases: root cause, pattern analysis, hypothesis, implementation. Always propose minimal single-file fix.' },
|
|
220
288
|
{ role: 'user', content: prompt },
|
|
221
289
|
],
|
|
290
|
+
max_output_tokens: 4000,
|
|
291
|
+
reasoning: { effort: 'low' },
|
|
222
292
|
}),
|
|
223
293
|
});
|
|
224
|
-
if (!
|
|
225
|
-
throw new Error(`LLM ${
|
|
226
|
-
const
|
|
227
|
-
|
|
294
|
+
if (!res2.ok)
|
|
295
|
+
throw new Error(`LLM ${res2.status} ${await res2.text().catch(() => '')}`);
|
|
296
|
+
const j2 = await res2.json();
|
|
297
|
+
// Extract text from responses output (opencode-go) or completions fallback
|
|
298
|
+
if (j2.output) {
|
|
299
|
+
const msg = j2.output.find((o) => o.type === 'message' && o.content?.[0]?.text);
|
|
300
|
+
if (msg)
|
|
301
|
+
return msg.content[0].text;
|
|
302
|
+
}
|
|
303
|
+
return j2.choices?.[0]?.message?.content ?? JSON.stringify(j2);
|
|
228
304
|
}
|
|
229
305
|
async function resolveLLMConfig() {
|
|
230
|
-
// Custom AI provider: env overrides, then
|
|
231
|
-
//
|
|
232
|
-
|
|
306
|
+
// Custom AI provider: env overrides, then settings.yaml (llm-pi-ai providers, DeepSeek suggested setup), then settings.json, then defaults
|
|
307
|
+
// Supports any provider/model the user configures via DeepSeek harness (opencode-go, omni-route, deepseek, openrouter, etc.)
|
|
308
|
+
let url = process.env.AI_API_URL ??
|
|
233
309
|
process.env.DEAPSEEK_API_URL ??
|
|
234
310
|
process.env.DEEPSEEK_API_URL ??
|
|
235
311
|
process.env.OPENAI_API_BASE ??
|
|
236
312
|
process.env.OMNI_ROUTE_API_URL ??
|
|
237
313
|
process.env.OPENCODE_API_URL ??
|
|
238
|
-
|
|
239
|
-
// Model: AI_MODEL / DEEPSEEK_MODEL / settings.json domains.review.model
|
|
240
|
-
let model = process.env.AI_MODEL ?? process.env.DEEPSEEK_MODEL ?? process.env.OPENAI_MODEL ??
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
314
|
+
null;
|
|
315
|
+
// Model: AI_MODEL / DEEPSEEK_MODEL / settings.json domains.review.model / default
|
|
316
|
+
let model = process.env.AI_MODEL ?? process.env.DEEPSEEK_MODEL ?? process.env.OPENAI_MODEL ?? null;
|
|
317
|
+
// Try settings.json first (review model is the user's current default model)
|
|
318
|
+
if (!model) {
|
|
319
|
+
try {
|
|
320
|
+
const { readFileSync } = await import('node:fs');
|
|
321
|
+
const { homedir } = await import('node:os');
|
|
322
|
+
const settingsPath = `${homedir()}/.dsh/maestro/settings.json`;
|
|
323
|
+
const raw = readFileSync(settingsPath, 'utf-8');
|
|
324
|
+
const j = JSON.parse(raw);
|
|
325
|
+
const m = j?.domains?.review?.model?.model ?? j?.domains?.review?.model;
|
|
326
|
+
if (typeof m === 'string')
|
|
327
|
+
model = m;
|
|
328
|
+
else if (m?.model && typeof m.model === 'string')
|
|
329
|
+
model = m.model;
|
|
330
|
+
}
|
|
331
|
+
catch { }
|
|
252
332
|
}
|
|
253
|
-
|
|
333
|
+
if (!model)
|
|
334
|
+
model = 'deepseek-chat';
|
|
335
|
+
// If url not set via env, try to resolve from ~/.dsh/settings.yaml llm-pi-ai providers (DeepSeek suggested setup)
|
|
336
|
+
if (!url) {
|
|
337
|
+
try {
|
|
338
|
+
const { readFileSync } = await import('node:fs');
|
|
339
|
+
const { homedir } = await import('node:os');
|
|
340
|
+
const yamlPath = `${homedir()}/.dsh/settings.yaml`;
|
|
341
|
+
const yaml = readFileSync(yamlPath, 'utf-8');
|
|
342
|
+
// Find provider that owns the current model (e.g. muse-spark -> opencode-go)
|
|
343
|
+
// settings.yaml structure: llm-pi-ai: providers: <provider>: { baseURL, models: [{id}] }
|
|
344
|
+
// Use regex to extract provider blocks
|
|
345
|
+
const providerBlocks = [...yaml.matchAll(/^ {4}(\S+):\s*\n([\s\S]*?)(?=^ {4}\S+:|\n\S)/gm)];
|
|
346
|
+
for (const [, provider, block] of providerBlocks) {
|
|
347
|
+
if (block.includes(`id: ${model}`)) {
|
|
348
|
+
const m = block.match(/baseURL:\s*(\S+)/);
|
|
349
|
+
if (m) {
|
|
350
|
+
url = m[1].trim();
|
|
351
|
+
break;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
// Also check agent-default-model provider
|
|
356
|
+
if (!url) {
|
|
357
|
+
const defM = yaml.match(/agent-default-model:\s*\n\s*provider:\s*(\S+)/);
|
|
358
|
+
if (defM) {
|
|
359
|
+
const prov = defM[1].trim();
|
|
360
|
+
const provBlock = yaml.match(new RegExp(`^ {4}${prov}:\\s*\\n([\\s\\S]*?)(?=^ {4}\\S+:|\\n\\S)`, 'm'));
|
|
361
|
+
if (provBlock) {
|
|
362
|
+
const m = provBlock[1].match(/baseURL:\s*(\S+)/);
|
|
363
|
+
if (m)
|
|
364
|
+
url = m[1].trim();
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
catch { }
|
|
370
|
+
}
|
|
371
|
+
if (!url)
|
|
372
|
+
url = 'https://api.deepseek.com/v1/chat/completions';
|
|
254
373
|
// Key: AI_API_KEY / DEEPSEEK_API_KEY / OMNI_ROUTE_API_KEY / OPENCODE_GO_API_KEY / OPENAI_API_KEY
|
|
255
374
|
let key = process.env.AI_API_KEY ??
|
|
256
375
|
process.env.DEEPSEEK_API_KEY ??
|
package/lib/health-poller.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ export interface HealthState {
|
|
|
3
3
|
httpCode?: number;
|
|
4
4
|
error?: string;
|
|
5
5
|
degraded?: boolean;
|
|
6
|
+
logTail?: string;
|
|
6
7
|
}
|
|
7
8
|
export interface PollHealthOpts {
|
|
8
9
|
fetch?: () => Promise<{
|
|
@@ -15,3 +16,4 @@ export interface PollHealthOpts {
|
|
|
15
16
|
timeoutMs?: number;
|
|
16
17
|
}
|
|
17
18
|
export declare function pollHealth(opts?: PollHealthOpts): Promise<HealthState>;
|
|
19
|
+
export declare function collectLogTail(): Promise<string>;
|
package/lib/health-poller.js
CHANGED
|
@@ -53,6 +53,7 @@ export async function pollHealth(opts = {}) {
|
|
|
53
53
|
httpCode,
|
|
54
54
|
error: logError ? `${fetchError} + ${logError}` : fetchError,
|
|
55
55
|
degraded: false,
|
|
56
|
+
logTail: logContent.slice(-5000),
|
|
56
57
|
};
|
|
57
58
|
}
|
|
58
59
|
if (logError) {
|
|
@@ -63,6 +64,7 @@ export async function pollHealth(opts = {}) {
|
|
|
63
64
|
httpCode,
|
|
64
65
|
error: logError,
|
|
65
66
|
degraded: true,
|
|
67
|
+
logTail: logContent.slice(-5000),
|
|
66
68
|
};
|
|
67
69
|
}
|
|
68
70
|
return {
|
|
@@ -70,6 +72,7 @@ export async function pollHealth(opts = {}) {
|
|
|
70
72
|
httpCode,
|
|
71
73
|
error: logError,
|
|
72
74
|
degraded: false,
|
|
75
|
+
logTail: logContent.slice(-5000),
|
|
73
76
|
};
|
|
74
77
|
}
|
|
75
78
|
// Also check psAlive as secondary signal — if fetch ok but ps dead, still down
|
|
@@ -82,7 +85,7 @@ export async function pollHealth(opts = {}) {
|
|
|
82
85
|
catch {
|
|
83
86
|
// ignore
|
|
84
87
|
}
|
|
85
|
-
return { up: httpCode === 200, httpCode };
|
|
88
|
+
return { up: httpCode === 200, httpCode, logTail: logContent.slice(-5000) };
|
|
86
89
|
}
|
|
87
90
|
function defaultFetch(url, timeoutMs) {
|
|
88
91
|
return async () => {
|
|
@@ -108,22 +111,50 @@ async function defaultPsAlive() {
|
|
|
108
111
|
return true;
|
|
109
112
|
}
|
|
110
113
|
}
|
|
111
|
-
async function
|
|
114
|
+
export async function collectLogTail() {
|
|
112
115
|
try {
|
|
113
|
-
const { readFileSync } = await import('node:fs');
|
|
116
|
+
const { readFileSync, existsSync, statSync } = await import('node:fs');
|
|
114
117
|
const { homedir } = await import('node:os');
|
|
115
|
-
const candidates = [
|
|
118
|
+
const candidates = [
|
|
119
|
+
`${homedir()}/.dsh/dsh-web.log`,
|
|
120
|
+
`${homedir()}/.dsh/.supervisor/supervisor.log`,
|
|
121
|
+
`${homedir()}/.dsh.log`,
|
|
122
|
+
];
|
|
116
123
|
for (const logPath of candidates) {
|
|
117
124
|
try {
|
|
125
|
+
if (!existsSync(logPath))
|
|
126
|
+
continue;
|
|
127
|
+
// avoid reading huge files fully — if >1MB, read tail via shell
|
|
128
|
+
try {
|
|
129
|
+
const sz = statSync(logPath).size;
|
|
130
|
+
if (sz > 1024 * 1024) {
|
|
131
|
+
const { execSync } = await import('node:child_process');
|
|
132
|
+
const out = execSync(`tail -c 5000 ${JSON.stringify(logPath)} 2>/dev/null || cat ${JSON.stringify(logPath)} 2>/dev/null | tail -c 5000`, { encoding: 'utf-8', timeout: 2000 });
|
|
133
|
+
if (out)
|
|
134
|
+
return out.slice(-5000);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch { }
|
|
118
138
|
const content = readFileSync(logPath, 'utf-8');
|
|
119
|
-
if (content)
|
|
139
|
+
if (content && content.trim())
|
|
120
140
|
return content.slice(-5000);
|
|
121
141
|
}
|
|
122
142
|
catch { }
|
|
123
143
|
}
|
|
144
|
+
// fallback: try journalctl for the dsh-web or supervisor units (if running via systemd)
|
|
145
|
+
try {
|
|
146
|
+
const { execSync } = await import('node:child_process');
|
|
147
|
+
const journal = execSync('journalctl --user -u dsh-web-supervisor --no-pager -n 100 2>/dev/null | tail -c 5000 || journalctl --user --no-pager -n 100 2>/dev/null | tail -c 5000 || true', { encoding: 'utf-8', timeout: 2000 });
|
|
148
|
+
if (journal && journal.trim())
|
|
149
|
+
return journal.slice(-5000);
|
|
150
|
+
}
|
|
151
|
+
catch { }
|
|
124
152
|
return '';
|
|
125
153
|
}
|
|
126
154
|
catch {
|
|
127
155
|
return '';
|
|
128
156
|
}
|
|
129
157
|
}
|
|
158
|
+
async function defaultLogTail() {
|
|
159
|
+
return collectLogTail();
|
|
160
|
+
}
|
package/lib/snapshot.d.ts
CHANGED
|
@@ -10,6 +10,9 @@ export declare function writeLKG(dshHome: string, lkgRoot: string): Promise<{
|
|
|
10
10
|
ts: string;
|
|
11
11
|
manifest: Manifest;
|
|
12
12
|
}>;
|
|
13
|
+
export declare function pruneByAge(root: string, maxAgeMs: number): Promise<void>;
|
|
14
|
+
export declare function pruneBySize(root: string, maxBytes: number): Promise<void>;
|
|
15
|
+
export declare function isDuplicateLKG(dshHome: string, lkgRoot: string): Promise<boolean>;
|
|
13
16
|
export declare function verifyLKG(lkgPath: string): Promise<boolean>;
|
|
14
17
|
export declare function rotateLKG(lkgRoot: string, keep?: number): Promise<void>;
|
|
15
18
|
export declare function writeFailed(dshHome: string, failedRoot: string): Promise<{
|
package/lib/snapshot.js
CHANGED
|
@@ -17,12 +17,29 @@ function walkFiles(dir, base = dir) {
|
|
|
17
17
|
return out;
|
|
18
18
|
}
|
|
19
19
|
export async function writeLKG(dshHome, lkgRoot) {
|
|
20
|
+
// Dedupe: skip snapshot if current state identical to latest LKG (prevents 5-min unconditional growth)
|
|
21
|
+
try {
|
|
22
|
+
if (await isDuplicateLKG(dshHome, lkgRoot)) {
|
|
23
|
+
const entries = fs.readdirSync(lkgRoot).filter((n) => {
|
|
24
|
+
try {
|
|
25
|
+
return fs.statSync(path.join(lkgRoot, n)).isDirectory();
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
}).sort();
|
|
31
|
+
const latest = entries[entries.length - 1];
|
|
32
|
+
const manifestPath = path.join(lkgRoot, latest, 'manifest.json');
|
|
33
|
+
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
|
|
34
|
+
return { ts: latest, manifest };
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
catch { }
|
|
20
38
|
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
21
39
|
const dest = path.join(lkgRoot, ts);
|
|
22
40
|
fs.mkdirSync(dest, { recursive: true });
|
|
23
|
-
// Copy DSH home contents (if exists, copy recursively)
|
|
41
|
+
// Copy DSH home contents (if exists, copy recursively) — skip .supervisor to avoid recursion
|
|
24
42
|
if (fs.existsSync(dshHome)) {
|
|
25
|
-
// Use cpSync if available
|
|
26
43
|
for (const entry of fs.readdirSync(dshHome)) {
|
|
27
44
|
if (entry === '.supervisor')
|
|
28
45
|
continue;
|
|
@@ -39,8 +56,117 @@ export async function writeLKG(dshHome, lkgRoot) {
|
|
|
39
56
|
.map(f => ({ path: f, sha256: sha256File(path.join(dest, f)) })),
|
|
40
57
|
};
|
|
41
58
|
fs.writeFileSync(path.join(dest, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
|
59
|
+
// Retention: keep only 3 most recent, plus age (7d) and size (5GB) caps — prevents unbounded 40GB+ growth
|
|
60
|
+
await rotateLKG(lkgRoot, 3).catch(() => { });
|
|
61
|
+
await pruneByAge(lkgRoot, 7 * 24 * 60 * 60 * 1000).catch(() => { });
|
|
62
|
+
await pruneBySize(lkgRoot, 5 * 1024 * 1024 * 1024).catch(() => { });
|
|
42
63
|
return { ts, manifest };
|
|
43
64
|
}
|
|
65
|
+
export async function pruneByAge(root, maxAgeMs) {
|
|
66
|
+
if (!fs.existsSync(root))
|
|
67
|
+
return;
|
|
68
|
+
const now = Date.now();
|
|
69
|
+
const entries = fs.readdirSync(root).filter((n) => {
|
|
70
|
+
try {
|
|
71
|
+
return fs.statSync(path.join(root, n)).isDirectory();
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
for (const name of entries) {
|
|
78
|
+
try {
|
|
79
|
+
const tsStr = name.replace(/^(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d+)Z$/, '$1-$2-$3T$4:$5:$6.$7Z');
|
|
80
|
+
const ts = Date.parse(tsStr);
|
|
81
|
+
if (!isNaN(ts) && now - ts > maxAgeMs) {
|
|
82
|
+
fs.rmSync(path.join(root, name), { recursive: true, force: true });
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
catch { }
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
export async function pruneBySize(root, maxBytes) {
|
|
89
|
+
if (!fs.existsSync(root))
|
|
90
|
+
return;
|
|
91
|
+
const entries = fs.readdirSync(root).filter((n) => {
|
|
92
|
+
try {
|
|
93
|
+
return fs.statSync(path.join(root, n)).isDirectory();
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
}).sort();
|
|
99
|
+
let total = 0;
|
|
100
|
+
const sizes = [];
|
|
101
|
+
for (const name of entries) {
|
|
102
|
+
try {
|
|
103
|
+
const p = path.join(root, name);
|
|
104
|
+
let size = 0;
|
|
105
|
+
for (const f of walkFiles(p)) {
|
|
106
|
+
try {
|
|
107
|
+
size += fs.statSync(path.join(p, f)).size;
|
|
108
|
+
}
|
|
109
|
+
catch { }
|
|
110
|
+
}
|
|
111
|
+
sizes.push({ name, size });
|
|
112
|
+
total += size;
|
|
113
|
+
}
|
|
114
|
+
catch { }
|
|
115
|
+
}
|
|
116
|
+
for (const { name, size } of sizes) {
|
|
117
|
+
if (total <= maxBytes)
|
|
118
|
+
break;
|
|
119
|
+
try {
|
|
120
|
+
fs.rmSync(path.join(root, name), { recursive: true, force: true });
|
|
121
|
+
total -= size;
|
|
122
|
+
}
|
|
123
|
+
catch { }
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
export async function isDuplicateLKG(dshHome, lkgRoot) {
|
|
127
|
+
// Lightweight dedupe: if latest snapshot is <5 minutes old, skip (prevents 5-min unconditional growth)
|
|
128
|
+
// Full hash check is too heavy (would read 500MB+ each tick) and caused status timeouts
|
|
129
|
+
if (!fs.existsSync(lkgRoot))
|
|
130
|
+
return false;
|
|
131
|
+
const entries = fs.readdirSync(lkgRoot).filter((n) => {
|
|
132
|
+
try {
|
|
133
|
+
return fs.statSync(path.join(lkgRoot, n)).isDirectory();
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
return false;
|
|
137
|
+
}
|
|
138
|
+
}).sort();
|
|
139
|
+
if (!entries.length)
|
|
140
|
+
return false;
|
|
141
|
+
const latestName = entries[entries.length - 1];
|
|
142
|
+
try {
|
|
143
|
+
const tsStr = latestName.replace(/^(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d+)Z$/, '$1-$2-$3T$4:$5:$6.$7Z');
|
|
144
|
+
const ts = Date.parse(tsStr);
|
|
145
|
+
if (!isNaN(ts) && Date.now() - ts < 5 * 60 * 1000) {
|
|
146
|
+
// If latest is recent and DSH home hasn't changed in mtime, consider duplicate
|
|
147
|
+
// Quick check: compare latest snapshot's mtime vs DSH home's newest file mtime
|
|
148
|
+
const latestPath = path.join(lkgRoot, latestName);
|
|
149
|
+
const latestMtime = fs.statSync(latestPath).mtimeMs;
|
|
150
|
+
let newestFileMtime = 0;
|
|
151
|
+
if (fs.existsSync(dshHome)) {
|
|
152
|
+
for (const entry of fs.readdirSync(dshHome)) {
|
|
153
|
+
if (entry === '.supervisor')
|
|
154
|
+
continue;
|
|
155
|
+
try {
|
|
156
|
+
const s = fs.statSync(path.join(dshHome, entry));
|
|
157
|
+
if (s.mtimeMs > newestFileMtime)
|
|
158
|
+
newestFileMtime = s.mtimeMs;
|
|
159
|
+
}
|
|
160
|
+
catch { }
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
if (newestFileMtime > 0 && newestFileMtime < latestMtime)
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
catch { }
|
|
168
|
+
return false;
|
|
169
|
+
}
|
|
44
170
|
export async function verifyLKG(lkgPath) {
|
|
45
171
|
const manifestPath = path.join(lkgPath, 'manifest.json');
|
|
46
172
|
if (!fs.existsSync(manifestPath))
|
package/lib/supervisor.d.ts
CHANGED
|
@@ -13,6 +13,8 @@ export interface SupervisorDeps {
|
|
|
13
13
|
ts: string;
|
|
14
14
|
health: HealthState;
|
|
15
15
|
action: string;
|
|
16
|
+
logTail?: string;
|
|
17
|
+
gitDiff?: string;
|
|
16
18
|
}) => Promise<string>;
|
|
17
19
|
rollback: (ts?: string) => Promise<void>;
|
|
18
20
|
notify: (msg: string) => Promise<void>;
|
|
@@ -41,6 +43,7 @@ export declare class Supervisor {
|
|
|
41
43
|
constructor(deps: SupervisorDeps);
|
|
42
44
|
private getRunDebugAgent;
|
|
43
45
|
private getFindInterrupted;
|
|
46
|
+
private collectGitDiff;
|
|
44
47
|
private handleDebugResult;
|
|
45
48
|
tick(): Promise<void>;
|
|
46
49
|
start(): void;
|
package/lib/supervisor.js
CHANGED
|
@@ -16,6 +16,25 @@ export class Supervisor {
|
|
|
16
16
|
getFindInterrupted() {
|
|
17
17
|
return this.deps.findInterrupted ?? defaultFindInterrupted;
|
|
18
18
|
}
|
|
19
|
+
async collectGitDiff() {
|
|
20
|
+
try {
|
|
21
|
+
const { execSync } = await import('node:child_process');
|
|
22
|
+
const ws = process.env.MAESTRO_HARNESS_ROOT ?? '/home/kai/Work/htdocs/maestro-harness';
|
|
23
|
+
try {
|
|
24
|
+
const out = execSync(`git -C ${JSON.stringify(ws)} status --porcelain 2>/dev/null | head -n 50`, { encoding: 'utf-8', timeout: 2000 });
|
|
25
|
+
if (out.trim()) {
|
|
26
|
+
const diff = execSync(`git -C ${JSON.stringify(ws)} diff 2>/dev/null | head -n 200`, { encoding: 'utf-8', timeout: 2000 });
|
|
27
|
+
return diff || out;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
catch { }
|
|
31
|
+
const diff = execSync('git diff 2>/dev/null | head -n 200', { encoding: 'utf-8', timeout: 2000 });
|
|
32
|
+
return diff.trim() ? diff : '';
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return '';
|
|
36
|
+
}
|
|
37
|
+
}
|
|
19
38
|
handleDebugResult(reportPath, res) {
|
|
20
39
|
if (res.fixed) {
|
|
21
40
|
void this.deps.notify(`FIXED: debug-agent fixed ${reportPath} — ${res.reason}`).catch(() => { });
|
|
@@ -46,7 +65,9 @@ export class Supervisor {
|
|
|
46
65
|
this.lastDegradedNotify = now;
|
|
47
66
|
try {
|
|
48
67
|
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
49
|
-
const
|
|
68
|
+
const logTail = health.logTail ?? '';
|
|
69
|
+
const gitDiff = await this.collectGitDiff().catch(() => '');
|
|
70
|
+
const reportPath = await this.deps.writeReport({ ts, health, action: `degraded — ${health.error ?? 'plugin'}`, logTail, gitDiff }).catch(() => '');
|
|
50
71
|
await this.deps.notify(`DEGRADED: ${health.error ?? 'plugin'} (report: ${reportPath})`).catch(() => { });
|
|
51
72
|
// Phase 3: debug + resume — use injected fn if provided (even in VITEST), otherwise fire-and-forget real impl (skip in VITEST)
|
|
52
73
|
const runner = this.getRunDebugAgent();
|
|
@@ -100,7 +121,9 @@ export class Supervisor {
|
|
|
100
121
|
try {
|
|
101
122
|
const failed = await this.deps.writeFailed().catch(() => ({ ts: new Date().toISOString().replace(/[:.]/g, '-'), manifest: null }));
|
|
102
123
|
const ts = failed?.ts ?? new Date().toISOString().replace(/[:.]/g, '-');
|
|
103
|
-
const
|
|
124
|
+
const logTail = health.logTail ?? '';
|
|
125
|
+
const gitDiff = await this.collectGitDiff().catch(() => '');
|
|
126
|
+
const reportPath = await this.deps.writeReport({ ts, health, action: `rollback — ${health.error ?? 'down'}`, logTail, gitDiff }).catch(() => '');
|
|
104
127
|
await this.deps.rollback();
|
|
105
128
|
await this.deps.notify(`CRASH detected → rollback (report: ${reportPath}, error: ${health.error ?? 'down'})`).catch(() => { });
|
|
106
129
|
const runner = this.getRunDebugAgent();
|