@ddtcorex/dsh-maestro-supervisor 0.6.2 → 0.6.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/cli.js CHANGED
@@ -5,6 +5,7 @@ import * as fs from 'node:fs';
5
5
  import * as path from 'node:path';
6
6
  import * as os from 'node:os';
7
7
  import { resolveHarnessRoot, resolveDeepseekHarnessDir } from './paths.js';
8
+ import { buildKillStalePortsCommand, isSelfCopyError, checkPlannedRestart, writePlannedRestart } from './restart-guards.js';
8
9
  export async function runCli(args) {
9
10
  const cmd = args[2] ?? '--help';
10
11
  if (cmd === '--help' || cmd === '-h' || cmd === 'help') {
@@ -106,26 +107,95 @@ Commands:
106
107
  return writeReport({ reportsRoot, ts, health, gitDiff: diff, logTail: tail, action });
107
108
  },
108
109
  rollback: async () => {
109
- // Find latest LKG and restore
110
+ const { execSync } = await import('node:child_process');
110
111
  const entries = fs.existsSync(lkgRoot) ? fs.readdirSync(lkgRoot).sort() : [];
111
112
  if (!entries.length)
112
113
  throw new Error('no LKG to rollback to');
113
- const latest = entries[entries.length - 1];
114
- const src = path.join(lkgRoot, latest);
115
- // naive restore: copy files back
114
+ // Try newest to oldest (up to 3) to find a clean LKG for plugin failures
115
+ // Extract failing plugin from current log tail if possible
116
+ let failingPlugin;
117
+ try {
118
+ const tail = fs.readFileSync(path.join(os.homedir(), '.dsh/dsh-web.log'), 'utf8').slice(-5000);
119
+ const m = tail.match(/@ddtcorex\/dsh-maestro-[a-z0-9_-]+/i) ?? tail.match(/dsh-maestro-[a-z0-9_-]+/i);
120
+ if (m)
121
+ failingPlugin = m[0].replace(/^@ddtcorex\//, '');
122
+ }
123
+ catch { }
124
+ const candidates = [...entries].reverse().slice(0, 3);
125
+ let chosen;
126
+ for (const cand of candidates) {
127
+ if (!failingPlugin) {
128
+ chosen = cand;
129
+ break;
130
+ }
131
+ try {
132
+ const pkgPath = path.join(lkgRoot, cand, 'profiles/web/package.json');
133
+ if (!fs.existsSync(pkgPath)) {
134
+ chosen = cand;
135
+ break;
136
+ }
137
+ const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
138
+ const bundles = pkg?.dsh?.profile?.bundles ?? [];
139
+ const deps = pkg?.dependencies ?? {};
140
+ const hasFailing = bundles.some((b) => b.includes(failingPlugin)) || Object.keys(deps).some(k => k.includes(failingPlugin));
141
+ if (!hasFailing) {
142
+ chosen = cand;
143
+ break;
144
+ }
145
+ console.log(`[supervisor] skipping LKG ${cand} still contains failing plugin ${failingPlugin}`);
146
+ }
147
+ catch {
148
+ chosen = cand;
149
+ break;
150
+ }
151
+ }
152
+ const target = chosen ?? entries[entries.length - 1];
153
+ const src = path.join(lkgRoot, target);
116
154
  for (const entry of fs.readdirSync(src)) {
117
155
  if (entry === 'manifest.json')
118
156
  continue;
119
- fs.cpSync(path.join(src, entry), path.join(dshHome, entry), { recursive: true, force: true });
157
+ const srcPath = path.join(src, entry);
158
+ const destPath = path.join(dshHome, entry);
159
+ try {
160
+ // Skip if src and dest are the same file (e.g. symlink to same target like ~/.dsh/AGENTS.md)
161
+ try {
162
+ if (fs.existsSync(srcPath) && fs.existsSync(destPath) && fs.realpathSync(srcPath) === fs.realpathSync(destPath))
163
+ continue;
164
+ }
165
+ catch { }
166
+ fs.cpSync(srcPath, destPath, { recursive: true, force: true });
167
+ }
168
+ catch (e) {
169
+ if (isSelfCopyError(String(e?.message ?? '')))
170
+ continue;
171
+ throw e;
172
+ }
173
+ }
174
+ console.log(`[supervisor] rolled back to ${target}${failingPlugin ? ` (avoiding ${failingPlugin})` : ''}`);
175
+ // Reconcile node_modules from restored package.json (critical for link: deps)
176
+ try {
177
+ execSync('pnpm --dir ~/.dsh/profiles/web install --silent', { timeout: 30000, stdio: 'pipe' });
178
+ console.log('[supervisor] pnpm install reconciled profiles/web');
179
+ }
180
+ catch (e) {
181
+ console.log(`[supervisor] pnpm install failed: ${e?.message ?? String(e)}`);
120
182
  }
121
- console.log(`[supervisor] rolled back to ${latest}`);
122
183
  },
123
184
  restartWeb: async () => {
124
185
  const { execSync } = await import('node:child_process');
125
- // Kill stale MainThread holding 3080/3000 before any restart attempt
186
+ // Single-owner: mark planned restart 30s before any systemctl/nohup
187
+ // so pollHealth + tick suppress the transient down (no double restart).
188
+ try {
189
+ writePlannedRestart(30000);
190
+ }
191
+ catch { }
192
+ // Kill stale MainThread holding 3080 before any restart attempt
126
193
  // (EADDRINUSE crash leaves old pid alive with http 200; new start would fail)
194
+ // Scoped to :3080 only — an unfiltered `ss -tlnp` matches every
195
+ // listening process on the host, not just dsh web (regression: killed
196
+ // unrelated services like redis/horizon on every restart).
127
197
  try {
128
- execSync(`pids=$(ss -tlnp 2>/dev/null | sed -n 's/.*pid=\\([0-9]*\\).*/\\1/p' | sort -u); if [ -n "$pids" ]; then echo "[supervisor] killing stale pids $pids"; kill $pids 2>/dev/null || true; sleep 2; fi`, { timeout: 5000, stdio: 'pipe' });
198
+ execSync(buildKillStalePortsCommand(), { timeout: 5000, stdio: 'pipe' });
129
199
  }
130
200
  catch { }
131
201
  // Prefer systemd — if dsh-web.service is installed, restart/start it
@@ -146,6 +216,10 @@ Commands:
146
216
  try {
147
217
  const harnessRoot = resolveDeepseekHarnessDir();
148
218
  const logPath = path.join(os.homedir(), '.dsh/dsh-web.log');
219
+ try {
220
+ writePlannedRestart(30000);
221
+ }
222
+ catch { }
149
223
  execSync(`setsid nohup bash -c 'export NVM_DIR="$HOME/.nvm"; [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; cd ${JSON.stringify(harnessRoot)} && exec node --import tsx/esm apps/cli/src/bin.ts web --no-open >> ${JSON.stringify(logPath)} 2>&1' &`, { timeout: 5000 });
150
224
  console.log('[supervisor] started dsh-web via nohup fallback (direct node, portable)');
151
225
  }
@@ -154,9 +228,9 @@ Commands:
154
228
  }
155
229
  },
156
230
  notify: async (msg) => console.log(`[notify] ${msg}`),
157
- intervalMs: 3000,
231
+ isPlannedRestartActive: () => checkPlannedRestart(),
158
232
  });
159
- supervisor.start();
233
+ await supervisor.start();
160
234
  // keep process alive
161
235
  await new Promise(() => { });
162
236
  }
@@ -0,0 +1 @@
1
+ export declare function readSupervisorConfig(): Promise<Record<string, any>>;
package/lib/config.js ADDED
@@ -0,0 +1,18 @@
1
+ import { load, readFlat } from '@ddtcorex/dsh-maestro-config-lib';
2
+ export async function readSupervisorConfig() {
3
+ try {
4
+ const doc = await load();
5
+ if (doc?.domains?.supervisor && typeof doc.domains.supervisor === 'object' && !Array.isArray(doc.domains.supervisor)) {
6
+ return doc.domains.supervisor;
7
+ }
8
+ }
9
+ catch { }
10
+ try {
11
+ const f = await readFlat().catch(() => ({}));
12
+ if (f?.supervisor && typeof f.supervisor === 'object' && !Array.isArray(f.supervisor)) {
13
+ return f.supervisor;
14
+ }
15
+ }
16
+ catch { }
17
+ return {};
18
+ }
@@ -316,9 +316,9 @@ async function resolveLLMConfig() {
316
316
  process.env.OMNI_ROUTE_API_URL ??
317
317
  process.env.OPENCODE_API_URL ??
318
318
  null;
319
- // Model: AI_MODEL / DEEPSEEK_MODEL / settings.json domains.review.model / default
319
+ // Model: AI_MODEL / DEEPSEEK_MODEL / settings.json domains.supervisor.model -> domains.review.model / default
320
+ // Supervisor has its own picker; falls back to review model for backward compat, then DSH default.
320
321
  let model = process.env.AI_MODEL ?? process.env.DEEPSEEK_MODEL ?? process.env.OPENAI_MODEL ?? null;
321
- // Try settings.json first (review model is the user's current default model)
322
322
  if (!model) {
323
323
  try {
324
324
  const { readFileSync } = await import('node:fs');
@@ -326,11 +326,20 @@ async function resolveLLMConfig() {
326
326
  const settingsPath = `${homedir()}/.dsh/maestro/settings.json`;
327
327
  const raw = readFileSync(settingsPath, 'utf-8');
328
328
  const j = JSON.parse(raw);
329
- const m = j?.domains?.review?.model?.model ?? j?.domains?.review?.model;
330
- if (typeof m === 'string')
329
+ // Prefer supervisor model, fall back to review model (so old installs keep working)
330
+ const sup = j?.domains?.supervisor?.model?.model ?? j?.domains?.supervisor?.model;
331
+ const rev = j?.domains?.review?.model?.model ?? j?.domains?.review?.model;
332
+ let m = null;
333
+ if (typeof sup === 'string')
334
+ m = sup;
335
+ else if (sup?.model && typeof sup.model === 'string')
336
+ m = sup.model;
337
+ else if (typeof rev === 'string')
338
+ m = rev;
339
+ else if (rev?.model && typeof rev.model === 'string')
340
+ m = rev.model;
341
+ if (typeof m === 'string' && m.trim() !== '')
331
342
  model = m;
332
- else if (m?.model && typeof m.model === 'string')
333
- model = m.model;
334
343
  }
335
344
  catch { }
336
345
  }
@@ -5,6 +5,8 @@ export interface HealthState {
5
5
  degraded?: boolean;
6
6
  logTail?: string;
7
7
  }
8
+ export declare function getActiveEnterMs(): number | undefined;
9
+ export declare function getActiveEnterWallMs(): number | undefined;
8
10
  export interface PollHealthOpts {
9
11
  fetch?: () => Promise<{
10
12
  status: number;
@@ -14,6 +16,8 @@ export interface PollHealthOpts {
14
16
  logTail?: () => Promise<string>;
15
17
  url?: string;
16
18
  timeoutMs?: number;
19
+ /** injectable for tests — overrides systemctl lookup */
20
+ getActiveEnterMs?: () => number | undefined;
17
21
  }
18
22
  export declare function pollHealth(opts?: PollHealthOpts): Promise<HealthState>;
19
23
  export declare function collectLogTail(): Promise<string>;
@@ -1,3 +1,55 @@
1
+ import { checkPlannedRestart } from './restart-guards.js';
2
+ import { execSync as execSyncImpl } from 'node:child_process';
3
+ export function getActiveEnterMs() {
4
+ if (process.env.VITEST)
5
+ return undefined;
6
+ try {
7
+ const out = execSyncImpl('systemctl --user show -p ActiveEnterTimestampMonotonic dsh-web.service 2>/dev/null', { encoding: 'utf8' });
8
+ const m = out.match(/ActiveEnterTimestampMonotonic=(\d+)/);
9
+ if (m)
10
+ return Number(m[1]);
11
+ }
12
+ catch { }
13
+ return undefined;
14
+ }
15
+ export function getActiveEnterWallMs() {
16
+ if (process.env.VITEST)
17
+ return undefined;
18
+ try {
19
+ const out = execSyncImpl('systemctl --user show -p ActiveEnterTimestamp dsh-web.service 2>/dev/null', { encoding: 'utf8' });
20
+ const m = out.match(/ActiveEnterTimestamp=(.+)/);
21
+ if (m) {
22
+ const s = m[1].trim();
23
+ if (!s || s === 'n/a')
24
+ return undefined;
25
+ const ms = Date.parse(s);
26
+ if (!Number.isNaN(ms))
27
+ return ms;
28
+ }
29
+ }
30
+ catch { }
31
+ return undefined;
32
+ }
33
+ function isRecentlyStarted(opts, wallMs) {
34
+ if (process.env.VITEST)
35
+ return false;
36
+ const now = Date.now();
37
+ // 30s grace after ActiveEnterTimestamp (wall clock) — covers manual systemctl start without marker
38
+ const wall = wallMs ?? (() => {
39
+ try {
40
+ const fn = opts.getActiveEnterWallMs;
41
+ if (typeof fn === 'function')
42
+ return fn();
43
+ return getActiveEnterWallMs();
44
+ }
45
+ catch {
46
+ return undefined;
47
+ }
48
+ })();
49
+ if (typeof wall === 'number' && now - wall < 30000)
50
+ return true;
51
+ return false;
52
+ }
1
53
  const ERROR_PATTERNS = [
2
54
  'ERR_MODULE_NOT_FOUND',
3
55
  'ERR_PNPM',
@@ -16,7 +68,12 @@ const ERROR_PATTERNS = [
16
68
  'address already in use',
17
69
  ];
18
70
  export async function pollHealth(opts = {}) {
19
- const fetchFn = opts.fetch ?? defaultFetch(opts.url ?? 'http://127.0.0.1:3080/', opts.timeoutMs ?? 5000);
71
+ // 5s was too tight for a busy plugin-tree boot: a lone AbortError from a slow
72
+ // (but otherwise fine) response was indistinguishable from a real crash, and
73
+ // combined with a low down-threshold this caused a self-sustaining restart
74
+ // loop (see Supervisor.downThreshold). 12s gives boot room without masking
75
+ // a genuinely dead process for long.
76
+ const fetchFn = opts.fetch ?? defaultFetch(opts.url ?? 'http://127.0.0.1:3080/', opts.timeoutMs ?? 12000);
20
77
  const psAliveFn = opts.psAlive ?? defaultPsAlive;
21
78
  const logTailFn = opts.logTail ?? defaultLogTail;
22
79
  let httpCode;
@@ -24,7 +81,8 @@ export async function pollHealth(opts = {}) {
24
81
  try {
25
82
  const res = await fetchFn();
26
83
  httpCode = res.status;
27
- if (res.status !== 200) {
84
+ // 401 is healthy: dsh web is up but requires browser token (since 0.1.2). Only 5xx / network errors are down.
85
+ if (res.status !== 200 && res.status !== 401) {
28
86
  fetchError = `http ${res.status}`;
29
87
  }
30
88
  }
@@ -38,20 +96,72 @@ export async function pollHealth(opts = {}) {
38
96
  catch {
39
97
  // ignore log read errors
40
98
  }
99
+ // Suppression gate: during 30s planned-restart window, boot transients are expected.
100
+ // Fetch failures are treated as suppressed (up:true), and log scanning is windowed.
101
+ const suppressed = checkPlannedRestart();
102
+ const recentlyStarted = isRecentlyStarted(opts);
103
+ const graceActive = suppressed || recentlyStarted;
104
+ // ActiveEnterTimestampMonotonic lookup — primary filter source; fallback to last success window
105
+ let activeEnterMs;
106
+ try {
107
+ const fn = opts.getActiveEnterMs ?? getActiveEnterMs;
108
+ activeEnterMs = fn();
109
+ void activeEnterMs;
110
+ }
111
+ catch {
112
+ activeEnterMs = undefined;
113
+ }
41
114
  let logError;
42
- // Find most recent error line (not first) and ignore stale errors that are
43
- // followed by a successful boot (log tail is append-only, old EADDRINUSE stays forever).
44
- // We check last occurrence and ensure no "dsh web: http" success after it.
115
+ // Filter logTail to only consider lines after ActiveEnterTimestamp.
116
+ // Fallback: last 200 lines after last "dsh web: http" success marker (log is append-only).
45
117
  const lines = logContent.split('\n');
46
118
  const lowerLines = lines.map(l => l.toLowerCase());
119
+ // find last success marker
120
+ let lastSuccessIdx = -1;
121
+ for (let i = lowerLines.length - 1; i >= 0; i--) {
122
+ if (lowerLines[i].includes('dsh web: http')) {
123
+ lastSuccessIdx = i;
124
+ break;
125
+ }
126
+ }
127
+ let scanLines;
128
+ let scanLower;
129
+ if (lastSuccessIdx !== -1) {
130
+ const after = lines.slice(lastSuccessIdx + 1);
131
+ const afterLower = lowerLines.slice(lastSuccessIdx + 1);
132
+ // keep only last 200 lines after success (fallback window)
133
+ if (after.length > 200) {
134
+ scanLines = after.slice(-200);
135
+ scanLower = afterLower.slice(-200);
136
+ }
137
+ else {
138
+ scanLines = after;
139
+ scanLower = afterLower;
140
+ }
141
+ // If ActiveEnter is available, the same window applies — old EADDRINUSE before restart
142
+ // is before the success marker and thus excluded. No extra timestamp->line mapping needed.
143
+ }
144
+ else {
145
+ // no success marker: consider last 200 lines total (both primary and fallback)
146
+ if (lines.length > 200) {
147
+ scanLines = lines.slice(-200);
148
+ scanLower = lowerLines.slice(-200);
149
+ }
150
+ else {
151
+ scanLines = lines;
152
+ scanLower = lowerLines;
153
+ }
154
+ }
155
+ // If ActiveEnter lookup succeeded/failed, we already applied the fallback window.
156
+ // When system has no success marker and no ActiveEnter, scanLines is still last 200.
47
157
  let lastErrorIdx = -1;
48
158
  let matchedLine = '';
49
- for (let i = lines.length - 1; i >= 0; i--) {
50
- const lower = lowerLines[i];
159
+ for (let i = scanLines.length - 1; i >= 0; i--) {
160
+ const lower = scanLower[i];
51
161
  for (const pat of ERROR_PATTERNS) {
52
162
  if (lower.includes(pat.toLowerCase())) {
53
163
  lastErrorIdx = i;
54
- matchedLine = lines[i].trim().slice(0, 500);
164
+ matchedLine = scanLines[i].trim().slice(0, 500);
55
165
  break;
56
166
  }
57
167
  }
@@ -59,10 +169,11 @@ export async function pollHealth(opts = {}) {
59
169
  break;
60
170
  }
61
171
  if (lastErrorIdx !== -1) {
62
- // If a successful boot line appears after the last error, error is stale (already recovered)
172
+ // Within the windowed view, if a success appears after the error it would have been before the slice,
173
+ // but check anyway for safety (error before success within window)
63
174
  let hasSuccessAfter = false;
64
- for (let i = lastErrorIdx + 1; i < lines.length; i++) {
65
- if (lowerLines[i].includes('dsh web: http')) {
175
+ for (let i = lastErrorIdx + 1; i < scanLines.length; i++) {
176
+ if (scanLower[i].includes('dsh web: http')) {
66
177
  hasSuccessAfter = true;
67
178
  break;
68
179
  }
@@ -70,8 +181,23 @@ export async function pollHealth(opts = {}) {
70
181
  if (!hasSuccessAfter)
71
182
  logError = matchedLine;
72
183
  }
184
+ else if (suppressed) {
185
+ // suppressed window with no windowed error — ensure stale errors before success are ignored
186
+ // (already handled by windowing)
187
+ }
188
+ // Legacy full-scan fallback for hasSuccessAfter across original lines when no windowed error
189
+ // but original had error before success — already suppressed by windowing, no need to re-check.
73
190
  // Distinguish FULL (http !=200) vs DEGRADED (http 200 but log has plugin error)
191
+ // Suppression: during 30s grace (planned-restart OR recently started), transient fetch failures are not a crash
74
192
  if (fetchError) {
193
+ if (graceActive) {
194
+ // treat fetch failed / http 404 as up:true suppressed (don't write error) — also doubles effective downThreshold
195
+ return {
196
+ up: true,
197
+ httpCode,
198
+ logTail: logContent.slice(-5000),
199
+ };
200
+ }
75
201
  return {
76
202
  up: false,
77
203
  httpCode,
@@ -80,8 +206,18 @@ export async function pollHealth(opts = {}) {
80
206
  logTail: logContent.slice(-5000),
81
207
  };
82
208
  }
209
+ // During grace, a windowed logError that is still present is considered stale/boot transient
210
+ // as well — treat as up to avoid double restart. The fallback window already filters pre-restart errors.
211
+ if (graceActive && logError) {
212
+ // Effective downThreshold doubling is handled by supervisor, but health also suppresses log tail
213
+ return {
214
+ up: true,
215
+ httpCode,
216
+ logTail: logContent.slice(-5000),
217
+ };
218
+ }
83
219
  if (logError) {
84
- // EADDRINUSE is fatal even with http 200 — old process still holds 3080/3000
220
+ // EADDRINUSE is fatal even with http 200 — old process still holds 3080
85
221
  // and new start failed; treat as FULL down so supervisor kills + restarts.
86
222
  const lowerErr = logError.toLowerCase();
87
223
  const isFatalPortError = lowerErr.includes('eaddrinuse') || lowerErr.includes('address already in use');
@@ -122,7 +258,7 @@ export async function pollHealth(opts = {}) {
122
258
  catch {
123
259
  // ignore
124
260
  }
125
- return { up: httpCode === 200, httpCode, logTail: logContent.slice(-5000) };
261
+ return { up: httpCode === 200 || httpCode === 401, httpCode, logTail: logContent.slice(-5000) };
126
262
  }
127
263
  function defaultFetch(url, timeoutMs) {
128
264
  return async () => {
package/lib/notifier.d.ts CHANGED
@@ -1,6 +1,11 @@
1
1
  export interface NotifierOpts {
2
2
  send?: (msg: string) => Promise<void>;
3
3
  }
4
+ export declare function notifyAutoRestart(reason: string, opts?: NotifierOpts & {
5
+ httpCode?: string | number;
6
+ lkgId?: string;
7
+ reportPath?: string;
8
+ }): Promise<void>;
4
9
  export declare function notify(msg: string, opts?: NotifierOpts): Promise<void>;
5
10
  export declare function notifyCrash(reportPath: string, error: string, opts?: NotifierOpts): Promise<void>;
6
11
  export declare function notifyDegraded(id: string, error: string, opts?: NotifierOpts): Promise<void>;
package/lib/notifier.js CHANGED
@@ -1,3 +1,13 @@
1
+ import { checkPlannedRestart } from './restart-guards.js';
2
+ export async function notifyAutoRestart(reason, opts = {}) {
3
+ if (checkPlannedRestart())
4
+ return;
5
+ const httpCode = opts.httpCode ?? 'n/a';
6
+ const lkgId = opts.lkgId ?? opts.lkg ?? 'n/a';
7
+ const reportPath = opts.reportPath ?? opts.report ?? 'n/a';
8
+ const msg = `🔄 dsh web auto-restart — ${reason} — ${new Date().toISOString()} — up:${httpCode} — LKG:${lkgId} — report:${reportPath}`;
9
+ await notify(msg, opts);
10
+ }
1
11
  export async function notify(msg, opts = {}) {
2
12
  const send = opts.send ?? defaultSend;
3
13
  try {
package/lib/plugin.js CHANGED
@@ -202,15 +202,56 @@ export async function resumeInterrupted(ctx, ids) {
202
202
  ctx.logger?.warn?.(`[supervisor] auto-resume: no live agent available for ${id}`);
203
203
  continue;
204
204
  }
205
+ // Ensure bash tool is registered before followup — initial resume header with
206
+ // only 11 tools (missing bash) broke 36646045... etc. Wait briefly for
207
+ // ctx.tools to populate (preset mount + shell). Best-effort: poll up to 5s.
208
+ try {
209
+ const tools = ctx.get?.('tools') ?? ctx.tools;
210
+ const hasBash = () => {
211
+ try {
212
+ if (typeof tools?.get === 'function')
213
+ return tools.get('bash') !== undefined;
214
+ if (typeof tools?.has === 'function')
215
+ return tools.has('bash');
216
+ if (Array.isArray(tools?.list?.()))
217
+ return tools.list().some((t) => t?.name === 'bash' || t === 'bash');
218
+ // Fallback: check systemPrompt assembly indirectly via tools registry size
219
+ return true;
220
+ }
221
+ catch {
222
+ return true;
223
+ }
224
+ };
225
+ if (!hasBash()) {
226
+ for (let i = 0; i < 10; i++) {
227
+ await new Promise(r => setTimeout(r, 500));
228
+ if (hasBash())
229
+ break;
230
+ }
231
+ if (!hasBash())
232
+ ctx.logger?.warn?.(`[supervisor] auto-resume: bash tool still not ready for ${id} — continuing anyway`);
233
+ }
234
+ }
235
+ catch { }
205
236
  const { createUserMessage } = await import('@deepseek-ai/dsh-llm').catch(() => ({
206
237
  createUserMessage: (input) => ({ ...input, role: 'user', id: crypto.randomUUID() }),
207
238
  }));
239
+ // Use an explicit recovery prompt instead of bare "continue" — the synthetic
240
+ // TOOL_OUTCOME_UNKNOWN / TOOL_NOT_STARTED result (repair.ts:104) tells the
241
+ // model to verify external state before retrying. A bare "continue" made
242
+ // the model reply with text instead of re-issuing bash, leaving the
243
+ // session stuck after every crash (36646045..., 31ae53a2...).
244
+ const resumeMessage = 'The previous turn was interrupted by a crash and the harness has synthesized a tool result with TOOL_OUTCOME_UNKNOWN / TOOL_NOT_STARTED. ' +
245
+ 'Outcome of the last tool call is unknown — it may or may not have had side effects. ' +
246
+ 'Verify external state with bash (e.g., ls, cat, git status) before retrying. ' +
247
+ 'Retry only if the operation is read-only or idempotent; if it may have side effects, verify first or ask the user. ' +
248
+ 'Then continue the original task from where it was interrupted — re-issue the next bash/tool call that the plan requires.';
208
249
  agent.followup(createUserMessage({
209
- content: [{ type: 'text', text: 'continue' }],
250
+ content: [{ type: 'text', text: resumeMessage }],
210
251
  source: { kind: 'user' },
211
252
  }));
212
253
  resumed.push(id);
213
- ctx.logger?.info?.(`[supervisor] auto-resume: sent continue trigger for ${id}`);
254
+ ctx.logger?.info?.(`[supervisor] auto-resume: sent recovery continue for ${id}`);
214
255
  }
215
256
  catch (e) {
216
257
  ctx.logger?.warn?.(`[supervisor] auto-resume failed ${id}: ${e?.message ?? String(e)}`);
@@ -239,12 +280,91 @@ export function createResumeRpcHandler(ctx, opts = {}) {
239
280
  return { ok: true, value: { resumed: await resume(ctx, ids) } };
240
281
  };
241
282
  }
283
+ function ensureSystemdKeepalive(ctx) {
284
+ // Best-effort: ensure dsh-web-keepalive.service exists and is enabled, and linger is on.
285
+ // This is the user-level auto-fix for the 11:42:58 crash where manager session 97
286
+ // Removed caused user manager to exit despite Linger, killing dsh-web.
287
+ // Runs inside dsh web (as the current user) so systemctl --user bus is available when manager is alive.
288
+ // Failures are swallowed — never throw from apply().
289
+ try {
290
+ const home = os.homedir();
291
+ const systemdDir = path.join(home, '.config/systemd/user');
292
+ const keepalivePath = path.join(systemdDir, 'dsh-web-keepalive.service');
293
+ // Migrate old keepalive.service (pre-prefix) if present
294
+ const oldKeepalivePath = path.join(systemdDir, 'keepalive.service');
295
+ if (fs.existsSync(oldKeepalivePath) && !fs.existsSync(keepalivePath)) {
296
+ try {
297
+ fs.renameSync(oldKeepalivePath, keepalivePath);
298
+ ctx.logger?.info?.('[supervisor] migrated keepalive.service → dsh-web-keepalive.service');
299
+ }
300
+ catch {
301
+ try {
302
+ fs.copyFileSync(oldKeepalivePath, keepalivePath);
303
+ }
304
+ catch { }
305
+ }
306
+ }
307
+ if (!fs.existsSync(keepalivePath)) {
308
+ try {
309
+ fs.mkdirSync(systemdDir, { recursive: true });
310
+ // Inline template — avoids needing to resolve package's systemd/dsh-web-keepalive.service.template at runtime
311
+ const content = `[Unit]\nDescription=Keepalive for user manager linger — prevents systemd --user exit on manager session close\nAfter=default.target\n\n[Service]\nType=simple\nExecStart=/bin/sleep infinity\nRestart=always\nRestartSec=3\n\n[Install]\nWantedBy=default.target\n`;
312
+ fs.writeFileSync(keepalivePath, content, 'utf-8');
313
+ ctx.logger?.info?.('[supervisor] dsh-web-keepalive.service installed');
314
+ }
315
+ catch (e) {
316
+ ctx.logger?.warn?.(`[supervisor] keepalive install failed: ${e?.message ?? String(e)}`);
317
+ return;
318
+ }
319
+ }
320
+ // Try to enable linger + enable/start units best-effort (may need sudo for linger, ignore failure)
321
+ try {
322
+ const { execSync } = require('node:child_process');
323
+ const username = (() => { try {
324
+ return os.userInfo().username;
325
+ }
326
+ catch {
327
+ return process.env.USER || process.env.LOGNAME || '';
328
+ } })();
329
+ const lingerPath = username ? `/var/lib/systemd/linger/${username}` : '';
330
+ if (lingerPath && !fs.existsSync(lingerPath)) {
331
+ try {
332
+ execSync(`loginctl enable-linger ${username} 2>/dev/null || true`, { timeout: 3000, stdio: 'ignore' });
333
+ }
334
+ catch { }
335
+ }
336
+ try {
337
+ execSync('systemctl --user daemon-reload 2>/dev/null || true', { timeout: 3000, stdio: 'ignore' });
338
+ }
339
+ catch { }
340
+ try {
341
+ execSync('systemctl --user enable dsh-web-keepalive.service 2>/dev/null || true', { timeout: 3000, stdio: 'ignore' });
342
+ }
343
+ catch { }
344
+ try {
345
+ execSync('systemctl --user start dsh-web-keepalive.service 2>/dev/null || true', { timeout: 3000, stdio: 'ignore' });
346
+ }
347
+ catch { }
348
+ // Also ensure dsh-web + supervisor are enabled (in case user installed plugin but never ran install-systemd.sh)
349
+ try {
350
+ execSync('systemctl --user enable dsh-web.service dsh-web-supervisor.service 2>/dev/null || true', { timeout: 3000, stdio: 'ignore' });
351
+ }
352
+ catch { }
353
+ }
354
+ catch { }
355
+ }
356
+ catch { }
357
+ }
242
358
  export function apply(ctx, config = {}) {
243
359
  // The whole body is wrapped: per AGENTS.md, apply() must never throw
244
360
  // synchronously or let a rejected promise escape, no matter what fails
245
361
  // (ctx.effect missing/throwing, RPC registration throwing, or even the
246
362
  // error-reporting logger call itself throwing).
247
363
  try {
364
+ try {
365
+ ensureSystemdKeepalive(ctx);
366
+ }
367
+ catch { }
248
368
  ctx.effect(() => {
249
369
  let disposed = false;
250
370
  let timer = null;
@@ -0,0 +1,8 @@
1
+ export declare function buildKillStalePortsCommand(ports?: number[]): string;
2
+ export declare function isSelfCopyError(message: string): boolean;
3
+ export declare const PLANNED_RESTART_TTL_MS = 180000;
4
+ export declare function isPlannedRestartFresh(mtimeMs: number, nowMs: number, ttlMs?: number): boolean;
5
+ export declare function plannedRestartPath(): string;
6
+ export declare function writePlannedRestart(ttlMs?: number): void;
7
+ export declare function checkPlannedRestart(markerPath?: string): boolean;
8
+ export declare function clearPlannedRestart(): void;
@@ -0,0 +1,94 @@
1
+ import * as fs from 'node:fs';
2
+ import * as path from 'node:path';
3
+ import * as os from 'node:os';
4
+ // Regression guard: PR #17 added a "kill stale pid holding :3080/:3000 before
5
+ // restart" step (EADDRINUSE recovery), but the original `ss -tlnp` call had no
6
+ // port filter — it matched every listening process on the host, so every
7
+ // restart also killed unrelated services (redis, php-fpm, horizon, ssh, ...).
8
+ // The dsh-web MainThread holds both 3080 and 3000 (see AGENTS.md Known Issues),
9
+ // so filtering `ss` itself to those ports is both correct and sufficient.
10
+ export function buildKillStalePortsCommand(ports = [3080]) {
11
+ const filter = ports.map(p => `sport = :${p}`).join(' or ');
12
+ return `pids=$(ss -tlnp '( ${filter} )' 2>/dev/null | sed -n 's/.*pid=\\([0-9]*\\).*/\\1/p' | sort -u); if [ -n "$pids" ]; then echo "[supervisor] killing stale pids $pids"; kill $pids 2>/dev/null || true; sleep 2; fi`;
13
+ }
14
+ // LKG rollback copies each entry with fs.cpSync — an entry that resolves back
15
+ // into itself (e.g. a symlink cycle reachable from ~/.dsh, observed in
16
+ // production pointing into ~/.npm/_npx/.../node_modules/unist-util-position)
17
+ // always throws here. Node's wording for this varies by version ("cannot be
18
+ // the same", "subdirectory of itself"/"of self") — match all known phrasings
19
+ // so the entry is skipped instead of aborting the whole rollback.
20
+ export function isSelfCopyError(message) {
21
+ const lower = message.toLowerCase();
22
+ return lower.includes('cannot be the same') || lower.includes('subdirectory of');
23
+ }
24
+ // Coordination contract with dsh-safe-web-update's restart-dsh-web.sh (see
25
+ // <workspace-root>/docs/specs/2026-08-28-supervisor-planned-restart-design.md):
26
+ // that script writes this marker right before it intentionally takes dsh-web
27
+ // down, so the supervisor's own health poll does not mistake a deliberate
28
+ // restart (kill -> dry-boot -> relaunch, up to ~130s) for a crash and race it
29
+ // with its own rollback + restartWeb(). Presence + freshness is authoritative;
30
+ // the marker's content is never parsed.
31
+ export const PLANNED_RESTART_TTL_MS = 180_000;
32
+ export function isPlannedRestartFresh(mtimeMs, nowMs, ttlMs = PLANNED_RESTART_TTL_MS) {
33
+ return nowMs - mtimeMs < ttlMs;
34
+ }
35
+ export function plannedRestartPath() {
36
+ return path.join(os.homedir(), '.dsh/.supervisor/planned-restart.json');
37
+ }
38
+ export function writePlannedRestart(ttlMs = 30000) {
39
+ const p = plannedRestartPath();
40
+ fs.mkdirSync(path.dirname(p), { recursive: true });
41
+ fs.writeFileSync(p, JSON.stringify({ ts: Date.now(), ttl: ttlMs }), { mode: 0o600 });
42
+ try {
43
+ fs.chmodSync(p, 0o600);
44
+ }
45
+ catch { }
46
+ }
47
+ export function checkPlannedRestart(markerPath) {
48
+ // Legacy path explicit: check mtime of that file
49
+ if (markerPath) {
50
+ try {
51
+ const stat = fs.statSync(markerPath);
52
+ return isPlannedRestartFresh(stat.mtimeMs, Date.now());
53
+ }
54
+ catch {
55
+ return false;
56
+ }
57
+ }
58
+ // New JSON marker with {ts, ttl}
59
+ try {
60
+ const raw = fs.readFileSync(plannedRestartPath(), 'utf8');
61
+ const j = JSON.parse(raw);
62
+ if (typeof j.ts === 'number' && typeof j.ttl === 'number') {
63
+ return Date.now() - j.ts < j.ttl;
64
+ }
65
+ }
66
+ catch { }
67
+ // Fallback: legacy plain file written by dsh-safe-web-update (no .json, mtime-based)
68
+ try {
69
+ const legacy = path.join(os.homedir(), '.dsh/.supervisor/planned-restart');
70
+ const stat = fs.statSync(legacy);
71
+ return isPlannedRestartFresh(stat.mtimeMs, Date.now());
72
+ }
73
+ catch { }
74
+ return false;
75
+ }
76
+ export function clearPlannedRestart() {
77
+ try {
78
+ fs.unlinkSync(plannedRestartPath());
79
+ }
80
+ catch { }
81
+ // also clear legacy plain file if present (best-effort, avoids stale suppression)
82
+ try {
83
+ const legacy = path.join(os.homedir(), '.dsh/.supervisor/planned-restart');
84
+ if (fs.existsSync(legacy) && legacy !== plannedRestartPath()) {
85
+ // only remove legacy if it was created as test artifact; keep conservative
86
+ // but clearing both ensures checkPlannedRestart() returns false after clear
87
+ try {
88
+ fs.unlinkSync(legacy);
89
+ }
90
+ catch { }
91
+ }
92
+ }
93
+ catch { }
94
+ }
@@ -21,7 +21,11 @@ export interface SupervisorDeps {
21
21
  notify: (msg: string) => Promise<void>;
22
22
  intervalMs?: number;
23
23
  debounceMs?: number;
24
+ downThreshold?: number;
24
25
  getTime?: () => number;
26
+ isPlannedRestartActive?: () => boolean | Promise<boolean>;
27
+ writePlannedRestart?: (ttlMs?: number) => void;
28
+ checkPlannedRestart?: () => boolean;
25
29
  runDebugAgent?: (opts: {
26
30
  reportPath: string;
27
31
  health: HealthState;
@@ -46,18 +50,26 @@ export declare class Supervisor {
46
50
  private rollingBack;
47
51
  private lastLKGWrite;
48
52
  private lastDegradedNotify;
53
+ private consecutiveDown;
54
+ private consecutiveDegraded;
49
55
  private timer;
50
56
  constructor(deps: SupervisorDeps);
57
+ private getWritePlannedRestart;
58
+ private getCheckPlannedRestart;
59
+ restartWeb(): Promise<void>;
51
60
  private getRunDebugAgent;
52
61
  private getFindInterrupted;
53
62
  private getResumeSessions;
54
63
  private getAutoResumeEnabled;
55
64
  private getResumeWithinMs;
65
+ private getEffectiveIntervalMs;
66
+ private getEffectiveDownThreshold;
56
67
  private findInterruptedRecent;
57
68
  private collectGitDiff;
58
69
  private attemptAutoResume;
59
70
  private handleDebugResult;
60
71
  tick(): Promise<void>;
61
- start(): void;
72
+ start(): Promise<void>;
73
+ startSync(): void;
62
74
  stop(): void;
63
75
  }
package/lib/supervisor.js CHANGED
@@ -4,6 +4,9 @@ import * as fs from 'node:fs';
4
4
  import * as path from 'node:path';
5
5
  import * as os from 'node:os';
6
6
  import { resolveHarnessRoot } from './paths.js';
7
+ import { readSupervisorConfig } from './config.js';
8
+ import { writePlannedRestart as defaultWritePlannedRestart, checkPlannedRestart as defaultCheckPlannedRestart } from './restart-guards.js';
9
+ import { buildKillStalePortsCommand } from './restart-guards.js';
7
10
  export async function resumeViaRpc(ids, fetchFn = globalThis.fetch) {
8
11
  const rpcId = crypto.randomUUID();
9
12
  const response = await fetchFn('http://127.0.0.1:3080/dsh-maestro-supervisor-resume/resume', {
@@ -30,10 +33,45 @@ export class Supervisor {
30
33
  rollingBack = false;
31
34
  lastLKGWrite = 0;
32
35
  lastDegradedNotify = 0;
36
+ consecutiveDown = 0;
37
+ consecutiveDegraded = 0;
33
38
  timer = null;
34
39
  constructor(deps) {
35
40
  this.deps = deps;
36
41
  }
42
+ getWritePlannedRestart() {
43
+ return this.deps.writePlannedRestart ?? defaultWritePlannedRestart;
44
+ }
45
+ getCheckPlannedRestart() {
46
+ return this.deps.checkPlannedRestart ?? defaultCheckPlannedRestart;
47
+ }
48
+ async restartWeb() {
49
+ this.getWritePlannedRestart()(30000);
50
+ if (this.deps.restartWeb) {
51
+ await this.deps.restartWeb();
52
+ return;
53
+ }
54
+ // Fallback systemctl path (mirrors cli.ts) — kept for standalone use
55
+ const { execSync } = await import('node:child_process');
56
+ try {
57
+ execSync(buildKillStalePortsCommand(), { timeout: 5000, stdio: 'pipe' });
58
+ }
59
+ catch { }
60
+ try {
61
+ execSync('systemctl --user is-active --quiet dsh-web.service && systemctl --user restart dsh-web.service || systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
62
+ return;
63
+ }
64
+ catch { }
65
+ try {
66
+ execSync('systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
67
+ return;
68
+ }
69
+ catch { }
70
+ const { resolveDeepseekHarnessDir } = await import('./paths.js');
71
+ const harnessRoot = resolveDeepseekHarnessDir();
72
+ const logPath = path.join(os.homedir(), '.dsh/dsh-web.log');
73
+ execSync(`setsid nohup bash -c 'export NVM_DIR="$HOME/.nvm"; [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; cd ${JSON.stringify(harnessRoot)} && exec node --import tsx/esm apps/cli/src/bin.ts web --no-open >> ${JSON.stringify(logPath)} 2>&1' &`, { timeout: 5000 });
74
+ }
37
75
  getRunDebugAgent() {
38
76
  return this.deps.runDebugAgent ?? runDebugAgent;
39
77
  }
@@ -139,6 +177,28 @@ export class Supervisor {
139
177
  catch { }
140
178
  return 5 * 60 * 1000; // default 5 minutes
141
179
  }
180
+ async getEffectiveIntervalMs() {
181
+ if (this.deps.intervalMs !== undefined)
182
+ return this.deps.intervalMs;
183
+ try {
184
+ const cfg = await readSupervisorConfig();
185
+ if (typeof cfg.intervalMs === 'number' && cfg.intervalMs > 0)
186
+ return cfg.intervalMs;
187
+ }
188
+ catch { }
189
+ return 3000;
190
+ }
191
+ async getEffectiveDownThreshold() {
192
+ if (this.deps.downThreshold !== undefined)
193
+ return this.deps.downThreshold;
194
+ try {
195
+ const cfg = await readSupervisorConfig();
196
+ if (typeof cfg.downThreshold === 'number' && cfg.downThreshold > 0)
197
+ return cfg.downThreshold;
198
+ }
199
+ catch { }
200
+ return 3;
201
+ }
142
202
  async findInterruptedRecent(withinMs) {
143
203
  const ms = withinMs ?? this.getResumeWithinMs();
144
204
  // Prefer injected mock for testability
@@ -231,44 +291,119 @@ export class Supervisor {
231
291
  }
232
292
  async tick() {
233
293
  const health = await this.deps.pollHealth();
234
- // DEGRADED: http 200 but log has plugin error → report, notify, no rollback
294
+ // DEGRADED: http 200 but log has plugin error → report + notify, rollback after consecutive threshold
235
295
  if (health.degraded) {
296
+ this.consecutiveDown = 0;
297
+ // Check suppression first — don't count degraded during planned restart grace
298
+ let suppressedByMarkerDeg = false;
299
+ try {
300
+ suppressedByMarkerDeg = this.getCheckPlannedRestart()();
301
+ }
302
+ catch {
303
+ suppressedByMarkerDeg = false;
304
+ }
305
+ if (suppressedByMarkerDeg) {
306
+ this.consecutiveDegraded = 0;
307
+ return;
308
+ }
309
+ if (this.deps.isPlannedRestartActive) {
310
+ let planned = false;
311
+ try {
312
+ planned = await Promise.resolve(this.deps.isPlannedRestartActive());
313
+ }
314
+ catch {
315
+ planned = false;
316
+ }
317
+ if (planned) {
318
+ this.consecutiveDegraded = 0;
319
+ return;
320
+ }
321
+ }
236
322
  const now = this.deps.getTime ? this.deps.getTime() : Date.now();
237
- if (now - this.lastDegradedNotify < 60000)
323
+ // degraded needs 3 consecutive (not downThreshold which tests set to 1) to avoid flapping
324
+ const degradedThreshold = 3;
325
+ this.consecutiveDegraded++;
326
+ if (this.consecutiveDegraded < degradedThreshold) {
327
+ if (now - this.lastDegradedNotify < 60000)
328
+ return;
329
+ this.lastDegradedNotify = now;
330
+ try {
331
+ const ts = new Date().toISOString().replace(/[:.]/g, '-');
332
+ const logTail = health.logTail ?? '';
333
+ const gitDiff = await this.collectGitDiff().catch(() => '');
334
+ const reportPath = await this.deps.writeReport({ ts, health, action: `degraded — ${health.error ?? 'plugin'}`, logTail, gitDiff }).catch(() => '');
335
+ await this.deps.notify(`DEGRADED: ${health.error ?? 'plugin'} (report: ${reportPath})`).catch(() => { });
336
+ const runner = this.getRunDebugAgent();
337
+ const isInjected = !!this.deps.runDebugAgent;
338
+ if (isInjected) {
339
+ void runner({ reportPath, health }).then(res => this.handleDebugResult(reportPath, res)).catch(() => { });
340
+ setTimeout(() => {
341
+ this.findInterruptedRecent().then(r => {
342
+ if (r.interrupted.length)
343
+ void this.attemptAutoResume(r.interrupted).catch(() => { });
344
+ }).catch(() => { });
345
+ }, 0);
346
+ }
347
+ else if (!process.env.VITEST) {
348
+ void runner({ reportPath, health }).then(res => this.handleDebugResult(reportPath, res)).catch(() => { });
349
+ setTimeout(() => {
350
+ this.findInterruptedRecent().then(r => {
351
+ if (r.interrupted.length)
352
+ void this.attemptAutoResume(r.interrupted).catch(() => { });
353
+ }).catch(() => { });
354
+ }, 0);
355
+ }
356
+ }
357
+ catch { }
238
358
  return;
239
- this.lastDegradedNotify = now;
359
+ }
360
+ // consecutive degraded threshold reached → treat as down: rollback + restart
361
+ this.consecutiveDegraded = 0;
362
+ const degradedError = health.error ?? 'degraded plugin error';
363
+ if (this.rollingBack)
364
+ return;
365
+ const now2 = this.deps.getTime ? this.deps.getTime() : Date.now();
366
+ const debounceMs2 = this.deps.debounceMs ?? 60000;
367
+ if (now2 - this.lastRollback < debounceMs2)
368
+ return;
369
+ this.rollingBack = true;
370
+ this.lastRollback = now2;
240
371
  try {
241
- const ts = new Date().toISOString().replace(/[:.]/g, '-');
242
- const logTail = health.logTail ?? '';
243
- const gitDiff = await this.collectGitDiff().catch(() => '');
244
- const reportPath = await this.deps.writeReport({ ts, health, action: `degraded — ${health.error ?? 'plugin'}`, logTail, gitDiff }).catch(() => '');
245
- await this.deps.notify(`DEGRADED: ${health.error ?? 'plugin'} (report: ${reportPath})`).catch(() => { });
246
- // Phase 3: debug + resume use injected fn if provided (even in VITEST), otherwise fire-and-forget real impl (skip in VITEST)
247
- const runner = this.getRunDebugAgent();
248
- const isInjected = !!this.deps.runDebugAgent;
249
- if (isInjected) {
250
- void runner({ reportPath, health }).then(res => this.handleDebugResult(reportPath, res)).catch(() => { });
251
- setTimeout(() => {
252
- this.findInterruptedRecent().then(r => {
253
- if (r.interrupted.length)
254
- void this.attemptAutoResume(r.interrupted).catch(() => { });
255
- }).catch(() => { });
256
- }, 0);
372
+ const failed = await this.deps.writeFailed().catch(() => ({ ts: new Date().toISOString().replace(/[:.]/g, '-'), manifest: null }));
373
+ const ts2 = failed?.ts ?? new Date().toISOString().replace(/[:.]/g, '-');
374
+ const logTail2 = health.logTail ?? '';
375
+ const gitDiff2 = await this.collectGitDiff().catch(() => '');
376
+ const degradedHealth = { up: false, httpCode: health.httpCode, error: `degraded down: ${degradedError}`, logTail: logTail2, degraded: false };
377
+ const reportPath2 = await this.deps.writeReport({ ts: ts2, health: degradedHealth, action: `rollback degraded: ${degradedError}`, logTail: logTail2, gitDiff: gitDiff2 }).catch(() => '');
378
+ try {
379
+ await this.deps.rollback();
380
+ }
381
+ catch (e) {
382
+ await this.deps.notify(`rollback failed: ${e?.message ?? String(e)} (report: ${reportPath2})`).catch(() => { });
257
383
  }
258
- else if (!process.env.VITEST) {
259
- void runner({ reportPath, health }).then(res => this.handleDebugResult(reportPath, res)).catch(() => { });
260
- setTimeout(() => {
261
- this.findInterruptedRecent().then(r => {
262
- if (r.interrupted.length)
263
- void this.attemptAutoResume(r.interrupted).catch(() => { });
264
- }).catch(() => { });
265
- }, 0);
384
+ if (this.deps.restartWeb) {
385
+ try {
386
+ await this.deps.restartWeb();
387
+ await this.deps.notify(`restarted dsh-web after rollback (report: ${reportPath2})`).catch(() => { });
388
+ }
389
+ catch (e) {
390
+ await this.deps.notify(`restart dsh-web failed: ${e?.message ?? String(e)} (report: ${reportPath2})`).catch(() => { });
391
+ }
266
392
  }
393
+ await this.deps.notify(`DEGRADED → rollback (report: ${reportPath2}, error: ${degradedError})`).catch(() => { });
394
+ try {
395
+ await this.deps.notify(`🔄 dsh web auto-restart — degraded: ${degradedError} — ${new Date().toISOString()}`).catch(() => { });
396
+ }
397
+ catch { }
398
+ }
399
+ finally {
400
+ this.rollingBack = false;
267
401
  }
268
- catch { }
269
402
  return;
270
403
  }
271
404
  if (health.up) {
405
+ this.consecutiveDown = 0;
406
+ this.consecutiveDegraded = 0;
272
407
  // Throttle LKG writes to at most once per 5 minutes
273
408
  const now = this.deps.getTime ? this.deps.getTime() : Date.now();
274
409
  if (now - this.lastLKGWrite > 5 * 60 * 1000) {
@@ -282,6 +417,57 @@ export class Supervisor {
282
417
  }
283
418
  return;
284
419
  }
420
+ // Single-owner 30s marker: planned restart in progress — suppress crash
421
+ // handling entirely for 30s (no writeFailed/writeReport/rollback).
422
+ // Check both the new JSON marker (writePlannedRestart) and the legacy
423
+ // isPlannedRestartActive (dsh-safe-web-update plain file) for compat.
424
+ let suppressedByMarker = false;
425
+ try {
426
+ suppressedByMarker = this.getCheckPlannedRestart()();
427
+ }
428
+ catch {
429
+ suppressedByMarker = false;
430
+ }
431
+ if (suppressedByMarker) {
432
+ this.consecutiveDown = 0;
433
+ return;
434
+ }
435
+ // Down while an intentional restart (e.g. dsh-safe-web-update) is in
436
+ // flight is expected, not a crash — never race it with our own
437
+ // rollback/restartWeb.
438
+ if (this.deps.isPlannedRestartActive) {
439
+ let planned = false;
440
+ try {
441
+ planned = await Promise.resolve(this.deps.isPlannedRestartActive());
442
+ }
443
+ catch {
444
+ planned = false;
445
+ }
446
+ if (planned) {
447
+ this.consecutiveDown = 0;
448
+ return;
449
+ }
450
+ }
451
+ // Down — require consecutive confirmations before treating as a crash.
452
+ // A lone timed-out poll (e.g. a slow plugin-tree boot) must not trigger
453
+ // rollback/restart: that restart produces its own transient errors on
454
+ // the next poll, which would otherwise re-trigger this same path forever.
455
+ this.consecutiveDown++;
456
+ let downThreshold = await this.getEffectiveDownThreshold();
457
+ // When a planned restart marker is active, double the threshold (3→6 at
458
+ // 3s interval ≈ 9s→18s). Health-poller already suppresses fetch failed in
459
+ // this window, but supervisor also doubles so a transient that escapes
460
+ // health still needs longer to trigger. The early return above already
461
+ // suppresses fully for 30s; doubling remains for the legacy
462
+ // isPlannedRestartActive path and for any race where the marker was
463
+ // written between the early check and this line.
464
+ try {
465
+ if (this.getCheckPlannedRestart()())
466
+ downThreshold *= 2;
467
+ }
468
+ catch { }
469
+ if (this.consecutiveDown < downThreshold)
470
+ return;
285
471
  // Down — check debounce and rolling state
286
472
  if (this.rollingBack)
287
473
  return;
@@ -290,6 +476,7 @@ export class Supervisor {
290
476
  if (now - this.lastRollback < debounceMs)
291
477
  return;
292
478
  this.rollingBack = true;
479
+ this.consecutiveDown = 0;
293
480
  this.lastRollback = now;
294
481
  try {
295
482
  const failed = await this.deps.writeFailed().catch(() => ({ ts: new Date().toISOString().replace(/[:.]/g, '-'), manifest: null }));
@@ -339,14 +526,18 @@ export class Supervisor {
339
526
  this.rollingBack = false;
340
527
  }
341
528
  }
342
- start() {
529
+ async start() {
343
530
  if (this.timer)
344
531
  return;
345
- const intervalMs = this.deps.intervalMs ?? 3000;
532
+ const intervalMs = await this.getEffectiveIntervalMs();
346
533
  this.timer = setInterval(() => { this.tick().catch(() => { }); }, intervalMs);
347
534
  // Immediate tick so a reboot is recovered in ~0-3s, not 3s
348
535
  this.tick().catch(() => { });
349
536
  }
537
+ // Synchronous start wrapper for callers that do not await (daemon cli fire-and-forget)
538
+ startSync() {
539
+ void this.start();
540
+ }
350
541
  stop() {
351
542
  if (this.timer) {
352
543
  clearInterval(this.timer);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ddtcorex/dsh-maestro-supervisor",
3
- "version": "0.6.2",
3
+ "version": "0.6.6",
4
4
  "description": "Supervisor daemon for DSH Web resilience — auto-detect crashes, rollback to LKG, report",
5
5
  "type": "module",
6
6
  "main": "./lib/index.js",
@@ -36,6 +36,9 @@
36
36
  "README.md",
37
37
  "cordis.patch.yml"
38
38
  ],
39
+ "dependencies": {
40
+ "@ddtcorex/dsh-maestro-config-lib": "^0.1.2"
41
+ },
39
42
  "devDependencies": {
40
43
  "@types/node": "^26.3.0",
41
44
  "typescript": "^5.9.3",
@@ -44,7 +47,7 @@
44
47
  "scripts": {
45
48
  "build": "tsc -p tsconfig.json && tsc -p tsconfig.client.json && node scripts/build-client.mjs",
46
49
  "verify": "tsc --noEmit && tsc -p tsconfig.client.json --noEmit",
47
- "test": "vitest run",
50
+ "test": "vitest run --testTimeout=10000",
48
51
  "test:watch": "vitest"
49
52
  }
50
53
  }