@ddtcorex/dsh-maestro-supervisor 0.7.10 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/snapshot.d.ts CHANGED
@@ -1,22 +1,59 @@
1
- interface ManifestEntry {
2
- path: string;
3
- sha256: string;
4
- }
5
- interface Manifest {
1
+ export interface SnapshotResult {
6
2
  ts: string;
7
- files: ManifestEntry[];
3
+ /** Files (and symlinks) copied into this snapshot. */
4
+ files: number;
5
+ /** Entries that could not be copied; the snapshot stays usable either way. */
6
+ skipped: Array<{
7
+ path: string;
8
+ reason: string;
9
+ }>;
8
10
  }
9
- export declare function writeLKG(dshHome: string, lkgRoot: string): Promise<{
10
- ts: string;
11
- manifest: Manifest;
12
- }>;
11
+ export interface SnapshotDeps {
12
+ /**
13
+ * Copy one regular file, preserving `mode`. Injectable so a failing entry
14
+ * (the EACCES a read-only attachment object produces) can be table-tested
15
+ * without depending on the host's uid or filesystem quirks.
16
+ */
17
+ copyFile?: (src: string, dest: string, mode: number) => void;
18
+ }
19
+ /**
20
+ * DSH-home entries the last-known-good snapshot deliberately never copies.
21
+ *
22
+ * The LKG exists to recover a boot that fails while loading the plugin tree, so
23
+ * it holds boot **configuration**: `profiles/` (the plugin tree with its
24
+ * lockfile, `cordis.patch.yml` and sidecars), the per-plugin config directories
25
+ * and the settings documents. Everything below is runtime **data** — it is
26
+ * written continuously while `dsh web` runs, so restoring a snapshot of it can
27
+ * only lose newer state, and some of it is hostile to a bulk copy:
28
+ *
29
+ * - `sessions/` — append-only transcripts. Restoring a stale copy over live
30
+ * sessions drops every turn recorded after the snapshot, i.e. the recovery
31
+ * path would lose the log it is supposed to protect.
32
+ * - `attachments/` — content-addressed blobs stored mode `0400`. That read-only
33
+ * bit is exactly what made the 2026-09-13 rollback abort with
34
+ * `EACCES: Permission denied '.../attachments/v1/objects/f8'`.
35
+ * - `plugins-src/` — plugin source cache, re-fetched on demand (~400 MB host).
36
+ * - `.supervisor/` — the LKG root itself lives inside the DSH home, so copying
37
+ * it would recurse into every retained snapshot.
38
+ *
39
+ * This list is data, not a heuristic: the copy loop and the restore loop both
40
+ * consult `isLkgExcluded()`, and the table test pins the rule.
41
+ */
42
+ export declare const LKG_EXCLUDED_ENTRIES: readonly string[];
43
+ /**
44
+ * True when a DSH-home-relative path must never enter (or leave) the LKG.
45
+ *
46
+ * The named entries match the first path segment; `*.log` matches by basename
47
+ * anywhere, because append-only logs are data at any depth and a restored stale
48
+ * `dsh-web.log` would poison the boot-boundary scan.
49
+ */
50
+ export declare function isLkgExcluded(relPath: string): boolean;
51
+ /** Human-readable, bounded reason for a per-entry copy failure. */
52
+ export declare function failureReason(e: unknown): string;
53
+ export declare function writeLKG(dshHome: string, lkgRoot: string, deps?: SnapshotDeps): Promise<SnapshotResult>;
13
54
  export declare function pruneByAge(root: string, maxAgeMs: number): Promise<void>;
14
55
  export declare function pruneBySize(root: string, maxBytes: number): Promise<void>;
15
56
  export declare function isDuplicateLKG(dshHome: string, lkgRoot: string): Promise<boolean>;
16
57
  export declare function verifyLKG(lkgPath: string): Promise<boolean>;
17
58
  export declare function rotateLKG(lkgRoot: string, keep?: number): Promise<void>;
18
- export declare function writeFailed(dshHome: string, failedRoot: string): Promise<{
19
- ts: string;
20
- manifest: Manifest;
21
- }>;
22
- export {};
59
+ export declare function writeFailed(dshHome: string, failedRoot: string): Promise<SnapshotResult>;
package/lib/snapshot.js CHANGED
@@ -1,6 +1,51 @@
1
1
  import * as fs from 'node:fs';
2
2
  import * as path from 'node:path';
3
3
  import * as crypto from 'node:crypto';
4
+ /**
5
+ * DSH-home entries the last-known-good snapshot deliberately never copies.
6
+ *
7
+ * The LKG exists to recover a boot that fails while loading the plugin tree, so
8
+ * it holds boot **configuration**: `profiles/` (the plugin tree with its
9
+ * lockfile, `cordis.patch.yml` and sidecars), the per-plugin config directories
10
+ * and the settings documents. Everything below is runtime **data** — it is
11
+ * written continuously while `dsh web` runs, so restoring a snapshot of it can
12
+ * only lose newer state, and some of it is hostile to a bulk copy:
13
+ *
14
+ * - `sessions/` — append-only transcripts. Restoring a stale copy over live
15
+ * sessions drops every turn recorded after the snapshot, i.e. the recovery
16
+ * path would lose the log it is supposed to protect.
17
+ * - `attachments/` — content-addressed blobs stored mode `0400`. That read-only
18
+ * bit is exactly what made the 2026-09-13 rollback abort with
19
+ * `EACCES: Permission denied '.../attachments/v1/objects/f8'`.
20
+ * - `plugins-src/` — plugin source cache, re-fetched on demand (~400 MB host).
21
+ * - `.supervisor/` — the LKG root itself lives inside the DSH home, so copying
22
+ * it would recurse into every retained snapshot.
23
+ *
24
+ * This list is data, not a heuristic: the copy loop and the restore loop both
25
+ * consult `isLkgExcluded()`, and the table test pins the rule.
26
+ */
27
+ export const LKG_EXCLUDED_ENTRIES = [
28
+ 'sessions',
29
+ 'attachments',
30
+ 'plugins-src',
31
+ '.supervisor',
32
+ ];
33
+ /**
34
+ * True when a DSH-home-relative path must never enter (or leave) the LKG.
35
+ *
36
+ * The named entries match the first path segment; `*.log` matches by basename
37
+ * anywhere, because append-only logs are data at any depth and a restored stale
38
+ * `dsh-web.log` would poison the boot-boundary scan.
39
+ */
40
+ export function isLkgExcluded(relPath) {
41
+ const normalized = relPath.replace(/\\/g, '/').replace(/^\.\//, '').replace(/^\/+/, '');
42
+ const segments = normalized.split('/').filter(s => s.length > 0 && s !== '.');
43
+ if (!segments.length)
44
+ return false;
45
+ if (LKG_EXCLUDED_ENTRIES.includes(segments[0]))
46
+ return true;
47
+ return segments[segments.length - 1].endsWith('.log');
48
+ }
4
49
  function sha256File(filePath) {
5
50
  const data = fs.readFileSync(filePath);
6
51
  return crypto.createHash('sha256').update(data).digest('hex');
@@ -16,7 +61,78 @@ function walkFiles(dir, base = dir) {
16
61
  }
17
62
  return out;
18
63
  }
19
- export async function writeLKG(dshHome, lkgRoot) {
64
+ /** Human-readable, bounded reason for a per-entry copy failure. */
65
+ export function failureReason(e) {
66
+ const err = e;
67
+ const code = typeof err?.code === 'string' && err.code ? `${err.code}: ` : '';
68
+ const message = typeof err?.message === 'string' ? err.message : String(e);
69
+ return `${code}${message}`.slice(0, 300);
70
+ }
71
+ /**
72
+ * Copy one snapshot entry, collecting — never throwing — per-entry failures
73
+ * (D2). `fs.cpSync` aborts the whole snapshot on the first unreadable object,
74
+ * which is precisely how one 0400 attachment file stopped the rollback that
75
+ * exists to rescue a broken boot.
76
+ */
77
+ function copyEntry(state, src, dest, rel) {
78
+ if (isLkgExcluded(rel))
79
+ return;
80
+ let st;
81
+ try {
82
+ st = fs.lstatSync(src);
83
+ }
84
+ catch (e) {
85
+ state.skipped.push({ path: rel, reason: failureReason(e) });
86
+ return;
87
+ }
88
+ if (st.isDirectory()) {
89
+ let names;
90
+ try {
91
+ // Directory modes are replicated only in their permission-to-traverse
92
+ // sense: the snapshot itself must stay readable/removable even when the
93
+ // source directory is not (a 0000 source dir must not produce a snapshot
94
+ // that nothing — including retention — can delete).
95
+ fs.mkdirSync(dest, { recursive: true, mode: (st.mode & 0o7777) | 0o700 });
96
+ names = fs.readdirSync(src);
97
+ }
98
+ catch (e) {
99
+ state.skipped.push({ path: rel, reason: failureReason(e) });
100
+ return;
101
+ }
102
+ for (const name of names) {
103
+ copyEntry(state, path.join(src, name), path.join(dest, name), rel ? `${rel}/${name}` : name);
104
+ }
105
+ return;
106
+ }
107
+ if (st.isSymbolicLink()) {
108
+ try {
109
+ const link = fs.readlinkSync(src);
110
+ fs.mkdirSync(path.dirname(dest), { recursive: true });
111
+ try {
112
+ fs.unlinkSync(dest);
113
+ }
114
+ catch { }
115
+ fs.symlinkSync(link, dest);
116
+ state.copied++;
117
+ }
118
+ catch (e) {
119
+ state.skipped.push({ path: rel, reason: failureReason(e) });
120
+ }
121
+ return;
122
+ }
123
+ if (st.isFile()) {
124
+ try {
125
+ fs.mkdirSync(path.dirname(dest), { recursive: true });
126
+ state.copyFile(src, dest, st.mode & 0o7777);
127
+ state.copied++;
128
+ }
129
+ catch (e) {
130
+ state.skipped.push({ path: rel, reason: failureReason(e) });
131
+ }
132
+ }
133
+ // Sockets/FIFOs/devices are not boot configuration: ignored on purpose.
134
+ }
135
+ export async function writeLKG(dshHome, lkgRoot, deps = {}) {
20
136
  // Dedupe: skip snapshot if current state identical to latest LKG (prevents 5-min unconditional growth)
21
137
  try {
22
138
  if (await isDuplicateLKG(dshHome, lkgRoot)) {
@@ -31,36 +147,68 @@ export async function writeLKG(dshHome, lkgRoot) {
31
147
  const latest = entries[entries.length - 1];
32
148
  const manifestPath = path.join(lkgRoot, latest, 'manifest.json');
33
149
  const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
34
- return { ts: latest, manifest };
150
+ return { ts: latest, files: manifest.files?.length ?? 0, skipped: [] };
35
151
  }
36
152
  }
37
153
  catch { }
38
154
  const ts = new Date().toISOString().replace(/[:.]/g, '-');
39
155
  const dest = path.join(lkgRoot, ts);
40
- fs.mkdirSync(dest, { recursive: true });
41
- // Copy DSH home contents (if exists, copy recursively) skip .supervisor to avoid recursion
42
- if (fs.existsSync(dshHome)) {
43
- for (const entry of fs.readdirSync(dshHome)) {
44
- if (entry === '.supervisor')
45
- continue;
46
- const src = path.join(dshHome, entry);
47
- const dst = path.join(dest, entry);
48
- fs.cpSync(src, dst, { recursive: true });
49
- }
50
- }
51
- const files = fs.existsSync(dest) ? walkFiles(dest) : [];
156
+ try {
157
+ fs.mkdirSync(dest, { recursive: true });
158
+ }
159
+ catch (e) {
160
+ // No snapshot is better than an exception thrown into the rollback path.
161
+ return { ts, files: 0, skipped: [{ path: lkgRoot, reason: failureReason(e) }] };
162
+ }
163
+ const state = {
164
+ dshHome,
165
+ copyFile: deps.copyFile ?? ((src, dst, mode) => {
166
+ fs.copyFileSync(src, dst);
167
+ try {
168
+ fs.chmodSync(dst, mode);
169
+ }
170
+ catch { }
171
+ }),
172
+ copied: 0,
173
+ skipped: [],
174
+ };
175
+ // Copy only the boot configuration (D1), one entry at a time so a single
176
+ // unreadable file is recorded and skipped instead of aborting the snapshot (D2).
177
+ let topLevel = [];
178
+ try {
179
+ if (fs.existsSync(dshHome))
180
+ topLevel = fs.readdirSync(dshHome);
181
+ }
182
+ catch (e) {
183
+ state.skipped.push({ path: '.', reason: failureReason(e) });
184
+ }
185
+ for (const entry of topLevel) {
186
+ copyEntry(state, path.join(dshHome, entry), path.join(dest, entry), entry);
187
+ }
188
+ let fileList = [];
189
+ try {
190
+ fileList = fs.existsSync(dest) ? walkFiles(dest) : [];
191
+ }
192
+ catch (e) {
193
+ state.skipped.push({ path: 'manifest', reason: failureReason(e) });
194
+ }
52
195
  const manifest = {
53
196
  ts,
54
- files: files
197
+ files: fileList
55
198
  .filter(f => f !== 'manifest.json')
56
199
  .map(f => ({ path: f, sha256: sha256File(path.join(dest, f)) })),
57
200
  };
58
- fs.writeFileSync(path.join(dest, 'manifest.json'), JSON.stringify(manifest, null, 2));
201
+ try {
202
+ fs.writeFileSync(path.join(dest, 'manifest.json'), JSON.stringify(manifest, null, 2));
203
+ }
204
+ catch (e) {
205
+ state.skipped.push({ path: 'manifest.json', reason: failureReason(e) });
206
+ }
59
207
  // Retention: keep only 3 most recent, plus age (7d) and size (5GB) caps — prevents unbounded 40GB+ growth
60
208
  await rotateLKG(lkgRoot, 3).catch(() => { });
61
209
  await pruneByAge(lkgRoot, 7 * 24 * 60 * 60 * 1000).catch(() => { });
62
210
  await pruneBySize(lkgRoot, 5 * 1024 * 1024 * 1024).catch(() => { });
63
- return { ts, manifest };
211
+ return { ts, files: state.copied, skipped: state.skipped };
64
212
  }
65
213
  export async function pruneByAge(root, maxAgeMs) {
66
214
  if (!fs.existsSync(root))
@@ -150,7 +298,9 @@ export async function isDuplicateLKG(dshHome, lkgRoot) {
150
298
  let newestFileMtime = 0;
151
299
  if (fs.existsSync(dshHome)) {
152
300
  for (const entry of fs.readdirSync(dshHome)) {
153
- if (entry === '.supervisor')
301
+ // Same scope as the copy loop: runtime data changes constantly and
302
+ // must not defeat the dedupe for the configuration being snapshotted.
303
+ if (isLkgExcluded(entry))
154
304
  continue;
155
305
  try {
156
306
  const s = fs.statSync(path.join(dshHome, entry));
@@ -1,15 +1,25 @@
1
- import type { HealthState } from './health-poller.js';
1
+ import { type HealthState } from './health-poller.js';
2
2
  import type { RestartRequest } from './restart-guards.js';
3
+ import { type RestartOutcome } from './intents.js';
3
4
  import { type MintCookieOpts } from './dsh-session.js';
5
+ /** Partial-restore report a rollback may hand back (cli.ts wires RestoreResult). */
6
+ export interface RollbackSummary {
7
+ target?: string;
8
+ restored?: number;
9
+ skipped?: Array<{
10
+ path: string;
11
+ reason: string;
12
+ }>;
13
+ }
4
14
  export interface SupervisorDeps {
5
15
  pollHealth: () => Promise<HealthState>;
6
16
  writeLKG: () => Promise<{
7
17
  ts: string;
8
- manifest: any;
18
+ manifest?: any;
9
19
  }>;
10
20
  writeFailed: () => Promise<{
11
21
  ts: string;
12
- manifest: any;
22
+ manifest?: any;
13
23
  }>;
14
24
  writeReport: (opts: {
15
25
  ts: string;
@@ -18,12 +28,13 @@ export interface SupervisorDeps {
18
28
  logTail?: string;
19
29
  gitDiff?: string;
20
30
  }) => Promise<string>;
21
- rollback: (ts?: string) => Promise<void>;
31
+ rollback: (ts?: string) => Promise<RollbackSummary | void>;
22
32
  restartWeb?: () => Promise<void>;
23
33
  notify: (msg: string) => Promise<void>;
24
34
  intervalMs?: number;
25
35
  debounceMs?: number;
26
36
  downThreshold?: number;
37
+ bootGraceMs?: number;
27
38
  getTime?: () => number;
28
39
  isPlannedRestartActive?: () => boolean | Promise<boolean>;
29
40
  writePlannedRestart?: (ttlMs?: number) => void;
@@ -45,7 +56,11 @@ export interface SupervisorDeps {
45
56
  }>;
46
57
  readRestartRequest?: () => RestartRequest | undefined;
47
58
  onRestartRequestHandled?: (req: RestartRequest) => void;
59
+ writeOutcome?: (sessionId: string, outcome: RestartOutcome) => void;
60
+ listenerPid?: (port: number) => number | undefined;
48
61
  }
62
+ /** PID holding a 127.0.0.1 listener on the port, or undefined. Never throws. */
63
+ export declare function defaultListenerPid(port: number): number | undefined;
49
64
  export declare function resumeViaRpc(ids: string[], fetchFn?: (url: string, init: RequestInit) => Promise<Response>, extraHeaders?: Record<string, string>): Promise<{
50
65
  resumed: string[];
51
66
  }>;
@@ -86,9 +101,16 @@ export declare class Supervisor {
86
101
  private getEffectiveDownThreshold;
87
102
  private getEffectiveDegradedThreshold;
88
103
  private getEffectivePollTimeoutMs;
104
+ private getEffectiveBootGraceMs;
89
105
  private findInterruptedRecent;
90
106
  private collectGitDiff;
91
107
  private attemptAutoResume;
108
+ /**
109
+ * D4: a restore that could not put every entry back must be surfaced loudly.
110
+ * Without this, a half-restored tree reads exactly like a clean recovery in
111
+ * the log and in the operator's Telegram feed.
112
+ */
113
+ private reportRollbackResult;
92
114
  private handleDebugResult;
93
115
  tick(): Promise<void>;
94
116
  start(): Promise<void>;
package/lib/supervisor.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { classifyFetchFailure } from './health-poller.js';
1
2
  import { runDebugAgent } from './debug-agent.js';
2
3
  import { findInterrupted as defaultFindInterrupted, parseDuration } from './resume.js';
3
4
  import * as fs from 'node:fs';
@@ -5,9 +6,23 @@ import * as path from 'node:path';
5
6
  import * as os from 'node:os';
6
7
  import { resolveHarnessRoot } from './paths.js';
7
8
  import { readSupervisorConfig } from './config.js';
8
- import { writePlannedRestart as defaultWritePlannedRestart, checkPlannedRestart as defaultCheckPlannedRestart, clearPlannedRestart, PLANNED_RESTART_TTL_MS } from './restart-guards.js';
9
+ import { writePlannedRestart as defaultWritePlannedRestart, checkPlannedRestart as defaultCheckPlannedRestart, clearPlannedRestart } from './restart-guards.js';
10
+ import { writeRestartOutcome as defaultWriteOutcome } from './intents.js';
11
+ import { execFileSync } from 'node:child_process';
9
12
  import { mintDshSessionCookie } from './dsh-session.js';
10
- import { buildKillStalePortsCommand } from './restart-guards.js';
13
+ /** PID holding a 127.0.0.1 listener on the port, or undefined. Never throws. */
14
+ export function defaultListenerPid(port) {
15
+ try {
16
+ const out = execFileSync('ss', ['-tlnp'], { encoding: 'utf8' });
17
+ for (const line of out.split('\n')) {
18
+ const m = new RegExp(`127\\.0\\.0\\.1:${port}\\s[^]*?pid=(\\d+)`).exec(line);
19
+ if (m)
20
+ return Number(m[1]);
21
+ }
22
+ }
23
+ catch { }
24
+ return undefined;
25
+ }
11
26
  export async function resumeViaRpc(ids, fetchFn = globalThis.fetch, extraHeaders = {}) {
12
27
  const rpcId = crypto.randomUUID();
13
28
  const response = await fetchFn('http://127.0.0.1:3080/dsh-maestro-supervisor-resume/resume', {
@@ -71,31 +86,22 @@ export class Supervisor {
71
86
  return this.deps.clearPlannedRestart ?? clearPlannedRestart;
72
87
  }
73
88
  async restartWeb() {
74
- this.getWritePlannedRestart()(30000);
89
+ const grace = await this.getEffectiveBootGraceMs();
90
+ // Marker first, and its TTL is the boot budget — not 30 s. The observed
91
+ // boot took 1m52s: after +30 s every poll was judged as a crash, which is
92
+ // exactly how the 2026-09-13 rollback report was produced (D5).
93
+ this.getWritePlannedRestart()(grace);
75
94
  if (this.deps.restartWeb) {
95
+ // Injected implementation (cli.ts daemon wiring, tests): it owns the
96
+ // boot lock, so the single-flight guarantee stays in one place.
76
97
  await this.deps.restartWeb();
77
98
  return;
78
99
  }
79
- // Fallback systemctl path (mirrors cli.ts) — kept for standalone use
80
- const { execSync } = await import('node:child_process');
81
- try {
82
- execSync(buildKillStalePortsCommand(), { timeout: 5000, stdio: 'pipe' });
83
- }
84
- catch { }
85
- try {
86
- execSync('systemctl --user is-active --quiet dsh-web.service && systemctl --user restart dsh-web.service || systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
87
- return;
88
- }
89
- catch { }
90
- try {
91
- execSync('systemctl --user start dsh-web.service', { timeout: 15000, stdio: 'pipe' });
92
- return;
100
+ const { performSingleBootRestart } = await import('./restart-web.js');
101
+ const res = await performSingleBootRestart({ bootGraceMs: grace });
102
+ if (!res.restarted) {
103
+ await this.deps.notify(`restart skipped: ${res.reason ?? 'boot lock held'}`).catch(() => { });
93
104
  }
94
- catch { }
95
- const { resolveDeepseekHarnessDir } = await import('./paths.js');
96
- const harnessRoot = resolveDeepseekHarnessDir();
97
- const logPath = path.join(os.homedir(), '.dsh/dsh-web.log');
98
- execSync(`setsid nohup bash -c 'export NVM_DIR="$HOME/.nvm"; [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; cd ${JSON.stringify(harnessRoot)} && exec node --import tsx/esm apps/cli/src/bin.ts web --no-open >> ${JSON.stringify(logPath)} 2>&1' &`, { timeout: 5000 });
99
105
  }
100
106
  getRunDebugAgent() {
101
107
  return this.deps.runDebugAgent ?? runDebugAgent;
@@ -242,6 +248,17 @@ export class Supervisor {
242
248
  catch { }
243
249
  return 20000;
244
250
  }
251
+ async getEffectiveBootGraceMs() {
252
+ if (this.deps.bootGraceMs !== undefined)
253
+ return this.deps.bootGraceMs;
254
+ try {
255
+ const cfg = await readSupervisorConfig();
256
+ if (typeof cfg.bootGraceMs === 'number' && cfg.bootGraceMs > 0)
257
+ return cfg.bootGraceMs;
258
+ }
259
+ catch { }
260
+ return 180_000;
261
+ }
245
262
  async findInterruptedRecent(withinMs) {
246
263
  const ms = withinMs ?? this.getResumeWithinMs();
247
264
  // Prefer injected mock for testability
@@ -312,6 +329,17 @@ export class Supervisor {
312
329
  await this.deps.notify(`RESUME FAILED: ${ids.length} interrupted sessions (${ids.slice(0, 3).join(', ')}) — ${e?.message ?? String(e)}`).catch(() => { });
313
330
  }
314
331
  }
332
+ /**
333
+ * D4: a restore that could not put every entry back must be surfaced loudly.
334
+ * Without this, a half-restored tree reads exactly like a clean recovery in
335
+ * the log and in the operator's Telegram feed.
336
+ */
337
+ async reportRollbackResult(summary, reportPath) {
338
+ if (!summary || !summary.skipped?.length)
339
+ return;
340
+ const first = summary.skipped.slice(0, 3).map(s => `${s.path} (${s.reason})`).join('; ');
341
+ await this.deps.notify(`ROLLBACK PARTIAL: ${summary.restored ?? 0} restored, ${summary.skipped.length} skipped — ${first} (report: ${reportPath})`).catch(() => { });
342
+ }
315
343
  handleDebugResult(reportPath, res) {
316
344
  if (res.fixed) {
317
345
  void this.deps.notify(`FIXED: debug-agent fixed ${reportPath} — ${res.reason}`).catch(() => { });
@@ -352,13 +380,24 @@ export class Supervisor {
352
380
  // own 30s TTL runs out.
353
381
  this.lastRollback = this.deps.getTime ? this.deps.getTime() : Date.now();
354
382
  await this.restartWeb();
355
- // Supervisor.restartWeb() wrote a 30s marker; extend it past a slow
356
- // boot. It is cleared only once the boot proves healthy below.
357
- this.getWritePlannedRestart()(PLANNED_RESTART_TTL_MS);
383
+ // restartWeb() already wrote the marker with the boot budget as its
384
+ // TTL; the marker is cleared on the first healthy poll instead.
358
385
  await this.deps.notify(`restarted dsh-web after self-restart by session ${restartReq.callerSessionId}`);
359
386
  }
360
387
  catch (e) {
361
388
  await this.deps.notify(`self-restart dsh-web failed: ${e?.message ?? String(e)}`).catch(() => { });
389
+ if (restartReq.callerSessionId) {
390
+ try {
391
+ ;
392
+ (this.deps.writeOutcome ?? defaultWriteOutcome)(restartReq.callerSessionId, {
393
+ state: 'failed',
394
+ oldPid: restartReq.oldPid,
395
+ swappedAt: this.deps.getTime ? this.deps.getTime() : Date.now(),
396
+ error: e?.message ?? String(e),
397
+ });
398
+ }
399
+ catch { }
400
+ }
362
401
  }
363
402
  finally {
364
403
  // Hold the marker and latch until health.up: clearing here would
@@ -375,6 +414,14 @@ export class Supervisor {
375
414
  // DEGRADED: http 200 but log has plugin error → report + notify, rollback after consecutive threshold
376
415
  if (health.degraded) {
377
416
  this.consecutiveDown = 0;
417
+ // D1: a degraded verdict while the current boot is unproven is a boot
418
+ // transient, not a plugin error — the incident's report line
419
+ // "rollback — degraded: This operation was aborted" was produced exactly
420
+ // here, five polls after a slow boot.
421
+ if (health.bootPhase === 'booting') {
422
+ this.consecutiveDegraded = 0;
423
+ return;
424
+ }
378
425
  // Check suppression first — don't count degraded during planned restart grace
379
426
  let suppressedByMarkerDeg = false;
380
427
  try {
@@ -457,7 +504,8 @@ export class Supervisor {
457
504
  const degradedHealth = { up: false, httpCode: health.httpCode, error: `degraded → down: ${degradedError}`, logTail: logTail2, degraded: false };
458
505
  const reportPath2 = await this.deps.writeReport({ ts: ts2, health: degradedHealth, action: `rollback — degraded: ${degradedError}`, logTail: logTail2, gitDiff: gitDiff2 }).catch(() => '');
459
506
  try {
460
- await this.deps.rollback();
507
+ const summary = await this.deps.rollback();
508
+ await this.reportRollbackResult(summary, reportPath2);
461
509
  }
462
510
  catch (e) {
463
511
  await this.deps.notify(`rollback failed: ${e?.message ?? String(e)} (report: ${reportPath2})`).catch(() => { });
@@ -485,20 +533,33 @@ export class Supervisor {
485
533
  if (health.up) {
486
534
  this.consecutiveDown = 0;
487
535
  this.consecutiveDegraded = 0;
488
- // Post-self-restart boot proved healthy: clear the suppression marker,
489
- // run the post-restart session-scan hook and re-arm the single-flight
490
- // latch. A failed clear keeps the latch set so the same marker is never
491
- // re-handled into a second restart.
536
+ // D5: a healthy poll is the authoritative "the restart succeeded"
537
+ // signal — clear the suppression marker here instead of waiting for its
538
+ // TTL, so a stale marker can never mute a later real crash.
539
+ let markerClearedThisTick = false;
540
+ try {
541
+ if (this.getCheckPlannedRestart()()) {
542
+ this.getClearPlannedRestart()();
543
+ markerClearedThisTick = true;
544
+ }
545
+ }
546
+ catch { }
547
+ // Post-self-restart boot proved healthy: run the post-restart
548
+ // session-scan hook and re-arm the single-flight latch. A failed clear
549
+ // keeps the latch set so the same marker is never re-handled into a
550
+ // second restart.
492
551
  if (this.awaitingHealthyBoot) {
493
552
  this.awaitingHealthyBoot = false;
494
553
  const req = this.pendingRestartRequest;
495
554
  this.pendingRestartRequest = undefined;
496
- let cleared = false;
497
- try {
498
- this.getClearPlannedRestart()();
499
- cleared = true;
555
+ let cleared = markerClearedThisTick;
556
+ if (!cleared) {
557
+ try {
558
+ this.getClearPlannedRestart()();
559
+ cleared = true;
560
+ }
561
+ catch { }
500
562
  }
501
- catch { }
502
563
  if (cleared)
503
564
  this.restartRequestHandled = false;
504
565
  if (req) {
@@ -506,6 +567,19 @@ export class Supervisor {
506
567
  this.deps.onRestartRequestHandled?.(req);
507
568
  }
508
569
  catch { }
570
+ if (req.callerSessionId) {
571
+ try {
572
+ const listen = this.deps.listenerPid ?? defaultListenerPid;
573
+ (this.deps.writeOutcome ?? defaultWriteOutcome)(req.callerSessionId, {
574
+ state: 'ok',
575
+ oldPid: req.oldPid,
576
+ newPid: listen(3082),
577
+ httpStatus: health?.httpCode ?? 200,
578
+ swappedAt: this.deps.getTime ? this.deps.getTime() : Date.now(),
579
+ });
580
+ }
581
+ catch { }
582
+ }
509
583
  }
510
584
  }
511
585
  // Throttle LKG writes to at most once per 5 minutes
@@ -556,6 +630,13 @@ export class Supervisor {
556
630
  // A lone timed-out poll (e.g. a slow plugin-tree boot) must not trigger
557
631
  // rollback/restart: that restart produces its own transient errors on
558
632
  // the next poll, which would otherwise re-trigger this same path forever.
633
+ // D1/D2: while the boot is unproven only a refused connection may advance
634
+ // the down counter — nothing is listening, so the process is gone. A
635
+ // timeout/abort (or anything unattributable) is a slow boot, not a crash.
636
+ if (health.bootPhase === 'booting' && classifyFetchFailure(health.error) !== 'refused') {
637
+ this.consecutiveDown = 0;
638
+ return;
639
+ }
559
640
  this.consecutiveDown++;
560
641
  let downThreshold = await this.getEffectiveDownThreshold();
561
642
  // When a planned restart marker is active, double the threshold (3→6 at
@@ -589,7 +670,8 @@ export class Supervisor {
589
670
  const gitDiff = await this.collectGitDiff().catch(() => '');
590
671
  const reportPath = await this.deps.writeReport({ ts, health, action: `rollback — ${health.error ?? 'down'}`, logTail, gitDiff }).catch(() => '');
591
672
  try {
592
- await this.deps.rollback();
673
+ const summary = await this.deps.rollback();
674
+ await this.reportRollbackResult(summary, reportPath);
593
675
  }
594
676
  catch (e) {
595
677
  await this.deps.notify(`rollback failed: ${e?.message ?? String(e)} (report: ${reportPath})`).catch(() => { });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ddtcorex/dsh-maestro-supervisor",
3
- "version": "0.7.10",
3
+ "version": "0.8.2",
4
4
  "description": "Supervisor daemon for DSH Web resilience — auto-detect crashes, rollback to LKG, report",
5
5
  "type": "module",
6
6
  "main": "./lib/index.js",