@ddtcorex/dsh-maestro-supervisor 0.8.0 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,7 +12,7 @@
12
12
  * callerSessionId) that the supervisor daemon owns and acts on
13
13
  * (out-of-band). This tool NEVER restarts the host in-tree.
14
14
  */
15
- import { writeRestartRequest } from './restart-guards.js';
15
+ import { writeRestartRequest, readRestartRequest } from './restart-guards.js';
16
16
  /**
17
17
  * Copy a live profile tree for an isolated dry-boot. A naive recursive copy
18
18
  * breaks `link:` installs: their node_modules entries are relative symlinks
@@ -115,6 +115,7 @@ export declare function registerRestartTool(ctx: any, deps?: {
115
115
  sessionIdOf?: (exec: any) => string | undefined;
116
116
  dryBoot?: typeof dryBootVerify;
117
117
  writeRestartRequest?: typeof writeRestartRequest;
118
+ readRestartRequest?: typeof readRestartRequest;
118
119
  harnessRoot?: string;
119
120
  gcReaders?: GcReaders;
120
121
  killPid?: (pid: number, sig: string) => void;
@@ -17,7 +17,7 @@ import { mkdtempSync, rmSync, cpSync, existsSync, readFileSync, readdirSync, sta
17
17
  import { tmpdir, homedir } from 'node:os';
18
18
  import { spawn, execFileSync } from 'node:child_process';
19
19
  import { createRequire } from 'node:module';
20
- import { writeRestartRequest } from './restart-guards.js';
20
+ import { writeRestartRequest, readRestartRequest } from './restart-guards.js';
21
21
  import { intentPath, readIntent, readRestartOutcome } from './intents.js';
22
22
  /**
23
23
  * Copy a live profile tree for an isolated dry-boot. A naive recursive copy
@@ -306,6 +306,7 @@ function currentSessionId(exec, fallback) {
306
306
  export function registerRestartTool(ctx, deps = {}) {
307
307
  const doDryBoot = deps.dryBoot ?? dryBootVerify;
308
308
  const doWrite = deps.writeRestartRequest ?? writeRestartRequest;
309
+ const doRead = deps.readRestartRequest ?? readRestartRequest;
309
310
  const doSessionId = deps.sessionIdOf ?? currentSessionId;
310
311
  let dispose;
311
312
  let disposeDryboot;
@@ -328,6 +329,23 @@ export function registerRestartTool(ctx, deps = {}) {
328
329
  render: (_args, value) => [{ type: 'text', text: value.detail }],
329
330
  },
330
331
  execute: async (args, exec) => {
332
+ // Serialize: a fresh restart-request marker means a restart is
333
+ // already in flight (another session, the daemon, or an earlier
334
+ // call of this tool). Scheduling another one overlaps the ~90s
335
+ // SIGTERM stop and crash-loops on EADDRINUSE — refuse fast, before
336
+ // the dry-boot gate, and point at the status tool instead.
337
+ let inflight;
338
+ try {
339
+ inflight = doRead();
340
+ }
341
+ catch {
342
+ inflight = undefined;
343
+ }
344
+ if (inflight !== undefined) {
345
+ const by = inflight.callerSessionId ?? 'unknown session';
346
+ const why = inflight.reason ? `: ${inflight.reason.slice(0, 120)}` : '';
347
+ return { ok: false, detail: `restart already in progress (requested by ${by}${why}) — check dsh_web_restart_status instead of scheduling another` };
348
+ }
331
349
  const harnessRoot = deps.harnessRoot ?? (await import('./paths.js')).resolveDeepseekHarnessDir();
332
350
  const lkgDir = join(homedir(), '.dsh/.supervisor/lkg');
333
351
  const changed = args.pluginChanged === true || (args.pluginChanged !== false && isPluginTreeChanged(harnessRoot, lkgDir));
@@ -0,0 +1,26 @@
1
+ import { type BootLockDeps } from './boot-lock.js';
2
+ /** Boot budget: how long a boot may take before its failures are judged. */
3
+ export declare const DEFAULT_BOOT_GRACE_MS = 180000;
4
+ export interface RestartWebDeps {
5
+ exec?: (cmd: string, opts?: {
6
+ timeout?: number;
7
+ }) => void;
8
+ writeMarker?: (ttlMs?: number) => void;
9
+ serializedRestart?: () => Promise<void>;
10
+ startViaSystemd?: () => void;
11
+ spawnNohup?: () => void | Promise<void>;
12
+ unitExists?: () => boolean;
13
+ bootGraceMs?: number;
14
+ lock?: BootLockDeps;
15
+ }
16
+ export interface RestartWebResult {
17
+ restarted: boolean;
18
+ reason?: string;
19
+ }
20
+ /**
21
+ * The single implementation of a supervised dsh web restart. Single-flight:
22
+ * boot.lock is held from before the marker is written until the port answers,
23
+ * so a racing tick or a second rollback skips instead of producing the second
24
+ * systemd start that turned the incident's log tail toxic (2026-09-13).
25
+ */
26
+ export declare function performSingleBootRestart(deps?: RestartWebDeps): Promise<RestartWebResult>;
@@ -0,0 +1,58 @@
1
+ import * as os from 'node:os';
2
+ import * as path from 'node:path';
3
+ import { buildKillStalePortsCommand, writePlannedRestart } from './restart-guards.js';
4
+ import { serializedSystemdRestart, shouldUseNohupFallback, systemdUnitExists } from './restart-exec.js';
5
+ import { withBootLock } from './boot-lock.js';
6
+ import { resolveDeepseekHarnessDir } from './paths.js';
7
+ /** Boot budget: how long a boot may take before its failures are judged. */
8
+ export const DEFAULT_BOOT_GRACE_MS = 180_000;
9
+ /**
10
+ * The single implementation of a supervised dsh web restart. Single-flight:
11
+ * boot.lock is held from before the marker is written until the port answers,
12
+ * so a racing tick or a second rollback skips instead of producing the second
13
+ * systemd start that turned the incident's log tail toxic (2026-09-13).
14
+ */
15
+ export async function performSingleBootRestart(deps = {}) {
16
+ const grace = deps.bootGraceMs ?? DEFAULT_BOOT_GRACE_MS;
17
+ const outcome = await withBootLock(async () => {
18
+ // Marker first, and inside the lock: every poll during the boot must know a
19
+ // restart is in flight, and the TTL is the boot budget — not 30 s (D5).
20
+ ;
21
+ (deps.writeMarker ?? writePlannedRestart)(grace);
22
+ const { execSync } = await import('node:child_process');
23
+ const exec = deps.exec ?? ((cmd, opts) => {
24
+ execSync(cmd, { timeout: opts?.timeout ?? 15_000, stdio: 'pipe' });
25
+ });
26
+ // A stale MainThread holding :3080/:3082 survives an EADDRINUSE crash with
27
+ // http 200 still served, so the new start would lose the race.
28
+ try {
29
+ exec(buildKillStalePortsCommand(), { timeout: 5000 });
30
+ }
31
+ catch { }
32
+ try {
33
+ await (deps.serializedRestart ?? (() => serializedSystemdRestart()))();
34
+ return;
35
+ }
36
+ catch { }
37
+ try {
38
+ ;
39
+ (deps.startViaSystemd ?? (() => exec('systemctl --user start dsh-web.service', { timeout: 15_000 })))();
40
+ return;
41
+ }
42
+ catch { }
43
+ const unitExists = (deps.unitExists ?? systemdUnitExists)();
44
+ if (!shouldUseNohupFallback(unitExists)) {
45
+ throw new Error('systemd manages dsh-web.service but start failed — refusing the direct-node fallback (it would create a second boot)');
46
+ }
47
+ await (deps.spawnNohup ?? defaultNohup)();
48
+ }, deps.lock ?? {});
49
+ if (!outcome.acquired)
50
+ return { restarted: false, reason: 'another boot already holds boot.lock' };
51
+ return { restarted: true };
52
+ }
53
+ async function defaultNohup() {
54
+ const { execSync } = await import('node:child_process');
55
+ const harnessRoot = resolveDeepseekHarnessDir();
56
+ const logPath = path.join(os.homedir(), '.dsh/dsh-web.log');
57
+ execSync(`setsid nohup bash -c 'export NVM_DIR="$HOME/.nvm"; [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; cd ${JSON.stringify(harnessRoot)} && exec node --import tsx/esm apps/cli/src/bin.ts web --no-open >> ${JSON.stringify(logPath)} 2>&1' &`, { timeout: 5000 });
58
+ }
package/lib/resume.d.ts CHANGED
@@ -6,6 +6,16 @@ export interface FindInterruptedOpts {
6
6
  withinMs?: number;
7
7
  sinceMs?: number;
8
8
  }
9
+ /**
10
+ * Resolve the readable session log inside one session directory across
11
+ * format generations. `session.v3.jsonl.zstd` is the current successor;
12
+ * `session.jsonl.zstd` / `session.jsonl` remain for committed older
13
+ * generations (migrations never move or delete them). Prefer v3 when
14
+ * present — live sessions persist there, and a scan that only knows the
15
+ * old names silently skips every current session (2026-09-11: auto-resume
16
+ * found nothing after a restart, so no recovery continue was triggered).
17
+ */
18
+ export declare function resolveSessionLogPath(dir: string): string | undefined;
9
19
  export declare function findInterrupted(dshHome?: string, opts?: FindInterruptedOpts): Promise<ResumeResult>;
10
20
  /**
11
21
  * Detect sessions whose raw log ends with a `turn/start` that has no
package/lib/resume.js CHANGED
@@ -1,6 +1,27 @@
1
1
  import * as fs from 'node:fs';
2
2
  import * as path from 'node:path';
3
3
  import * as os from 'node:os';
4
+ /**
5
+ * Resolve the readable session log inside one session directory across
6
+ * format generations. `session.v3.jsonl.zstd` is the current successor;
7
+ * `session.jsonl.zstd` / `session.jsonl` remain for committed older
8
+ * generations (migrations never move or delete them). Prefer v3 when
9
+ * present — live sessions persist there, and a scan that only knows the
10
+ * old names silently skips every current session (2026-09-11: auto-resume
11
+ * found nothing after a restart, so no recovery continue was triggered).
12
+ */
13
+ export function resolveSessionLogPath(dir) {
14
+ const v3 = path.join(dir, 'session.v3.jsonl.zstd');
15
+ if (fs.existsSync(v3))
16
+ return v3;
17
+ const zstd = path.join(dir, 'session.jsonl.zstd');
18
+ if (fs.existsSync(zstd))
19
+ return zstd;
20
+ const jsonl = path.join(dir, 'session.jsonl');
21
+ if (fs.existsSync(jsonl))
22
+ return jsonl;
23
+ return undefined;
24
+ }
4
25
  /**
5
26
  * Read the last ~100 lines of one session's raw log, applying the mtime
6
27
  * pre-filter before any (potentially expensive) zstd decompression: a
@@ -13,27 +34,26 @@ import * as os from 'node:os';
13
34
  * @returns `undefined` when the session has no log file, or is filtered
14
35
  * out by `sinceMs` — callers must treat that the same as "nothing found".
15
36
  */
16
- async function readSessionTailLines(zstdPath, jsonlPath, sinceMs) {
37
+ async function readSessionTailLines(logPath, sinceMs) {
38
+ if (logPath === undefined)
39
+ return undefined;
17
40
  if (sinceMs !== undefined) {
18
41
  try {
19
- const statPath = fs.existsSync(zstdPath) ? zstdPath : (fs.existsSync(jsonlPath) ? jsonlPath : undefined);
20
- if (statPath) {
21
- const mtimeMs = fs.statSync(statPath).mtimeMs;
22
- if (mtimeMs < sinceMs)
23
- return undefined;
24
- }
42
+ if (fs.statSync(logPath).mtimeMs < sinceMs)
43
+ return undefined;
25
44
  }
26
45
  catch { }
27
46
  }
28
- if (fs.existsSync(zstdPath)) {
47
+ if (logPath.endsWith('.zstd')) {
29
48
  const { execSync } = await import('node:child_process');
30
- const out = execSync(`zstd -d -c ${JSON.stringify(zstdPath)} 2>/dev/null | tail -100`, { encoding: 'utf-8' });
49
+ const out = execSync(`zstd -d -c ${JSON.stringify(logPath)} 2>/dev/null | tail -100`, { encoding: 'utf-8' });
31
50
  return out.split('\n').filter(Boolean);
32
51
  }
33
- if (fs.existsSync(jsonlPath)) {
34
- const content = fs.readFileSync(jsonlPath, 'utf-8');
52
+ try {
53
+ const content = fs.readFileSync(logPath, 'utf-8');
35
54
  return content.trim().split('\n').slice(-100);
36
55
  }
56
+ catch { }
37
57
  return undefined;
38
58
  }
39
59
  export async function findInterrupted(dshHome, opts) {
@@ -55,10 +75,9 @@ export async function findInterrupted(dshHome, opts) {
55
75
  if (!s.isDirectory())
56
76
  continue;
57
77
  scanned++;
58
- const zstdPath = path.join(groupPath, s.name, 'session.jsonl.zstd');
59
- const jsonlPath = path.join(groupPath, s.name, 'session.jsonl');
78
+ const logPath = resolveSessionLogPath(path.join(groupPath, s.name));
60
79
  try {
61
- const lines = await readSessionTailLines(zstdPath, jsonlPath, sinceMs);
80
+ const lines = await readSessionTailLines(logPath, sinceMs);
62
81
  if (lines === undefined)
63
82
  continue;
64
83
  let found = false;
@@ -103,31 +122,30 @@ export async function findInterrupted(dshHome, opts) {
103
122
  * The mtime pre-filter ensures this full decompression runs only for
104
123
  * recent sessions (within 5m, typically 1-2 files), not for all 425.
105
124
  */
106
- async function readSessionAllLines(zstdPath, jsonlPath, sinceMs) {
125
+ async function readSessionAllLines(logPath, sinceMs) {
126
+ if (logPath === undefined)
127
+ return undefined;
107
128
  if (sinceMs !== undefined) {
108
129
  try {
109
- const statPath = fs.existsSync(zstdPath) ? zstdPath : (fs.existsSync(jsonlPath) ? jsonlPath : undefined);
110
- if (statPath) {
111
- const mtimeMs = fs.statSync(statPath).mtimeMs;
112
- if (mtimeMs < sinceMs)
113
- return undefined;
114
- }
130
+ if (fs.statSync(logPath).mtimeMs < sinceMs)
131
+ return undefined;
115
132
  }
116
133
  catch { }
117
134
  }
118
- if (fs.existsSync(zstdPath)) {
135
+ if (logPath.endsWith('.zstd')) {
119
136
  const { execSync } = await import('node:child_process');
120
137
  // maxBuffer must exceed the decompressed size of any real session log —
121
138
  // worker sessions decode to 8-23MB while execSync's default 1MB would
122
139
  // throw ENOBUFS and silently drop the session from every scan that needs
123
140
  // the full file (findDanglingOpenTurns). Use 64MB to leave headroom.
124
- const out = execSync(`zstd -d -c ${JSON.stringify(zstdPath)} 2>/dev/null`, { encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024 });
141
+ const out = execSync(`zstd -d -c ${JSON.stringify(logPath)} 2>/dev/null`, { encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024 });
125
142
  return out.split('\n').filter(Boolean);
126
143
  }
127
- if (fs.existsSync(jsonlPath)) {
128
- const content = fs.readFileSync(jsonlPath, 'utf-8');
144
+ try {
145
+ const content = fs.readFileSync(logPath, 'utf-8');
129
146
  return content.trim().split('\n').filter(Boolean);
130
147
  }
148
+ catch { }
131
149
  return undefined;
132
150
  }
133
151
  /**
@@ -162,10 +180,9 @@ export async function findDanglingOpenTurns(dshHome, opts) {
162
180
  if (!s.isDirectory())
163
181
  continue;
164
182
  scanned++;
165
- const zstdPath = path.join(groupPath, s.name, 'session.jsonl.zstd');
166
- const jsonlPath = path.join(groupPath, s.name, 'session.jsonl');
183
+ const logPath = resolveSessionLogPath(path.join(groupPath, s.name));
167
184
  try {
168
- const lines = await readSessionAllLines(zstdPath, jsonlPath, sinceMs);
185
+ const lines = await readSessionAllLines(logPath, sinceMs);
169
186
  if (lines === undefined)
170
187
  continue;
171
188
  let openTurn;
package/lib/snapshot.d.ts CHANGED
@@ -1,22 +1,59 @@
1
- interface ManifestEntry {
2
- path: string;
3
- sha256: string;
4
- }
5
- interface Manifest {
1
+ export interface SnapshotResult {
6
2
  ts: string;
7
- files: ManifestEntry[];
3
+ /** Files (and symlinks) copied into this snapshot. */
4
+ files: number;
5
+ /** Entries that could not be copied; the snapshot stays usable either way. */
6
+ skipped: Array<{
7
+ path: string;
8
+ reason: string;
9
+ }>;
8
10
  }
9
- export declare function writeLKG(dshHome: string, lkgRoot: string): Promise<{
10
- ts: string;
11
- manifest: Manifest;
12
- }>;
11
+ export interface SnapshotDeps {
12
+ /**
13
+ * Copy one regular file, preserving `mode`. Injectable so a failing entry
14
+ * (the EACCES a read-only attachment object produces) can be table-tested
15
+ * without depending on the host's uid or filesystem quirks.
16
+ */
17
+ copyFile?: (src: string, dest: string, mode: number) => void;
18
+ }
19
+ /**
20
+ * DSH-home entries the last-known-good snapshot deliberately never copies.
21
+ *
22
+ * The LKG exists to recover a boot that fails while loading the plugin tree, so
23
+ * it holds boot **configuration**: `profiles/` (the plugin tree with its
24
+ * lockfile, `cordis.patch.yml` and sidecars), the per-plugin config directories
25
+ * and the settings documents. Everything below is runtime **data** — it is
26
+ * written continuously while `dsh web` runs, so restoring a snapshot of it can
27
+ * only lose newer state, and some of it is hostile to a bulk copy:
28
+ *
29
+ * - `sessions/` — append-only transcripts. Restoring a stale copy over live
30
+ * sessions drops every turn recorded after the snapshot, i.e. the recovery
31
+ * path would lose the log it is supposed to protect.
32
+ * - `attachments/` — content-addressed blobs stored mode `0400`. That read-only
33
+ * bit is exactly what made the 2026-09-13 rollback abort with
34
+ * `EACCES: Permission denied '.../attachments/v1/objects/f8'`.
35
+ * - `plugins-src/` — plugin source cache, re-fetched on demand (~400 MB host).
36
+ * - `.supervisor/` — the LKG root itself lives inside the DSH home, so copying
37
+ * it would recurse into every retained snapshot.
38
+ *
39
+ * This list is data, not a heuristic: the copy loop and the restore loop both
40
+ * consult `isLkgExcluded()`, and the table test pins the rule.
41
+ */
42
+ export declare const LKG_EXCLUDED_ENTRIES: readonly string[];
43
+ /**
44
+ * True when a DSH-home-relative path must never enter (or leave) the LKG.
45
+ *
46
+ * The named entries match the first path segment; `*.log` matches by basename
47
+ * anywhere, because append-only logs are data at any depth and a restored stale
48
+ * `dsh-web.log` would poison the boot-boundary scan.
49
+ */
50
+ export declare function isLkgExcluded(relPath: string): boolean;
51
+ /** Human-readable, bounded reason for a per-entry copy failure. */
52
+ export declare function failureReason(e: unknown): string;
53
+ export declare function writeLKG(dshHome: string, lkgRoot: string, deps?: SnapshotDeps): Promise<SnapshotResult>;
13
54
  export declare function pruneByAge(root: string, maxAgeMs: number): Promise<void>;
14
55
  export declare function pruneBySize(root: string, maxBytes: number): Promise<void>;
15
56
  export declare function isDuplicateLKG(dshHome: string, lkgRoot: string): Promise<boolean>;
16
57
  export declare function verifyLKG(lkgPath: string): Promise<boolean>;
17
58
  export declare function rotateLKG(lkgRoot: string, keep?: number): Promise<void>;
18
- export declare function writeFailed(dshHome: string, failedRoot: string): Promise<{
19
- ts: string;
20
- manifest: Manifest;
21
- }>;
22
- export {};
59
+ export declare function writeFailed(dshHome: string, failedRoot: string): Promise<SnapshotResult>;
package/lib/snapshot.js CHANGED
@@ -1,6 +1,51 @@
1
1
  import * as fs from 'node:fs';
2
2
  import * as path from 'node:path';
3
3
  import * as crypto from 'node:crypto';
4
+ /**
5
+ * DSH-home entries the last-known-good snapshot deliberately never copies.
6
+ *
7
+ * The LKG exists to recover a boot that fails while loading the plugin tree, so
8
+ * it holds boot **configuration**: `profiles/` (the plugin tree with its
9
+ * lockfile, `cordis.patch.yml` and sidecars), the per-plugin config directories
10
+ * and the settings documents. Everything below is runtime **data** — it is
11
+ * written continuously while `dsh web` runs, so restoring a snapshot of it can
12
+ * only lose newer state, and some of it is hostile to a bulk copy:
13
+ *
14
+ * - `sessions/` — append-only transcripts. Restoring a stale copy over live
15
+ * sessions drops every turn recorded after the snapshot, i.e. the recovery
16
+ * path would lose the log it is supposed to protect.
17
+ * - `attachments/` — content-addressed blobs stored mode `0400`. That read-only
18
+ * bit is exactly what made the 2026-09-13 rollback abort with
19
+ * `EACCES: Permission denied '.../attachments/v1/objects/f8'`.
20
+ * - `plugins-src/` — plugin source cache, re-fetched on demand (~400 MB host).
21
+ * - `.supervisor/` — the LKG root itself lives inside the DSH home, so copying
22
+ * it would recurse into every retained snapshot.
23
+ *
24
+ * This list is data, not a heuristic: the copy loop and the restore loop both
25
+ * consult `isLkgExcluded()`, and the table test pins the rule.
26
+ */
27
+ export const LKG_EXCLUDED_ENTRIES = [
28
+ 'sessions',
29
+ 'attachments',
30
+ 'plugins-src',
31
+ '.supervisor',
32
+ ];
33
+ /**
34
+ * True when a DSH-home-relative path must never enter (or leave) the LKG.
35
+ *
36
+ * The named entries match the first path segment; `*.log` matches by basename
37
+ * anywhere, because append-only logs are data at any depth and a restored stale
38
+ * `dsh-web.log` would poison the boot-boundary scan.
39
+ */
40
+ export function isLkgExcluded(relPath) {
41
+ const normalized = relPath.replace(/\\/g, '/').replace(/^\.\//, '').replace(/^\/+/, '');
42
+ const segments = normalized.split('/').filter(s => s.length > 0 && s !== '.');
43
+ if (!segments.length)
44
+ return false;
45
+ if (LKG_EXCLUDED_ENTRIES.includes(segments[0]))
46
+ return true;
47
+ return segments[segments.length - 1].endsWith('.log');
48
+ }
4
49
  function sha256File(filePath) {
5
50
  const data = fs.readFileSync(filePath);
6
51
  return crypto.createHash('sha256').update(data).digest('hex');
@@ -16,7 +61,78 @@ function walkFiles(dir, base = dir) {
16
61
  }
17
62
  return out;
18
63
  }
19
- export async function writeLKG(dshHome, lkgRoot) {
64
+ /** Human-readable, bounded reason for a per-entry copy failure. */
65
+ export function failureReason(e) {
66
+ const err = e;
67
+ const code = typeof err?.code === 'string' && err.code ? `${err.code}: ` : '';
68
+ const message = typeof err?.message === 'string' ? err.message : String(e);
69
+ return `${code}${message}`.slice(0, 300);
70
+ }
71
+ /**
72
+ * Copy one snapshot entry, collecting — never throwing — per-entry failures
73
+ * (D2). `fs.cpSync` aborts the whole snapshot on the first unreadable object,
74
+ * which is precisely how one 0400 attachment file stopped the rollback that
75
+ * exists to rescue a broken boot.
76
+ */
77
+ function copyEntry(state, src, dest, rel) {
78
+ if (isLkgExcluded(rel))
79
+ return;
80
+ let st;
81
+ try {
82
+ st = fs.lstatSync(src);
83
+ }
84
+ catch (e) {
85
+ state.skipped.push({ path: rel, reason: failureReason(e) });
86
+ return;
87
+ }
88
+ if (st.isDirectory()) {
89
+ let names;
90
+ try {
91
+ // Directory modes are replicated only in their permission-to-traverse
92
+ // sense: the snapshot itself must stay readable/removable even when the
93
+ // source directory is not (a 0000 source dir must not produce a snapshot
94
+ // that nothing — including retention — can delete).
95
+ fs.mkdirSync(dest, { recursive: true, mode: (st.mode & 0o7777) | 0o700 });
96
+ names = fs.readdirSync(src);
97
+ }
98
+ catch (e) {
99
+ state.skipped.push({ path: rel, reason: failureReason(e) });
100
+ return;
101
+ }
102
+ for (const name of names) {
103
+ copyEntry(state, path.join(src, name), path.join(dest, name), rel ? `${rel}/${name}` : name);
104
+ }
105
+ return;
106
+ }
107
+ if (st.isSymbolicLink()) {
108
+ try {
109
+ const link = fs.readlinkSync(src);
110
+ fs.mkdirSync(path.dirname(dest), { recursive: true });
111
+ try {
112
+ fs.unlinkSync(dest);
113
+ }
114
+ catch { }
115
+ fs.symlinkSync(link, dest);
116
+ state.copied++;
117
+ }
118
+ catch (e) {
119
+ state.skipped.push({ path: rel, reason: failureReason(e) });
120
+ }
121
+ return;
122
+ }
123
+ if (st.isFile()) {
124
+ try {
125
+ fs.mkdirSync(path.dirname(dest), { recursive: true });
126
+ state.copyFile(src, dest, st.mode & 0o7777);
127
+ state.copied++;
128
+ }
129
+ catch (e) {
130
+ state.skipped.push({ path: rel, reason: failureReason(e) });
131
+ }
132
+ }
133
+ // Sockets/FIFOs/devices are not boot configuration: ignored on purpose.
134
+ }
135
+ export async function writeLKG(dshHome, lkgRoot, deps = {}) {
20
136
  // Dedupe: skip snapshot if current state identical to latest LKG (prevents 5-min unconditional growth)
21
137
  try {
22
138
  if (await isDuplicateLKG(dshHome, lkgRoot)) {
@@ -31,36 +147,68 @@ export async function writeLKG(dshHome, lkgRoot) {
31
147
  const latest = entries[entries.length - 1];
32
148
  const manifestPath = path.join(lkgRoot, latest, 'manifest.json');
33
149
  const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
34
- return { ts: latest, manifest };
150
+ return { ts: latest, files: manifest.files?.length ?? 0, skipped: [] };
35
151
  }
36
152
  }
37
153
  catch { }
38
154
  const ts = new Date().toISOString().replace(/[:.]/g, '-');
39
155
  const dest = path.join(lkgRoot, ts);
40
- fs.mkdirSync(dest, { recursive: true });
41
- // Copy DSH home contents (if exists, copy recursively) skip .supervisor to avoid recursion
42
- if (fs.existsSync(dshHome)) {
43
- for (const entry of fs.readdirSync(dshHome)) {
44
- if (entry === '.supervisor')
45
- continue;
46
- const src = path.join(dshHome, entry);
47
- const dst = path.join(dest, entry);
48
- fs.cpSync(src, dst, { recursive: true });
49
- }
50
- }
51
- const files = fs.existsSync(dest) ? walkFiles(dest) : [];
156
+ try {
157
+ fs.mkdirSync(dest, { recursive: true });
158
+ }
159
+ catch (e) {
160
+ // No snapshot is better than an exception thrown into the rollback path.
161
+ return { ts, files: 0, skipped: [{ path: lkgRoot, reason: failureReason(e) }] };
162
+ }
163
+ const state = {
164
+ dshHome,
165
+ copyFile: deps.copyFile ?? ((src, dst, mode) => {
166
+ fs.copyFileSync(src, dst);
167
+ try {
168
+ fs.chmodSync(dst, mode);
169
+ }
170
+ catch { }
171
+ }),
172
+ copied: 0,
173
+ skipped: [],
174
+ };
175
+ // Copy only the boot configuration (D1), one entry at a time so a single
176
+ // unreadable file is recorded and skipped instead of aborting the snapshot (D2).
177
+ let topLevel = [];
178
+ try {
179
+ if (fs.existsSync(dshHome))
180
+ topLevel = fs.readdirSync(dshHome);
181
+ }
182
+ catch (e) {
183
+ state.skipped.push({ path: '.', reason: failureReason(e) });
184
+ }
185
+ for (const entry of topLevel) {
186
+ copyEntry(state, path.join(dshHome, entry), path.join(dest, entry), entry);
187
+ }
188
+ let fileList = [];
189
+ try {
190
+ fileList = fs.existsSync(dest) ? walkFiles(dest) : [];
191
+ }
192
+ catch (e) {
193
+ state.skipped.push({ path: 'manifest', reason: failureReason(e) });
194
+ }
52
195
  const manifest = {
53
196
  ts,
54
- files: files
197
+ files: fileList
55
198
  .filter(f => f !== 'manifest.json')
56
199
  .map(f => ({ path: f, sha256: sha256File(path.join(dest, f)) })),
57
200
  };
58
- fs.writeFileSync(path.join(dest, 'manifest.json'), JSON.stringify(manifest, null, 2));
201
+ try {
202
+ fs.writeFileSync(path.join(dest, 'manifest.json'), JSON.stringify(manifest, null, 2));
203
+ }
204
+ catch (e) {
205
+ state.skipped.push({ path: 'manifest.json', reason: failureReason(e) });
206
+ }
59
207
  // Retention: keep only 3 most recent, plus age (7d) and size (5GB) caps — prevents unbounded 40GB+ growth
60
208
  await rotateLKG(lkgRoot, 3).catch(() => { });
61
209
  await pruneByAge(lkgRoot, 7 * 24 * 60 * 60 * 1000).catch(() => { });
62
210
  await pruneBySize(lkgRoot, 5 * 1024 * 1024 * 1024).catch(() => { });
63
- return { ts, manifest };
211
+ return { ts, files: state.copied, skipped: state.skipped };
64
212
  }
65
213
  export async function pruneByAge(root, maxAgeMs) {
66
214
  if (!fs.existsSync(root))
@@ -150,7 +298,9 @@ export async function isDuplicateLKG(dshHome, lkgRoot) {
150
298
  let newestFileMtime = 0;
151
299
  if (fs.existsSync(dshHome)) {
152
300
  for (const entry of fs.readdirSync(dshHome)) {
153
- if (entry === '.supervisor')
301
+ // Same scope as the copy loop: runtime data changes constantly and
302
+ // must not defeat the dedupe for the configuration being snapshotted.
303
+ if (isLkgExcluded(entry))
154
304
  continue;
155
305
  try {
156
306
  const s = fs.statSync(path.join(dshHome, entry));
@@ -1,16 +1,25 @@
1
- import type { HealthState } from './health-poller.js';
1
+ import { type HealthState } from './health-poller.js';
2
2
  import type { RestartRequest } from './restart-guards.js';
3
3
  import { type RestartOutcome } from './intents.js';
4
4
  import { type MintCookieOpts } from './dsh-session.js';
5
+ /** Partial-restore report a rollback may hand back (cli.ts wires RestoreResult). */
6
+ export interface RollbackSummary {
7
+ target?: string;
8
+ restored?: number;
9
+ skipped?: Array<{
10
+ path: string;
11
+ reason: string;
12
+ }>;
13
+ }
5
14
  export interface SupervisorDeps {
6
15
  pollHealth: () => Promise<HealthState>;
7
16
  writeLKG: () => Promise<{
8
17
  ts: string;
9
- manifest: any;
18
+ manifest?: any;
10
19
  }>;
11
20
  writeFailed: () => Promise<{
12
21
  ts: string;
13
- manifest: any;
22
+ manifest?: any;
14
23
  }>;
15
24
  writeReport: (opts: {
16
25
  ts: string;
@@ -19,12 +28,13 @@ export interface SupervisorDeps {
19
28
  logTail?: string;
20
29
  gitDiff?: string;
21
30
  }) => Promise<string>;
22
- rollback: (ts?: string) => Promise<void>;
31
+ rollback: (ts?: string) => Promise<RollbackSummary | void>;
23
32
  restartWeb?: () => Promise<void>;
24
33
  notify: (msg: string) => Promise<void>;
25
34
  intervalMs?: number;
26
35
  debounceMs?: number;
27
36
  downThreshold?: number;
37
+ bootGraceMs?: number;
28
38
  getTime?: () => number;
29
39
  isPlannedRestartActive?: () => boolean | Promise<boolean>;
30
40
  writePlannedRestart?: (ttlMs?: number) => void;
@@ -91,9 +101,16 @@ export declare class Supervisor {
91
101
  private getEffectiveDownThreshold;
92
102
  private getEffectiveDegradedThreshold;
93
103
  private getEffectivePollTimeoutMs;
104
+ private getEffectiveBootGraceMs;
94
105
  private findInterruptedRecent;
95
106
  private collectGitDiff;
96
107
  private attemptAutoResume;
108
+ /**
109
+ * D4: a restore that could not put every entry back must be surfaced loudly.
110
+ * Without this, a half-restored tree reads exactly like a clean recovery in
111
+ * the log and in the operator's Telegram feed.
112
+ */
113
+ private reportRollbackResult;
97
114
  private handleDebugResult;
98
115
  tick(): Promise<void>;
99
116
  start(): Promise<void>;