@ddtcorex/dsh-maestro-supervisor 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/boot-guard.d.ts +33 -0
- package/lib/boot-guard.js +68 -0
- package/lib/boot-lock.d.ts +56 -0
- package/lib/boot-lock.js +155 -0
- package/lib/cli.d.ts +22 -5
- package/lib/cli.js +120 -54
- package/lib/health-poller.d.ts +68 -1
- package/lib/health-poller.js +191 -133
- package/lib/plugin.js +15 -16
- package/lib/restart-exec.d.ts +35 -0
- package/lib/restart-exec.js +60 -0
- package/lib/restart-guards.d.ts +18 -0
- package/lib/restart-guards.js +33 -0
- package/lib/restart-tool.d.ts +2 -1
- package/lib/restart-tool.js +19 -1
- package/lib/restart-web.d.ts +26 -0
- package/lib/restart-web.js +58 -0
- package/lib/resume.d.ts +10 -0
- package/lib/resume.js +45 -28
- package/lib/snapshot.d.ts +52 -15
- package/lib/snapshot.js +168 -18
- package/lib/supervisor.d.ts +21 -4
- package/lib/supervisor.js +78 -36
- package/package.json +1 -1
- package/skills/dsh-safe-restart/SKILL.md +32 -2
- package/skills/dsh-safe-restart/scripts/restart-dsh-web.sh +162 -8
package/lib/restart-tool.d.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* callerSessionId) that the supervisor daemon owns and acts on
|
|
13
13
|
* (out-of-band). This tool NEVER restarts the host in-tree.
|
|
14
14
|
*/
|
|
15
|
-
import { writeRestartRequest } from './restart-guards.js';
|
|
15
|
+
import { writeRestartRequest, readRestartRequest } from './restart-guards.js';
|
|
16
16
|
/**
|
|
17
17
|
* Copy a live profile tree for an isolated dry-boot. A naive recursive copy
|
|
18
18
|
* breaks `link:` installs: their node_modules entries are relative symlinks
|
|
@@ -115,6 +115,7 @@ export declare function registerRestartTool(ctx: any, deps?: {
|
|
|
115
115
|
sessionIdOf?: (exec: any) => string | undefined;
|
|
116
116
|
dryBoot?: typeof dryBootVerify;
|
|
117
117
|
writeRestartRequest?: typeof writeRestartRequest;
|
|
118
|
+
readRestartRequest?: typeof readRestartRequest;
|
|
118
119
|
harnessRoot?: string;
|
|
119
120
|
gcReaders?: GcReaders;
|
|
120
121
|
killPid?: (pid: number, sig: string) => void;
|
package/lib/restart-tool.js
CHANGED
|
@@ -17,7 +17,7 @@ import { mkdtempSync, rmSync, cpSync, existsSync, readFileSync, readdirSync, sta
|
|
|
17
17
|
import { tmpdir, homedir } from 'node:os';
|
|
18
18
|
import { spawn, execFileSync } from 'node:child_process';
|
|
19
19
|
import { createRequire } from 'node:module';
|
|
20
|
-
import { writeRestartRequest } from './restart-guards.js';
|
|
20
|
+
import { writeRestartRequest, readRestartRequest } from './restart-guards.js';
|
|
21
21
|
import { intentPath, readIntent, readRestartOutcome } from './intents.js';
|
|
22
22
|
/**
|
|
23
23
|
* Copy a live profile tree for an isolated dry-boot. A naive recursive copy
|
|
@@ -306,6 +306,7 @@ function currentSessionId(exec, fallback) {
|
|
|
306
306
|
export function registerRestartTool(ctx, deps = {}) {
|
|
307
307
|
const doDryBoot = deps.dryBoot ?? dryBootVerify;
|
|
308
308
|
const doWrite = deps.writeRestartRequest ?? writeRestartRequest;
|
|
309
|
+
const doRead = deps.readRestartRequest ?? readRestartRequest;
|
|
309
310
|
const doSessionId = deps.sessionIdOf ?? currentSessionId;
|
|
310
311
|
let dispose;
|
|
311
312
|
let disposeDryboot;
|
|
@@ -328,6 +329,23 @@ export function registerRestartTool(ctx, deps = {}) {
|
|
|
328
329
|
render: (_args, value) => [{ type: 'text', text: value.detail }],
|
|
329
330
|
},
|
|
330
331
|
execute: async (args, exec) => {
|
|
332
|
+
// Serialize: a fresh restart-request marker means a restart is
|
|
333
|
+
// already in flight (another session, the daemon, or an earlier
|
|
334
|
+
// call of this tool). Scheduling another one overlaps the ~90s
|
|
335
|
+
// SIGTERM stop and crash-loops on EADDRINUSE — refuse fast, before
|
|
336
|
+
// the dry-boot gate, and point at the status tool instead.
|
|
337
|
+
let inflight;
|
|
338
|
+
try {
|
|
339
|
+
inflight = doRead();
|
|
340
|
+
}
|
|
341
|
+
catch {
|
|
342
|
+
inflight = undefined;
|
|
343
|
+
}
|
|
344
|
+
if (inflight !== undefined) {
|
|
345
|
+
const by = inflight.callerSessionId ?? 'unknown session';
|
|
346
|
+
const why = inflight.reason ? `: ${inflight.reason.slice(0, 120)}` : '';
|
|
347
|
+
return { ok: false, detail: `restart already in progress (requested by ${by}${why}) — check dsh_web_restart_status instead of scheduling another` };
|
|
348
|
+
}
|
|
331
349
|
const harnessRoot = deps.harnessRoot ?? (await import('./paths.js')).resolveDeepseekHarnessDir();
|
|
332
350
|
const lkgDir = join(homedir(), '.dsh/.supervisor/lkg');
|
|
333
351
|
const changed = args.pluginChanged === true || (args.pluginChanged !== false && isPluginTreeChanged(harnessRoot, lkgDir));
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type BootLockDeps } from './boot-lock.js';
|
|
2
|
+
/** Boot budget: how long a boot may take before its failures are judged. */
|
|
3
|
+
export declare const DEFAULT_BOOT_GRACE_MS = 180000;
|
|
4
|
+
export interface RestartWebDeps {
|
|
5
|
+
exec?: (cmd: string, opts?: {
|
|
6
|
+
timeout?: number;
|
|
7
|
+
}) => void;
|
|
8
|
+
writeMarker?: (ttlMs?: number) => void;
|
|
9
|
+
serializedRestart?: () => Promise<void>;
|
|
10
|
+
startViaSystemd?: () => void;
|
|
11
|
+
spawnNohup?: () => void | Promise<void>;
|
|
12
|
+
unitExists?: () => boolean;
|
|
13
|
+
bootGraceMs?: number;
|
|
14
|
+
lock?: BootLockDeps;
|
|
15
|
+
}
|
|
16
|
+
export interface RestartWebResult {
|
|
17
|
+
restarted: boolean;
|
|
18
|
+
reason?: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The single implementation of a supervised dsh web restart. Single-flight:
|
|
22
|
+
* boot.lock is held from before the marker is written until the port answers,
|
|
23
|
+
* so a racing tick or a second rollback skips instead of producing the second
|
|
24
|
+
* systemd start that turned the incident's log tail toxic (2026-09-13).
|
|
25
|
+
*/
|
|
26
|
+
export declare function performSingleBootRestart(deps?: RestartWebDeps): Promise<RestartWebResult>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import * as os from 'node:os';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { buildKillStalePortsCommand, writePlannedRestart } from './restart-guards.js';
|
|
4
|
+
import { serializedSystemdRestart, shouldUseNohupFallback, systemdUnitExists } from './restart-exec.js';
|
|
5
|
+
import { withBootLock } from './boot-lock.js';
|
|
6
|
+
import { resolveDeepseekHarnessDir } from './paths.js';
|
|
7
|
+
/** Boot budget: how long a boot may take before its failures are judged. */
|
|
8
|
+
export const DEFAULT_BOOT_GRACE_MS = 180_000;
|
|
9
|
+
/**
|
|
10
|
+
* The single implementation of a supervised dsh web restart. Single-flight:
|
|
11
|
+
* boot.lock is held from before the marker is written until the port answers,
|
|
12
|
+
* so a racing tick or a second rollback skips instead of producing the second
|
|
13
|
+
* systemd start that turned the incident's log tail toxic (2026-09-13).
|
|
14
|
+
*/
|
|
15
|
+
export async function performSingleBootRestart(deps = {}) {
|
|
16
|
+
const grace = deps.bootGraceMs ?? DEFAULT_BOOT_GRACE_MS;
|
|
17
|
+
const outcome = await withBootLock(async () => {
|
|
18
|
+
// Marker first, and inside the lock: every poll during the boot must know a
|
|
19
|
+
// restart is in flight, and the TTL is the boot budget — not 30 s (D5).
|
|
20
|
+
;
|
|
21
|
+
(deps.writeMarker ?? writePlannedRestart)(grace);
|
|
22
|
+
const { execSync } = await import('node:child_process');
|
|
23
|
+
const exec = deps.exec ?? ((cmd, opts) => {
|
|
24
|
+
execSync(cmd, { timeout: opts?.timeout ?? 15_000, stdio: 'pipe' });
|
|
25
|
+
});
|
|
26
|
+
// A stale MainThread holding :3080/:3082 survives an EADDRINUSE crash with
|
|
27
|
+
// http 200 still served, so the new start would lose the race.
|
|
28
|
+
try {
|
|
29
|
+
exec(buildKillStalePortsCommand(), { timeout: 5000 });
|
|
30
|
+
}
|
|
31
|
+
catch { }
|
|
32
|
+
try {
|
|
33
|
+
await (deps.serializedRestart ?? (() => serializedSystemdRestart()))();
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
catch { }
|
|
37
|
+
try {
|
|
38
|
+
;
|
|
39
|
+
(deps.startViaSystemd ?? (() => exec('systemctl --user start dsh-web.service', { timeout: 15_000 })))();
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
catch { }
|
|
43
|
+
const unitExists = (deps.unitExists ?? systemdUnitExists)();
|
|
44
|
+
if (!shouldUseNohupFallback(unitExists)) {
|
|
45
|
+
throw new Error('systemd manages dsh-web.service but start failed — refusing the direct-node fallback (it would create a second boot)');
|
|
46
|
+
}
|
|
47
|
+
await (deps.spawnNohup ?? defaultNohup)();
|
|
48
|
+
}, deps.lock ?? {});
|
|
49
|
+
if (!outcome.acquired)
|
|
50
|
+
return { restarted: false, reason: 'another boot already holds boot.lock' };
|
|
51
|
+
return { restarted: true };
|
|
52
|
+
}
|
|
53
|
+
async function defaultNohup() {
|
|
54
|
+
const { execSync } = await import('node:child_process');
|
|
55
|
+
const harnessRoot = resolveDeepseekHarnessDir();
|
|
56
|
+
const logPath = path.join(os.homedir(), '.dsh/dsh-web.log');
|
|
57
|
+
execSync(`setsid nohup bash -c 'export NVM_DIR="$HOME/.nvm"; [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; cd ${JSON.stringify(harnessRoot)} && exec node --import tsx/esm apps/cli/src/bin.ts web --no-open >> ${JSON.stringify(logPath)} 2>&1' &`, { timeout: 5000 });
|
|
58
|
+
}
|
package/lib/resume.d.ts
CHANGED
|
@@ -6,6 +6,16 @@ export interface FindInterruptedOpts {
|
|
|
6
6
|
withinMs?: number;
|
|
7
7
|
sinceMs?: number;
|
|
8
8
|
}
|
|
9
|
+
/**
|
|
10
|
+
* Resolve the readable session log inside one session directory across
|
|
11
|
+
* format generations. `session.v3.jsonl.zstd` is the current successor;
|
|
12
|
+
* `session.jsonl.zstd` / `session.jsonl` remain for committed older
|
|
13
|
+
* generations (migrations never move or delete them). Prefer v3 when
|
|
14
|
+
* present — live sessions persist there, and a scan that only knows the
|
|
15
|
+
* old names silently skips every current session (2026-09-11: auto-resume
|
|
16
|
+
* found nothing after a restart, so no recovery continue was triggered).
|
|
17
|
+
*/
|
|
18
|
+
export declare function resolveSessionLogPath(dir: string): string | undefined;
|
|
9
19
|
export declare function findInterrupted(dshHome?: string, opts?: FindInterruptedOpts): Promise<ResumeResult>;
|
|
10
20
|
/**
|
|
11
21
|
* Detect sessions whose raw log ends with a `turn/start` that has no
|
package/lib/resume.js
CHANGED
|
@@ -1,6 +1,27 @@
|
|
|
1
1
|
import * as fs from 'node:fs';
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import * as os from 'node:os';
|
|
4
|
+
/**
|
|
5
|
+
* Resolve the readable session log inside one session directory across
|
|
6
|
+
* format generations. `session.v3.jsonl.zstd` is the current successor;
|
|
7
|
+
* `session.jsonl.zstd` / `session.jsonl` remain for committed older
|
|
8
|
+
* generations (migrations never move or delete them). Prefer v3 when
|
|
9
|
+
* present — live sessions persist there, and a scan that only knows the
|
|
10
|
+
* old names silently skips every current session (2026-09-11: auto-resume
|
|
11
|
+
* found nothing after a restart, so no recovery continue was triggered).
|
|
12
|
+
*/
|
|
13
|
+
export function resolveSessionLogPath(dir) {
|
|
14
|
+
const v3 = path.join(dir, 'session.v3.jsonl.zstd');
|
|
15
|
+
if (fs.existsSync(v3))
|
|
16
|
+
return v3;
|
|
17
|
+
const zstd = path.join(dir, 'session.jsonl.zstd');
|
|
18
|
+
if (fs.existsSync(zstd))
|
|
19
|
+
return zstd;
|
|
20
|
+
const jsonl = path.join(dir, 'session.jsonl');
|
|
21
|
+
if (fs.existsSync(jsonl))
|
|
22
|
+
return jsonl;
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
4
25
|
/**
|
|
5
26
|
* Read the last ~100 lines of one session's raw log, applying the mtime
|
|
6
27
|
* pre-filter before any (potentially expensive) zstd decompression: a
|
|
@@ -13,27 +34,26 @@ import * as os from 'node:os';
|
|
|
13
34
|
* @returns `undefined` when the session has no log file, or is filtered
|
|
14
35
|
* out by `sinceMs` — callers must treat that the same as "nothing found".
|
|
15
36
|
*/
|
|
16
|
-
async function readSessionTailLines(
|
|
37
|
+
async function readSessionTailLines(logPath, sinceMs) {
|
|
38
|
+
if (logPath === undefined)
|
|
39
|
+
return undefined;
|
|
17
40
|
if (sinceMs !== undefined) {
|
|
18
41
|
try {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
const mtimeMs = fs.statSync(statPath).mtimeMs;
|
|
22
|
-
if (mtimeMs < sinceMs)
|
|
23
|
-
return undefined;
|
|
24
|
-
}
|
|
42
|
+
if (fs.statSync(logPath).mtimeMs < sinceMs)
|
|
43
|
+
return undefined;
|
|
25
44
|
}
|
|
26
45
|
catch { }
|
|
27
46
|
}
|
|
28
|
-
if (
|
|
47
|
+
if (logPath.endsWith('.zstd')) {
|
|
29
48
|
const { execSync } = await import('node:child_process');
|
|
30
|
-
const out = execSync(`zstd -d -c ${JSON.stringify(
|
|
49
|
+
const out = execSync(`zstd -d -c ${JSON.stringify(logPath)} 2>/dev/null | tail -100`, { encoding: 'utf-8' });
|
|
31
50
|
return out.split('\n').filter(Boolean);
|
|
32
51
|
}
|
|
33
|
-
|
|
34
|
-
const content = fs.readFileSync(
|
|
52
|
+
try {
|
|
53
|
+
const content = fs.readFileSync(logPath, 'utf-8');
|
|
35
54
|
return content.trim().split('\n').slice(-100);
|
|
36
55
|
}
|
|
56
|
+
catch { }
|
|
37
57
|
return undefined;
|
|
38
58
|
}
|
|
39
59
|
export async function findInterrupted(dshHome, opts) {
|
|
@@ -55,10 +75,9 @@ export async function findInterrupted(dshHome, opts) {
|
|
|
55
75
|
if (!s.isDirectory())
|
|
56
76
|
continue;
|
|
57
77
|
scanned++;
|
|
58
|
-
const
|
|
59
|
-
const jsonlPath = path.join(groupPath, s.name, 'session.jsonl');
|
|
78
|
+
const logPath = resolveSessionLogPath(path.join(groupPath, s.name));
|
|
60
79
|
try {
|
|
61
|
-
const lines = await readSessionTailLines(
|
|
80
|
+
const lines = await readSessionTailLines(logPath, sinceMs);
|
|
62
81
|
if (lines === undefined)
|
|
63
82
|
continue;
|
|
64
83
|
let found = false;
|
|
@@ -103,31 +122,30 @@ export async function findInterrupted(dshHome, opts) {
|
|
|
103
122
|
* The mtime pre-filter ensures this full decompression runs only for
|
|
104
123
|
* recent sessions (within 5m, typically 1-2 files), not for all 425.
|
|
105
124
|
*/
|
|
106
|
-
async function readSessionAllLines(
|
|
125
|
+
async function readSessionAllLines(logPath, sinceMs) {
|
|
126
|
+
if (logPath === undefined)
|
|
127
|
+
return undefined;
|
|
107
128
|
if (sinceMs !== undefined) {
|
|
108
129
|
try {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
const mtimeMs = fs.statSync(statPath).mtimeMs;
|
|
112
|
-
if (mtimeMs < sinceMs)
|
|
113
|
-
return undefined;
|
|
114
|
-
}
|
|
130
|
+
if (fs.statSync(logPath).mtimeMs < sinceMs)
|
|
131
|
+
return undefined;
|
|
115
132
|
}
|
|
116
133
|
catch { }
|
|
117
134
|
}
|
|
118
|
-
if (
|
|
135
|
+
if (logPath.endsWith('.zstd')) {
|
|
119
136
|
const { execSync } = await import('node:child_process');
|
|
120
137
|
// maxBuffer must exceed the decompressed size of any real session log —
|
|
121
138
|
// worker sessions decode to 8-23MB while execSync's default 1MB would
|
|
122
139
|
// throw ENOBUFS and silently drop the session from every scan that needs
|
|
123
140
|
// the full file (findDanglingOpenTurns). Use 64MB to leave headroom.
|
|
124
|
-
const out = execSync(`zstd -d -c ${JSON.stringify(
|
|
141
|
+
const out = execSync(`zstd -d -c ${JSON.stringify(logPath)} 2>/dev/null`, { encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024 });
|
|
125
142
|
return out.split('\n').filter(Boolean);
|
|
126
143
|
}
|
|
127
|
-
|
|
128
|
-
const content = fs.readFileSync(
|
|
144
|
+
try {
|
|
145
|
+
const content = fs.readFileSync(logPath, 'utf-8');
|
|
129
146
|
return content.trim().split('\n').filter(Boolean);
|
|
130
147
|
}
|
|
148
|
+
catch { }
|
|
131
149
|
return undefined;
|
|
132
150
|
}
|
|
133
151
|
/**
|
|
@@ -162,10 +180,9 @@ export async function findDanglingOpenTurns(dshHome, opts) {
|
|
|
162
180
|
if (!s.isDirectory())
|
|
163
181
|
continue;
|
|
164
182
|
scanned++;
|
|
165
|
-
const
|
|
166
|
-
const jsonlPath = path.join(groupPath, s.name, 'session.jsonl');
|
|
183
|
+
const logPath = resolveSessionLogPath(path.join(groupPath, s.name));
|
|
167
184
|
try {
|
|
168
|
-
const lines = await readSessionAllLines(
|
|
185
|
+
const lines = await readSessionAllLines(logPath, sinceMs);
|
|
169
186
|
if (lines === undefined)
|
|
170
187
|
continue;
|
|
171
188
|
let openTurn;
|
package/lib/snapshot.d.ts
CHANGED
|
@@ -1,22 +1,59 @@
|
|
|
1
|
-
interface
|
|
2
|
-
path: string;
|
|
3
|
-
sha256: string;
|
|
4
|
-
}
|
|
5
|
-
interface Manifest {
|
|
1
|
+
export interface SnapshotResult {
|
|
6
2
|
ts: string;
|
|
7
|
-
|
|
3
|
+
/** Files (and symlinks) copied into this snapshot. */
|
|
4
|
+
files: number;
|
|
5
|
+
/** Entries that could not be copied; the snapshot stays usable either way. */
|
|
6
|
+
skipped: Array<{
|
|
7
|
+
path: string;
|
|
8
|
+
reason: string;
|
|
9
|
+
}>;
|
|
8
10
|
}
|
|
9
|
-
export
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
export interface SnapshotDeps {
|
|
12
|
+
/**
|
|
13
|
+
* Copy one regular file, preserving `mode`. Injectable so a failing entry
|
|
14
|
+
* (the EACCES a read-only attachment object produces) can be table-tested
|
|
15
|
+
* without depending on the host's uid or filesystem quirks.
|
|
16
|
+
*/
|
|
17
|
+
copyFile?: (src: string, dest: string, mode: number) => void;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* DSH-home entries the last-known-good snapshot deliberately never copies.
|
|
21
|
+
*
|
|
22
|
+
* The LKG exists to recover a boot that fails while loading the plugin tree, so
|
|
23
|
+
* it holds boot **configuration**: `profiles/` (the plugin tree with its
|
|
24
|
+
* lockfile, `cordis.patch.yml` and sidecars), the per-plugin config directories
|
|
25
|
+
* and the settings documents. Everything below is runtime **data** — it is
|
|
26
|
+
* written continuously while `dsh web` runs, so restoring a snapshot of it can
|
|
27
|
+
* only lose newer state, and some of it is hostile to a bulk copy:
|
|
28
|
+
*
|
|
29
|
+
* - `sessions/` — append-only transcripts. Restoring a stale copy over live
|
|
30
|
+
* sessions drops every turn recorded after the snapshot, i.e. the recovery
|
|
31
|
+
* path would lose the log it is supposed to protect.
|
|
32
|
+
* - `attachments/` — content-addressed blobs stored mode `0400`. That read-only
|
|
33
|
+
* bit is exactly what made the 2026-09-13 rollback abort with
|
|
34
|
+
* `EACCES: Permission denied '.../attachments/v1/objects/f8'`.
|
|
35
|
+
* - `plugins-src/` — plugin source cache, re-fetched on demand (~400 MB host).
|
|
36
|
+
* - `.supervisor/` — the LKG root itself lives inside the DSH home, so copying
|
|
37
|
+
* it would recurse into every retained snapshot.
|
|
38
|
+
*
|
|
39
|
+
* This list is data, not a heuristic: the copy loop and the restore loop both
|
|
40
|
+
* consult `isLkgExcluded()`, and the table test pins the rule.
|
|
41
|
+
*/
|
|
42
|
+
export declare const LKG_EXCLUDED_ENTRIES: readonly string[];
|
|
43
|
+
/**
|
|
44
|
+
* True when a DSH-home-relative path must never enter (or leave) the LKG.
|
|
45
|
+
*
|
|
46
|
+
* The named entries match the first path segment; `*.log` matches by basename
|
|
47
|
+
* anywhere, because append-only logs are data at any depth and a restored stale
|
|
48
|
+
* `dsh-web.log` would poison the boot-boundary scan.
|
|
49
|
+
*/
|
|
50
|
+
export declare function isLkgExcluded(relPath: string): boolean;
|
|
51
|
+
/** Human-readable, bounded reason for a per-entry copy failure. */
|
|
52
|
+
export declare function failureReason(e: unknown): string;
|
|
53
|
+
export declare function writeLKG(dshHome: string, lkgRoot: string, deps?: SnapshotDeps): Promise<SnapshotResult>;
|
|
13
54
|
export declare function pruneByAge(root: string, maxAgeMs: number): Promise<void>;
|
|
14
55
|
export declare function pruneBySize(root: string, maxBytes: number): Promise<void>;
|
|
15
56
|
export declare function isDuplicateLKG(dshHome: string, lkgRoot: string): Promise<boolean>;
|
|
16
57
|
export declare function verifyLKG(lkgPath: string): Promise<boolean>;
|
|
17
58
|
export declare function rotateLKG(lkgRoot: string, keep?: number): Promise<void>;
|
|
18
|
-
export declare function writeFailed(dshHome: string, failedRoot: string): Promise<
|
|
19
|
-
ts: string;
|
|
20
|
-
manifest: Manifest;
|
|
21
|
-
}>;
|
|
22
|
-
export {};
|
|
59
|
+
export declare function writeFailed(dshHome: string, failedRoot: string): Promise<SnapshotResult>;
|
package/lib/snapshot.js
CHANGED
|
@@ -1,6 +1,51 @@
|
|
|
1
1
|
import * as fs from 'node:fs';
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import * as crypto from 'node:crypto';
|
|
4
|
+
/**
|
|
5
|
+
* DSH-home entries the last-known-good snapshot deliberately never copies.
|
|
6
|
+
*
|
|
7
|
+
* The LKG exists to recover a boot that fails while loading the plugin tree, so
|
|
8
|
+
* it holds boot **configuration**: `profiles/` (the plugin tree with its
|
|
9
|
+
* lockfile, `cordis.patch.yml` and sidecars), the per-plugin config directories
|
|
10
|
+
* and the settings documents. Everything below is runtime **data** — it is
|
|
11
|
+
* written continuously while `dsh web` runs, so restoring a snapshot of it can
|
|
12
|
+
* only lose newer state, and some of it is hostile to a bulk copy:
|
|
13
|
+
*
|
|
14
|
+
* - `sessions/` — append-only transcripts. Restoring a stale copy over live
|
|
15
|
+
* sessions drops every turn recorded after the snapshot, i.e. the recovery
|
|
16
|
+
* path would lose the log it is supposed to protect.
|
|
17
|
+
* - `attachments/` — content-addressed blobs stored mode `0400`. That read-only
|
|
18
|
+
* bit is exactly what made the 2026-09-13 rollback abort with
|
|
19
|
+
* `EACCES: Permission denied '.../attachments/v1/objects/f8'`.
|
|
20
|
+
* - `plugins-src/` — plugin source cache, re-fetched on demand (~400 MB host).
|
|
21
|
+
* - `.supervisor/` — the LKG root itself lives inside the DSH home, so copying
|
|
22
|
+
* it would recurse into every retained snapshot.
|
|
23
|
+
*
|
|
24
|
+
* This list is data, not a heuristic: the copy loop and the restore loop both
|
|
25
|
+
* consult `isLkgExcluded()`, and the table test pins the rule.
|
|
26
|
+
*/
|
|
27
|
+
export const LKG_EXCLUDED_ENTRIES = [
|
|
28
|
+
'sessions',
|
|
29
|
+
'attachments',
|
|
30
|
+
'plugins-src',
|
|
31
|
+
'.supervisor',
|
|
32
|
+
];
|
|
33
|
+
/**
|
|
34
|
+
* True when a DSH-home-relative path must never enter (or leave) the LKG.
|
|
35
|
+
*
|
|
36
|
+
* The named entries match the first path segment; `*.log` matches by basename
|
|
37
|
+
* anywhere, because append-only logs are data at any depth and a restored stale
|
|
38
|
+
* `dsh-web.log` would poison the boot-boundary scan.
|
|
39
|
+
*/
|
|
40
|
+
export function isLkgExcluded(relPath) {
|
|
41
|
+
const normalized = relPath.replace(/\\/g, '/').replace(/^\.\//, '').replace(/^\/+/, '');
|
|
42
|
+
const segments = normalized.split('/').filter(s => s.length > 0 && s !== '.');
|
|
43
|
+
if (!segments.length)
|
|
44
|
+
return false;
|
|
45
|
+
if (LKG_EXCLUDED_ENTRIES.includes(segments[0]))
|
|
46
|
+
return true;
|
|
47
|
+
return segments[segments.length - 1].endsWith('.log');
|
|
48
|
+
}
|
|
4
49
|
function sha256File(filePath) {
|
|
5
50
|
const data = fs.readFileSync(filePath);
|
|
6
51
|
return crypto.createHash('sha256').update(data).digest('hex');
|
|
@@ -16,7 +61,78 @@ function walkFiles(dir, base = dir) {
|
|
|
16
61
|
}
|
|
17
62
|
return out;
|
|
18
63
|
}
|
|
19
|
-
|
|
64
|
+
/** Human-readable, bounded reason for a per-entry copy failure. */
|
|
65
|
+
export function failureReason(e) {
|
|
66
|
+
const err = e;
|
|
67
|
+
const code = typeof err?.code === 'string' && err.code ? `${err.code}: ` : '';
|
|
68
|
+
const message = typeof err?.message === 'string' ? err.message : String(e);
|
|
69
|
+
return `${code}${message}`.slice(0, 300);
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Copy one snapshot entry, collecting — never throwing — per-entry failures
|
|
73
|
+
* (D2). `fs.cpSync` aborts the whole snapshot on the first unreadable object,
|
|
74
|
+
* which is precisely how one 0400 attachment file stopped the rollback that
|
|
75
|
+
* exists to rescue a broken boot.
|
|
76
|
+
*/
|
|
77
|
+
function copyEntry(state, src, dest, rel) {
|
|
78
|
+
if (isLkgExcluded(rel))
|
|
79
|
+
return;
|
|
80
|
+
let st;
|
|
81
|
+
try {
|
|
82
|
+
st = fs.lstatSync(src);
|
|
83
|
+
}
|
|
84
|
+
catch (e) {
|
|
85
|
+
state.skipped.push({ path: rel, reason: failureReason(e) });
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
if (st.isDirectory()) {
|
|
89
|
+
let names;
|
|
90
|
+
try {
|
|
91
|
+
// Directory modes are replicated only in their permission-to-traverse
|
|
92
|
+
// sense: the snapshot itself must stay readable/removable even when the
|
|
93
|
+
// source directory is not (a 0000 source dir must not produce a snapshot
|
|
94
|
+
// that nothing — including retention — can delete).
|
|
95
|
+
fs.mkdirSync(dest, { recursive: true, mode: (st.mode & 0o7777) | 0o700 });
|
|
96
|
+
names = fs.readdirSync(src);
|
|
97
|
+
}
|
|
98
|
+
catch (e) {
|
|
99
|
+
state.skipped.push({ path: rel, reason: failureReason(e) });
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
for (const name of names) {
|
|
103
|
+
copyEntry(state, path.join(src, name), path.join(dest, name), rel ? `${rel}/${name}` : name);
|
|
104
|
+
}
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
if (st.isSymbolicLink()) {
|
|
108
|
+
try {
|
|
109
|
+
const link = fs.readlinkSync(src);
|
|
110
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
111
|
+
try {
|
|
112
|
+
fs.unlinkSync(dest);
|
|
113
|
+
}
|
|
114
|
+
catch { }
|
|
115
|
+
fs.symlinkSync(link, dest);
|
|
116
|
+
state.copied++;
|
|
117
|
+
}
|
|
118
|
+
catch (e) {
|
|
119
|
+
state.skipped.push({ path: rel, reason: failureReason(e) });
|
|
120
|
+
}
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
if (st.isFile()) {
|
|
124
|
+
try {
|
|
125
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
126
|
+
state.copyFile(src, dest, st.mode & 0o7777);
|
|
127
|
+
state.copied++;
|
|
128
|
+
}
|
|
129
|
+
catch (e) {
|
|
130
|
+
state.skipped.push({ path: rel, reason: failureReason(e) });
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// Sockets/FIFOs/devices are not boot configuration: ignored on purpose.
|
|
134
|
+
}
|
|
135
|
+
export async function writeLKG(dshHome, lkgRoot, deps = {}) {
|
|
20
136
|
// Dedupe: skip snapshot if current state identical to latest LKG (prevents 5-min unconditional growth)
|
|
21
137
|
try {
|
|
22
138
|
if (await isDuplicateLKG(dshHome, lkgRoot)) {
|
|
@@ -31,36 +147,68 @@ export async function writeLKG(dshHome, lkgRoot) {
|
|
|
31
147
|
const latest = entries[entries.length - 1];
|
|
32
148
|
const manifestPath = path.join(lkgRoot, latest, 'manifest.json');
|
|
33
149
|
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
|
|
34
|
-
return { ts: latest, manifest };
|
|
150
|
+
return { ts: latest, files: manifest.files?.length ?? 0, skipped: [] };
|
|
35
151
|
}
|
|
36
152
|
}
|
|
37
153
|
catch { }
|
|
38
154
|
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
39
155
|
const dest = path.join(lkgRoot, ts);
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
156
|
+
try {
|
|
157
|
+
fs.mkdirSync(dest, { recursive: true });
|
|
158
|
+
}
|
|
159
|
+
catch (e) {
|
|
160
|
+
// No snapshot is better than an exception thrown into the rollback path.
|
|
161
|
+
return { ts, files: 0, skipped: [{ path: lkgRoot, reason: failureReason(e) }] };
|
|
162
|
+
}
|
|
163
|
+
const state = {
|
|
164
|
+
dshHome,
|
|
165
|
+
copyFile: deps.copyFile ?? ((src, dst, mode) => {
|
|
166
|
+
fs.copyFileSync(src, dst);
|
|
167
|
+
try {
|
|
168
|
+
fs.chmodSync(dst, mode);
|
|
169
|
+
}
|
|
170
|
+
catch { }
|
|
171
|
+
}),
|
|
172
|
+
copied: 0,
|
|
173
|
+
skipped: [],
|
|
174
|
+
};
|
|
175
|
+
// Copy only the boot configuration (D1), one entry at a time so a single
|
|
176
|
+
// unreadable file is recorded and skipped instead of aborting the snapshot (D2).
|
|
177
|
+
let topLevel = [];
|
|
178
|
+
try {
|
|
179
|
+
if (fs.existsSync(dshHome))
|
|
180
|
+
topLevel = fs.readdirSync(dshHome);
|
|
181
|
+
}
|
|
182
|
+
catch (e) {
|
|
183
|
+
state.skipped.push({ path: '.', reason: failureReason(e) });
|
|
184
|
+
}
|
|
185
|
+
for (const entry of topLevel) {
|
|
186
|
+
copyEntry(state, path.join(dshHome, entry), path.join(dest, entry), entry);
|
|
187
|
+
}
|
|
188
|
+
let fileList = [];
|
|
189
|
+
try {
|
|
190
|
+
fileList = fs.existsSync(dest) ? walkFiles(dest) : [];
|
|
191
|
+
}
|
|
192
|
+
catch (e) {
|
|
193
|
+
state.skipped.push({ path: 'manifest', reason: failureReason(e) });
|
|
194
|
+
}
|
|
52
195
|
const manifest = {
|
|
53
196
|
ts,
|
|
54
|
-
files:
|
|
197
|
+
files: fileList
|
|
55
198
|
.filter(f => f !== 'manifest.json')
|
|
56
199
|
.map(f => ({ path: f, sha256: sha256File(path.join(dest, f)) })),
|
|
57
200
|
};
|
|
58
|
-
|
|
201
|
+
try {
|
|
202
|
+
fs.writeFileSync(path.join(dest, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
|
203
|
+
}
|
|
204
|
+
catch (e) {
|
|
205
|
+
state.skipped.push({ path: 'manifest.json', reason: failureReason(e) });
|
|
206
|
+
}
|
|
59
207
|
// Retention: keep only 3 most recent, plus age (7d) and size (5GB) caps — prevents unbounded 40GB+ growth
|
|
60
208
|
await rotateLKG(lkgRoot, 3).catch(() => { });
|
|
61
209
|
await pruneByAge(lkgRoot, 7 * 24 * 60 * 60 * 1000).catch(() => { });
|
|
62
210
|
await pruneBySize(lkgRoot, 5 * 1024 * 1024 * 1024).catch(() => { });
|
|
63
|
-
return { ts,
|
|
211
|
+
return { ts, files: state.copied, skipped: state.skipped };
|
|
64
212
|
}
|
|
65
213
|
export async function pruneByAge(root, maxAgeMs) {
|
|
66
214
|
if (!fs.existsSync(root))
|
|
@@ -150,7 +298,9 @@ export async function isDuplicateLKG(dshHome, lkgRoot) {
|
|
|
150
298
|
let newestFileMtime = 0;
|
|
151
299
|
if (fs.existsSync(dshHome)) {
|
|
152
300
|
for (const entry of fs.readdirSync(dshHome)) {
|
|
153
|
-
|
|
301
|
+
// Same scope as the copy loop: runtime data changes constantly and
|
|
302
|
+
// must not defeat the dedupe for the configuration being snapshotted.
|
|
303
|
+
if (isLkgExcluded(entry))
|
|
154
304
|
continue;
|
|
155
305
|
try {
|
|
156
306
|
const s = fs.statSync(path.join(dshHome, entry));
|
package/lib/supervisor.d.ts
CHANGED
|
@@ -1,16 +1,25 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type HealthState } from './health-poller.js';
|
|
2
2
|
import type { RestartRequest } from './restart-guards.js';
|
|
3
3
|
import { type RestartOutcome } from './intents.js';
|
|
4
4
|
import { type MintCookieOpts } from './dsh-session.js';
|
|
5
|
+
/** Partial-restore report a rollback may hand back (cli.ts wires RestoreResult). */
|
|
6
|
+
export interface RollbackSummary {
|
|
7
|
+
target?: string;
|
|
8
|
+
restored?: number;
|
|
9
|
+
skipped?: Array<{
|
|
10
|
+
path: string;
|
|
11
|
+
reason: string;
|
|
12
|
+
}>;
|
|
13
|
+
}
|
|
5
14
|
export interface SupervisorDeps {
|
|
6
15
|
pollHealth: () => Promise<HealthState>;
|
|
7
16
|
writeLKG: () => Promise<{
|
|
8
17
|
ts: string;
|
|
9
|
-
manifest
|
|
18
|
+
manifest?: any;
|
|
10
19
|
}>;
|
|
11
20
|
writeFailed: () => Promise<{
|
|
12
21
|
ts: string;
|
|
13
|
-
manifest
|
|
22
|
+
manifest?: any;
|
|
14
23
|
}>;
|
|
15
24
|
writeReport: (opts: {
|
|
16
25
|
ts: string;
|
|
@@ -19,12 +28,13 @@ export interface SupervisorDeps {
|
|
|
19
28
|
logTail?: string;
|
|
20
29
|
gitDiff?: string;
|
|
21
30
|
}) => Promise<string>;
|
|
22
|
-
rollback: (ts?: string) => Promise<void>;
|
|
31
|
+
rollback: (ts?: string) => Promise<RollbackSummary | void>;
|
|
23
32
|
restartWeb?: () => Promise<void>;
|
|
24
33
|
notify: (msg: string) => Promise<void>;
|
|
25
34
|
intervalMs?: number;
|
|
26
35
|
debounceMs?: number;
|
|
27
36
|
downThreshold?: number;
|
|
37
|
+
bootGraceMs?: number;
|
|
28
38
|
getTime?: () => number;
|
|
29
39
|
isPlannedRestartActive?: () => boolean | Promise<boolean>;
|
|
30
40
|
writePlannedRestart?: (ttlMs?: number) => void;
|
|
@@ -91,9 +101,16 @@ export declare class Supervisor {
|
|
|
91
101
|
private getEffectiveDownThreshold;
|
|
92
102
|
private getEffectiveDegradedThreshold;
|
|
93
103
|
private getEffectivePollTimeoutMs;
|
|
104
|
+
private getEffectiveBootGraceMs;
|
|
94
105
|
private findInterruptedRecent;
|
|
95
106
|
private collectGitDiff;
|
|
96
107
|
private attemptAutoResume;
|
|
108
|
+
/**
|
|
109
|
+
* D4: a restore that could not put every entry back must be surfaced loudly.
|
|
110
|
+
* Without this, a half-restored tree reads exactly like a clean recovery in
|
|
111
|
+
* the log and in the operator's Telegram feed.
|
|
112
|
+
*/
|
|
113
|
+
private reportRollbackResult;
|
|
97
114
|
private handleDebugResult;
|
|
98
115
|
tick(): Promise<void>;
|
|
99
116
|
start(): Promise<void>;
|