@ddtcorex/dsh-maestro-supervisor 0.1.0

package/README.md

@@ -0,0 +1,44 @@
+# dsh-maestro-supervisor
+
+Supervisor daemon for DSH Web resilience — Phase 1 Guard & Report.
+
+Runs **outside** the `pnpm → sh → node` tree. Polls `:3080` every 3s, manages last-known-good (LKG) snapshots in `~/.dsh/.supervisor/lkg/`, auto-rollbacks on crash, and writes `report-<ts>.md` for the next session. Telegram via `dsh-maestro-notifier`.
+
+## Install
+
+```bash
+pnpm --dir packages/dsh-maestro-supervisor install
+pnpm --dir packages/dsh-maestro-supervisor build
+
+# systemd (recommended)
+bash packages/dsh-maestro-supervisor/scripts/install-systemd.sh
+systemctl --user daemon-reload && systemctl --user enable --now dsh-web-supervisor
+
+# or manual sidecar
+setsid node packages/dsh-maestro-supervisor/lib/index.js daemon &
+```
+
+## CLI
+
+```bash
+node packages/dsh-maestro-supervisor/lib/index.js --help
+node packages/dsh-maestro-supervisor/lib/index.js status
+node packages/dsh-maestro-supervisor/lib/index.js daemon
+```
+
+Reports: `~/.dsh/.supervisor/reports/report-<ts>.md`, LKG: `~/.dsh/.supervisor/lkg/<ts>/`, failed: `~/.dsh/.supervisor/failed/<ts>/`
+
+## Telegram
+
+`src/host/notifier.ts` is loose by default: tries `import('@ddtcorex/dsh-maestro-notifier')`, then `TELEGRAM_BOT_TOKEN`/`TELEGRAM_CHAT_ID` env, then `console.log`. No hard dependency, daemon never blocks on Telegram.
+
+- **Enable:** `systemctl --user edit dsh-web-supervisor` → uncomment `Environment=TELEGRAM_BOT_TOKEN`/`TELEGRAM_CHAT_ID` in `systemd/dsh-web-supervisor.service.template` → `systemctl --user daemon-reload && systemctl --user restart dsh-web-supervisor`.
+- **Hard mode (optional):** `package.json` add `"@ddtcorex/dsh-maestro-notifier": "workspace:^0.1.0"` + `pnpm-workspace.yaml` `packages: ["../dsh-maestro-notifier"]` — then `pnpm install` links it and every `notify()` hits the hard import.
+
+See `AGENTS.md` §Dependency patterns for details and for interdependent Cordis plugins (A ↔ B) — never mutual `inject`, use shared lib C / one-way + events / isolate+RPC.
+
+## Integration test
+
+```bash
+DSH_INTEGRATION=1 pnpm --dir packages/dsh-maestro-supervisor test -- tests/integration.test.ts
+```

package/lib/cli.d.ts

@@ -0,0 +1 @@
+export declare function runCli(args: string[]): Promise<void>;

package/lib/cli.js

@@ -0,0 +1,74 @@
+import { Supervisor } from './supervisor.js';
+import { pollHealth } from './health-poller.js';
+import { writeLKG, verifyLKG } from './snapshot.js';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+export async function runCli(args) {
+    const cmd = args[2] ?? '--help';
+    if (cmd === '--help' || cmd === '-h' || cmd === 'help') {
+        console.log(`Usage: dsh-web-supervisor <command>
+
+Commands:
+  daemon     Run supervisor daemon (poll every 3s)
+  status     Show health + LKG status
+  logs       Tail supervisor reports
+  rollback --to <ts>  Rollback to LKG <ts>
+`);
+        return;
+    }
+    if (cmd === 'status') {
+        const health = await pollHealth();
+        console.log(`up: ${health.up}, httpCode: ${health.httpCode}, error: ${health.error ?? 'none'}`);
+        const lkgRoot = path.join(os.homedir(), '.dsh/.supervisor/lkg');
+        if (fs.existsSync(lkgRoot)) {
+            const entries = fs.readdirSync(lkgRoot).sort();
+            console.log(`LKG: ${entries.length} snapshots, latest: ${entries[entries.length - 1] ?? 'none'}`);
+            if (entries.length) {
+                const ok = await verifyLKG(path.join(lkgRoot, entries[entries.length - 1])).catch(() => false);
+                console.log(`latest LKG valid: ${ok}`);
+            }
+        }
+        else {
+            console.log('LKG: none');
+        }
+        return;
+    }
+    if (cmd === 'daemon') {
+        console.log('[supervisor] starting daemon — poll every 3s, Ctrl+C to stop');
+        const dshHome = path.join(os.homedir(), '.dsh');
+        const lkgRoot = path.join(os.homedir(), '.dsh/.supervisor/lkg');
+        const failedRoot = path.join(os.homedir(), '.dsh/.supervisor/failed');
+        const reportsRoot = path.join(os.homedir(), '.dsh/.supervisor/reports');
+        const supervisor = new Supervisor({
+            pollHealth: () => pollHealth(),
+            writeLKG: () => writeLKG(dshHome, lkgRoot),
+            writeFailed: () => writeLKG(dshHome, failedRoot),
+            writeReport: async ({ ts, health, action }) => {
+                const { writeReport } = await import('./report.js');
+                return writeReport({ reportsRoot, ts, health, gitDiff: '', logTail: '', action });
+            },
+            rollback: async () => {
+                // Find latest LKG and restore
+                const entries = fs.existsSync(lkgRoot) ? fs.readdirSync(lkgRoot).sort() : [];
+                if (!entries.length)
+                    throw new Error('no LKG to rollback to');
+                const latest = entries[entries.length - 1];
+                const src = path.join(lkgRoot, latest);
+                // naive restore: copy files back
+                for (const entry of fs.readdirSync(src)) {
+                    if (entry === 'manifest.json')
+                        continue;
+                    fs.cpSync(path.join(src, entry), path.join(dshHome, entry), { recursive: true, force: true });
+                }
+                console.log(`[supervisor] rolled back to ${latest}`);
+            },
+            notify: async (msg) => console.log(`[notify] ${msg}`),
+            intervalMs: 3000,
+        });
+        supervisor.start();
+        // keep process alive
+        await new Promise(() => { });
+    }
+    console.log(`unknown command: ${cmd} — try --help`);
+}

package/lib/health-poller.d.ts

@@ -0,0 +1,17 @@
+export interface HealthState {
+    up: boolean;
+    httpCode?: number;
+    error?: string;
+    degraded?: boolean;
+}
+export interface PollHealthOpts {
+    fetch?: () => Promise<{
+        status: number;
+        text: () => Promise<string>;
+    }>;
+    psAlive?: () => Promise<boolean>;
+    logTail?: () => Promise<string>;
+    url?: string;
+    timeoutMs?: number;
+}
+export declare function pollHealth(opts?: PollHealthOpts): Promise<HealthState>;

package/lib/health-poller.js

@@ -0,0 +1,105 @@
+const ERROR_PATTERNS = [
+    'ERR_MODULE_NOT_FOUND',
+    'ERR_PNPM',
+    'assertChannel',
+    'unhandledRejection',
+    'SyntaxError',
+    'YAMLParseError',
+    'ParseError',
+    'YAML',
+    'JSON',
+    'corrupted',
+    'allowBuilds',
+    'Cannot find module',
+    'Failed to load',
+];
+export async function pollHealth(opts = {}) {
+    const fetchFn = opts.fetch ?? defaultFetch(opts.url ?? 'http://127.0.0.1:3080/', opts.timeoutMs ?? 2000);
+    const psAliveFn = opts.psAlive ?? defaultPsAlive;
+    const logTailFn = opts.logTail ?? defaultLogTail;
+    let httpCode;
+    let fetchError;
+    try {
+        const res = await fetchFn();
+        httpCode = res.status;
+        if (res.status !== 200) {
+            fetchError = `http ${res.status}`;
+        }
+    }
+    catch (e) {
+        fetchError = e?.message ?? String(e);
+    }
+    let logContent = '';
+    try {
+        logContent = await logTailFn();
+    }
+    catch {
+        // ignore log read errors
+    }
+    let logError;
+    const lowerLog = logContent.toLowerCase();
+    for (const pat of ERROR_PATTERNS) {
+        if (lowerLog.includes(pat.toLowerCase())) {
+            // extract line containing pattern (case-insensitive)
+            const line = logContent.split('\n').find(l => l.toLowerCase().includes(pat.toLowerCase())) ?? pat;
+            logError = line.trim().slice(0, 500);
+            break;
+        }
+    }
+    // If either fetch failed or log has error, consider down
+    if (fetchError || logError) {
+        return {
+            up: false,
+            httpCode,
+            error: logError ?? fetchError,
+            degraded: !!logError && httpCode === 200,
+        };
+    }
+    // Also check psAlive as secondary signal — if fetch ok but ps dead, still down
+    try {
+        const alive = await psAliveFn();
+        if (!alive && httpCode === 200) {
+            // fetch succeeded but ps says dead — likely stale, still consider up if http 200
+        }
+    }
+    catch {
+        // ignore
+    }
+    return { up: httpCode === 200, httpCode };
+}
+function defaultFetch(url, timeoutMs) {
+    return async () => {
+        const controller = new AbortController();
+        const t = setTimeout(() => controller.abort(), timeoutMs);
+        try {
+            const res = await fetch(url, { signal: controller.signal });
+            return { status: res.status, text: async () => res.text() };
+        }
+        finally {
+            clearTimeout(t);
+        }
+    };
+}
+async function defaultPsAlive() {
+    // Check if any listener on 3080 exists via ss — fallback to true if ss unavailable
+    try {
+        const { execSync } = await import('node:child_process');
+        const out = execSync('ss -tln 2>/dev/null || true', { encoding: 'utf-8' });
+        return out.includes(':3080');
+    }
+    catch {
+        return true;
+    }
+}
+async function defaultLogTail() {
+    try {
+        const { readFileSync } = await import('node:fs');
+        const { homedir } = await import('node:os');
+        const logPath = `${homedir()}/.dsh.log`;
+        const content = readFileSync(logPath, 'utf-8');
+        return content.slice(-5000);
+    }
+    catch {
+        return '';
+    }
+}

package/lib/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/lib/index.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { runCli } from './cli.js';
+runCli(process.argv).catch(e => { console.error(e); process.exit(1); });

package/lib/notifier.d.ts

@@ -0,0 +1,7 @@
+export interface NotifierOpts {
+    send?: (msg: string) => Promise<void>;
+}
+export declare function notify(msg: string, opts?: NotifierOpts): Promise<void>;
+export declare function notifyCrash(reportPath: string, error: string, opts?: NotifierOpts): Promise<void>;
+export declare function notifyDegraded(id: string, error: string, opts?: NotifierOpts): Promise<void>;
+export declare function notifyFixed(branch: string, sessions: string[], opts?: NotifierOpts): Promise<void>;

package/lib/notifier.js

@@ -0,0 +1,52 @@
+export async function notify(msg, opts = {}) {
+    const send = opts.send ?? defaultSend;
+    try {
+        await send(msg);
+    }
+    catch {
+        // never block caller — rollback/report must not fail due to notifier
+    }
+}
+async function defaultSend(msg) {
+    // 1) Try hard dependency: @ddtcorex/dsh-maestro-notifier if installed (workspace:^)
+    try {
+        // dynamic import so daemon still runs when notifier is not installed (loose mode)
+        // @ts-ignore — optional hard dep, may not be installed
+        const mod = await import('@ddtcorex/dsh-maestro-notifier').catch(() => null);
+        if (mod?.sendTelegram || mod?.notify) {
+            const fn = mod.sendTelegram ?? mod.notify;
+            await fn(msg);
+            return;
+        }
+    }
+    catch {
+        // ignore and fall through
+    }
+    // 2) Try env Telegram token (TELEGRAM_BOT_TOKEN + TELEGRAM_CHAT_ID)
+    const token = process.env.TELEGRAM_BOT_TOKEN;
+    const chatId = process.env.TELEGRAM_CHAT_ID;
+    if (token && chatId) {
+        try {
+            await fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ chat_id: chatId, text: msg, parse_mode: 'Markdown' }),
+            });
+            return;
+        }
+        catch {
+            // fall through to log
+        }
+    }
+    // 3) Fallback — log only, never throw
+    console.log(`[supervisor notify] ${msg}`);
+}
+export async function notifyCrash(reportPath, error, opts = {}) {
+    await notify(`CRASH detected → rollback (report: ${reportPath}, error: ${error})`, opts);
+}
+export async function notifyDegraded(id, error, opts = {}) {
+    await notify(`DEGRADED: ${id} failed — ${error}`, opts);
+}
+export async function notifyFixed(branch, sessions, opts = {}) {
+    await notify(`FIXED: ${branch}, sessions resumed: [${sessions.join(', ')}]`, opts);
+}

package/lib/report.d.ts

@@ -0,0 +1,11 @@
+import type { HealthState } from './health-poller.js';
+export interface ReportOpts {
+    reportsRoot: string;
+    ts: string;
+    health: HealthState;
+    gitDiff: string;
+    logTail: string;
+    action: string;
+}
+export declare function writeReport(opts: ReportOpts): Promise<string>;
+export declare function collectGitDiff(workspaceRoot: string): Promise<string>;

package/lib/report.js

@@ -0,0 +1,56 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+export async function writeReport(opts) {
+    fs.mkdirSync(opts.reportsRoot, { recursive: true });
+    const fileName = `report-${opts.ts}.md`;
+    const filePath = path.join(opts.reportsRoot, fileName);
+    const content = `# DSH Web Crash Report — ${opts.ts}
+
+**Action:** ${opts.action}
+
+## Health Snapshot
+
+- up: ${opts.health.up}
+- httpCode: ${opts.health.httpCode ?? 'n/a'}
+- error: ${opts.health.error ?? 'none'}
+- degraded: ${opts.health.degraded ?? false}
+
+## Git Diff (link plugins)
+
+\`\`\`diff
+${opts.gitDiff || '(no changes)'}
+\`\`\`
+
+## Log Tail (last 200 lines)
+
+\`\`\`
+${opts.logTail.slice(-5000) || '(empty)'}
+\`\`\`
+
+## Next Steps
+
+- Check \`~/.dsh/.supervisor/lkg/\` for last-known-good
+- Review \`failed/${opts.ts}/\` snapshot
+- If degraded, fix the listed plugin row and run \`pnpm verify\` + dry-boot
+
+---
+*Generated by dsh-web-supervisor*
+`;
+    fs.writeFileSync(filePath, content);
+    return filePath;
+}
+export async function collectGitDiff(workspaceRoot) {
+    try {
+        const { execSync } = await import('node:child_process');
+        // Find all link: plugins from profiles
+        const out = execSync('git -C ' + JSON.stringify(workspaceRoot) + ' status --porcelain 2>/dev/null || true', { encoding: 'utf-8' });
+        if (out.trim()) {
+            const diff = execSync('git -C ' + JSON.stringify(workspaceRoot) + ' diff 2>/dev/null | head -n 200', { encoding: 'utf-8' });
+            return diff || out;
+        }
+        return '';
+    }
+    catch {
+        return '';
+    }
+}

package/lib/snapshot.d.ts

@@ -0,0 +1,19 @@
+interface ManifestEntry {
+    path: string;
+    sha256: string;
+}
+interface Manifest {
+    ts: string;
+    files: ManifestEntry[];
+}
+export declare function writeLKG(dshHome: string, lkgRoot: string): Promise<{
+    ts: string;
+    manifest: Manifest;
+}>;
+export declare function verifyLKG(lkgPath: string): Promise<boolean>;
+export declare function rotateLKG(lkgRoot: string, keep?: number): Promise<void>;
+export declare function writeFailed(dshHome: string, failedRoot: string): Promise<{
+    ts: string;
+    manifest: Manifest;
+}>;
+export {};

package/lib/snapshot.js

@@ -0,0 +1,82 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as crypto from 'node:crypto';
+function sha256File(filePath) {
+    const data = fs.readFileSync(filePath);
+    return crypto.createHash('sha256').update(data).digest('hex');
+}
+function walkFiles(dir, base = dir) {
+    const out = [];
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+        const full = path.join(dir, entry.name);
+        if (entry.isDirectory())
+            out.push(...walkFiles(full, base));
+        else if (entry.isFile())
+            out.push(path.relative(base, full));
+    }
+    return out;
+}
+export async function writeLKG(dshHome, lkgRoot) {
+    const ts = new Date().toISOString().replace(/[:.]/g, '-');
+    const dest = path.join(lkgRoot, ts);
+    fs.mkdirSync(dest, { recursive: true });
+    // Copy DSH home contents (if exists, copy recursively)
+    if (fs.existsSync(dshHome)) {
+        // Use cpSync if available
+        for (const entry of fs.readdirSync(dshHome)) {
+            if (entry === '.supervisor')
+                continue;
+            const src = path.join(dshHome, entry);
+            const dst = path.join(dest, entry);
+            fs.cpSync(src, dst, { recursive: true });
+        }
+    }
+    const files = fs.existsSync(dest) ? walkFiles(dest) : [];
+    const manifest = {
+        ts,
+        files: files
+            .filter(f => f !== 'manifest.json')
+            .map(f => ({ path: f, sha256: sha256File(path.join(dest, f)) })),
+    };
+    fs.writeFileSync(path.join(dest, 'manifest.json'), JSON.stringify(manifest, null, 2));
+    return { ts, manifest };
+}
+export async function verifyLKG(lkgPath) {
+    const manifestPath = path.join(lkgPath, 'manifest.json');
+    if (!fs.existsSync(manifestPath))
+        return false;
+    try {
+        const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+        for (const entry of manifest.files) {
+            const filePath = path.join(lkgPath, entry.path);
+            if (!fs.existsSync(filePath))
+                return false;
+            const hash = sha256File(filePath);
+            if (hash !== entry.sha256)
+                return false;
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function rotateLKG(lkgRoot, keep = 3) {
+    if (!fs.existsSync(lkgRoot))
+        return;
+    const entries = fs.readdirSync(lkgRoot).filter((n) => {
+        try {
+            return fs.statSync(path.join(lkgRoot, n)).isDirectory();
+        }
+        catch {
+            return false;
+        }
+    }).sort();
+    const toDelete = entries.slice(0, Math.max(0, entries.length - keep));
+    for (const name of toDelete) {
+        fs.rmSync(path.join(lkgRoot, name), { recursive: true, force: true });
+    }
+}
+export async function writeFailed(dshHome, failedRoot) {
+    return writeLKG(dshHome, failedRoot);
+}

package/lib/supervisor.d.ts

@@ -0,0 +1,33 @@
+import type { HealthState } from './health-poller.js';
+export interface SupervisorDeps {
+    pollHealth: () => Promise<HealthState>;
+    writeLKG: () => Promise<{
+        ts: string;
+        manifest: any;
+    }>;
+    writeFailed: () => Promise<{
+        ts: string;
+        manifest: any;
+    }>;
+    writeReport: (opts: {
+        ts: string;
+        health: HealthState;
+        action: string;
+    }) => Promise<string>;
+    rollback: (ts?: string) => Promise<void>;
+    notify: (msg: string) => Promise<void>;
+    intervalMs?: number;
+    debounceMs?: number;
+    getTime?: () => number;
+}
+export declare class Supervisor {
+    private deps;
+    private lastRollback;
+    private rollingBack;
+    private lastLKGWrite;
+    private timer;
+    constructor(deps: SupervisorDeps);
+    tick(): Promise<void>;
+    start(): void;
+    stop(): void;
+}

package/lib/supervisor.js

@@ -0,0 +1,58 @@
+export class Supervisor {
+    deps;
+    lastRollback = 0;
+    rollingBack = false;
+    lastLKGWrite = 0;
+    timer = null;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    async tick() {
+        const health = await this.deps.pollHealth();
+        if (health.up) {
+            // Throttle LKG writes to at most once per 5 minutes
+            const now = this.deps.getTime ? this.deps.getTime() : Date.now();
+            if (now - this.lastLKGWrite > 5 * 60 * 1000) {
+                try {
+                    await this.deps.writeLKG();
+                    this.lastLKGWrite = now;
+                }
+                catch {
+                    // ignore snapshot errors
+                }
+            }
+            return;
+        }
+        // Down — check debounce and rolling state
+        if (this.rollingBack)
+            return;
+        const now = this.deps.getTime ? this.deps.getTime() : Date.now();
+        const debounceMs = this.deps.debounceMs ?? 60000;
+        if (now - this.lastRollback < debounceMs)
+            return;
+        this.rollingBack = true;
+        this.lastRollback = now;
+        try {
+            const failed = await this.deps.writeFailed().catch(() => ({ ts: new Date().toISOString().replace(/[:.]/g, '-'), manifest: null }));
+            const ts = failed?.ts ?? new Date().toISOString().replace(/[:.]/g, '-');
+            const reportPath = await this.deps.writeReport({ ts, health, action: `rollback — ${health.error ?? 'down'}` }).catch(() => '');
+            await this.deps.rollback();
+            await this.deps.notify(`CRASH detected → rollback (report: ${reportPath}, error: ${health.error ?? 'down'})`).catch(() => { });
+        }
+        finally {
+            this.rollingBack = false;
+        }
+    }
+    start() {
+        if (this.timer)
+            return;
+        const intervalMs = this.deps.intervalMs ?? 3000;
+        this.timer = setInterval(() => { this.tick().catch(() => { }); }, intervalMs);
+    }
+    stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+    }
+}

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@ddtcorex/dsh-maestro-supervisor",
+  "version": "0.1.0",
+  "description": "Supervisor daemon for DSH Web resilience — auto-detect crashes, rollback to LKG, report",
+  "type": "module",
+  "bin": {
+    "dsh-web-supervisor": "./lib/index.js"
+  },
+  "files": [
+    "lib",
+    "README.md"
+  ],
+  "devDependencies": {
+    "@types/node": "^26.3.0",
+    "typescript": "^5.9.3",
+    "vitest": "^3.2.4"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "verify": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}