@loom-framework/core 0.1.0-alpha.19 → 0.1.0-alpha.190

package/dist/backend/lifecycle/manager.js

@@ -0,0 +1,684 @@
+/**
+ * LifecycleManager — manages long-running lifecycle scripts with
+ * auto-restart, exponential backoff, memory limits, and graceful shutdown.
+ */
+import { spawn } from 'node:child_process';
+import { execSync } from 'node:child_process';
+import fs from 'fs/promises';
+import path from 'path';
+import { deriveLifecycleNames } from '../process/process-helpers.js';
+// ---------------------------------------------------------------------------
+// Defaults
+// ---------------------------------------------------------------------------
+const DEFAULT_SHUTDOWN_TIMEOUT_MS = 5_000;
+const DEFAULT_RESTART = true;
+const DEFAULT_MAX_RESTARTS = 5;
+const DEFAULT_RESTART_WINDOW_MS = 60_000;
+const DEFAULT_BACKOFF_INITIAL_MS = 1_000;
+const DEFAULT_BACKOFF_MAX_MS = 30_000;
+const DEFAULT_BACKOFF_MULTIPLIER = 2.0;
+const DEFAULT_MEMORY_LIMIT_MB = 512;
+const DEFAULT_ENABLED = true;
+const MEMORY_CHECK_INTERVAL_MS = 5_000;
+const PID_DIR = '.loom/.pids';
+// ---------------------------------------------------------------------------
+// LifecycleManager
+// ---------------------------------------------------------------------------
+export class LifecycleManager {
+    config;
+    eventBus;
+    projectRoot;
+    serverPort;
+    processes = new Map();
+    /** Index: command → primary script name currently running that command */
+    commandPrimary = new Map();
+    pidDir;
+    constructor(config, eventBus, projectRoot, serverPort) {
+        this.config = config;
+        this.eventBus = eventBus;
+        this.projectRoot = projectRoot;
+        this.serverPort = serverPort ?? Number(process.env.LOOM_SERVER_PORT ?? 3000);
+        this.pidDir = path.join(projectRoot, PID_DIR);
+    }
+    // -----------------------------------------------------------------------
+    // Public API
+    // -----------------------------------------------------------------------
+    /** Start all enabled lifecycle scripts. Cleans up orphan processes from previous runs first. */
+    async startAll() {
+        await this.cleanOrphanProcesses();
+        for (const proc of this.processes.values()) {
+            if (proc.config.enabled === false) {
+                continue;
+            }
+            this.spawnProcess(proc);
+        }
+    }
+    /** Stop all running lifecycle scripts in parallel. */
+    async stopAll() {
+        const stops = [];
+        for (const proc of this.processes.values()) {
+            if (proc.status === 'running' || proc.status === 'starting' || proc.status === 'backoff') {
+                stops.push(this.stopProcess(proc));
+            }
+        }
+        await Promise.all(stops);
+    }
+    /** Resolve to the primary script — follow mirrorsPrimary if set. */
+    resolvePrimary(name) {
+        const proc = this.processes.get(name);
+        if (!proc)
+            return undefined;
+        if (proc.mirrorsPrimary) {
+            return this.processes.get(proc.mirrorsPrimary) ?? proc;
+        }
+        return proc;
+    }
+    /** Manually start a single script by name. */
+    async start(name) {
+        const proc = this.processes.get(name);
+        if (!proc) {
+            throw new Error(`Lifecycle script "${name}" not found`);
+        }
+        if (proc.status === 'running' || proc.status === 'starting') {
+            return; // already running
+        }
+        this.spawnProcess(proc);
+    }
+    /** Manually stop a single script by name. If mirroring, stops the primary. */
+    async stop(name) {
+        const primary = this.resolvePrimary(name);
+        if (!primary) {
+            throw new Error(`Lifecycle script "${name}" not found`);
+        }
+        // Clear mirror links for all scripts with the same command before stopping
+        const command = primary.config.command;
+        for (const other of this.processes.values()) {
+            if (other.mirrorsPrimary === primary.name) {
+                other.mirrorsPrimary = undefined;
+                other.status = 'stopped';
+                other.startedAt = undefined;
+                other.pid = undefined;
+            }
+        }
+        await this.stopProcess(primary);
+    }
+    /** Restart a single script by name. If mirroring, restarts the primary. */
+    async restart(name) {
+        const primary = this.resolvePrimary(name);
+        if (!primary) {
+            throw new Error(`Lifecycle script "${name}" not found`);
+        }
+        await this.stopProcess(primary);
+        this.spawnProcess(primary);
+    }
+    /** Return status info for all managed scripts. */
+    listStatus() {
+        const result = [];
+        for (const proc of this.processes.values()) {
+            result.push(this.toScriptInfo(proc));
+        }
+        return result;
+    }
+    /** Return status info for a single script. */
+    getStatus(name) {
+        const proc = this.processes.get(name);
+        if (!proc)
+            return undefined;
+        return this.toScriptInfo(proc);
+    }
+    /** Dynamically add a script at runtime. Engine parameters are filled from global config. */
+    addScript(script) {
+        if (this.processes.has(script.name)) {
+            return; // already exists with same name, skip silently
+        }
+        const fullScript = {
+            ...script,
+            restart: this.config.restart,
+            maxRestarts: this.config.maxRestarts,
+            restartWindowMs: this.config.restartWindowMs,
+            backoffInitialMs: this.config.backoffInitialMs,
+            backoffMaxMs: this.config.backoffMaxMs,
+            backoffMultiplier: this.config.backoffMultiplier,
+            memoryLimitMb: this.config.memoryLimitMb,
+            shutdownTimeoutMs: this.config.shutdownTimeoutMs,
+        };
+        this.processes.set(script.name, {
+            name: script.name,
+            config: fullScript,
+            childProcess: null,
+            status: 'stopped',
+            restartCount: 0,
+            windowStart: Date.now(),
+            currentBackoffMs: 0,
+        });
+    }
+    /**
+     * Sync lifecycle scripts from ProcessRegistry.
+     * For each Process with sources that has an event binding
+     * in subscriptions, auto-register LifecycleScripts.
+     * Uses deriveLifecycleNames() for multi-source naming.
+     * Skips if a script with that name already exists.
+     */
+    syncFromProcessRegistry(processes, subscriptions = []) {
+        for (const proc of processes) {
+            if (proc.sources.length === 0)
+                continue;
+            if (proc.paused)
+                continue;
+            const hasEventBinding = subscriptions.some(s => s.handler === 'process' && s.config.process === proc.name);
+            if (hasEventBinding) {
+                const names = deriveLifecycleNames(proc.name, proc.sources);
+                proc.sources.forEach((source, i) => {
+                    this.addScript({
+                        name: names[i],
+                        command: source.command,
+                        env: source.env,
+                    });
+                });
+            }
+        }
+    }
+    /** Dynamically remove a script and stop it if running. */
+    async removeScript(name) {
+        const proc = this.processes.get(name);
+        if (!proc) {
+            throw new Error(`Lifecycle script "${name}" not found`);
+        }
+        // If this script is a primary being removed, disconnect all mirrors
+        for (const other of this.processes.values()) {
+            if (other.mirrorsPrimary === name) {
+                other.mirrorsPrimary = undefined;
+                other.status = 'stopped';
+                other.startedAt = undefined;
+                other.pid = undefined;
+            }
+        }
+        await this.stopProcess(proc);
+        this.processes.delete(name);
+    }
+    // -----------------------------------------------------------------------
+    // Process lifecycle
+    // -----------------------------------------------------------------------
+    spawnProcess(proc) {
+        if (proc.childProcess && !proc.childProcess.killed && proc.status === 'running') {
+            return; // already running
+        }
+        // Command dedup: if another script with the same command is already running,
+        // mark this as a mirror — no second process spawned.
+        const existingPrimary = this.commandPrimary.get(proc.config.command);
+        if (existingPrimary && existingPrimary !== proc.name) {
+            const primary = this.processes.get(existingPrimary);
+            if (primary && (primary.status === 'running' || primary.status === 'starting' || primary.status === 'backoff')) {
+                console.log(`[lifecycle] Script "${proc.name}" mirrors "${existingPrimary}" (same command: "${proc.config.command}")`);
+                proc.mirrorsPrimary = existingPrimary;
+                return;
+            }
+        }
+        proc.mirrorsPrimary = undefined;
+        proc.status = 'starting';
+        // Build environment
+        const mergedEnv = {
+            ...process.env,
+            LOOM_SERVER_URL: `http://localhost:${this.serverPort}`,
+            LOOM_PROJECT_ROOT: this.projectRoot,
+            LOOM_SCRIPT_NAME: proc.name,
+        };
+        // Merge user-configured env (user overrides take precedence)
+        if (proc.config.env) {
+            Object.assign(mergedEnv, proc.config.env);
+        }
+        const child = spawn(proc.config.command, {
+            shell: true,
+            cwd: this.projectRoot,
+            env: mergedEnv,
+            detached: true,
+        });
+        proc.childProcess = child;
+        // Pipe stdout/stderr with prefix
+        child.stdout?.on('data', (data) => {
+            const lines = data.toString().split('\n');
+            for (const line of lines) {
+                if (line.length > 0) {
+                    console.log(`[lifecycle:${proc.name}] ${line}`);
+                }
+            }
+        });
+        child.stderr?.on('data', (data) => {
+            const lines = data.toString().split('\n');
+            for (const line of lines) {
+                if (line.length > 0) {
+                    console.error(`[lifecycle:${proc.name}] ${line}`);
+                }
+            }
+        });
+        // Handle spawn errors
+        child.on('error', (err) => {
+            console.error(`[lifecycle:${proc.name}] spawn error: ${err.message}`);
+            proc.status = 'failed';
+            proc.lastError = err.message;
+            this.eventBus?.emit(`lifecycle:failed:${proc.name}`, {
+                name: proc.name,
+                error: err.message,
+            });
+        });
+        // Set running state once process is confirmed alive
+        if (child.pid) {
+            proc.status = 'running';
+            proc.startedAt = new Date().toISOString();
+            proc.pid = child.pid;
+            proc.lastError = undefined;
+            this.commandPrimary.set(proc.config.command, proc.name);
+            this.writePidFile(proc.name, child.pid);
+            this.eventBus?.emit(`lifecycle:started:${proc.name}`, {
+                name: proc.name,
+                pid: child.pid,
+            });
+            this.startMemoryCheck(proc);
+        }
+        else {
+            // For shell mode, pid may be set asynchronously
+            child.on('spawn', () => {
+                proc.status = 'running';
+                proc.startedAt = new Date().toISOString();
+                proc.pid = child.pid ?? undefined;
+                proc.lastError = undefined;
+                if (child.pid) {
+                    this.commandPrimary.set(proc.config.command, proc.name);
+                    this.writePidFile(proc.name, child.pid);
+                }
+                this.eventBus?.emit(`lifecycle:started:${proc.name}`, {
+                    name: proc.name,
+                    pid: child.pid,
+                });
+                this.startMemoryCheck(proc);
+            });
+        }
+        // Handle exit
+        child.on('exit', (code, signal) => {
+            this.clearTimers(proc);
+            this.removePidFile(proc.name);
+            proc.pid = undefined;
+            if (proc.status === 'stopping') {
+                // We initiated the stop — do not restart
+                this.markStopped(proc);
+                this.eventBus?.emit(`lifecycle:stopped:${proc.name}`, {
+                    name: proc.name,
+                    code,
+                    signal,
+                });
+                return;
+            }
+            if (code === 0) {
+                // Normal exit (e.g. script finished on its own) — do not restart
+                this.markStopped(proc);
+                this.eventBus?.emit(`lifecycle:stopped:${proc.name}`, {
+                    name: proc.name,
+                    code,
+                    signal,
+                });
+            }
+            else {
+                // Abnormal exit (non-zero code or killed by signal) — handle as crash (may restart)
+                this.handleCrash(proc, code, signal);
+            }
+        });
+    }
+    // -----------------------------------------------------------------------
+    // Crash handling with exponential backoff
+    // -----------------------------------------------------------------------
+    handleCrash(proc, code, signal) {
+        const exitReason = signal
+            ? `killed by signal ${signal}`
+            : `exited with code ${code}`;
+        const errorMsg = `Process ${exitReason}`;
+        proc.lastError = errorMsg;
+        proc.status = 'failed';
+        this.eventBus?.emit(`lifecycle:failed:${proc.name}`, {
+            name: proc.name,
+            code,
+            signal,
+            error: errorMsg,
+        });
+        const shouldRestart = proc.config.restart ?? DEFAULT_RESTART;
+        if (!shouldRestart) {
+            return;
+        }
+        // Sliding window reset
+        const windowMs = proc.config.restartWindowMs ?? DEFAULT_RESTART_WINDOW_MS;
+        if (Date.now() - proc.windowStart > windowMs) {
+            proc.restartCount = 0;
+            proc.windowStart = Date.now();
+        }
+        proc.restartCount++;
+        const maxRestarts = proc.config.maxRestarts ?? DEFAULT_MAX_RESTARTS;
+        if (proc.restartCount > maxRestarts) {
+            proc.status = 'failed';
+            this.eventBus?.emit(`lifecycle:circuit_open:${proc.name}`, {
+                name: proc.name,
+                restartCount: proc.restartCount,
+                maxRestarts,
+            });
+            return;
+        }
+        // Calculate exponential backoff
+        const initialMs = proc.config.backoffInitialMs ?? DEFAULT_BACKOFF_INITIAL_MS;
+        const maxMs = proc.config.backoffMaxMs ?? DEFAULT_BACKOFF_MAX_MS;
+        const multiplier = proc.config.backoffMultiplier ?? DEFAULT_BACKOFF_MULTIPLIER;
+        const backoffMs = Math.min(initialMs * Math.pow(multiplier, proc.restartCount - 1), maxMs);
+        proc.currentBackoffMs = backoffMs;
+        proc.status = 'backoff';
+        this.eventBus?.emit(`lifecycle:restart:${proc.name}`, {
+            name: proc.name,
+            restartCount: proc.restartCount,
+            backoffMs,
+        });
+        // Schedule restart after backoff
+        proc.backoffTimer = setTimeout(() => {
+            proc.backoffTimer = undefined;
+            this.spawnProcess(proc);
+        }, backoffMs);
+    }
+    // -----------------------------------------------------------------------
+    // Graceful stop
+    // -----------------------------------------------------------------------
+    async stopProcess(proc) {
+        this.clearTimers(proc);
+        this.removePidFile(proc.name);
+        if (!proc.childProcess || proc.childProcess.killed) {
+            this.markStopped(proc);
+            proc.childProcess = null;
+            return;
+        }
+        const wasRunning = proc.status === 'running' || proc.status === 'starting' || proc.status === 'backoff';
+        if (!wasRunning) {
+            return;
+        }
+        proc.status = 'stopping';
+        const shutdownMs = proc.config.shutdownTimeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS;
+        return new Promise((resolve) => {
+            const child = proc.childProcess;
+            let resolved = false;
+            const done = () => {
+                if (!resolved) {
+                    resolved = true;
+                    this.markStopped(proc);
+                    proc.childProcess = null;
+                    resolve();
+                }
+            };
+            // Listen for exit
+            child.on('exit', () => {
+                done();
+            });
+            // Send SIGTERM to entire process group
+            try {
+                if (child.pid) {
+                    process.kill(-child.pid, 'SIGTERM');
+                }
+                else {
+                    child.kill('SIGTERM');
+                }
+            }
+            catch {
+                // Process may have already exited
+                done();
+                return;
+            }
+            // Force kill after timeout
+            const forceTimer = setTimeout(() => {
+                if (!resolved && !child.killed) {
+                    try {
+                        if (child.pid) {
+                            process.kill(-child.pid, 'SIGKILL');
+                        }
+                        else {
+                            child.kill('SIGKILL');
+                        }
+                    }
+                    catch {
+                        // already dead
+                    }
+                    done();
+                }
+            }, shutdownMs);
+            // If process exits before timeout, clean up timer
+            child.on('exit', () => {
+                clearTimeout(forceTimer);
+            });
+        });
+    }
+    // -----------------------------------------------------------------------
+    // Memory check
+    // -----------------------------------------------------------------------
+    startMemoryCheck(proc) {
+        this.clearMemoryCheck(proc);
+        proc.memoryCheckTimer = setInterval(() => {
+            this.checkMemory(proc);
+        }, MEMORY_CHECK_INTERVAL_MS);
+    }
+    checkMemory(proc) {
+        const pid = proc.pid;
+        if (!pid || !proc.childProcess || proc.childProcess.killed) {
+            return;
+        }
+        const limitMb = proc.config.memoryLimitMb ?? DEFAULT_MEMORY_LIMIT_MB;
+        const limitKb = limitMb * 1024;
+        try {
+            const output = execSync(`ps -o rss= -p ${pid}`, {
+                encoding: 'utf-8',
+                timeout: 3_000,
+            }).trim();
+            const rssKb = parseInt(output, 10);
+            if (isNaN(rssKb)) {
+                return;
+            }
+            if (rssKb > limitKb) {
+                console.error(`[lifecycle:${proc.name}] memory limit exceeded: ${Math.round(rssKb / 1024)}MB > ${limitMb}MB, killing process`);
+                this.eventBus?.emit(`lifecycle:memory_exceeded:${proc.name}`, {
+                    name: proc.name,
+                    rssMb: Math.round(rssKb / 1024),
+                    limitMb,
+                });
+                // Mark as stopping so the exit handler doesn't treat this as a crash
+                proc.status = 'stopping';
+                // Graceful kill: SIGTERM then SIGKILL after timeout
+                const shutdownMs = proc.config.shutdownTimeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS;
+                try {
+                    if (proc.childProcess.pid) {
+                        process.kill(-proc.childProcess.pid, 'SIGTERM');
+                    }
+                    else {
+                        proc.childProcess.kill('SIGTERM');
+                    }
+                }
+                catch {
+                    // already dead
+                }
+                setTimeout(() => {
+                    if (proc.childProcess && !proc.childProcess.killed) {
+                        try {
+                            if (proc.childProcess.pid) {
+                                process.kill(-proc.childProcess.pid, 'SIGKILL');
+                            }
+                            else {
+                                proc.childProcess.kill('SIGKILL');
+                            }
+                        }
+                        catch {
+                            // already dead
+                        }
+                    }
+                }, shutdownMs);
+            }
+        }
+        catch {
+            // ps command failed — process likely already exited, skip check
+        }
+    }
+    // -----------------------------------------------------------------------
+    // Helpers
+    // -----------------------------------------------------------------------
+    clearTimers(proc) {
+        this.clearBackoff(proc);
+        this.clearMemoryCheck(proc);
+    }
+    clearBackoff(proc) {
+        if (proc.backoffTimer) {
+            clearTimeout(proc.backoffTimer);
+            proc.backoffTimer = undefined;
+        }
+    }
+    clearMemoryCheck(proc) {
+        if (proc.memoryCheckTimer) {
+            clearInterval(proc.memoryCheckTimer);
+            proc.memoryCheckTimer = undefined;
+        }
+    }
+    /** Mark a process as stopped and clear start-time metadata. */
+    markStopped(proc) {
+        // Clear command primary if this was the primary
+        if (this.commandPrimary.get(proc.config.command) === proc.name) {
+            this.commandPrimary.delete(proc.config.command);
+        }
+        proc.status = 'stopped';
+        proc.startedAt = undefined;
+        proc.pid = undefined;
+    }
+    // -----------------------------------------------------------------------
+    // PID file management & orphan cleanup
+    // -----------------------------------------------------------------------
+    /** Write PID file so we can clean up orphans after server restart. */
+    async writePidFile(name, pid) {
+        try {
+            await fs.mkdir(this.pidDir, { recursive: true });
+            await fs.writeFile(path.join(this.pidDir, `${name}.pid`), String(pid), 'utf-8');
+        }
+        catch {
+            // Non-critical — best effort
+        }
+    }
+    /** Remove PID file on clean stop/exit. */
+    async removePidFile(name) {
+        try {
+            await fs.unlink(path.join(this.pidDir, `${name}.pid`)).catch(() => { });
+        }
+        catch {
+            // Non-critical
+        }
+    }
+    /**
+     * Kill orphan processes from a previous server run.
+     * Reads PID files from .loom/.pids/, verifies the process still exists,
+     * and kills the entire process group before spawning new ones.
+     */
+    async cleanOrphanProcesses() {
+        try {
+            const entries = await fs.readdir(this.pidDir).catch(() => []);
+            const pidFiles = entries.filter(f => f.endsWith('.pid'));
+            for (const pidFile of pidFiles) {
+                const filePath = path.join(this.pidDir, pidFile);
+                try {
+                    const pidStr = await fs.readFile(filePath, 'utf-8');
+                    const pid = parseInt(pidStr.trim(), 10);
+                    if (isNaN(pid)) {
+                        await fs.unlink(filePath).catch(() => { });
+                        continue;
+                    }
+                    // Check if the process still exists
+                    try {
+                        process.kill(pid, 0); // signal 0 = existence check
+                    }
+                    catch {
+                        // Process doesn't exist anymore — just clean up the PID file
+                        await fs.unlink(filePath).catch(() => { });
+                        continue;
+                    }
+                    // Process is still alive — kill the entire process group
+                    console.log(`[lifecycle] Cleaning up orphan process: ${pidFile.replace('.pid', '')} (PID ${pid}, PGID ${pid})`);
+                    try {
+                        process.kill(-pid, 'SIGTERM');
+                    }
+                    catch {
+                        // Process group may not exist, try killing just the process
+                        try {
+                            process.kill(pid, 'SIGTERM');
+                        }
+                        catch {
+                            // Already dead
+                        }
+                    }
+                    // Give it a moment, then force kill if still alive
+                    await new Promise((resolve) => {
+                        const forceTimer = setTimeout(() => {
+                            try {
+                                process.kill(-pid, 'SIGKILL');
+                            }
+                            catch { /* already dead */ }
+                            try {
+                                process.kill(pid, 'SIGKILL');
+                            }
+                            catch { /* already dead */ }
+                            resolve();
+                        }, 3_000);
+                        // Poll for process exit
+                        const checkInterval = setInterval(() => {
+                            try {
+                                process.kill(pid, 0);
+                            }
+                            catch {
+                                // Process is gone
+                                clearInterval(checkInterval);
+                                clearTimeout(forceTimer);
+                                resolve();
+                            }
+                        }, 200);
+                    });
+                    await fs.unlink(filePath).catch(() => { });
+                }
+                catch (err) {
+                    console.error(`[lifecycle] Error cleaning up orphan ${pidFile}:`, err instanceof Error ? err.message : String(err));
+                }
+            }
+            if (pidFiles.length > 0) {
+                // Ensure PID directory exists for new runs
+                await fs.mkdir(this.pidDir, { recursive: true });
+            }
+        }
+        catch {
+            // Non-critical — best effort
+        }
+    }
+    toScriptInfo(proc) {
+        // If mirroring another script, return the primary's runtime status
+        if (proc.mirrorsPrimary) {
+            const primary = this.processes.get(proc.mirrorsPrimary);
+            if (primary) {
+                return {
+                    name: proc.name,
+                    command: proc.config.command,
+                    status: primary.status,
+                    pid: primary.pid,
+                    restartCount: primary.restartCount,
+                    currentBackoffMs: primary.currentBackoffMs,
+                    lastError: primary.lastError,
+                    startedAt: primary.startedAt,
+                    enabled: proc.config.enabled ?? DEFAULT_ENABLED,
+                };
+            }
+            // Primary gone — clear mirror and return stopped
+            proc.mirrorsPrimary = undefined;
+        }
+        return {
+            name: proc.name,
+            command: proc.config.command,
+            status: proc.status,
+            pid: proc.pid,
+            restartCount: proc.restartCount,
+            currentBackoffMs: proc.currentBackoffMs,
+            lastError: proc.lastError,
+            startedAt: proc.startedAt,
+            enabled: proc.config.enabled ?? DEFAULT_ENABLED,
+        };
+    }
+}
+//# sourceMappingURL=manager.js.map