@orkify/cli 1.0.0-beta.5

package/dist/daemon/ManagedProcess.js

@@ -0,0 +1,1020 @@
+import { watch } from 'chokidar';
+import { fork } from 'node:child_process';
+import { randomBytes } from 'node:crypto';
+import { EventEmitter } from 'node:events';
+import { existsSync, mkdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import pidusage from 'pidusage';
+import { ExecMode, LAUNCH_TIMEOUT, LOGS_DIR, MEMORY_RESTART_COOLDOWN, METRICS_PROBE_IMPORT, ProcessStatus, } from '../constants.js';
+import { RotatingWriter } from './RotatingWriter.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+export class ManagedProcess extends EventEmitter {
+    id;
+    config;
+    // For fork mode: single child process
+    forkProcess = null;
+    // For cluster mode: primary process that manages workers
+    clusterPrimary = null;
+    clusterWorkers = new Map();
+    slotRestarts = new Map();
+    slotCrashes = new Map();
+    launchTimers = new Map();
+    outWriter = null;
+    errWriter = null;
+    watcher = null;
+    isShuttingDown = false;
+    isReloading = false;
+    statsInterval = null;
+    forkRestarts = 0;
+    forkCrashes = 0;
+    forkCreatedAt = 0;
+    forkReady = false;
+    forkLaunchTimer = null;
+    detectedPort;
+    cronSecret;
+    primaryRestarts = 0;
+    lastMemoryRestart = 0;
+    workerMemoryCooldowns = new Map();
+    memoryRestartingWorkers = new Set();
+    // Ring buffer of recent stderr lines for crash diagnostics
+    recentStderr = [];
+    static STDERR_BUFFER_SIZE = 10;
+    forkStats = {
+        memory: 0,
+        cpu: 0,
+        heapUsed: 0,
+        heapTotal: 0,
+        external: 0,
+        arrayBuffers: 0,
+        eventLoopLag: 0,
+        eventLoopLagP95: 0,
+        activeHandles: 0,
+    };
+    constructor(id, config) {
+        super();
+        this.id = id;
+        this.config = config;
+        if (config.cron?.length) {
+            this.cronSecret = randomBytes(32).toString('hex');
+        }
+        this.setupLogStreams();
+    }
+    setupLogStreams() {
+        if (!existsSync(LOGS_DIR)) {
+            mkdirSync(LOGS_DIR, { recursive: true });
+        }
+        const outPath = join(LOGS_DIR, `${this.config.name}.stdout.log`);
+        const errPath = join(LOGS_DIR, `${this.config.name}.stderr.log`);
+        this.outWriter = new RotatingWriter(outPath, this.config.logMaxSize, this.config.logMaxFiles, this.config.logMaxAge);
+        this.errWriter = new RotatingWriter(errPath, this.config.logMaxSize, this.config.logMaxFiles, this.config.logMaxAge);
+    }
+    async start() {
+        if (this.isShuttingDown) {
+            return;
+        }
+        if (this.config.execMode === ExecMode.CLUSTER) {
+            await this.startCluster();
+        }
+        else {
+            await this.startFork();
+        }
+        if (this.config.watch) {
+            this.setupWatcher();
+        }
+        this.startStatsCollection();
+    }
+    async startFork() {
+        const env = {
+            ...process.env,
+            ...this.config.env,
+            ORKIFY_PROCESS_ID: String(this.id),
+            ORKIFY_WORKER_ID: '0',
+            ORKIFY_PROCESS_NAME: this.config.name,
+            ORKIFY_EXEC_MODE: 'fork',
+        };
+        if (this.config.healthCheck) {
+            env.ORKIFY_HEALTH_CHECK = this.config.healthCheck;
+        }
+        if (this.config.port !== undefined) {
+            env.ORKIFY_PORT = String(this.config.port);
+        }
+        if (this.cronSecret) {
+            env.ORKIFY_CRON_SECRET = this.cronSecret;
+        }
+        // Prepend --import for the metrics probe so it runs inside the child
+        const execArgv = [METRICS_PROBE_IMPORT, ...this.config.nodeArgs];
+        // windowsHide is supported by fork() but not in TypeScript types
+        // See: https://github.com/nodejs/node/issues/17370
+        this.forkProcess = fork(this.config.script, this.config.args, {
+            cwd: this.config.cwd,
+            env,
+            execArgv,
+            stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
+            detached: false,
+            windowsHide: true, // Hide subprocess console window on Windows
+        });
+        this.forkCreatedAt = Date.now();
+        this.forkReady = false;
+        this.recentStderr = [];
+        this.setupForkHandlers(this.forkProcess);
+        this.startForkLaunchTimer();
+    }
+    startForkLaunchTimer() {
+        this.clearForkLaunchTimer();
+        this.forkLaunchTimer = setTimeout(() => {
+            this.forkLaunchTimer = null;
+            if (!this.forkReady && this.forkProcess && !this.isShuttingDown) {
+                console.error(`[ERROR] ${this.config.name}: process failed to start — not listening and no ready signal after ${LAUNCH_TIMEOUT / 1000}s.\n` +
+                    `  Common causes:\n` +
+                    `  - Application crashed or hung during startup\n` +
+                    `  - Missing process.send('ready') for apps that don't bind a port`);
+                this.emit('worker:error', {
+                    workerId: 0,
+                    error: new Error(`Fork process launch timeout after ${LAUNCH_TIMEOUT / 1000}s`),
+                });
+            }
+        }, LAUNCH_TIMEOUT);
+    }
+    clearForkLaunchTimer() {
+        if (this.forkLaunchTimer) {
+            clearTimeout(this.forkLaunchTimer);
+            this.forkLaunchTimer = null;
+        }
+    }
+    setupForkHandlers(child) {
+        child.stdout?.on('data', (data) => {
+            this.handleLog('out', 0, data);
+        });
+        child.stderr?.on('data', (data) => {
+            this.handleLog('err', 0, data);
+        });
+        child.on('message', (message) => {
+            // Handle metrics probe messages from the injected MetricsProbe preload
+            const msg = message;
+            if (msg?.__orkify && msg.type === 'metrics' && msg.data) {
+                const d = msg.data;
+                this.forkStats.heapUsed = d.heapUsed ?? 0;
+                this.forkStats.heapTotal = d.heapTotal ?? 0;
+                this.forkStats.external = d.external ?? 0;
+                this.forkStats.arrayBuffers = d.arrayBuffers ?? 0;
+                this.forkStats.eventLoopLag = d.eventLoopLag ?? 0;
+                this.forkStats.eventLoopLagP95 = d.eventLoopLagP95 ?? 0;
+                this.forkStats.activeHandles = d.activeHandles ?? 0;
+                if (d.cacheSize !== undefined) {
+                    this.forkStats.cacheSize = d.cacheSize;
+                    this.forkStats.cacheTotalBytes = d.cacheTotalBytes;
+                    this.forkStats.cacheHits = d.cacheHits;
+                    this.forkStats.cacheMisses = d.cacheMisses;
+                    this.forkStats.cacheHitRate = d.cacheHitRate;
+                }
+                return;
+            }
+            if (msg?.__orkify && msg.type === 'error' && msg.data) {
+                this.emit('worker:error:captured', {
+                    workerId: 0,
+                    error: msg.data,
+                });
+                return;
+            }
+            // Port auto-detection from the metrics probe's net.Server.listen hook
+            if (msg?.__orkify && msg.type === 'listening' && msg.data) {
+                const port = msg.data.port;
+                if (port && !this.detectedPort) {
+                    this.detectedPort = port;
+                }
+                if (!this.forkReady) {
+                    this.forkReady = true;
+                    this.clearForkLaunchTimer();
+                    const effectivePort = this.config.port ?? this.detectedPort;
+                    if (this.config.healthCheck && effectivePort) {
+                        this.checkHealth(effectivePort, this.config.healthCheck)
+                            .then(() => this.emit('worker:ready', 0))
+                            .catch((err) => this.emit('worker:error', { workerId: 0, error: err }));
+                    }
+                    else {
+                        this.emit('worker:ready', 0);
+                    }
+                }
+                return;
+            }
+            if (message === 'ready') {
+                this.forkReady = true;
+                this.clearForkLaunchTimer();
+                if (this.config.healthCheck && this.config.port) {
+                    this.checkHealth(this.config.port, this.config.healthCheck)
+                        .then(() => this.emit('worker:ready', 0))
+                        .catch((err) => this.emit('worker:error', { workerId: 0, error: err }));
+                }
+                else {
+                    this.emit('worker:ready', 0);
+                }
+            }
+            else {
+                this.emit('message', { workerId: 0, message });
+            }
+        });
+        child.on('exit', (code, signal) => {
+            this.clearForkLaunchTimer();
+            this.emit('worker:exit', { workerId: 0, code, signal });
+            if (!this.isShuttingDown) {
+                // Clean exit (code 0, no signal) is not a crash — don't restart or count
+                if (code === 0 && !signal) {
+                    this.forkProcess = null;
+                    this.emit('process:finished', { code, signal });
+                    return;
+                }
+                const uptime = Date.now() - this.forkCreatedAt;
+                this.forkCrashes++;
+                if (this.forkRestarts < this.config.maxRestarts) {
+                    if (uptime < this.config.minUptime) {
+                        const stderrContext = this.recentStderr.length
+                            ? `\n  Last stderr:\n    ${this.recentStderr.join('\n    ')}`
+                            : '';
+                        console.error(`[${this.config.name}] Process crashed after ${uptime}ms (exit code ${code ?? 'null'}, signal ${signal ?? 'none'})${stderrContext}`);
+                    }
+                    this.forkRestarts++;
+                    // Exponential backoff: delay * 2^(restarts-1), capped at 15s
+                    const backoffDelay = Math.min(this.config.restartDelay * Math.pow(2, this.forkRestarts - 1), 15000);
+                    setTimeout(() => {
+                        if (!this.isShuttingDown) {
+                            this.startFork();
+                        }
+                    }, backoffDelay);
+                }
+                else {
+                    const stderrContext = this.recentStderr.length
+                        ? `\n  Last stderr:\n    ${this.recentStderr.join('\n    ')}`
+                        : '';
+                    console.error(`[${this.config.name}] Max restarts exceeded${stderrContext}`);
+                    this.forkProcess = null;
+                    this.emit('worker:maxRestarts', 0);
+                    this.emit('process:finished', { code, signal });
+                }
+            }
+            else {
+                // Process exited during shutdown
+                this.forkProcess = null;
+            }
+        });
+        child.on('error', (err) => {
+            console.error(`[${this.config.name}] Process error:`, err.message);
+            this.emit('worker:error', { workerId: 0, error: err });
+        });
+    }
+    async startCluster() {
+        const clusterWrapperPath = join(__dirname, '..', 'cluster', 'ClusterWrapper.js');
+        const env = {
+            ...process.env,
+            ...this.config.env,
+            ORKIFY_SCRIPT: this.config.script,
+            ORKIFY_WORKERS: String(this.config.workerCount),
+            ORKIFY_PROCESS_NAME: this.config.name,
+            ORKIFY_PROCESS_ID: String(this.id),
+            ORKIFY_KILL_TIMEOUT: String(this.config.killTimeout),
+            ORKIFY_STICKY: String(this.config.sticky),
+            ORKIFY_RELOAD_RETRIES: String(this.config.reloadRetries ?? 3),
+        };
+        // Set sticky port for TCP-level session routing
+        if (this.config.sticky && this.config.port) {
+            env.ORKIFY_STICKY_PORT = String(this.config.port);
+        }
+        // Pass health check config to ClusterWrapper
+        if (this.config.healthCheck) {
+            env.ORKIFY_HEALTH_CHECK = this.config.healthCheck;
+        }
+        if (this.config.port !== undefined) {
+            env.ORKIFY_PORT = String(this.config.port);
+        }
+        if (this.cronSecret) {
+            env.ORKIFY_CRON_SECRET = this.cronSecret;
+        }
+        if (this.config.args.length > 0) {
+            env.ORKIFY_ARGS = JSON.stringify(this.config.args);
+        }
+        // Spawn the cluster wrapper as the primary
+        // windowsHide is supported by fork() but not in TypeScript types
+        // See: https://github.com/nodejs/node/issues/17370
+        this.clusterPrimary = fork(clusterWrapperPath, [], {
+            cwd: this.config.cwd,
+            env,
+            execArgv: this.config.nodeArgs,
+            stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
+            detached: false,
+            windowsHide: true, // Hide subprocess console window on Windows
+        });
+        this.recentStderr = [];
+        this.setupClusterHandlers(this.clusterPrimary);
+        // Wait for primary to be ready
+        await this.waitForPrimaryReady();
+    }
+    setupClusterHandlers(primary) {
+        // With silent: true, per-worker output arrives via IPC (worker:output).
+        // Primary stdout still carries the ClusterWrapper's own log() lines — capture as primary (-1).
+        primary.stdout?.on('data', (data) => {
+            this.handleLog('out', -1, data);
+        });
+        primary.stderr?.on('data', (data) => {
+            this.handleLog('err', -1, data);
+        });
+        primary.on('message', (message) => {
+            const msg = message;
+            switch (msg.type) {
+                case 'primary:ready':
+                    this.emit('primary:ready');
+                    break;
+                case 'worker:ready': {
+                    const readyWorkerId = msg.workerId;
+                    this.clearLaunchTimer(readyWorkerId);
+                    this.updateWorkerState(readyWorkerId, { ready: true, status: ProcessStatus.ONLINE });
+                    this.emit('worker:ready', readyWorkerId);
+                    break;
+                }
+                case 'worker:listening': {
+                    const listeningWorkerId = msg.workerId;
+                    const addr = msg.address;
+                    if (addr?.port && !this.detectedPort) {
+                        this.detectedPort = addr.port;
+                    }
+                    this.clearLaunchTimer(listeningWorkerId);
+                    this.updateWorkerState(listeningWorkerId, { ready: true, status: ProcessStatus.ONLINE });
+                    this.emit('worker:ready', listeningWorkerId);
+                    break;
+                }
+                case 'worker:online': {
+                    const onlineWorkerId = msg.workerId;
+                    const onlinePid = msg.pid;
+                    this.updateWorkerState(onlineWorkerId, {
+                        pid: onlinePid,
+                        status: ProcessStatus.LAUNCHING,
+                        ready: false,
+                        createdAt: Date.now(),
+                    });
+                    this.startLaunchTimer(onlineWorkerId);
+                    break;
+                }
+                case 'worker:exit': {
+                    const exitedWorkerId = msg.workerId;
+                    const exitedPid = msg.pid;
+                    this.clearLaunchTimer(exitedWorkerId);
+                    const existing = this.clusterWorkers.get(exitedWorkerId);
+                    if (existing) {
+                        const newRestarts = (existing.restarts ?? 0) + 1;
+                        this.slotRestarts.set(exitedWorkerId, newRestarts);
+                        // Only delete if PID matches — during reload, a new worker already holds this slot
+                        if (!exitedPid || existing.pid === exitedPid) {
+                            // Tracked worker exited — count as error if not a deliberate shutdown, reload, or memory restart
+                            if (!this.isShuttingDown &&
+                                !this.isReloading &&
+                                !this.memoryRestartingWorkers.has(exitedWorkerId)) {
+                                const newCrashes = (existing.crashes ?? 0) + 1;
+                                this.slotCrashes.set(exitedWorkerId, newCrashes);
+                            }
+                            this.memoryRestartingWorkers.delete(exitedWorkerId);
+                            this.clusterWorkers.delete(exitedWorkerId);
+                        }
+                        else {
+                            // PID mismatch: new worker already holds slot — carry the counter forward
+                            existing.restarts = newRestarts;
+                        }
+                    }
+                    this.emit('worker:exit', {
+                        workerId: exitedWorkerId,
+                        code: msg.code,
+                        signal: msg.signal,
+                    });
+                    break;
+                }
+                case 'worker:metrics': {
+                    const metricsWorkerId = msg.workerId;
+                    const metricsData = msg.data;
+                    const worker = this.clusterWorkers.get(metricsWorkerId);
+                    if (worker && metricsData) {
+                        worker.heapUsed = metricsData.heapUsed ?? 0;
+                        worker.heapTotal = metricsData.heapTotal ?? 0;
+                        worker.external = metricsData.external ?? 0;
+                        worker.arrayBuffers = metricsData.arrayBuffers ?? 0;
+                        worker.eventLoopLag = metricsData.eventLoopLag ?? 0;
+                        worker.eventLoopLagP95 = metricsData.eventLoopLagP95 ?? 0;
+                        worker.activeHandles = metricsData.activeHandles ?? 0;
+                        if (metricsData.cacheSize !== undefined) {
+                            worker.cacheSize = metricsData.cacheSize;
+                            worker.cacheTotalBytes = metricsData.cacheTotalBytes;
+                            worker.cacheHits = metricsData.cacheHits;
+                            worker.cacheMisses = metricsData.cacheMisses;
+                            worker.cacheHitRate = metricsData.cacheHitRate;
+                        }
+                        // Recover from launch timeout: if a worker is sending metrics,
+                        // it's alive and its event loop is responsive. The 30s launch
+                        // timeout already fired but the process didn't crash — it just
+                        // took longer than expected to start (e.g. Next.js compilation).
+                        if (worker.status === ProcessStatus.ERRORED) {
+                            worker.status = ProcessStatus.ONLINE;
+                            worker.ready = true;
+                        }
+                    }
+                    break;
+                }
+                case 'worker:output': {
+                    const outputWorkerId = msg.workerId;
+                    const outputStream = msg.stream;
+                    const outputData = msg.data;
+                    this.handleLog(outputStream, outputWorkerId, Buffer.from(outputData));
+                    break;
+                }
+                case 'worker:error:captured': {
+                    this.emit('worker:error:captured', {
+                        workerId: msg.workerId,
+                        error: msg.data,
+                    });
+                    break;
+                }
+                case 'reload:complete': {
+                    // Update stale flags from per-slot results
+                    const results = (msg.results ?? []);
+                    for (const result of results) {
+                        if (result.status === 'stale') {
+                            const worker = this.clusterWorkers.get(result.slotId);
+                            if (worker) {
+                                worker.stale = true;
+                            }
+                        }
+                    }
+                    // Clear stale flags if all slots succeeded
+                    if (results.length > 0 && results.every((r) => r.status === 'success')) {
+                        for (const worker of this.clusterWorkers.values()) {
+                            worker.stale = false;
+                        }
+                    }
+                    this.emit('reload:complete', { results });
+                    break;
+                }
+                case 'restart-worker-failed': {
+                    this.memoryRestartingWorkers.delete(msg.workerId);
+                    break;
+                }
+                default:
+                    this.emit('message', { workerId: -1, message: msg });
+            }
+        });
+        primary.on('exit', (code, signal) => {
+            this.clusterWorkers.clear();
+            this.emit('primary:exit', { code, signal });
+            if (!this.isShuttingDown) {
+                this.primaryRestarts++;
+                if (this.primaryRestarts <= this.config.maxRestarts) {
+                    const stderrContext = this.recentStderr.length
+                        ? `\n  Last stderr:\n    ${this.recentStderr.join('\n    ')}`
+                        : '';
+                    console.error(`[${this.config.name}] Cluster primary exited unexpectedly (exit code ${code ?? 'null'}, signal ${signal ?? 'none'}), restarting... (${this.primaryRestarts}/${this.config.maxRestarts})${stderrContext}`);
+                    const backoffDelay = Math.min(this.config.restartDelay * Math.pow(2, this.primaryRestarts - 1), 15000);
+                    setTimeout(() => {
+                        if (!this.isShuttingDown) {
+                            this.startCluster();
+                        }
+                    }, backoffDelay);
+                }
+                else {
+                    const stderrContext = this.recentStderr.length
+                        ? `\n  Last stderr:\n    ${this.recentStderr.join('\n    ')}`
+                        : '';
+                    console.error(`[${this.config.name}] Cluster primary max restarts exceeded${stderrContext}`);
+                    this.clusterPrimary = null;
+                    this.emit('worker:maxRestarts', -1);
+                    this.emit('process:finished', { code, signal });
+                }
+            }
+        });
+        primary.on('error', (err) => {
+            console.error(`[${this.config.name}] Cluster primary error:`, err.message);
+        });
+    }
+    waitForPrimaryReady() {
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                this.off('primary:ready', onReady);
+                reject(new Error(`Cluster primary failed to start within 10s`));
+            }, 10000);
+            const onReady = () => {
+                clearTimeout(timeout);
+                this.off('primary:ready', onReady);
+                resolve();
+            };
+            this.on('primary:ready', onReady);
+        });
+    }
+    updateWorkerState(workerId, updates) {
+        let state = this.clusterWorkers.get(workerId);
+        if (!state) {
+            state = {
+                id: workerId,
+                pid: 0,
+                status: ProcessStatus.LAUNCHING,
+                ready: false,
+                stale: false,
+                restarts: this.slotRestarts.get(workerId) ?? 0,
+                crashes: this.slotCrashes.get(workerId) ?? 0,
+                createdAt: Date.now(),
+                memory: 0,
+                cpu: 0,
+                heapUsed: 0,
+                heapTotal: 0,
+                external: 0,
+                arrayBuffers: 0,
+                eventLoopLag: 0,
+                eventLoopLagP95: 0,
+                activeHandles: 0,
+            };
+            this.clusterWorkers.set(workerId, state);
+        }
+        Object.assign(state, updates);
+    }
+    startLaunchTimer(workerId) {
+        this.clearLaunchTimer(workerId);
+        const timer = setTimeout(() => {
+            this.launchTimers.delete(workerId);
+            const worker = this.clusterWorkers.get(workerId);
+            if (worker && !worker.ready) {
+                worker.status = ProcessStatus.ERRORED;
+                console.error(`[ERROR] ${this.config.name}: worker ${workerId} failed to start — not listening and no ready signal after ${LAUNCH_TIMEOUT / 1000}s.\n` +
+                    `  Common causes:\n` +
+                    `  - Application crashed or hung during startup\n` +
+                    `  - Running a dev server in cluster mode (e.g., Next.js dev with -w 0)\n` +
+                    `  - Missing process.send('ready') for apps that don't bind a port`);
+                this.emit('worker:error', {
+                    workerId,
+                    error: new Error(`Worker ${workerId} launch timeout after ${LAUNCH_TIMEOUT / 1000}s`),
+                });
+            }
+        }, LAUNCH_TIMEOUT);
+        this.launchTimers.set(workerId, timer);
+    }
+    clearLaunchTimer(workerId) {
+        const timer = this.launchTimers.get(workerId);
+        if (timer) {
+            clearTimeout(timer);
+            this.launchTimers.delete(workerId);
+        }
+    }
+    clearAllLaunchTimers() {
+        for (const timer of this.launchTimers.values()) {
+            clearTimeout(timer);
+        }
+        this.launchTimers.clear();
+    }
+    async checkHealth(port, path) {
+        const url = `http://localhost:${port}${path}`;
+        for (let attempt = 0; attempt < 3; attempt++) {
+            try {
+                const resp = await fetch(url, { signal: AbortSignal.timeout(5000) });
+                if (resp.status >= 200 && resp.status < 300)
+                    return;
+            }
+            catch {
+                // Retry — app may still be booting
+            }
+            if (attempt < 2)
+                await new Promise((r) => setTimeout(r, 1000));
+        }
+        throw new Error(`Health check failed: ${url}`);
+    }
+    handleLog(type, workerId, data) {
+        const line = data.toString();
+        const timestamp = new Date().toISOString();
+        const workerLabel = workerId === -1 ? 'primary' : workerId;
+        const logLine = `[${timestamp}] [${this.config.name}:${workerLabel}] ${line}`;
+        if (type === 'out') {
+            this.outWriter?.write(logLine);
+        }
+        else {
+            this.errWriter?.write(logLine);
+            // Buffer recent stderr for crash diagnostics
+            const trimmed = line.trimEnd();
+            if (trimmed) {
+                this.recentStderr.push(trimmed);
+                if (this.recentStderr.length > ManagedProcess.STDERR_BUFFER_SIZE) {
+                    this.recentStderr.shift();
+                }
+            }
+        }
+        this.emit('log', { type, workerId, data: line });
+    }
+    setupWatcher() {
+        const paths = this.config.watchPaths || [this.config.cwd];
+        this.watcher = watch(paths, {
+            ignored: /(^|[/\\])\.|node_modules/,
+            persistent: true,
+            ignoreInitial: true,
+        });
+        let reloadTimeout = null;
+        this.watcher.on('change', (path) => {
+            this.emit('watch:change', path);
+            if (reloadTimeout) {
+                clearTimeout(reloadTimeout);
+            }
+            reloadTimeout = setTimeout(() => {
+                this.emit('watch:reload');
+            }, 300);
+        });
+    }
+    startStatsCollection() {
+        this.statsInterval = setInterval(async () => {
+            await this.collectStats();
+        }, 1000);
+    }
+    async collectStats() {
+        if (this.config.execMode === ExecMode.FORK && this.forkProcess?.pid) {
+            try {
+                const stats = await pidusage(this.forkProcess.pid);
+                // Store stats for getInfo() — preserve probe metrics
+                this.forkStats.memory = stats.memory;
+                this.forkStats.cpu = stats.cpu;
+            }
+            catch {
+                // Process might have exited
+            }
+        }
+        else if (this.config.execMode === ExecMode.CLUSTER) {
+            // Collect stats for each worker
+            for (const [_workerId, state] of this.clusterWorkers) {
+                if (state.pid) {
+                    try {
+                        const stats = await pidusage(state.pid);
+                        state.memory = stats.memory;
+                        state.cpu = stats.cpu;
+                    }
+                    catch {
+                        // Worker might have exited
+                    }
+                }
+            }
+        }
+        this.checkMemoryThreshold();
+    }
+    checkMemoryThreshold() {
+        const limit = this.config.restartOnMemory;
+        if (!limit)
+            return;
+        if (this.isShuttingDown)
+            return;
+        if (this.config.execMode === ExecMode.FORK) {
+            // Cooldown: skip checks for 30s after a memory-triggered restart
+            if (Date.now() - this.lastMemoryRestart < MEMORY_RESTART_COOLDOWN)
+                return;
+            if (this.forkStats.memory > limit) {
+                console.log(`[${this.config.name}] RSS ${formatBytes(this.forkStats.memory)} exceeds ${formatBytes(limit)}, restarting`);
+                this.lastMemoryRestart = Date.now();
+                this.forkRestarts++;
+                this.emit('worker:memoryRestart', {
+                    workerId: 0,
+                    memory: this.forkStats.memory,
+                    limit,
+                });
+                this.memoryRestartFork().catch((err) => {
+                    console.error(`[${this.config.name}] Memory-triggered restart failed:`, err.message);
+                });
+            }
+        }
+        else {
+            // Cluster: check each worker individually
+            for (const [workerId, state] of this.clusterWorkers) {
+                const lastRestart = this.workerMemoryCooldowns.get(workerId) ?? 0;
+                if (Date.now() - lastRestart < MEMORY_RESTART_COOLDOWN)
+                    continue;
+                if (state.memory > limit) {
+                    console.log(`[${this.config.name}] Worker ${workerId} RSS ${formatBytes(state.memory)} exceeds ${formatBytes(limit)}, restarting worker`);
+                    this.workerMemoryCooldowns.set(workerId, Date.now());
+                    this.emit('worker:memoryRestart', {
+                        workerId,
+                        memory: state.memory,
+                        limit,
+                    });
+                    this.memoryRestartWorker(workerId);
+                }
+            }
+        }
+    }
+    async memoryRestartFork() {
+        await this.stop();
+        this.isShuttingDown = false;
+        this.forkReady = false;
+        this.setupLogStreams();
+        await this.start();
+    }
+    memoryRestartWorker(workerId) {
+        if (this.isReloading || !this.clusterPrimary?.connected)
+            return;
+        this.memoryRestartingWorkers.add(workerId);
+        this.clusterPrimary.send({ type: 'restart-worker', workerId });
+    }
+    async stop(opts) {
+        this.isShuttingDown = true;
+        this.clearForkLaunchTimer();
+        this.clearAllLaunchTimers();
+        if (this.statsInterval) {
+            clearInterval(this.statsInterval);
+            this.statsInterval = null;
+        }
+        if (this.watcher) {
+            await this.watcher.close();
+            this.watcher = null;
+        }
+        if (this.config.execMode === ExecMode.FORK) {
+            await this.stopFork(opts);
+        }
+        else {
+            await this.stopCluster(opts);
+        }
+        this.outWriter?.end();
+        this.errWriter?.end();
+    }
+    async flushLogs() {
+        await Promise.all([this.outWriter?.flush(), this.errWriter?.flush()]);
+    }
+    /**
+     * Immediately SIGKILL all child processes without waiting for graceful shutdown.
+     */
+    forceKill() {
+        this.isShuttingDown = true;
+        this.clearForkLaunchTimer();
+        this.clearAllLaunchTimers();
+        if (this.statsInterval) {
+            clearInterval(this.statsInterval);
+            this.statsInterval = null;
+        }
+        if (this.config.execMode === ExecMode.FORK) {
+            if (this.forkProcess) {
+                this.forkProcess.kill('SIGKILL');
+                this.forkProcess = null;
+            }
+        }
+        else {
+            if (this.clusterPrimary) {
+                this.clusterPrimary.kill('SIGKILL');
+                this.clusterPrimary = null;
+                this.clusterWorkers.clear();
+            }
+        }
+        this.outWriter?.end();
+        this.errWriter?.end();
+    }
+    async stopFork(opts) {
+        const child = this.forkProcess;
+        if (!child)
+            return;
+        return new Promise((resolve) => {
+            const timeout = setTimeout(() => {
+                child.kill('SIGKILL');
+                resolve();
+            }, this.config.killTimeout);
+            child.once('exit', () => {
+                clearTimeout(timeout);
+                this.forkProcess = null;
+                resolve();
+            });
+            if (opts?.persistCache && child.connected) {
+                const flushTimer = setTimeout(() => child.kill('SIGTERM'), 2000);
+                child.once('message', (msg) => {
+                    const m = msg;
+                    if (m?.__orkify && m.type === 'cache:flushed') {
+                        clearTimeout(flushTimer);
+                    }
+                });
+                try {
+                    child.send({ __orkify: true, type: 'cache:flush' });
+                }
+                catch {
+                    clearTimeout(flushTimer);
+                    child.kill('SIGTERM');
+                }
+            }
+            else {
+                child.kill('SIGTERM');
+            }
+        });
+    }
+    async stopCluster(opts) {
+        const primary = this.clusterPrimary;
+        if (!primary)
+            return;
+        return new Promise((resolve) => {
+            const timeout = setTimeout(() => {
+                primary.kill('SIGKILL');
+                resolve();
+            }, this.config.killTimeout + 5000); // Extra time for workers
+            primary.once('exit', () => {
+                clearTimeout(timeout);
+                this.clusterPrimary = null;
+                this.clusterWorkers.clear();
+                resolve();
+            });
+            // Send shutdown command to cluster primary
+            if (primary.connected) {
+                primary.send({ type: 'shutdown', persistCache: opts?.persistCache });
+            }
+            else {
+                primary.kill('SIGTERM');
+            }
+        });
+    }
+    async restart() {
+        await this.stop();
+        this.isShuttingDown = false;
+        this.forkRestarts = 0;
+        this.forkCrashes = 0;
+        this.forkReady = false;
+        this.primaryRestarts = 0;
+        this.lastMemoryRestart = 0;
+        this.workerMemoryCooldowns.clear();
+        this.memoryRestartingWorkers.clear();
+        this.clusterWorkers.clear();
+        this.slotRestarts.clear();
+        this.slotCrashes.clear();
+        // Re-create log writers since stop() closed them
+        this.setupLogStreams();
+        await this.start();
+    }
+    async reload() {
+        if (this.config.execMode === ExecMode.FORK) {
+            // For fork mode, just restart
+            await this.restart();
+            return;
+        }
+        // For cluster mode, send reload command to primary
+        const primary = this.clusterPrimary;
+        if (!primary?.connected) {
+            throw new Error('Cluster primary not connected');
+        }
+        this.isReloading = true;
+        // Compute a timeout that accommodates the worst-case reload duration:
+        // - Success path: each worker waits up to LAUNCH_TIMEOUT to become ready + KILL_TIMEOUT to stop old
+        // - Failure path: one slot retries (reloadRetries+1) × LAUNCH_TIMEOUT before aborting
+        const retries = this.config.reloadRetries ?? 3;
+        const perSlotSuccess = LAUNCH_TIMEOUT + this.config.killTimeout;
+        const failurePath = (retries + 1) * LAUNCH_TIMEOUT + this.config.killTimeout;
+        const reloadTimeout = Math.max(this.config.workerCount * perSlotSuccess, failurePath) + 5000;
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                this.isReloading = false;
+                reject(new Error('Reload timeout'));
+            }, reloadTimeout);
+            const onComplete = () => {
+                clearTimeout(timeout);
+                this.off('reload:complete', onComplete);
+                this.isReloading = false;
+                resolve();
+            };
+            this.on('reload:complete', onComplete);
+            primary.send({ type: 'reload' });
+        });
+    }
+    getInfo() {
+        const workers = [];
+        if (this.config.execMode === ExecMode.FORK) {
+            const stats = this.forkStats;
+            if (this.forkProcess) {
+                workers.push({
+                    id: 0,
+                    pid: this.forkProcess.pid || 0,
+                    status: this.isShuttingDown ? ProcessStatus.STOPPING : ProcessStatus.ONLINE,
+                    restarts: this.forkRestarts,
+                    crashes: this.forkCrashes,
+                    uptime: Date.now() - this.forkCreatedAt,
+                    memory: stats.memory,
+                    cpu: stats.cpu,
+                    createdAt: this.forkCreatedAt,
+                    heapUsed: stats.heapUsed,
+                    heapTotal: stats.heapTotal,
+                    external: stats.external,
+                    arrayBuffers: stats.arrayBuffers,
+                    eventLoopLag: stats.eventLoopLag,
+                    eventLoopLagP95: stats.eventLoopLagP95,
+                    activeHandles: stats.activeHandles,
+                    ...(stats.cacheSize !== undefined && {
+                        cacheSize: stats.cacheSize,
+                        cacheTotalBytes: stats.cacheTotalBytes,
+                        cacheHits: stats.cacheHits,
+                        cacheMisses: stats.cacheMisses,
+                        cacheHitRate: stats.cacheHitRate,
+                    }),
+                });
+            }
+            else {
+                // Process has stopped - still show worker entry for restart count
+                workers.push({
+                    id: 0,
+                    pid: 0,
+                    status: ProcessStatus.STOPPED,
+                    restarts: this.forkRestarts,
+                    crashes: this.forkCrashes,
+                    uptime: 0,
+                    memory: 0,
+                    cpu: 0,
+                    createdAt: this.forkCreatedAt,
+                });
+            }
+        }
+        else {
+            for (const state of this.clusterWorkers.values()) {
+                workers.push({
+                    id: state.id,
+                    pid: state.pid,
+                    status: state.status,
+                    restarts: state.restarts,
+                    crashes: state.crashes,
+                    uptime: Date.now() - state.createdAt,
+                    memory: state.memory,
+                    cpu: state.cpu,
+                    createdAt: state.createdAt,
+                    stale: state.stale || undefined,
+                    heapUsed: state.heapUsed,
+                    heapTotal: state.heapTotal,
+                    external: state.external,
+                    arrayBuffers: state.arrayBuffers,
+                    eventLoopLag: state.eventLoopLag,
+                    eventLoopLagP95: state.eventLoopLagP95,
+                    activeHandles: state.activeHandles,
+                    ...(state.cacheSize !== undefined && {
+                        cacheSize: state.cacheSize,
+                        cacheTotalBytes: state.cacheTotalBytes,
+                        cacheHits: state.cacheHits,
+                        cacheMisses: state.cacheMisses,
+                        cacheHitRate: state.cacheHitRate,
+                    }),
+                });
+            }
+            workers.sort((a, b) => a.id - b.id);
+        }
+        return {
+            id: this.id,
+            name: this.config.name,
+            script: this.config.script,
+            cwd: this.config.cwd,
+            execMode: this.config.execMode,
+            workerCount: this.config.workerCount,
+            status: this.getStatus(),
+            workers,
+            pid: this.forkProcess?.pid || this.clusterPrimary?.pid,
+            createdAt: workers[0]?.createdAt || Date.now(),
+            watch: this.config.watch,
+            sticky: this.config.sticky,
+            port: this.config.port ?? this.detectedPort,
+        };
+    }
+    getStatus() {
+        if (this.config.execMode === ExecMode.FORK) {
+            // Check if process exists first - if null, it's stopped regardless of flags
+            if (!this.forkProcess) {
+                return ProcessStatus.STOPPED;
+            }
+            if (this.isShuttingDown) {
+                return ProcessStatus.STOPPING;
+            }
+            return ProcessStatus.ONLINE;
+        }
+        // Cluster mode - check if primary exists first
+        if (!this.clusterPrimary) {
+            return ProcessStatus.STOPPED;
+        }
+        if (this.isShuttingDown) {
+            return ProcessStatus.STOPPING;
+        }
+        if (this.clusterWorkers.size === 0) {
+            return ProcessStatus.LAUNCHING;
+        }
+        const statuses = Array.from(this.clusterWorkers.values()).map((w) => w.status);
+        if (statuses.every((s) => s === ProcessStatus.ONLINE)) {
+            return ProcessStatus.ONLINE;
+        }
+        if (statuses.some((s) => s === ProcessStatus.ERRORED)) {
+            return ProcessStatus.ERRORED;
+        }
+        return ProcessStatus.LAUNCHING;
+    }
+    getWorkerCount() {
+        if (this.config.execMode === ExecMode.FORK) {
+            return this.forkProcess ? 1 : 0;
+        }
+        return this.clusterWorkers.size;
+    }
+    isRunning() {
+        if (this.config.execMode === ExecMode.FORK) {
+            return !!this.forkProcess && !this.isShuttingDown;
+        }
+        return !!this.clusterPrimary && !this.isShuttingDown;
+    }
+}
+function formatBytes(bytes) {
+    if (bytes >= 1024 * 1024 * 1024) {
+        return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
+    }
+    if (bytes >= 1024 * 1024) {
+        return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+    }
+    if (bytes >= 1024) {
+        return `${(bytes / 1024).toFixed(1)} KB`;
+    }
+    return `${bytes} B`;
+}
+//# sourceMappingURL=ManagedProcess.js.map