immortal-js 1.0.0

package/packages/core/src/supervision/Supervisor.ts

@@ -0,0 +1,311 @@
+/**
+ * @file Supervisor.ts
+ * @description Erlang/OTP-inspired Worker Supervisor
+ *
+ * Engineering honesty about the restart limit:
+ * If a worker keeps crashing repeatedly within a time window, it's a
+ * programming error or unresolvable runtime condition. Infinite restart
+ * loops waste CPU and hide the real problem. After maxRestartsInWindow,
+ * the supervisor ESCALATES (stops retrying, fires an alert) instead of
+ * continuing to crash-loop.
+ *
+ * This is exactly what Erlang's supervisor does with max_restart / max_period.
+ */
+
+import type { SupervisorConfig, WorkerStatus } from "../types.js";
+import { WorkerEscalatedError } from "../types.js";
+import { DEFAULT_SUPERVISOR } from "../config/defaults.js";
+import { getEventBus } from "../event-bus.js";
+import { getLogger } from "../logger.js";
+import { computeRestartDecision } from "./RestartStrategy.js";
+import type { WorkerEntry } from "./RestartStrategy.js";
+import { calculateBackoff } from "../recovery/RetryEngine.js";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// TYPES
+// ─────────────────────────────────────────────────────────────────────────────
+
+export type SpawnFunction = () => { terminate?: () => Promise<void> | void };
+
+export interface ManagedWorker {
+  id: string;
+  index: number;
+  spawn: SpawnFunction;
+  instance?: ReturnType<SpawnFunction>;
+  status: WorkerStatus["state"];
+  restartCount: number;
+  restartTimestamps: number[]; // timestamps within current window
+  startedAt: number;
+  pid?: number;
+  memoryMb?: number;
+  cpuPercent?: number;
+}
+
+export interface SupervisorOptions {
+  config?: SupervisorConfig;
+  /** Called when a worker is escalated (for alerting) */
+  onEscalation?: (workerId: string, error: WorkerEscalatedError) => void;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// SUPERVISOR CLASS
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class Supervisor {
+  private readonly workers = new Map<string, ManagedWorker>();
+  private readonly config: Required<SupervisorConfig>;
+  private readonly onEscalation: ((workerId: string, error: WorkerEscalatedError) => void) | undefined;
+  private stopped = false;
+
+  constructor(options: SupervisorOptions = {}) {
+    this.config = { ...DEFAULT_SUPERVISOR, ...options.config };
+    this.onEscalation = options.onEscalation;
+  }
+
+  /**
+   * Register a worker with the supervisor.
+   * The supervisor will automatically restart it if it fails.
+   *
+   * @param id - Unique worker identifier
+   * @param spawnFn - Factory function to create/restart the worker
+   * @returns The initial worker instance
+   */
+  register(id: string, spawnFn: SpawnFunction): ReturnType<SpawnFunction> {
+    if (this.workers.has(id)) {
+      throw new Error(`Worker '${id}' is already registered with this supervisor`);
+    }
+
+    const worker: ManagedWorker = {
+      id,
+      index: this.workers.size,
+      spawn: spawnFn,
+      status: "running",
+      restartCount: 0,
+      restartTimestamps: [],
+      startedAt: Date.now(),
+    };
+
+    const instance = spawnFn();
+    worker.instance = instance;
+    this.workers.set(id, worker);
+
+    getEventBus().emit("worker:started", {
+      workerId: id,
+      data: { index: worker.index, strategy: this.config.strategy },
+    });
+
+    getLogger().info(`Supervisor: Worker '${id}' registered and started`);
+    return instance;
+  }
+
+  /**
+   * Notify the supervisor that a worker has exited/crashed.
+   * This triggers the restart logic based on the configured strategy.
+   *
+   * @param workerId - ID of the crashed worker
+   * @param exitCode - Exit code (0 = clean, non-zero = crash)
+   */
+  onWorkerExit(workerId: string, exitCode = 1): void {
+    if (this.stopped) return;
+
+    const worker = this.workers.get(workerId);
+    if (!worker) {
+      getLogger().warn(`Supervisor: Unknown worker '${workerId}' exited`);
+      return;
+    }
+
+    const bus = getEventBus();
+    const logger = getLogger();
+
+    bus.emit("worker:crashed", {
+      workerId,
+      data: { exitCode, restartCount: worker.restartCount },
+    });
+
+    logger.warn(`Supervisor: Worker '${workerId}' crashed (exit: ${exitCode})`);
+
+    // Clean up expired restart timestamps
+    const windowStart = Date.now() - this.config.windowMs;
+    worker.restartTimestamps = worker.restartTimestamps.filter((t) => t > windowStart);
+
+    // Check restart limit
+    if (worker.restartTimestamps.length >= this.config.maxRestartsInWindow) {
+      this.escalate(workerId, worker);
+      return;
+    }
+
+    // Determine which workers to restart
+    const workerEntries: WorkerEntry[] = Array.from(this.workers.values()).map((w) => ({
+      id: w.id,
+      index: w.index,
+      restart: () => this.restartWorker(w.id),
+    }));
+
+    const decision = computeRestartDecision(workerId, workerEntries, this.config.strategy);
+
+    // Execute restarts with exponential backoff
+    for (const idToRestart of decision.workersToRestart) {
+      const restartCount = this.workers.get(idToRestart)?.restartCount ?? 0;
+      const delay = calculateBackoff(
+        restartCount + 1,
+        this.config.baseRestartDelayMs,
+        30_000 // max 30s restart delay
+      );
+
+      logger.info(`Supervisor: Restarting '${idToRestart}' in ${delay}ms (attempt #${restartCount + 1})`);
+
+      setTimeout(() => {
+        if (!this.stopped) {
+          this.restartWorker(idToRestart);
+        }
+      }, delay).unref();
+    }
+  }
+
+  /**
+   * Gracefully stop a managed worker
+   */
+  async stop(workerId: string): Promise<void> {
+    const worker = this.workers.get(workerId);
+    if (!worker) return;
+
+    worker.status = "stopped";
+
+    if (worker.instance?.terminate) {
+      await worker.instance.terminate();
+    }
+
+    this.workers.delete(workerId);
+    getLogger().info(`Supervisor: Worker '${workerId}' stopped`);
+  }
+
+  /**
+   * Stop all workers (graceful shutdown)
+   */
+  async stopAll(): Promise<void> {
+    this.stopped = true;
+
+    const stopPromises = Array.from(this.workers.keys()).map((id) => this.stop(id));
+    await Promise.allSettled(stopPromises);
+
+    getLogger().info("Supervisor: All workers stopped");
+  }
+
+  /**
+   * Get current status of all managed workers
+   */
+  getWorkerStatuses(): WorkerStatus[] {
+    return Array.from(this.workers.values()).map((w) => {
+      const status: WorkerStatus = {
+        id: w.id,
+        state: w.status,
+        restartCount: w.restartCount,
+        uptime: Date.now() - w.startedAt,
+      };
+      if (w.pid !== undefined) status.pid = w.pid;
+      const lastRestart = w.restartTimestamps.at(-1);
+      if (lastRestart !== undefined) status.lastRestartTime = lastRestart;
+      if (w.memoryMb !== undefined) status.memoryMb = w.memoryMb;
+      if (w.cpuPercent !== undefined) status.cpuPercent = w.cpuPercent;
+      return status;
+    });
+  }
+
+  /**
+   * Update resource metrics for a worker (called by HealthMonitor)
+   */
+  updateWorkerMetrics(
+    workerId: string,
+    metrics: { memoryMb?: number; cpuPercent?: number; pid?: number }
+  ): void {
+    const worker = this.workers.get(workerId);
+    if (!worker) return;
+
+    if (metrics.memoryMb !== undefined) worker.memoryMb = metrics.memoryMb;
+    if (metrics.cpuPercent !== undefined) worker.cpuPercent = metrics.cpuPercent;
+    if (metrics.pid !== undefined) worker.pid = metrics.pid;
+  }
+
+  /**
+   * Request graceful restart of a specific worker (from HealthMonitor)
+   */
+  requestGracefulRestart(workerId: string, reason = "health-threshold"): void {
+    const worker = this.workers.get(workerId);
+    if (!worker || worker.status !== "running") return;
+
+    getLogger().info(`Supervisor: Graceful restart requested for '${workerId}' (reason: ${reason})`);
+    getEventBus().emit("memory:proactive-restart", {
+      workerId,
+      data: { reason },
+    });
+
+    // Graceful restart: stop current, allow supervisor to restart
+    if (worker.instance?.terminate) {
+      Promise.resolve(worker.instance.terminate()).catch(() => void 0);
+    }
+    // Simulate exit to trigger restart logic
+    this.onWorkerExit(workerId, 0);
+  }
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // PRIVATE
+  // ─────────────────────────────────────────────────────────────────────────
+
+  private restartWorker(workerId: string): void {
+    const worker = this.workers.get(workerId);
+    if (!worker || this.stopped) return;
+
+    try {
+      worker.status = "restarting";
+      worker.restartCount++;
+      worker.restartTimestamps.push(Date.now());
+      worker.startedAt = Date.now();
+
+      const instance = worker.spawn();
+      worker.instance = instance;
+      worker.status = "running";
+
+      getEventBus().emit("worker:restarted", {
+        workerId,
+        data: {
+          restartCount: worker.restartCount,
+          strategy: this.config.strategy,
+        },
+      });
+
+      getLogger().info(`Supervisor: Worker '${workerId}' restarted (count: ${worker.restartCount})`);
+    } catch (err) {
+      getLogger().error(`Supervisor: Failed to restart worker '${workerId}'`, {
+        error: (err as Error).message,
+      });
+      worker.status = "stopped";
+    }
+  }
+
+  private escalate(workerId: string, worker: ManagedWorker): void {
+    const bus = getEventBus();
+    const logger = getLogger();
+
+    worker.status = "escalated";
+    const error = new WorkerEscalatedError(workerId, worker.restartCount);
+
+    bus.emit("worker:escalated", {
+      workerId,
+      data: {
+        restartCount: worker.restartCount,
+        windowMs: this.config.windowMs,
+        maxRestartsInWindow: this.config.maxRestartsInWindow,
+      },
+      error: { name: error.name, message: error.message, isOperational: false },
+    });
+
+    logger.error(
+      `Supervisor: Worker '${workerId}' ESCALATED — ` +
+      `crashed ${worker.restartCount} times in ${this.config.windowMs}ms window. ` +
+      `Manual intervention required.`,
+      { workerId, restartCount: worker.restartCount }
+    );
+
+    this.onEscalation?.(workerId, error);
+  }
+}

package/packages/core/src/supervision/index.ts

@@ -0,0 +1,11 @@
+/**
+ * @file supervision/index.ts
+ * @description Supervision layer public exports
+ */
+
+export { Supervisor } from "./Supervisor.js";
+export type { ManagedWorker, SupervisorOptions, SpawnFunction } from "./Supervisor.js";
+export { ClusterManager } from "./ClusterManager.js";
+export type { ClusterManagerOptions } from "./ClusterManager.js";
+export { computeRestartDecision } from "./RestartStrategy.js";
+export type { WorkerEntry, RestartDecision } from "./RestartStrategy.js";