immortal-js 1.0.0

package/packages/core/src/safe-zone/SafeWrapper.ts

@@ -0,0 +1,317 @@
+/**
+ * @file SafeWrapper.ts
+ * @description Safe zone wrapping for route handlers and middleware
+ *
+ * This is the core mechanism that prevents a single failing route from
+ * crashing the entire server. It wraps every handler in:
+ * 1. An AsyncLocalStorage context (request tracking)
+ * 2. A try/catch that handles both sync and async errors
+ * 3. Error capture + fallback response dispatch
+ *
+ * The key insight: most Express/Node server crashes happen because
+ * async errors inside route handlers are never caught. SafeWrapper
+ * eliminates this class of failure entirely.
+ */
+
+import type { RequestContext, FallbackGenerator, SafeZoneConfig } from "../types.js";
+import { AsyncBoundary } from "./AsyncBoundary.js";
+import { ErrorTrap, serializeError } from "./ErrorTrap.js";
+import type { ImmortalEventBus } from "../event-bus.js";
+import { getLogger } from "../logger.js";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// GENERIC HANDLER TYPE (Framework-agnostic)
+// ─────────────────────────────────────────────────────────────────────────────
+
+export type AnyFunction = (...args: unknown[]) => unknown | Promise<unknown>;
+
+// Express-style types (without importing express)
+export type RequestLike = {
+  path?: string;
+  url?: string;
+  method?: string;
+  headers?: Record<string, string | string[] | undefined>;
+  [key: string]: unknown;
+};
+
+export type ResponseLike = {
+  headersSent?: boolean;
+  statusCode?: number;
+  status?: (code: number) => ResponseLike;
+  json?: (body: unknown) => ResponseLike;
+  send?: (body: unknown) => ResponseLike;
+  end?: (body?: string) => void;
+  setHeader?: (name: string, value: string) => void;
+  [key: string]: unknown;
+};
+
+export type NextLike = (error?: unknown) => void;
+
+export type RouteHandler = (
+  req: RequestLike,
+  res: ResponseLike,
+  next?: NextLike
+) => unknown | Promise<unknown>;
+
+export type Middleware = (
+  req: RequestLike,
+  res: ResponseLike,
+  next: NextLike
+) => unknown | Promise<unknown>;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// SAFE ZONE CONTEXT
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface SafeZoneContext {
+  errorTrap: ErrorTrap;
+  bus: ImmortalEventBus;
+  config: Required<SafeZoneConfig>;
+  fallback?: FallbackGenerator;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// SAFE WRAPPER IMPLEMENTATION
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Wrap a route handler in a safe execution boundary.
+ *
+ * @param handler - The route handler function to wrap
+ * @param ctx - Safe zone context (errorTrap, bus, config, fallback)
+ * @returns A new handler with full error containment
+ */
+export function wrapHandler(handler: RouteHandler, ctx: SafeZoneContext): RouteHandler {
+  const logger = getLogger();
+
+  return async function immortalWrappedHandler(
+    req: RequestLike,
+    res: ResponseLike,
+    next?: NextLike
+  ): Promise<void> {
+    const requestId = crypto.randomUUID();
+    const route = req.path ?? req.url ?? "unknown";
+    const method = req.method ?? "UNKNOWN";
+
+    const requestContext: RequestContext = {
+      requestId,
+      route,
+      method,
+      startedAt: Date.now(),
+    };
+
+    await AsyncBoundary.run(requestContext, async () => {
+      try {
+        await Promise.resolve(handler(req, res, next));
+      } catch (error) {
+        const duration = Date.now() - requestContext.startedAt;
+
+        logger.warn(`Error in handler [${method} ${route}]`, {
+          requestId,
+          durationMs: duration,
+          error: (error as Error).message,
+        });
+
+        // Capture the error for telemetry
+        ctx.errorTrap.capture(error, { requestId, route });
+
+        // Attempt recovery
+        await dispatchFallback(error, req, res, next, ctx, requestContext);
+      }
+    });
+  };
+}
+
+/**
+ * Wrap a middleware function in a safe execution boundary.
+ * Middlewares with errors are forwarded to next(err) instead of fallback.
+ */
+export function wrapMiddleware(middleware: Middleware, ctx: SafeZoneContext): Middleware {
+  const logger = getLogger();
+
+  return async function immortalWrappedMiddleware(
+    req: RequestLike,
+    res: ResponseLike,
+    next: NextLike
+  ): Promise<void> {
+    const requestId =
+      AsyncBoundary.getRequestId() ?? crypto.randomUUID();
+    const route = req.path ?? req.url ?? "unknown";
+
+    try {
+      await Promise.resolve(middleware(req, res, next));
+    } catch (error) {
+      logger.warn(`Error in middleware [${route}]`, {
+        requestId,
+        error: (error as Error).message,
+      });
+
+      ctx.errorTrap.capture(error, { requestId, route });
+
+      // For middleware, always forward to next(err)
+      next(error);
+    }
+  };
+}
+
+/**
+ * Wrap any async function for safe execution (non-HTTP context).
+ * Returns the result or undefined if an error occurs.
+ */
+export function wrapAsync<Args extends unknown[], R>(
+  fn: (...args: Args) => Promise<R>,
+  ctx: SafeZoneContext,
+  operationName = "anonymous"
+): (...args: Args) => Promise<R | undefined> {
+  const logger = getLogger();
+
+  return async (...args: Args): Promise<R | undefined> => {
+    const requestId = AsyncBoundary.getRequestId() ?? crypto.randomUUID();
+
+    try {
+      return await fn(...args);
+    } catch (error) {
+      logger.warn(`Error in async operation '${operationName}'`, {
+        requestId,
+        error: (error as Error).message,
+      });
+
+      ctx.errorTrap.capture(error, { requestId, operation: operationName });
+      return undefined;
+    }
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// FALLBACK DISPATCH
+// ─────────────────────────────────────────────────────────────────────────────
+
+async function dispatchFallback(
+  error: unknown,
+  req: RequestLike,
+  res: ResponseLike,
+  next: NextLike | undefined,
+  ctx: SafeZoneContext,
+  requestContext: RequestContext
+): Promise<void> {
+  // If response already started, can't send a new one — just log
+  if (res.headersSent) {
+    getLogger().warn("Cannot send fallback — response already sent", {
+      requestId: requestContext.requestId,
+    });
+    return;
+  }
+
+  // If a custom fallback is defined, use it
+  const fallbackFn = ctx.fallback ?? ctx.config.defaultFallback;
+  if (fallbackFn) {
+    try {
+      const fallbackResponse = await Promise.resolve(
+        fallbackFn(error, requestContext)
+      );
+
+      ctx.bus.emit("fallback:default-used", {
+        requestId: requestContext.requestId,
+        route: requestContext.route,
+        data: { hasCustomFallback: !!ctx.fallback },
+      });
+
+      if (res.setHeader) res.setHeader("X-Immortal-Fallback", "true");
+      if (res.setHeader) res.setHeader("X-Immortal-Request-Id", requestContext.requestId);
+
+      if (res.json) {
+        res.status?.(200).json?.(fallbackResponse);
+      } else if (res.send) {
+        res.status?.(200).send?.(JSON.stringify(fallbackResponse));
+      }
+      return;
+    } catch (fallbackError) {
+      getLogger().error("Fallback function itself threw an error", {
+        requestId: requestContext.requestId,
+        fallbackError: (fallbackError as Error).message,
+      });
+    }
+  }
+
+  // No fallback: forward to framework error handler via next(err)
+  if (next) {
+    next(error);
+    return;
+  }
+
+  // Last resort: send a generic 500 response
+  const serialized = serializeError(error);
+  const body: Record<string, unknown> = {
+    error: "Internal Server Error",
+    requestId: requestContext.requestId,
+    timestamp: new Date().toISOString(),
+  };
+
+  if (ctx.config.exposeErrors) {
+    body["details"] = serialized;
+  }
+
+  if (res.setHeader) res.setHeader("X-Immortal-Request-Id", requestContext.requestId);
+
+  if (res.json) {
+    res.status?.(500).json?.(body);
+  } else if (res.send) {
+    res.status?.(500).send?.(JSON.stringify(body));
+  } else if (res.end) {
+    res.end(JSON.stringify(body));
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// SAFE ZONE FACTORY
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface SafeZoneAPI {
+  /** Wrap a route handler */
+  handler: (fn: RouteHandler, fallback?: FallbackGenerator) => RouteHandler;
+  /** Wrap a middleware */
+  middleware: (fn: Middleware) => Middleware;
+  /** Wrap an async function */
+  async: <Args extends unknown[], R>(
+    fn: (...args: Args) => Promise<R>,
+    name?: string
+  ) => (...args: Args) => Promise<R | undefined>;
+  /** Create a decorator factory for class methods */
+  method: () => MethodDecorator;
+}
+
+/**
+ * Create a SafeZone API bound to specific config and error trap
+ */
+export function createSafeZone(ctx: SafeZoneContext): SafeZoneAPI {
+  return {
+    handler: (fn: RouteHandler, fallback?: FallbackGenerator) =>
+      wrapHandler(fn, fallback !== undefined ? { ...ctx, fallback } : ctx),
+
+    middleware: (fn: Middleware) =>
+      wrapMiddleware(fn, ctx),
+
+    async: <Args extends unknown[], R>(
+      fn: (...args: Args) => Promise<R>,
+      name?: string
+    ) => wrapAsync(fn, ctx, name),
+
+    method: (): MethodDecorator => {
+      return function (
+        _target: object,
+        propertyKey: string | symbol,
+        descriptor: PropertyDescriptor
+      ): PropertyDescriptor {
+        const originalMethod = descriptor.value as (...args: unknown[]) => Promise<unknown>;
+
+        descriptor.value = wrapAsync(
+          originalMethod,
+          ctx,
+          String(propertyKey)
+        );
+
+        return descriptor;
+      };
+    },
+  };
+}

package/packages/core/src/safe-zone/index.ts

@@ -0,0 +1,23 @@
+/**
+ * @file safe-zone/index.ts
+ * @description Safe Zone layer public exports
+ */
+
+export { AsyncBoundary, asyncBoundary, getRequestContext, getRequestId } from "./AsyncBoundary.js";
+export { ErrorTrap, classifyError, serializeError } from "./ErrorTrap.js";
+export type { CapturedError, ErrorClassification } from "./ErrorTrap.js";
+export {
+  wrapHandler,
+  wrapMiddleware,
+  wrapAsync,
+  createSafeZone,
+} from "./SafeWrapper.js";
+export type {
+  RouteHandler,
+  Middleware,
+  SafeZoneContext,
+  SafeZoneAPI,
+  RequestLike,
+  ResponseLike,
+  NextLike,
+} from "./SafeWrapper.js";

package/packages/core/src/supervision/ClusterManager.ts

@@ -0,0 +1,243 @@
+/**
+ * @file ClusterManager.ts
+ * @description Node.js Cluster API wrapper with Supervisor integration
+ * Manages multiple OS processes for multi-core utilization
+ * with automatic restart on worker failure
+ */
+
+import * as os from "node:os";
+import type { SupervisorConfig, WorkerStatus } from "../types.js";
+import { DEFAULT_SUPERVISOR } from "../config/defaults.js";
+import { getEventBus } from "../event-bus.js";
+import { getLogger } from "../logger.js";
+import { calculateBackoff } from "../recovery/RetryEngine.js";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// CLUSTER MANAGER
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface ClusterManagerOptions {
+  /** Number of workers (default: os.cpus().length) */
+  workerCount?: number;
+  /** Supervisor config for restart limits */
+  supervisorConfig?: SupervisorConfig;
+  /** Called when a worker exits */
+  onWorkerExit?: (workerId: number, exitCode: number, signal: string | null) => void;
+  /** Called when a worker is escalated */
+  onEscalation?: (workerId: number) => void;
+}
+
+interface ClusterWorkerMeta {
+  id: number;
+  pid: number | undefined;
+  restartCount: number;
+  restartTimestamps: number[];
+  startedAt: number;
+  status: WorkerStatus["state"];
+}
+
+export class ClusterManager {
+  private readonly workerCount: number;
+  private readonly config: Required<SupervisorConfig>;
+  private readonly workerMeta = new Map<number, ClusterWorkerMeta>();
+  private readonly options: ClusterManagerOptions;
+  private running = false;
+
+  constructor(options: ClusterManagerOptions = {}) {
+    this.options = options;
+    this.workerCount = options.workerCount ?? os.cpus().length;
+    this.config = { ...DEFAULT_SUPERVISOR, ...options.supervisorConfig };
+  }
+
+  /**
+   * Check if this process is the cluster primary (formerly "master")
+   * Uses dynamic import to avoid pulling cluster into non-cluster environments
+   */
+  static isPrimary(): boolean {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const cluster = require("node:cluster") as import("node:cluster").Cluster;
+    return cluster.isPrimary;
+  }
+
+  /**
+   * Start cluster mode — forks `workerCount` processes.
+   *
+   * @example
+   * if (ClusterManager.isPrimary()) {
+   *   const mgr = new ClusterManager({ workerCount: 4 });
+   *   await mgr.start();
+   * } else {
+   *   await startApp();
+   * }
+   */
+  async start(): Promise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const cluster = require("node:cluster") as import("node:cluster").Cluster;
+
+    if (!cluster.isPrimary) {
+      throw new Error("ClusterManager.start() must only be called in the primary process");
+    }
+
+    this.running = true;
+    const logger = getLogger();
+    const bus = getEventBus();
+
+    logger.info(`ClusterManager: Forking ${this.workerCount} workers...`);
+
+    for (let i = 0; i < this.workerCount; i++) {
+      this.forkWorkerInternal(cluster);
+    }
+
+    cluster.on("exit", (worker, code, signal) => {
+      const meta = this.workerMeta.get(worker.id);
+      if (!meta || !this.running) return;
+
+      logger.warn(`Cluster worker ${worker.id} (PID: ${String(worker.process.pid)}) exited`, {
+        code,
+        signal,
+        restartCount: meta.restartCount,
+      });
+
+      bus.emit("worker:crashed", {
+        workerId: worker.id,
+        data: { pid: worker.process.pid, exitCode: code, signal },
+      });
+
+      this.options.onWorkerExit?.(worker.id, code ?? -1, signal);
+
+      const windowStart = Date.now() - this.config.windowMs;
+      meta.restartTimestamps = meta.restartTimestamps.filter((t) => t > windowStart);
+
+      if (meta.restartTimestamps.length >= this.config.maxRestartsInWindow) {
+        meta.status = "escalated";
+        logger.error(
+          `Cluster worker ${worker.id} escalated — ` +
+          `crashed ${meta.restartCount} times in ${this.config.windowMs}ms`
+        );
+        bus.emit("worker:escalated", {
+          workerId: worker.id,
+          data: { restartCount: meta.restartCount },
+        });
+        this.options.onEscalation?.(worker.id);
+        return;
+      }
+
+      const delay = calculateBackoff(meta.restartCount + 1, this.config.baseRestartDelayMs, 30_000);
+
+      setTimeout(() => {
+        if (this.running) {
+          logger.info(`Cluster: Restarting worker ${worker.id} in ${delay}ms`);
+          this.forkWorkerInternal(cluster, worker.id);
+        }
+      }, delay).unref();
+    });
+
+    cluster.on("online", (worker) => {
+      const meta = this.workerMeta.get(worker.id);
+      if (meta) {
+        meta.pid = worker.process.pid;
+        meta.status = "running";
+      }
+
+      bus.emit("worker:started", {
+        workerId: worker.id,
+        data: { pid: worker.process.pid },
+      });
+
+      logger.info(`Cluster worker ${worker.id} (PID: ${String(worker.process.pid)}) online`);
+    });
+  }
+
+  /**
+   * Gracefully restart all cluster workers (zero-downtime rolling restart)
+   */
+  async rollingRestart(): Promise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const cluster = require("node:cluster") as import("node:cluster").Cluster;
+    const logger = getLogger();
+    logger.info("ClusterManager: Starting rolling restart...");
+
+    const workers = Array.from(Object.values(cluster.workers ?? {}));
+
+    for (const worker of workers) {
+      if (!worker) continue;
+
+      const newWorkerForked = new Promise<void>((resolve) => {
+        cluster.once("online", () => resolve());
+        this.forkWorkerInternal(cluster);
+      });
+
+      await newWorkerForked;
+      worker.kill("SIGTERM");
+      logger.info(`ClusterManager: Worker ${worker.id} replaced`);
+      await sleep(500);
+    }
+
+    logger.info("ClusterManager: Rolling restart complete");
+  }
+
+  /**
+   * Stop all cluster workers
+   */
+  async stop(): Promise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const cluster = require("node:cluster") as import("node:cluster").Cluster;
+    this.running = false;
+    const logger = getLogger();
+    logger.info("ClusterManager: Stopping all workers...");
+
+    const workers = Object.values(cluster.workers ?? {});
+    for (const worker of workers) {
+      worker?.kill("SIGTERM");
+    }
+
+    logger.info("ClusterManager: All workers signaled");
+  }
+
+  /**
+   * Get current status of all cluster workers
+   */
+  getWorkerStatuses(): WorkerStatus[] {
+    return Array.from(this.workerMeta.values()).map((meta) => {
+      const status: WorkerStatus = {
+        id: meta.id,
+        state: meta.status,
+        restartCount: meta.restartCount,
+        uptime: Date.now() - meta.startedAt,
+      };
+      if (meta.pid !== undefined) status.pid = meta.pid;
+      const lastRestart = meta.restartTimestamps.at(-1);
+      if (lastRestart !== undefined) status.lastRestartTime = lastRestart;
+      return status;
+    });
+  }
+
+  private forkWorkerInternal(
+    cluster: import("node:cluster").Cluster,
+    existingId?: number
+  ): import("node:cluster").Worker {
+    const worker = cluster.fork();
+
+    const meta: ClusterWorkerMeta = this.workerMeta.get(existingId ?? worker.id) ?? {
+      id: worker.id,
+      pid: undefined,
+      restartCount: 0,
+      restartTimestamps: [],
+      startedAt: Date.now(),
+      status: "restarting",
+    };
+
+    meta.id = worker.id;
+    meta.restartCount++;
+    meta.restartTimestamps.push(Date.now());
+    meta.startedAt = Date.now();
+
+    this.workerMeta.set(worker.id, meta);
+
+    return worker;
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms).unref());
+}

package/packages/core/src/supervision/RestartStrategy.ts

@@ -0,0 +1,68 @@
+/**
+ * @file RestartStrategy.ts
+ * @description Erlang/OTP-inspired restart strategies for Worker supervision
+ *
+ * one_for_one: Only the failed worker is restarted. Others continue unchanged.
+ *              Use when workers are independent (most common case).
+ *
+ * one_for_all: If any worker fails, ALL workers are restarted.
+ *              Use when workers share critical coupled state.
+ *
+ * rest_for_one: The failed worker AND all workers started after it are restarted.
+ *               Use when workers have ordered dependencies (A → B → C).
+ */
+
+import type { RestartStrategy } from "../types.js";
+
+export interface WorkerEntry {
+  id: string;
+  index: number; // Order of registration (used for rest_for_one)
+  restart: () => void;
+}
+
+export interface RestartDecision {
+  workersToRestart: string[]; // IDs of workers to restart
+  strategy: RestartStrategy;
+}
+
+/**
+ * Determine which workers to restart based on strategy and failed worker
+ */
+export function computeRestartDecision(
+  failedWorkerId: string,
+  allWorkers: WorkerEntry[],
+  strategy: RestartStrategy
+): RestartDecision {
+  const failed = allWorkers.find((w) => w.id === failedWorkerId);
+
+  if (!failed) {
+    // Unknown worker — safe to ignore
+    return { workersToRestart: [], strategy };
+  }
+
+  switch (strategy) {
+    case "one_for_one":
+      // Only restart the failed worker
+      return { workersToRestart: [failedWorkerId], strategy };
+
+    case "one_for_all":
+      // Restart ALL workers in the group
+      return {
+        workersToRestart: allWorkers.map((w) => w.id),
+        strategy,
+      };
+
+    case "rest_for_one":
+      // Restart failed worker + all workers with higher index (started after it)
+      return {
+        workersToRestart: allWorkers
+          .filter((w) => w.index >= failed.index)
+          .sort((a, b) => b.index - a.index) // Restart in reverse order (most dependent first)
+          .map((w) => w.id),
+        strategy,
+      };
+
+    default:
+      return { workersToRestart: [failedWorkerId], strategy };
+  }
+}