immortal-js 1.0.0

package/packages/core/src/isolation/WorkerSandbox.ts

@@ -0,0 +1,306 @@
+/**
+ * @file WorkerSandbox.ts
+ * @description Worker Thread sandbox for dangerous/heavy operations
+ *
+ * Use cases:
+ * - Executing user-supplied code (code evaluation)
+ * - Processing untrusted files (parsing, validation)
+ * - CPU-intensive operations that would block the event loop
+ * - Operations with unknown memory footprint
+ *
+ * If the sandboxed Worker crashes (stack overflow, out-of-memory, etc.),
+ * the Main Thread is completely unaffected. The Supervisor is notified
+ * to restart the Worker if needed.
+ *
+ * Communication via postMessage (structured clone serialization) — no shared mutable state.
+ */
+
+import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads";
+import { EventEmitter } from "node:events";
+import * as path from "node:path";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import { getLogger } from "../logger.js";
+import { getEventBus } from "../event-bus.js";
+import { TimeoutError } from "../types.js";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// MESSAGE PROTOCOL
+// ─────────────────────────────────────────────────────────────────────────────
+
+interface WorkerRequest {
+  id: string;
+  type: "execute";
+  code?: string;
+  args?: unknown[];
+  timeoutMs?: number;
+}
+
+interface WorkerResponse {
+  id: string;
+  type: "result" | "error" | "ready";
+  result?: unknown;
+  error?: { name: string; message: string; stack?: string };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// WORKER SANDBOX CLASS (Main Thread)
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface WorkerSandboxOptions {
+  /** Timeout for individual operations in ms (default: 30_000) */
+  operationTimeoutMs?: number;
+  /** Maximum memory for the worker in MB (enforced via --max-old-space-size) */
+  maxMemoryMb?: number;
+  /** Worker environment variables */
+  env?: Record<string, string>;
+  /** Shared resource limits */
+  resourceLimits?: {
+    maxOldGenerationSizeMb?: number;
+    maxYoungGenerationSizeMb?: number;
+    stackSizeMb?: number;
+    codeRangeSizeMb?: number;
+  };
+}
+
+export class WorkerSandbox extends EventEmitter {
+  private worker: Worker | null = null;
+  private pending = new Map<string, {
+    resolve: (value: unknown) => void;
+    reject: (reason: unknown) => void;
+    timeoutId: ReturnType<typeof setTimeout>;
+  }>();
+  private ready = false;
+  private readonly options: Required<WorkerSandboxOptions>;
+  private restartCount = 0;
+
+  constructor(options: WorkerSandboxOptions = {}) {
+    super();
+    this.options = {
+      operationTimeoutMs: options.operationTimeoutMs ?? 30_000,
+      maxMemoryMb: options.maxMemoryMb ?? 512,
+      env: options.env ?? {},
+      resourceLimits: options.resourceLimits ?? {},
+    };
+  }
+
+  /**
+   * Start the worker sandbox
+   */
+  async start(): Promise<void> {
+    await this.spawnWorker();
+    getLogger().info(`WorkerSandbox started (PID: ${this.worker?.threadId})`);
+  }
+
+  /**
+   * Execute code in the sandbox and return the result
+   */
+  async execute<T = unknown>(
+    code: string,
+    args?: unknown[],
+    timeoutMs?: number
+  ): Promise<T> {
+    if (!this.worker || !this.ready) {
+      throw new Error("WorkerSandbox is not running. Call start() first.");
+    }
+
+    const id = crypto.randomUUID();
+    const timeout = timeoutMs ?? this.options.operationTimeoutMs;
+
+    return new Promise<T>((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        this.pending.delete(id);
+        reject(new TimeoutError(`worker:execute`, timeout));
+      }, timeout);
+
+      this.pending.set(id, {
+        resolve: resolve as (value: unknown) => void,
+        reject,
+        timeoutId,
+      });
+
+      const request: WorkerRequest = { id, type: "execute", code, timeoutMs: timeout };
+      if (args !== undefined) request.args = args;
+      this.worker!.postMessage(request);
+    });
+  }
+
+  /**
+   * Gracefully terminate the worker
+   */
+  async terminate(): Promise<void> {
+    if (!this.worker) return;
+
+    // Reject all pending operations
+    for (const [id, pending] of this.pending) {
+      clearTimeout(pending.timeoutId);
+      pending.reject(new Error("WorkerSandbox terminated"));
+      this.pending.delete(id);
+    }
+
+    await this.worker.terminate();
+    this.worker = null;
+    this.ready = false;
+    getLogger().info("WorkerSandbox terminated");
+  }
+
+  /**
+   * Check if the sandbox is ready to accept work
+   */
+  isReady(): boolean {
+    return this.ready && this.worker !== null;
+  }
+
+  private async spawnWorker(): Promise<void> {
+    const workerScript = this.generateWorkerScript();
+    const tmpFile = path.join(os.tmpdir(), `immortal-sandbox-${Date.now()}.mjs`);
+
+    try {
+      fs.writeFileSync(tmpFile, workerScript, "utf-8");
+
+      this.worker = new Worker(tmpFile, {
+        workerData: { sandboxed: true },
+        env: { ...process.env, ...this.options.env } as Record<string, string>,
+        resourceLimits: {
+          maxOldGenerationSizeMb: this.options.maxMemoryMb,
+          maxYoungGenerationSizeMb: Math.floor(this.options.maxMemoryMb / 4),
+          ...this.options.resourceLimits,
+        },
+      });
+
+      this.worker.on("message", (msg: WorkerResponse) => this.handleMessage(msg));
+      this.worker.on("error", (err) => this.handleWorkerError(err));
+      this.worker.on("exit", (code) => this.handleWorkerExit(code));
+
+      // Wait for ready signal
+      await new Promise<void>((resolve, reject) => {
+        const timeout = setTimeout(() => reject(new Error("Worker failed to start")), 10_000);
+        this.once("ready", () => { clearTimeout(timeout); resolve(); });
+        this.once("error", (err) => { clearTimeout(timeout); reject(err); });
+      });
+
+    } finally {
+      // Clean up temp file
+      try { fs.unlinkSync(tmpFile); } catch { /* ignore */ }
+    }
+  }
+
+  private handleMessage(msg: WorkerResponse): void {
+    if (msg.type === "ready") {
+      this.ready = true;
+      this.emit("ready");
+      return;
+    }
+
+    const pending = this.pending.get(msg.id);
+    if (!pending) return;
+
+    clearTimeout(pending.timeoutId);
+    this.pending.delete(msg.id);
+
+    if (msg.type === "error" && msg.error) {
+      const err = new Error(msg.error.message);
+      err.name = msg.error.name;
+      if (msg.error.stack) err.stack = msg.error.stack;
+      pending.reject(err);
+    } else {
+      pending.resolve(msg.result);
+    }
+  }
+
+  private handleWorkerError(err: Error): void {
+    getLogger().error("WorkerSandbox error", { error: err.message });
+    getEventBus().emit("worker:crashed", { data: { error: err.message, restartCount: this.restartCount } });
+    this.emit("error", err);
+
+    // Reject all pending
+    for (const [id, pending] of this.pending) {
+      clearTimeout(pending.timeoutId);
+      pending.reject(err);
+      this.pending.delete(id);
+    }
+  }
+
+  private handleWorkerExit(code: number): void {
+    this.ready = false;
+    getLogger().warn(`WorkerSandbox exited with code ${code}`);
+    getEventBus().emit("worker:crashed", { data: { exitCode: code, restartCount: this.restartCount } });
+    this.emit("exit", code);
+  }
+
+  private generateWorkerScript(): string {
+    return `
+import { parentPort, workerData } from 'node:worker_threads';
+
+// Signal ready
+parentPort.postMessage({ type: 'ready' });
+
+parentPort.on('message', async (msg) => {
+  if (msg.type !== 'execute') return;
+
+  try {
+    // Execute code in sandbox using Function constructor
+    // This is intentionally restricted — no access to Node internals beyond what's allowed
+    const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor;
+    const fn = new AsyncFunction('args', msg.code);
+    const result = await fn(msg.args ?? []);
+    parentPort.postMessage({ id: msg.id, type: 'result', result });
+  } catch (err) {
+    parentPort.postMessage({
+      id: msg.id,
+      type: 'error',
+      error: {
+        name: err.name ?? 'Error',
+        message: err.message ?? String(err),
+        stack: err.stack,
+      },
+    });
+  }
+});
+
+process.on('uncaughtException', (err) => {
+  parentPort.postMessage({
+    id: 'uncaught',
+    type: 'error',
+    error: { name: err.name, message: err.message, stack: err.stack }
+  });
+});
+`;
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// WORKER THREAD SIDE HANDLER (re-exported for user Worker scripts)
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Use this in Worker Thread files to simplify the boilerplate
+ */
+export function setupWorkerThread(
+  handler: (data: unknown) => Promise<unknown>
+): void {
+  if (isMainThread) {
+    throw new Error("setupWorkerThread() must be called from a Worker Thread");
+  }
+
+  parentPort!.postMessage({ type: "ready" });
+
+  parentPort!.on("message", async (msg: WorkerRequest) => {
+    if (msg.type !== "execute") return;
+
+    try {
+      const result = await handler(workerData);
+      parentPort!.postMessage({ id: msg.id, type: "result", result });
+    } catch (err) {
+      parentPort!.postMessage({
+        id: msg.id,
+        type: "error",
+        error: {
+          name: (err as Error).name,
+          message: (err as Error).message,
+          stack: (err as Error).stack,
+        },
+      });
+    }
+  });
+}

package/packages/core/src/isolation/index.ts

@@ -0,0 +1,8 @@
+/**
+ * @file isolation/index.ts
+ * @description Isolation layer public exports
+ */
+
+export { BulkheadPool, BulkheadRegistry } from "./BulkheadPool.js";
+export { WorkerSandbox, setupWorkerThread } from "./WorkerSandbox.js";
+export type { WorkerSandboxOptions } from "./WorkerSandbox.js";

package/packages/core/src/lifecycle/GracefulShutdown.ts

@@ -0,0 +1,161 @@
+/**
+ * @file GracefulShutdown.ts
+ * @description Graceful shutdown orchestration
+ *
+ * 4-phase shutdown sequence:
+ * Phase 1: Stop accepting new connections (server.close())
+ * Phase 2: Grace period — wait for in-flight requests to complete
+ * Phase 3: Close external connections (DB, Redis, queues) in reverse dependency order
+ * Phase 4: Final cleanup + process exit
+ *
+ * If any in-flight request exceeds the grace period, it is forcibly terminated
+ * and logged as an exceptional event requiring review.
+ */
+
+import type { GracefulShutdownConfig, ShutdownHook } from "../types.js";
+import { DEFAULT_GRACEFUL_SHUTDOWN } from "../config/defaults.js";
+import { getLogger } from "../logger.js";
+import type { ImmortalEventBus } from "../event-bus.js";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// GRACEFUL SHUTDOWN CLASS
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class GracefulShutdown {
+  private readonly config: Required<GracefulShutdownConfig>;
+  private readonly bus: ImmortalEventBus;
+  private shutdownInProgress = false;
+  private readonly hooks: ShutdownHook[] = [];
+  private serverRef?: { close: (cb?: () => void) => void };
+
+  constructor(config: GracefulShutdownConfig = {}, bus: ImmortalEventBus) {
+    this.config = { ...DEFAULT_GRACEFUL_SHUTDOWN, ...config };
+    this.bus = bus;
+    this.hooks.push(...(config.onShutdown ?? []));
+
+    this.installSignalHandlers();
+  }
+
+  /**
+   * Register an HTTP server for graceful connection draining
+   */
+  registerServer(server: { close: (cb?: () => void) => void }): void {
+    this.serverRef = server;
+  }
+
+  /**
+   * Add a cleanup hook (run during shutdown in registration order)
+   */
+  addHook(hook: ShutdownHook): void {
+    this.hooks.push(hook);
+  }
+
+  /**
+   * Manually trigger graceful shutdown (useful for Supervisor-initiated restarts)
+   */
+  async execute(signal = "manual"): Promise<void> {
+    if (this.shutdownInProgress) {
+      getLogger().warn("Graceful shutdown already in progress — ignoring duplicate signal");
+      return;
+    }
+
+    this.shutdownInProgress = true;
+    const logger = getLogger();
+    const startTime = Date.now();
+
+    logger.info(`Graceful shutdown initiated (signal: ${signal})`);
+
+    this.bus.emit("shutdown:initiated", {
+      data: { signal, timeoutMs: this.config.timeoutMs },
+    });
+
+    // ── Phase 1: Stop accepting new connections ───────────────────────
+    if (this.serverRef) {
+      logger.info("Phase 1: Stopping server from accepting new connections...");
+      await new Promise<void>((resolve) => {
+        this.serverRef!.close(() => {
+          logger.info("Phase 1: Server closed to new connections");
+          resolve();
+        });
+      });
+    }
+
+    // ── Phase 2: Grace period for in-flight requests ──────────────────
+    logger.info(`Phase 2: Grace period — ${this.config.timeoutMs}ms for in-flight requests...`);
+
+    const gracefulComplete = this.runHooksWithTimeout();
+
+    let forced = false;
+    const forceTimeout = new Promise<"forced">((resolve) => {
+      setTimeout(() => {
+        forced = true;
+        resolve("forced");
+      }, this.config.timeoutMs).unref();
+    });
+
+    const raceResult = await Promise.race([gracefulComplete.then(() => "complete"), forceTimeout]);
+
+    if (raceResult === "forced" || forced) {
+      logger.error("Graceful shutdown timeout — forcing exit", {
+        elapsedMs: Date.now() - startTime,
+        timeoutMs: this.config.timeoutMs,
+      });
+
+      this.bus.emit("shutdown:forced", {
+        data: {
+          elapsedMs: Date.now() - startTime,
+          signal,
+          reason: "grace-period-exceeded",
+        },
+      });
+
+      process.exit(1);
+    }
+
+    // ── Phase 3 & 4: Hooks ran within grace period ────────────────────
+    const elapsed = Date.now() - startTime;
+    logger.info(`Graceful shutdown complete in ${elapsed}ms`);
+
+    this.bus.emit("shutdown:complete", {
+      data: { elapsedMs: elapsed, signal },
+    });
+
+    // Final exit
+    process.exit(0);
+  }
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // PRIVATE
+  // ─────────────────────────────────────────────────────────────────────────
+
+  private async runHooksWithTimeout(): Promise<void> {
+    const logger = getLogger();
+
+    // Run hooks in sequence (order matters for dependency cleanup)
+    for (const hook of this.hooks) {
+      try {
+        await Promise.resolve(hook(this.config.signals[0] ?? "SIGTERM"));
+        logger.debug("Shutdown hook completed");
+      } catch (err) {
+        logger.error("Shutdown hook failed", {
+          error: (err as Error).message,
+        });
+        // Continue with other hooks even if one fails
+      }
+    }
+  }
+
+  private installSignalHandlers(): void {
+    const logger = getLogger();
+
+    for (const signal of this.config.signals) {
+      process.once(signal as NodeJS.Signals, () => {
+        logger.info(`Received ${signal} — initiating graceful shutdown`);
+        this.execute(signal).catch((err) => {
+          logger.error("Graceful shutdown failed", { error: (err as Error).message });
+          process.exit(1);
+        });
+      });
+    }
+  }
+}

package/packages/core/src/logger.ts

@@ -0,0 +1,104 @@
+/**
+ * @file logger.ts
+ * @description Immortal.js internal structured logger
+ * Supports log levels, field redaction, and custom transports
+ */
+
+import type { LoggerConfig, LogTransport } from "./types.js";
+
+const LEVEL_RANK = { debug: 0, info: 1, warn: 2, error: 3, silent: 4 } as const;
+type Level = keyof typeof LEVEL_RANK;
+
+class ImmortalLogger implements LogTransport {
+  private level: Level;
+  private redact: Set<string>;
+  private transport?: LogTransport;
+
+  constructor(config: Required<LoggerConfig>) {
+    this.level = config.level;
+    this.redact = new Set(config.redact);
+    this.transport = config.transport;
+  }
+
+  private shouldLog(level: Level): boolean {
+    return LEVEL_RANK[level] >= LEVEL_RANK[this.level];
+  }
+
+  private sanitize(meta?: Record<string, unknown>): Record<string, unknown> | undefined {
+    if (!meta) return undefined;
+    const result: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(meta)) {
+      if (this.redact.has(key)) {
+        result[key] = "[REDACTED]";
+      } else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+        result[key] = this.sanitize(value as Record<string, unknown>);
+      } else {
+        result[key] = value;
+      }
+    }
+    return result;
+  }
+
+  private format(level: Level, message: string, meta?: Record<string, unknown>): string {
+    const ts = new Date().toISOString();
+    const sanitized = this.sanitize(meta);
+    const base = `[${ts}] [IMMORTAL] [${level.toUpperCase()}] ${message}`;
+    return sanitized ? `${base} ${JSON.stringify(sanitized)}` : base;
+  }
+
+  debug(message: string, meta?: Record<string, unknown>): void {
+    if (!this.shouldLog("debug")) return;
+    if (this.transport) {
+      this.transport.debug(message, this.sanitize(meta));
+    } else {
+      console.debug(this.format("debug", message, meta));
+    }
+  }
+
+  info(message: string, meta?: Record<string, unknown>): void {
+    if (!this.shouldLog("info")) return;
+    if (this.transport) {
+      this.transport.info(message, this.sanitize(meta));
+    } else {
+      console.info(this.format("info", message, meta));
+    }
+  }
+
+  warn(message: string, meta?: Record<string, unknown>): void {
+    if (!this.shouldLog("warn")) return;
+    if (this.transport) {
+      this.transport.warn(message, this.sanitize(meta));
+    } else {
+      console.warn(this.format("warn", message, meta));
+    }
+  }
+
+  error(message: string, meta?: Record<string, unknown>): void {
+    if (!this.shouldLog("error")) return;
+    if (this.transport) {
+      this.transport.error(message, this.sanitize(meta));
+    } else {
+      console.error(this.format("error", message, meta));
+    }
+  }
+
+  updateLevel(level: Level): void {
+    this.level = level;
+  }
+}
+
+let _logger: ImmortalLogger | null = null;
+
+export function createLogger(config: Required<LoggerConfig>): ImmortalLogger {
+  _logger = new ImmortalLogger(config);
+  return _logger;
+}
+
+export function getLogger(): ImmortalLogger {
+  if (!_logger) {
+    _logger = new ImmortalLogger({ level: "info", redact: [], transport: undefined as never });
+  }
+  return _logger;
+}
+
+export type { ImmortalLogger };