immortal-js 1.0.0

package/packages/core/src/monitoring/MemoryLeakGuard.ts

@@ -0,0 +1,340 @@
+/**
+ * @file MemoryLeakGuard.ts
+ * @description Memory leak detection and proactive restart
+ *
+ * Detection strategy (multi-signal, not just heap size):
+ * 1. Linear regression on heap growth across multiple windows
+ * 2. GC frequency trends (GC running more = struggling to reclaim)
+ * 3. Old generation growth rate (new gen is expected to be volatile)
+ * 4. RSS growth (overall process memory, includes native allocations)
+ * 5. WeakRef tracking for specific objects (optional, developer-controlled)
+ *
+ * Avoids false positives by:
+ * - Requiring N consecutive windows of growth (not just one spike)
+ * - Ignoring normal GC-driven sawtooth patterns
+ * - Using absolute thresholds as final override (when memory is genuinely critical)
+ *
+ * Proactive restart:
+ * Restarting before OOM is far safer than letting it crash:
+ * - Controlled: Supervisor orchestrates restart with zero-downtime
+ * - Uncontrolled: OOM kill leaves requests in-flight with no response
+ */
+
+import * as v8 from "node:v8";
+import type { MemoryGuardConfig } from "../types.js";
+import { DEFAULT_MEMORY_GUARD } from "../config/defaults.js";
+import { getLogger } from "../logger.js";
+import type { ImmortalEventBus } from "../event-bus.js";
+import * as os from "node:os";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// TYPES
+// ─────────────────────────────────────────────────────────────────────────────
+
+interface HeapWindow {
+  timestamp: number;
+  heapUsedMb: number;
+  heapTotalMb: number;
+  rssMb: number;
+  externalMb: number;
+}
+
+interface LeakSignal {
+  type: "heap-growth" | "rss-growth" | "gc-pressure" | "absolute-threshold";
+  severity: "warning" | "critical";
+  value: number;
+  threshold: number;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// MEMORY LEAK GUARD CLASS
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class MemoryLeakGuard {
+  private readonly config: Required<MemoryGuardConfig>;
+  private readonly bus: ImmortalEventBus;
+  private interval?: ReturnType<typeof setInterval>;
+  private running = false;
+  private windows: HeapWindow[] = [];
+  private consecutiveGrowthWindows = 0;
+  private lastRestartTime = 0;
+  private readonly minRestartIntervalMs = 5 * 60 * 1000; // Don't restart more than once per 5 minutes
+
+  // Adaptive thresholds based on available system RAM
+  private readonly adaptedWarningMb: number;
+  private readonly adaptedRestartMb: number;
+
+  constructor(config: MemoryGuardConfig = {}, bus: ImmortalEventBus) {
+    this.config = { ...DEFAULT_MEMORY_GUARD, ...config };
+    this.bus = bus;
+
+    // Adaptive thresholds: use configured values or 40%/60% of system RAM
+    if (this.config.adaptiveThresholds) {
+      const systemRamMb = os.totalmem() / 1024 / 1024;
+      this.adaptedWarningMb = Math.min(this.config.warningThresholdMb, systemRamMb * 0.40);
+      this.adaptedRestartMb = Math.min(this.config.restartThresholdMb, systemRamMb * 0.60);
+    } else {
+      this.adaptedWarningMb = this.config.warningThresholdMb;
+      this.adaptedRestartMb = this.config.restartThresholdMb;
+    }
+  }
+
+  /**
+   * Start memory monitoring
+   */
+  start(): void {
+    if (this.running) return;
+    this.running = true;
+
+    this.interval = setInterval(() => {
+      this.sample();
+    }, this.config.checkIntervalMs).unref();
+
+    getLogger().info("MemoryLeakGuard started", {
+      checkIntervalMs: this.config.checkIntervalMs,
+      warningThresholdMb: `${this.adaptedWarningMb.toFixed(0)}MB`,
+      restartThresholdMb: `${this.adaptedRestartMb.toFixed(0)}MB`,
+      adaptive: this.config.adaptiveThresholds,
+    });
+  }
+
+  /**
+   * Stop monitoring
+   */
+  stop(): void {
+    if (!this.running) return;
+    this.running = false;
+
+    if (this.interval) {
+      clearInterval(this.interval);
+      delete this.interval;
+    }
+
+    getLogger().info("MemoryLeakGuard stopped");
+  }
+
+  /**
+   * Track a specific object for GC verification (optional)
+   * If the object is NOT garbage collected within timeoutMs, logs a warning
+   */
+  trackObject(
+    obj: object,
+    name: string,
+    expectedGcMs = 30_000
+  ): void {
+    if (!this.config.enableObjectTracking) return;
+
+    const ref = new WeakRef(obj);
+    const registry = new FinalizationRegistry((objectName: string) => {
+      getLogger().debug(`MemoryLeakGuard: Object '${objectName}' was garbage collected ✓`);
+    });
+
+    registry.register(obj, name);
+
+    setTimeout(() => {
+      if (ref.deref() !== undefined) {
+        getLogger().warn(`MemoryLeakGuard: Object '${name}' was NOT garbage collected after ${expectedGcMs}ms`, {
+          hint: "This may indicate a memory leak (event listener not removed, connection not closed, etc.)",
+        });
+
+        this.bus.emit("memory:leak-detected", {
+          data: { objectName: name, timeoutMs: expectedGcMs },
+        });
+      }
+    }, expectedGcMs).unref();
+  }
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // SAMPLING LOGIC
+  // ─────────────────────────────────────────────────────────────────────────
+
+  private sample(): void {
+    const heapStats = v8.getHeapStatistics();
+    const memUsage = process.memoryUsage();
+
+    const window: HeapWindow = {
+      timestamp: Date.now(),
+      heapUsedMb: heapStats.used_heap_size / 1024 / 1024,
+      heapTotalMb: heapStats.heap_size_limit / 1024 / 1024,
+      rssMb: memUsage.rss / 1024 / 1024,
+      externalMb: memUsage.external / 1024 / 1024,
+    };
+
+    this.windows.push(window);
+
+    // Keep only last 30 windows
+    if (this.windows.length > 30) {
+      this.windows.shift();
+    }
+
+    const signals = this.detectLeakSignals(window);
+    this.processSignals(signals, window);
+  }
+
+  private detectLeakSignals(current: HeapWindow): LeakSignal[] {
+    const signals: LeakSignal[] = [];
+
+    // ── Signal 1: Absolute threshold (immediate response) ──────────────
+    if (current.heapUsedMb > this.adaptedRestartMb) {
+      signals.push({
+        type: "absolute-threshold",
+        severity: "critical",
+        value: current.heapUsedMb,
+        threshold: this.adaptedRestartMb,
+      });
+    } else if (current.heapUsedMb > this.adaptedWarningMb) {
+      signals.push({
+        type: "absolute-threshold",
+        severity: "warning",
+        value: current.heapUsedMb,
+        threshold: this.adaptedWarningMb,
+      });
+    }
+
+    // Need at least 5 windows for trend analysis
+    if (this.windows.length < 5) return signals;
+
+    // ── Signal 2: Heap growth trend (linear regression) ──────────────
+    const heapGrowthRate = this.computeGrowthRate(this.windows.map((w) => w.heapUsedMb));
+    if (heapGrowthRate > 50) {
+      // Growing > 50MB/hour
+      signals.push({
+        type: "heap-growth",
+        severity: "critical",
+        value: heapGrowthRate,
+        threshold: 50,
+      });
+    } else if (heapGrowthRate > 20) {
+      // Growing > 20MB/hour
+      signals.push({
+        type: "heap-growth",
+        severity: "warning",
+        value: heapGrowthRate,
+        threshold: 20,
+      });
+    }
+
+    // ── Signal 3: RSS growth ───────────────────────────────────────────
+    const rssGrowthRate = this.computeGrowthRate(this.windows.map((w) => w.rssMb));
+    if (rssGrowthRate > 100) {
+      signals.push({
+        type: "rss-growth",
+        severity: "critical",
+        value: rssGrowthRate,
+        threshold: 100,
+      });
+    }
+
+    return signals;
+  }
+
+  private processSignals(signals: LeakSignal[], current: HeapWindow): void {
+    const logger = getLogger();
+    const hasCritical = signals.some((s) => s.severity === "critical");
+    const hasWarning = signals.some((s) => s.severity === "warning");
+
+    if (hasCritical) {
+      this.consecutiveGrowthWindows++;
+    } else if (hasWarning) {
+      this.consecutiveGrowthWindows = Math.max(0, this.consecutiveGrowthWindows - 1);
+    } else {
+      this.consecutiveGrowthWindows = 0;
+    }
+
+    // Emit appropriate events
+    if (hasCritical) {
+      this.bus.emit("memory:critical", {
+        data: {
+          heapUsedMb: current.heapUsedMb,
+          rssMb: current.rssMb,
+          signals: signals.filter((s) => s.severity === "critical"),
+          consecutiveWindows: this.consecutiveGrowthWindows,
+        },
+      });
+
+      logger.error("Memory CRITICAL threshold reached", {
+        heapMb: `${current.heapUsedMb.toFixed(1)}MB`,
+        rssMb: `${current.rssMb.toFixed(1)}MB`,
+        signals: signals.map((s) => s.type),
+      });
+    } else if (hasWarning) {
+      this.bus.emit("memory:warning", {
+        data: {
+          heapUsedMb: current.heapUsedMb,
+          signals: signals.filter((s) => s.severity === "warning"),
+        },
+      });
+
+      logger.warn("Memory warning threshold reached", {
+        heapMb: `${current.heapUsedMb.toFixed(1)}MB`,
+      });
+    }
+
+    // Trigger proactive restart after N consecutive critical windows
+    const shouldRestart =
+      hasCritical &&
+      this.consecutiveGrowthWindows >= this.config.growthWindowCount &&
+      (Date.now() - this.lastRestartTime) > this.minRestartIntervalMs;
+
+    if (shouldRestart) {
+      this.triggerProactiveRestart(current, signals);
+    }
+  }
+
+  private triggerProactiveRestart(current: HeapWindow, signals: LeakSignal[]): void {
+    const logger = getLogger();
+    this.lastRestartTime = Date.now();
+
+    logger.error("MemoryLeakGuard: Triggering proactive restart before OOM", {
+      heapMb: `${current.heapUsedMb.toFixed(1)}MB`,
+      rssMb: `${current.rssMb.toFixed(1)}MB`,
+      consecutiveWindows: this.consecutiveGrowthWindows,
+      signals: signals.map((s) => `${s.type}:${s.severity}`),
+    });
+
+    this.bus.emit("memory:proactive-restart", {
+      data: {
+        heapUsedMb: current.heapUsedMb,
+        rssMb: current.rssMb,
+        signals,
+        reason: "memory-leak-guard",
+      },
+    });
+
+    // If no supervisor is listening to the restart event, force exit
+    // so the external process manager (PM2, K8s) handles the restart
+    setTimeout(() => {
+      logger.error("MemoryLeakGuard: No restart handler responded — forcing process exit");
+      process.exit(1);
+    }, 10_000).unref();
+  }
+
+  /**
+   * Simple linear regression to compute growth rate across time series
+   * Returns MB per hour
+   */
+  private computeGrowthRate(values: number[]): number {
+    const n = values.length;
+    if (n < 2) return 0;
+
+    let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
+    for (let i = 0; i < n; i++) {
+      sumX += i;
+      sumY += values[i]!;
+      sumXY += i * values[i]!;
+      sumX2 += i * i;
+    }
+
+    const denominator = n * sumX2 - sumX * sumX;
+    if (denominator === 0) return 0;
+
+    // slope = MB per interval
+    const slope = (n * sumXY - sumX * sumY) / denominator;
+
+    // Convert to MB per hour based on check interval
+    const intervalsPerHour = 3_600_000 / this.config.checkIntervalMs;
+    const ratePerHour = slope * intervalsPerHour;
+
+    return isFinite(ratePerHour) ? ratePerHour : 0;
+  }
+}

package/packages/core/src/monitoring/MetricsCollector.ts

@@ -0,0 +1,219 @@
+/**
+ * @file MetricsCollector.ts
+ * @description Central metrics aggregation hub
+ * Collects, stores, and exposes system-wide metrics as a coherent snapshot
+ */
+
+import * as v8 from "node:v8";
+import * as process from "node:process";
+import type {
+  SystemMetrics,
+  CircuitBreakerStatus,
+  BulkheadStatus,
+  WorkerStatus,
+} from "../types.js";
+
+export interface RequestMetrics {
+  total: number;
+  active: number;
+  errors: number;
+  timestamps: number[]; // for per-second calculation
+}
+
+export interface EventLoopMetrics {
+  p50: number;
+  p95: number;
+  p99: number;
+  max: number;
+}
+
+export interface CpuMetrics {
+  percentUser: number;
+  percentSystem: number;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// METRICS COLLECTOR
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class MetricsCollector {
+  private eventLoopMetrics: EventLoopMetrics = { p50: 0, p95: 0, p99: 0, max: 0 };
+  private cpuMetrics: CpuMetrics = { percentUser: 0, percentSystem: 0 };
+  private requestMetrics: RequestMetrics = {
+    total: 0,
+    active: 0,
+    errors: 0,
+    timestamps: [],
+  };
+  private circuits: Record<string, CircuitBreakerStatus> = {};
+  private bulkheads: Record<string, BulkheadStatus> = {};
+  private workers: WorkerStatus[] = [];
+  private lastCpuUsage = process.cpuUsage();
+  private lastCpuTime = Date.now();
+  private heapGrowthHistory: number[] = [];
+  private lastHeapMb = 0;
+  private gcCount = 0;
+  private lastGcCountTime = Date.now();
+
+  constructor() {
+    this.initGcTracking();
+  }
+
+  /** Update event loop lag metrics (called by HealthMonitor) */
+  updateEventLoopMetrics(metrics: EventLoopMetrics): void {
+    this.eventLoopMetrics = metrics;
+  }
+
+  /** Update CPU metrics */
+  updateCpuMetrics(): void {
+    const now = Date.now();
+    const elapsed = (now - this.lastCpuTime) / 1000;
+    const usage = process.cpuUsage(this.lastCpuUsage);
+
+    this.cpuMetrics = {
+      percentUser: (usage.user / 1000 / elapsed) / 100,
+      percentSystem: (usage.system / 1000 / elapsed) / 100,
+    };
+
+    this.lastCpuUsage = process.cpuUsage();
+    this.lastCpuTime = now;
+  }
+
+  /** Record a request started */
+  recordRequestStart(): void {
+    this.requestMetrics.total++;
+    this.requestMetrics.active++;
+    this.requestMetrics.timestamps.push(Date.now());
+
+    // Keep only last 60 seconds of timestamps
+    const cutoff = Date.now() - 60_000;
+    this.requestMetrics.timestamps = this.requestMetrics.timestamps.filter((t) => t > cutoff);
+  }
+
+  /** Record a request finished */
+  recordRequestEnd(wasError = false): void {
+    this.requestMetrics.active = Math.max(0, this.requestMetrics.active - 1);
+    if (wasError) this.requestMetrics.errors++;
+  }
+
+  /** Update circuit breaker statuses */
+  updateCircuits(circuits: Record<string, CircuitBreakerStatus>): void {
+    this.circuits = circuits;
+  }
+
+  /** Update bulkhead statuses */
+  updateBulkheads(bulkheads: Record<string, BulkheadStatus>): void {
+    this.bulkheads = bulkheads;
+  }
+
+  /** Update worker statuses */
+  updateWorkers(workers: WorkerStatus[]): void {
+    this.workers = workers;
+  }
+
+  /** Track heap growth for MemoryLeakGuard */
+  recordHeapSample(heapMb: number): number {
+    const growthMb = heapMb - this.lastHeapMb;
+    this.lastHeapMb = heapMb;
+
+    this.heapGrowthHistory.push(growthMb);
+    if (this.heapGrowthHistory.length > 20) {
+      this.heapGrowthHistory.shift();
+    }
+
+    return growthMb;
+  }
+
+  /** Get heap growth rate (MB/minute) using linear regression */
+  getHeapGrowthRate(): number {
+    if (this.heapGrowthHistory.length < 3) return 0;
+
+    const n = this.heapGrowthHistory.length;
+    let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
+
+    for (let i = 0; i < n; i++) {
+      sumX += i;
+      sumY += this.heapGrowthHistory[i]!;
+      sumXY += i * this.heapGrowthHistory[i]!;
+      sumX2 += i * i;
+    }
+
+    const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
+    return isFinite(slope) ? slope * 60 : 0; // Convert to per-minute
+  }
+
+  /** Get GC frequency (GCs per minute) */
+  getGcFrequency(): number {
+    const elapsed = (Date.now() - this.lastGcCountTime) / 60_000;
+    return elapsed > 0 ? this.gcCount / elapsed : 0;
+  }
+
+  /**
+   * Get a complete system metrics snapshot
+   */
+  getSnapshot(): SystemMetrics {
+    const heapStats = v8.getHeapStatistics();
+    const heapUsedMb = heapStats.used_heap_size / 1024 / 1024;
+    const heapTotalMb = heapStats.heap_size_limit / 1024 / 1024;
+    const memUsage = process.memoryUsage();
+
+    this.updateCpuMetrics();
+
+    // Per-second request rate (last 10 seconds)
+    const tenSecondsAgo = Date.now() - 10_000;
+    const recentRequests = this.requestMetrics.timestamps.filter((t) => t > tenSecondsAgo);
+    const perSecond = recentRequests.length / 10;
+
+    // Error rate
+    const errorRate = this.requestMetrics.total > 0
+      ? this.requestMetrics.errors / this.requestMetrics.total
+      : 0;
+
+    return {
+      timestamp: Date.now(),
+      eventLoopLag: this.eventLoopMetrics,
+      memory: {
+        heapUsedMb: Math.round(heapUsedMb * 100) / 100,
+        heapTotalMb: Math.round(heapTotalMb * 100) / 100,
+        externalMb: Math.round(memUsage.external / 1024 / 1024 * 100) / 100,
+        rssMb: Math.round(memUsage.rss / 1024 / 1024 * 100) / 100,
+        heapGrowthRate: Math.round(this.getHeapGrowthRate() * 100) / 100,
+        gcFrequency: Math.round(this.getGcFrequency() * 10) / 10,
+      },
+      cpu: {
+        percentUser: Math.round(this.cpuMetrics.percentUser * 100) / 100,
+        percentSystem: Math.round(this.cpuMetrics.percentSystem * 100) / 100,
+      },
+      handles: {
+        active: (process as NodeJS.Process & { _getActiveHandles?: () => unknown[] })._getActiveHandles?.().length ?? 0,
+      },
+      requests: {
+        total: this.requestMetrics.total,
+        active: this.requestMetrics.active,
+        perSecond: Math.round(perSecond * 10) / 10,
+        errorRate: Math.round(errorRate * 1000) / 1000,
+      },
+      circuits: this.circuits,
+      bulkheads: this.bulkheads,
+      workers: this.workers,
+    };
+  }
+
+  private initGcTracking(): void {
+    try {
+      // Use PerformanceObserver for GC events (Node 18+)
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      const { PerformanceObserver } = require("node:perf_hooks") as typeof import("perf_hooks");
+      const observer = new PerformanceObserver((list) => {
+        for (const entry of list.getEntries()) {
+          if (entry.entryType === "gc") {
+            this.gcCount++;
+          }
+        }
+      });
+      observer.observe({ entryTypes: ["gc"] });
+    } catch {
+      // GC tracking not available in this environment
+    }
+  }
+}

package/packages/core/src/monitoring/index.ts

@@ -0,0 +1,10 @@
+/**
+ * @file monitoring/index.ts
+ * @description Monitoring layer public exports
+ */
+
+export { HealthMonitor } from "./HealthMonitor.js";
+export { MemoryLeakGuard } from "./MemoryLeakGuard.js";
+export { MetricsCollector } from "./MetricsCollector.js";
+export type { RequestMetrics, EventLoopMetrics, CpuMetrics } from "./MetricsCollector.js";
+export { DiagnosticsChannel, IMMORTAL_CHANNELS } from "./DiagnosticsChannel.js";