immortal-js 1.0.0

package/packages/core/src/monitoring/DiagnosticsChannel.ts

@@ -0,0 +1,248 @@
+/**
+ * @file DiagnosticsChannel.ts
+ * @description Node.js diagnostics_channel integration + OpenTelemetry export bridge
+ *
+ * Provides:
+ * 1. Publishing Immortal events to Node.js diagnostics_channel (for APM tool integration)
+ * 2. Prometheus-compatible metrics exposition
+ * 3. OpenTelemetry OTLP metrics export (when configured)
+ */
+
+import diagnostics_channel from "node:diagnostics_channel";
+import type { ImmortalEvent, SystemMetrics } from "../types.js";
+import type { ImmortalEventBus } from "../event-bus.js";
+import { getLogger } from "../logger.js";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// CHANNEL NAMES
+// ─────────────────────────────────────────────────────────────────────────────
+
+export const IMMORTAL_CHANNELS = {
+  ERROR: "immortal:error",
+  CIRCUIT: "immortal:circuit",
+  RETRY: "immortal:retry",
+  BULKHEAD: "immortal:bulkhead",
+  WORKER: "immortal:worker",
+  MEMORY: "immortal:memory",
+  METRICS: "immortal:metrics",
+} as const;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// DIAGNOSTICS CHANNEL CLASS
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class DiagnosticsChannel {
+  private channels: Map<string, ReturnType<typeof diagnostics_channel.channel>>;
+  private readonly bus: ImmortalEventBus;
+  private metricsInterval?: ReturnType<typeof setInterval>;
+  private otlpEndpoint?: string;
+  private prometheusEnabled: boolean;
+  private metricsSnapshot: SystemMetrics | null = null;
+
+  constructor(bus: ImmortalEventBus, options: {
+    otlpEndpoint?: string;
+    enablePrometheus?: boolean;
+  } = {}) {
+    this.bus = bus;
+    if (options.otlpEndpoint !== undefined) this.otlpEndpoint = options.otlpEndpoint;
+    this.prometheusEnabled = options.enablePrometheus ?? true;
+
+    this.channels = new Map(
+      Object.values(IMMORTAL_CHANNELS).map((name) => [
+        name,
+        diagnostics_channel.channel(name),
+      ])
+    );
+
+    this.wireEventBus();
+  }
+
+  /**
+   * Update the current metrics snapshot (called by MetricsCollector)
+   */
+  updateMetrics(metrics: SystemMetrics): void {
+    this.metricsSnapshot = metrics;
+
+    if (this.channels.get(IMMORTAL_CHANNELS.METRICS)?.hasSubscribers) {
+      this.channels.get(IMMORTAL_CHANNELS.METRICS)?.publish(metrics);
+    }
+  }
+
+  /**
+   * Get Prometheus-format metrics text
+   * Compatible with Prometheus scraping and Grafana datasource
+   */
+  getPrometheusMetrics(): string {
+    if (!this.prometheusEnabled || !this.metricsSnapshot) {
+      return "# Immortal.js metrics not available\n";
+    }
+
+    const m = this.metricsSnapshot;
+    const lines: string[] = [
+      "# HELP immortal_event_loop_lag_ms Event loop lag percentiles in milliseconds",
+      "# TYPE immortal_event_loop_lag_ms gauge",
+      `immortal_event_loop_lag_ms{quantile="0.5"} ${m.eventLoopLag.p50}`,
+      `immortal_event_loop_lag_ms{quantile="0.95"} ${m.eventLoopLag.p95}`,
+      `immortal_event_loop_lag_ms{quantile="0.99"} ${m.eventLoopLag.p99}`,
+      `immortal_event_loop_lag_ms{quantile="max"} ${m.eventLoopLag.max}`,
+      "",
+      "# HELP immortal_heap_used_mb Heap used in megabytes",
+      "# TYPE immortal_heap_used_mb gauge",
+      `immortal_heap_used_mb ${m.memory.heapUsedMb}`,
+      "",
+      "# HELP immortal_heap_total_mb Heap total limit in megabytes",
+      "# TYPE immortal_heap_total_mb gauge",
+      `immortal_heap_total_mb ${m.memory.heapTotalMb}`,
+      "",
+      "# HELP immortal_rss_mb Resident Set Size in megabytes",
+      "# TYPE immortal_rss_mb gauge",
+      `immortal_rss_mb ${m.memory.rssMb}`,
+      "",
+      "# HELP immortal_heap_growth_rate_mb_per_min Heap growth rate",
+      "# TYPE immortal_heap_growth_rate_mb_per_min gauge",
+      `immortal_heap_growth_rate_mb_per_min ${m.memory.heapGrowthRate}`,
+      "",
+      "# HELP immortal_cpu_percent CPU utilization percentage",
+      "# TYPE immortal_cpu_percent gauge",
+      `immortal_cpu_percent{type="user"} ${m.cpu.percentUser}`,
+      `immortal_cpu_percent{type="system"} ${m.cpu.percentSystem}`,
+      "",
+      "# HELP immortal_requests_total Total HTTP requests handled",
+      "# TYPE immortal_requests_total counter",
+      `immortal_requests_total ${m.requests.total}`,
+      "",
+      "# HELP immortal_requests_active Currently active requests",
+      "# TYPE immortal_requests_active gauge",
+      `immortal_requests_active ${m.requests.active}`,
+      "",
+      "# HELP immortal_requests_per_second Request rate per second",
+      "# TYPE immortal_requests_per_second gauge",
+      `immortal_requests_per_second ${m.requests.perSecond}`,
+      "",
+      "# HELP immortal_error_rate Request error rate (0-1)",
+      "# TYPE immortal_error_rate gauge",
+      `immortal_error_rate ${m.requests.errorRate}`,
+      "",
+    ];
+
+    // Circuit breaker metrics
+    for (const [name, circuit] of Object.entries(m.circuits)) {
+      const stateValue = circuit.state === "CLOSED" ? 0 : circuit.state === "HALF_OPEN" ? 1 : 2;
+      lines.push(`immortal_circuit_state{circuit="${name}"} ${stateValue}`);
+      lines.push(`immortal_circuit_failure_rate{circuit="${name}"} ${circuit.failureRate}`);
+      lines.push(`immortal_circuit_total_requests{circuit="${name}"} ${circuit.totalRequests}`);
+    }
+
+    if (Object.keys(m.circuits).length > 0) {
+      lines.unshift(
+        "# HELP immortal_circuit_state Circuit state (0=CLOSED, 1=HALF_OPEN, 2=OPEN)",
+        "# TYPE immortal_circuit_state gauge"
+      );
+    }
+
+    // Bulkhead metrics
+    for (const [name, bh] of Object.entries(m.bulkheads)) {
+      lines.push(`immortal_bulkhead_active{pool="${name}"} ${bh.active}`);
+      lines.push(`immortal_bulkhead_queued{pool="${name}"} ${bh.queued}`);
+      lines.push(`immortal_bulkhead_rejected{pool="${name}"} ${bh.rejected}`);
+    }
+
+    return lines.join("\n") + "\n";
+  }
+
+  /**
+   * Subscribe to a specific diagnostics channel (for APM integration)
+   */
+  subscribe(channelName: string, callback: (data: unknown) => void): void {
+    diagnostics_channel.subscribe(channelName, callback);
+  }
+
+  /**
+   * Stop all periodic exports
+   */
+  stop(): void {
+    if (this.metricsInterval) {
+      clearInterval(this.metricsInterval);
+      delete this.metricsInterval;
+    }
+  }
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // PRIVATE
+  // ─────────────────────────────────────────────────────────────────────────
+
+  private wireEventBus(): void {
+    // Bridge Immortal events to diagnostics channels
+    this.bus.onAny((event: ImmortalEvent) => {
+      const channelName = this.getChannelForEvent(event.type);
+      const channel = this.channels.get(channelName);
+
+      if (channel?.hasSubscribers) {
+        channel.publish({ event, timestamp: Date.now() });
+      }
+    });
+  }
+
+  private getChannelForEvent(type: string): string {
+    if (type.startsWith("error:")) return IMMORTAL_CHANNELS.ERROR;
+    if (type.startsWith("circuit:")) return IMMORTAL_CHANNELS.CIRCUIT;
+    if (type.startsWith("retry:")) return IMMORTAL_CHANNELS.RETRY;
+    if (type.startsWith("bulkhead:")) return IMMORTAL_CHANNELS.BULKHEAD;
+    if (type.startsWith("worker:")) return IMMORTAL_CHANNELS.WORKER;
+    if (type.startsWith("memory:")) return IMMORTAL_CHANNELS.MEMORY;
+    return IMMORTAL_CHANNELS.METRICS;
+  }
+
+  /**
+   * Export metrics to OTLP endpoint (OpenTelemetry)
+   * Format: OTLP/HTTP JSON
+   */
+  private async exportToOTLP(metrics: SystemMetrics): Promise<void> {
+    if (!this.otlpEndpoint) return;
+
+    const otlpPayload = this.buildOTLPPayload(metrics);
+
+    try {
+      await fetch(this.otlpEndpoint, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(otlpPayload),
+      });
+    } catch (err) {
+      getLogger().debug("OTLP export failed", { error: (err as Error).message });
+    }
+  }
+
+  private buildOTLPPayload(metrics: SystemMetrics): Record<string, unknown> {
+    // Simplified OTLP/HTTP JSON format
+    return {
+      resourceMetrics: [{
+        resource: {
+          attributes: [
+            { key: "service.name", value: { stringValue: "immortal-js" } },
+            { key: "service.version", value: { stringValue: "1.0.0" } },
+          ],
+        },
+        scopeMetrics: [{
+          metrics: [
+            {
+              name: "immortal.event_loop.lag.p99",
+              unit: "ms",
+              gauge: { dataPoints: [{ asDouble: metrics.eventLoopLag.p99, timeUnixNano: metrics.timestamp * 1e6 }] },
+            },
+            {
+              name: "immortal.memory.heap_used",
+              unit: "MB",
+              gauge: { dataPoints: [{ asDouble: metrics.memory.heapUsedMb, timeUnixNano: metrics.timestamp * 1e6 }] },
+            },
+            {
+              name: "immortal.requests.per_second",
+              unit: "req/s",
+              gauge: { dataPoints: [{ asDouble: metrics.requests.perSecond, timeUnixNano: metrics.timestamp * 1e6 }] },
+            },
+          ],
+        }],
+      }],
+    };
+  }
+}

package/packages/core/src/monitoring/HealthMonitor.ts

@@ -0,0 +1,191 @@
+/**
+ * @file HealthMonitor.ts
+ * @description Real-time system health monitoring
+ *
+ * Monitors:
+ * - Event Loop Delay/Lag (via perf_hooks histogram — most accurate method)
+ * - CPU utilization (via process.cpuUsage)
+ * - Open handles count (via process._getActiveHandles)
+ * - Memory usage trends
+ *
+ * Key insight on event loop lag:
+ * High event loop lag means synchronous CPU work is blocking the loop —
+ * the server literally cannot respond to new requests during that time.
+ * Detecting > threshold is a signal to restart the worker (not just log).
+ */
+
+import { monitorEventLoopDelay } from "node:perf_hooks";
+import type { HealthMonitorConfig } from "../types.js";
+import { DEFAULT_HEALTH_MONITOR } from "../config/defaults.js";
+import { getLogger } from "../logger.js";
+import type { ImmortalEventBus } from "../event-bus.js";
+import type { MetricsCollector } from "./MetricsCollector.js";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// HEALTH MONITOR CLASS
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class HealthMonitor {
+  private readonly config: Required<HealthMonitorConfig>;
+  private readonly bus: ImmortalEventBus;
+  private readonly metrics: MetricsCollector;
+  private interval?: ReturnType<typeof setInterval>;
+  private histogram: ReturnType<typeof monitorEventLoopDelay>;
+  private running = false;
+  private consecutiveCriticalLag = 0;
+  private consecutiveCriticalCpu = 0;
+  private restartRequested = false;
+  private supervisorRef?: { requestGracefulRestart: (id: string, reason?: string) => void };
+
+  constructor(
+    config: HealthMonitorConfig = {},
+    bus: ImmortalEventBus,
+    metrics: MetricsCollector
+  ) {
+    this.config = { ...DEFAULT_HEALTH_MONITOR, ...config };
+    this.bus = bus;
+    this.metrics = metrics;
+
+    // Initialize event loop histogram (20ms resolution = balance of accuracy vs overhead)
+    this.histogram = monitorEventLoopDelay({ resolution: 20 });
+  }
+
+  /**
+   * Wire a supervisor reference for proactive restarts
+   */
+  setSupervisor(supervisor: { requestGracefulRestart: (id: string, reason?: string) => void }): void {
+    this.supervisorRef = supervisor;
+  }
+
+  /**
+   * Start monitoring
+   */
+  start(): void {
+    if (this.running) return;
+
+    this.histogram.enable();
+    this.running = true;
+
+    this.interval = setInterval(() => {
+      this.collect();
+    }, this.config.intervalMs).unref();
+
+    getLogger().info("HealthMonitor started", {
+      intervalMs: this.config.intervalMs,
+      eventLoopLagThreshold: `${this.config.eventLoopLagMs}ms`,
+      criticalLagThreshold: `${this.config.criticalLagMs}ms`,
+    });
+  }
+
+  /**
+   * Stop monitoring
+   */
+  stop(): void {
+    if (!this.running) return;
+
+    this.running = false;
+
+    if (this.interval) {
+      clearInterval(this.interval);
+      delete this.interval;
+    }
+
+    this.histogram.disable();
+    getLogger().info("HealthMonitor stopped");
+  }
+
+  /**
+   * Force an immediate health check (for testing)
+   */
+  checkNow(): ReturnType<MetricsCollector["getSnapshot"]> {
+    this.collect();
+    return this.metrics.getSnapshot();
+  }
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // COLLECTION LOGIC
+  // ─────────────────────────────────────────────────────────────────────────
+
+  private collect(): void {
+    const logger = getLogger();
+
+    // ── Event Loop Lag ────────────────────────────────────────────────────
+    const p50 = this.histogram.percentile(50) / 1e6;   // Convert nanoseconds to ms
+    const p95 = this.histogram.percentile(95) / 1e6;
+    const p99 = this.histogram.percentile(99) / 1e6;
+    const maxLag = this.histogram.max / 1e6;
+
+    this.metrics.updateEventLoopMetrics({ p50, p95, p99, max: maxLag });
+    this.histogram.reset();
+
+    // ── Event Loop Thresholds ─────────────────────────────────────────────
+    if (p99 > this.config.criticalLagMs) {
+      this.consecutiveCriticalLag++;
+
+      this.bus.emit("eventloop:lag-critical", {
+        data: { p50, p95, p99, max: maxLag, consecutive: this.consecutiveCriticalLag },
+      });
+
+      logger.error("Event loop CRITICAL lag detected", {
+        p99: `${p99.toFixed(1)}ms`,
+        threshold: `${this.config.criticalLagMs}ms`,
+        consecutive: this.consecutiveCriticalLag,
+      });
+
+      // After 2 consecutive critical readings, trigger a proactive restart
+      if (this.consecutiveCriticalLag >= 2 && !this.restartRequested) {
+        this.restartRequested = true;
+        logger.error("Requesting graceful restart due to sustained critical event loop lag");
+        this.supervisorRef?.requestGracefulRestart(
+          `worker-${process.pid}`,
+          "critical-event-loop-lag"
+        );
+        // Reset after request
+        setTimeout(() => { this.restartRequested = false; }, 30_000).unref();
+      }
+    } else if (p99 > this.config.eventLoopLagMs) {
+      this.consecutiveCriticalLag = 0;
+
+      this.bus.emit("eventloop:lag-warning", {
+        data: { p50, p95, p99, max: maxLag },
+      });
+
+      logger.warn("Event loop lag elevated", {
+        p99: `${p99.toFixed(1)}ms`,
+        threshold: `${this.config.eventLoopLagMs}ms`,
+      });
+    } else {
+      this.consecutiveCriticalLag = 0;
+    }
+
+    // ── CPU Usage ────────────────────────────────────────────────────────
+    this.metrics.updateCpuMetrics();
+    const snapshot = this.metrics.getSnapshot();
+    const totalCpu = snapshot.cpu.percentUser + snapshot.cpu.percentSystem;
+
+    if (totalCpu > this.config.cpuCriticalPercent) {
+      this.consecutiveCriticalCpu++;
+
+      this.bus.emit("cpu:critical", {
+        data: { totalCpu, user: snapshot.cpu.percentUser, system: snapshot.cpu.percentSystem },
+      });
+
+      if (this.consecutiveCriticalCpu >= 3) {
+        logger.error("Sustained critical CPU usage", {
+          cpu: `${totalCpu.toFixed(1)}%`,
+          consecutive: this.consecutiveCriticalCpu,
+        });
+      }
+    } else if (totalCpu > this.config.cpuWarningPercent) {
+      this.consecutiveCriticalCpu = 0;
+
+      this.bus.emit("cpu:warning", {
+        data: { totalCpu },
+      });
+
+      logger.warn("High CPU usage", { cpu: `${totalCpu.toFixed(1)}%` });
+    } else {
+      this.consecutiveCriticalCpu = 0;
+    }
+  }
+}